?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
soft-float: add float32_log2() and float64_log2()
[qemu/mini2440/sniper_sniper_test.git] / exec.c
blob302da34f04b8f9fbd899db7d6a9f4a558a665104
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA
19 */
20 #include "config.h"
21 #ifdef _WIN32
22 #define WIN32_LEAN_AND_MEAN
23 #include <windows.h>
24 #else
25 #include <sys/types.h>
26 #include <sys/mman.h>
27 #endif
28 #include <stdlib.h>
29 #include <stdio.h>
30 #include <stdarg.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <unistd.h>
34 #include <inttypes.h>
35
36 #include "cpu.h"
37 #include "exec-all.h"
38 #include "qemu-common.h"
39 #include "tcg.h"
40 #include "hw/hw.h"
41 #include "osdep.h"
42 #include "kvm.h"
43 #if defined(CONFIG_USER_ONLY)
44 #include <qemu.h>
45 #endif
46
47 //#define DEBUG_TB_INVALIDATE
48 //#define DEBUG_FLUSH
49 //#define DEBUG_TLB
50 //#define DEBUG_UNASSIGNED
51
52 /* make various TB consistency checks */
53 //#define DEBUG_TB_CHECK
54 //#define DEBUG_TLB_CHECK
55
56 //#define DEBUG_IOPORT
57 //#define DEBUG_SUBPAGE
58
59 #if !defined(CONFIG_USER_ONLY)
60 /* TB consistency checks only implemented for usermode emulation. */
61 #undef DEBUG_TB_CHECK
62 #endif
63
64 #define SMC_BITMAP_USE_THRESHOLD 10
65
66 #define MMAP_AREA_START 0x00000000
67 #define MMAP_AREA_END 0xa8000000
68
69 #if defined(TARGET_SPARC64)
70 #define TARGET_PHYS_ADDR_SPACE_BITS 41
71 #elif defined(TARGET_SPARC)
72 #define TARGET_PHYS_ADDR_SPACE_BITS 36
73 #elif defined(TARGET_ALPHA)
74 #define TARGET_PHYS_ADDR_SPACE_BITS 42
75 #define TARGET_VIRT_ADDR_SPACE_BITS 42
76 #elif defined(TARGET_PPC64)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 42
78 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
80 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
81 #define TARGET_PHYS_ADDR_SPACE_BITS 36
82 #else
83 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
84 #define TARGET_PHYS_ADDR_SPACE_BITS 32
85 #endif
86
87 static TranslationBlock *tbs;
88 int code_gen_max_blocks;
89 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
90 static int nb_tbs;
91 /* any access to the tbs or the page table must use this lock */
92 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
93
94 #if defined(__arm__) || defined(__sparc_v9__)
95 /* The prologue must be reachable with a direct jump. ARM and Sparc64
96 have limited branch ranges (possibly also PPC) so place it in a
97 section close to code segment. */
98 #define code_gen_section \
99 __attribute__((__section__(".gen_code"))) \
100 __attribute__((aligned (32)))
101 #else
102 #define code_gen_section \
103 __attribute__((aligned (32)))
104 #endif
105
106 uint8_t code_gen_prologue[1024] code_gen_section;
107 static uint8_t *code_gen_buffer;
108 static unsigned long code_gen_buffer_size;
109 /* threshold to flush the translated code buffer */
110 static unsigned long code_gen_buffer_max_size;
111 uint8_t *code_gen_ptr;
112
113 #if !defined(CONFIG_USER_ONLY)
114 ram_addr_t phys_ram_size;
115 int phys_ram_fd;
116 uint8_t *phys_ram_base;
117 uint8_t *phys_ram_dirty;
118 static int in_migration;
119 static ram_addr_t phys_ram_alloc_offset = 0;
120 #endif
121
122 CPUState *first_cpu;
123 /* current CPU in the current thread. It is only valid inside
124 cpu_exec() */
125 CPUState *cpu_single_env;
126 /* 0 = Do not count executed instructions.
127 1 = Precise instruction counting.
128 2 = Adaptive rate instruction counting. */
129 int use_icount = 0;
130 /* Current instruction counter. While executing translated code this may
131 include some instructions that have not yet been executed. */
132 int64_t qemu_icount;
133
134 typedef struct PageDesc {
135 /* list of TBs intersecting this ram page */
136 TranslationBlock *first_tb;
137 /* in order to optimize self modifying code, we count the number
138 of lookups we do to a given page to use a bitmap */
139 unsigned int code_write_count;
140 uint8_t *code_bitmap;
141 #if defined(CONFIG_USER_ONLY)
142 unsigned long flags;
143 #endif
144 } PageDesc;
145
146 typedef struct PhysPageDesc {
147 /* offset in host memory of the page + io_index in the low bits */
148 ram_addr_t phys_offset;
149 ram_addr_t region_offset;
150 } PhysPageDesc;
151
152 #define L2_BITS 10
153 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
154 /* XXX: this is a temporary hack for alpha target.
155 * In the future, this is to be replaced by a multi-level table
156 * to actually be able to handle the complete 64 bits address space.
157 */
158 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
159 #else
160 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
161 #endif
162
163 #define L1_SIZE (1 << L1_BITS)
164 #define L2_SIZE (1 << L2_BITS)
165
166 unsigned long qemu_real_host_page_size;
167 unsigned long qemu_host_page_bits;
168 unsigned long qemu_host_page_size;
169 unsigned long qemu_host_page_mask;
170
171 /* XXX: for system emulation, it could just be an array */
172 static PageDesc *l1_map[L1_SIZE];
173 static PhysPageDesc **l1_phys_map;
174
175 #if !defined(CONFIG_USER_ONLY)
176 static void io_mem_init(void);
177
178 /* io memory support */
179 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
180 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
181 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
182 static int io_mem_nb;
183 static int io_mem_watch;
184 #endif
185
186 /* log support */
187 static const char *logfilename = "/tmp/qemu.log";
188 FILE *logfile;
189 int loglevel;
190 static int log_append = 0;
191
192 /* statistics */
193 static int tlb_flush_count;
194 static int tb_flush_count;
195 static int tb_phys_invalidate_count;
196
197 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
198 typedef struct subpage_t {
199 target_phys_addr_t base;
200 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
201 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
202 void *opaque[TARGET_PAGE_SIZE][2][4];
203 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
204 } subpage_t;
205
206 #ifdef _WIN32
207 static void map_exec(void *addr, long size)
208 {
209 DWORD old_protect;
210 VirtualProtect(addr, size,
211 PAGE_EXECUTE_READWRITE, &old_protect);
212
213 }
214 #else
215 static void map_exec(void *addr, long size)
216 {
217 unsigned long start, end, page_size;
218
219 page_size = getpagesize();
220 start = (unsigned long)addr;
221 start &= ~(page_size - 1);
222
223 end = (unsigned long)addr + size;
224 end += page_size - 1;
225 end &= ~(page_size - 1);
226
227 mprotect((void *)start, end - start,
228 PROT_READ | PROT_WRITE | PROT_EXEC);
229 }
230 #endif
231
232 static void page_init(void)
233 {
234 /* NOTE: we can always suppose that qemu_host_page_size >=
235 TARGET_PAGE_SIZE */
236 #ifdef _WIN32
237 {
238 SYSTEM_INFO system_info;
239
240 GetSystemInfo(&system_info);
241 qemu_real_host_page_size = system_info.dwPageSize;
242 }
243 #else
244 qemu_real_host_page_size = getpagesize();
245 #endif
246 if (qemu_host_page_size == 0)
247 qemu_host_page_size = qemu_real_host_page_size;
248 if (qemu_host_page_size < TARGET_PAGE_SIZE)
249 qemu_host_page_size = TARGET_PAGE_SIZE;
250 qemu_host_page_bits = 0;
251 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
252 qemu_host_page_bits++;
253 qemu_host_page_mask = ~(qemu_host_page_size - 1);
254 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
255 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
256
257 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
258 {
259 long long startaddr, endaddr;
260 FILE *f;
261 int n;
262
263 mmap_lock();
264 last_brk = (unsigned long)sbrk(0);
265 f = fopen("/proc/self/maps", "r");
266 if (f) {
267 do {
268 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
269 if (n == 2) {
270 startaddr = MIN(startaddr,
271 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
272 endaddr = MIN(endaddr,
273 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
274 page_set_flags(startaddr & TARGET_PAGE_MASK,
275 TARGET_PAGE_ALIGN(endaddr),
276 PAGE_RESERVED);
277 }
278 } while (!feof(f));
279 fclose(f);
280 }
281 mmap_unlock();
282 }
283 #endif
284 }
285
286 static inline PageDesc **page_l1_map(target_ulong index)
287 {
288 #if TARGET_LONG_BITS > 32
289 /* Host memory outside guest VM. For 32-bit targets we have already
290 excluded high addresses. */
291 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
292 return NULL;
293 #endif
294 return &l1_map[index >> L2_BITS];
295 }
296
297 static inline PageDesc *page_find_alloc(target_ulong index)
298 {
299 PageDesc **lp, *p;
300 lp = page_l1_map(index);
301 if (!lp)
302 return NULL;
303
304 p = *lp;
305 if (!p) {
306 /* allocate if not found */
307 #if defined(CONFIG_USER_ONLY)
308 size_t len = sizeof(PageDesc) * L2_SIZE;
309 /* Don't use qemu_malloc because it may recurse. */
310 p = mmap(0, len, PROT_READ | PROT_WRITE,
311 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
312 *lp = p;
313 if (h2g_valid(p)) {
314 unsigned long addr = h2g(p);
315 page_set_flags(addr & TARGET_PAGE_MASK,
316 TARGET_PAGE_ALIGN(addr + len),
317 PAGE_RESERVED);
318 }
319 #else
320 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
321 *lp = p;
322 #endif
323 }
324 return p + (index & (L2_SIZE - 1));
325 }
326
327 static inline PageDesc *page_find(target_ulong index)
328 {
329 PageDesc **lp, *p;
330 lp = page_l1_map(index);
331 if (!lp)
332 return NULL;
333
334 p = *lp;
335 if (!p)
336 return 0;
337 return p + (index & (L2_SIZE - 1));
338 }
339
340 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
341 {
342 void **lp, **p;
343 PhysPageDesc *pd;
344
345 p = (void **)l1_phys_map;
346 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
347
348 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
349 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
350 #endif
351 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
352 p = *lp;
353 if (!p) {
354 /* allocate if not found */
355 if (!alloc)
356 return NULL;
357 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
358 memset(p, 0, sizeof(void *) * L1_SIZE);
359 *lp = p;
360 }
361 #endif
362 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
363 pd = *lp;
364 if (!pd) {
365 int i;
366 /* allocate if not found */
367 if (!alloc)
368 return NULL;
369 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
370 *lp = pd;
371 for (i = 0; i < L2_SIZE; i++)
372 pd[i].phys_offset = IO_MEM_UNASSIGNED;
373 }
374 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
375 }
376
377 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
378 {
379 return phys_page_find_alloc(index, 0);
380 }
381
382 #if !defined(CONFIG_USER_ONLY)
383 static void tlb_protect_code(ram_addr_t ram_addr);
384 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
385 target_ulong vaddr);
386 #define mmap_lock() do { } while(0)
387 #define mmap_unlock() do { } while(0)
388 #endif
389
390 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
391
392 #if defined(CONFIG_USER_ONLY)
393 /* Currently it is not recommanded to allocate big chunks of data in
394 user mode. It will change when a dedicated libc will be used */
395 #define USE_STATIC_CODE_GEN_BUFFER
396 #endif
397
398 #ifdef USE_STATIC_CODE_GEN_BUFFER
399 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
400 #endif
401
402 static void code_gen_alloc(unsigned long tb_size)
403 {
404 #ifdef USE_STATIC_CODE_GEN_BUFFER
405 code_gen_buffer = static_code_gen_buffer;
406 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
407 map_exec(code_gen_buffer, code_gen_buffer_size);
408 #else
409 code_gen_buffer_size = tb_size;
410 if (code_gen_buffer_size == 0) {
411 #if defined(CONFIG_USER_ONLY)
412 /* in user mode, phys_ram_size is not meaningful */
413 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
414 #else
415 /* XXX: needs ajustments */
416 code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
417 #endif
418 }
419 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
420 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
421 /* The code gen buffer location may have constraints depending on
422 the host cpu and OS */
423 #if defined(__linux__)
424 {
425 int flags;
426 void *start = NULL;
427
428 flags = MAP_PRIVATE | MAP_ANONYMOUS;
429 #if defined(__x86_64__)
430 flags |= MAP_32BIT;
431 /* Cannot map more than that */
432 if (code_gen_buffer_size > (800 * 1024 * 1024))
433 code_gen_buffer_size = (800 * 1024 * 1024);
434 #elif defined(__sparc_v9__)
435 // Map the buffer below 2G, so we can use direct calls and branches
436 flags |= MAP_FIXED;
437 start = (void *) 0x60000000UL;
438 if (code_gen_buffer_size > (512 * 1024 * 1024))
439 code_gen_buffer_size = (512 * 1024 * 1024);
440 #elif defined(__arm__)
441 /* Map the buffer below 32M, so we can use direct calls and branches */
442 flags |= MAP_FIXED;
443 start = (void *) 0x01000000UL;
444 if (code_gen_buffer_size > 16 * 1024 * 1024)
445 code_gen_buffer_size = 16 * 1024 * 1024;
446 #endif
447 code_gen_buffer = mmap(start, code_gen_buffer_size,
448 PROT_WRITE | PROT_READ | PROT_EXEC,
449 flags, -1, 0);
450 if (code_gen_buffer == MAP_FAILED) {
451 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
452 exit(1);
453 }
454 }
455 #elif defined(__FreeBSD__)
456 {
457 int flags;
458 void *addr = NULL;
459 flags = MAP_PRIVATE | MAP_ANONYMOUS;
460 #if defined(__x86_64__)
461 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
462 * 0x40000000 is free */
463 flags |= MAP_FIXED;
464 addr = (void *)0x40000000;
465 /* Cannot map more than that */
466 if (code_gen_buffer_size > (800 * 1024 * 1024))
467 code_gen_buffer_size = (800 * 1024 * 1024);
468 #endif
469 code_gen_buffer = mmap(addr, code_gen_buffer_size,
470 PROT_WRITE | PROT_READ | PROT_EXEC,
471 flags, -1, 0);
472 if (code_gen_buffer == MAP_FAILED) {
473 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
474 exit(1);
475 }
476 }
477 #else
478 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
479 if (!code_gen_buffer) {
480 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
481 exit(1);
482 }
483 map_exec(code_gen_buffer, code_gen_buffer_size);
484 #endif
485 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
486 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
487 code_gen_buffer_max_size = code_gen_buffer_size -
488 code_gen_max_block_size();
489 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
490 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
491 }
492
493 /* Must be called before using the QEMU cpus. 'tb_size' is the size
494 (in bytes) allocated to the translation buffer. Zero means default
495 size. */
496 void cpu_exec_init_all(unsigned long tb_size)
497 {
498 cpu_gen_init();
499 code_gen_alloc(tb_size);
500 code_gen_ptr = code_gen_buffer;
501 page_init();
502 #if !defined(CONFIG_USER_ONLY)
503 io_mem_init();
504 #endif
505 }
506
507 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
508
509 #define CPU_COMMON_SAVE_VERSION 1
510
511 static void cpu_common_save(QEMUFile *f, void *opaque)
512 {
513 CPUState *env = opaque;
514
515 qemu_put_be32s(f, &env->halted);
516 qemu_put_be32s(f, &env->interrupt_request);
517 }
518
519 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
520 {
521 CPUState *env = opaque;
522
523 if (version_id != CPU_COMMON_SAVE_VERSION)
524 return -EINVAL;
525
526 qemu_get_be32s(f, &env->halted);
527 qemu_get_be32s(f, &env->interrupt_request);
528 tlb_flush(env, 1);
529
530 return 0;
531 }
532 #endif
533
534 void cpu_exec_init(CPUState *env)
535 {
536 CPUState **penv;
537 int cpu_index;
538
539 env->next_cpu = NULL;
540 penv = &first_cpu;
541 cpu_index = 0;
542 while (*penv != NULL) {
543 penv = (CPUState **)&(*penv)->next_cpu;
544 cpu_index++;
545 }
546 env->cpu_index = cpu_index;
547 TAILQ_INIT(&env->breakpoints);
548 TAILQ_INIT(&env->watchpoints);
549 *penv = env;
550 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
551 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
552 cpu_common_save, cpu_common_load, env);
553 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
554 cpu_save, cpu_load, env);
555 #endif
556 }
557
558 static inline void invalidate_page_bitmap(PageDesc *p)
559 {
560 if (p->code_bitmap) {
561 qemu_free(p->code_bitmap);
562 p->code_bitmap = NULL;
563 }
564 p->code_write_count = 0;
565 }
566
567 /* set to NULL all the 'first_tb' fields in all PageDescs */
568 static void page_flush_tb(void)
569 {
570 int i, j;
571 PageDesc *p;
572
573 for(i = 0; i < L1_SIZE; i++) {
574 p = l1_map[i];
575 if (p) {
576 for(j = 0; j < L2_SIZE; j++) {
577 p->first_tb = NULL;
578 invalidate_page_bitmap(p);
579 p++;
580 }
581 }
582 }
583 }
584
585 /* flush all the translation blocks */
586 /* XXX: tb_flush is currently not thread safe */
587 void tb_flush(CPUState *env1)
588 {
589 CPUState *env;
590 #if defined(DEBUG_FLUSH)
591 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
592 (unsigned long)(code_gen_ptr - code_gen_buffer),
593 nb_tbs, nb_tbs > 0 ?
594 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
595 #endif
596 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
597 cpu_abort(env1, "Internal error: code buffer overflow\n");
598
599 nb_tbs = 0;
600
601 for(env = first_cpu; env != NULL; env = env->next_cpu) {
602 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
603 }
604
605 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
606 page_flush_tb();
607
608 code_gen_ptr = code_gen_buffer;
609 /* XXX: flush processor icache at this point if cache flush is
610 expensive */
611 tb_flush_count++;
612 }
613
614 #ifdef DEBUG_TB_CHECK
615
616 static void tb_invalidate_check(target_ulong address)
617 {
618 TranslationBlock *tb;
619 int i;
620 address &= TARGET_PAGE_MASK;
621 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
622 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
623 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
624 address >= tb->pc + tb->size)) {
625 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
626 address, (long)tb->pc, tb->size);
627 }
628 }
629 }
630 }
631
632 /* verify that all the pages have correct rights for code */
633 static void tb_page_check(void)
634 {
635 TranslationBlock *tb;
636 int i, flags1, flags2;
637
638 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
639 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
640 flags1 = page_get_flags(tb->pc);
641 flags2 = page_get_flags(tb->pc + tb->size - 1);
642 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
643 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
644 (long)tb->pc, tb->size, flags1, flags2);
645 }
646 }
647 }
648 }
649
650 static void tb_jmp_check(TranslationBlock *tb)
651 {
652 TranslationBlock *tb1;
653 unsigned int n1;
654
655 /* suppress any remaining jumps to this TB */
656 tb1 = tb->jmp_first;
657 for(;;) {
658 n1 = (long)tb1 & 3;
659 tb1 = (TranslationBlock *)((long)tb1 & ~3);
660 if (n1 == 2)
661 break;
662 tb1 = tb1->jmp_next[n1];
663 }
664 /* check end of list */
665 if (tb1 != tb) {
666 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
667 }
668 }
669
670 #endif
671
672 /* invalidate one TB */
673 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
674 int next_offset)
675 {
676 TranslationBlock *tb1;
677 for(;;) {
678 tb1 = *ptb;
679 if (tb1 == tb) {
680 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
681 break;
682 }
683 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
684 }
685 }
686
687 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
688 {
689 TranslationBlock *tb1;
690 unsigned int n1;
691
692 for(;;) {
693 tb1 = *ptb;
694 n1 = (long)tb1 & 3;
695 tb1 = (TranslationBlock *)((long)tb1 & ~3);
696 if (tb1 == tb) {
697 *ptb = tb1->page_next[n1];
698 break;
699 }
700 ptb = &tb1->page_next[n1];
701 }
702 }
703
704 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
705 {
706 TranslationBlock *tb1, **ptb;
707 unsigned int n1;
708
709 ptb = &tb->jmp_next[n];
710 tb1 = *ptb;
711 if (tb1) {
712 /* find tb(n) in circular list */
713 for(;;) {
714 tb1 = *ptb;
715 n1 = (long)tb1 & 3;
716 tb1 = (TranslationBlock *)((long)tb1 & ~3);
717 if (n1 == n && tb1 == tb)
718 break;
719 if (n1 == 2) {
720 ptb = &tb1->jmp_first;
721 } else {
722 ptb = &tb1->jmp_next[n1];
723 }
724 }
725 /* now we can suppress tb(n) from the list */
726 *ptb = tb->jmp_next[n];
727
728 tb->jmp_next[n] = NULL;
729 }
730 }
731
732 /* reset the jump entry 'n' of a TB so that it is not chained to
733 another TB */
734 static inline void tb_reset_jump(TranslationBlock *tb, int n)
735 {
736 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
737 }
738
739 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
740 {
741 CPUState *env;
742 PageDesc *p;
743 unsigned int h, n1;
744 target_phys_addr_t phys_pc;
745 TranslationBlock *tb1, *tb2;
746
747 /* remove the TB from the hash list */
748 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
749 h = tb_phys_hash_func(phys_pc);
750 tb_remove(&tb_phys_hash[h], tb,
751 offsetof(TranslationBlock, phys_hash_next));
752
753 /* remove the TB from the page list */
754 if (tb->page_addr[0] != page_addr) {
755 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
756 tb_page_remove(&p->first_tb, tb);
757 invalidate_page_bitmap(p);
758 }
759 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
760 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
761 tb_page_remove(&p->first_tb, tb);
762 invalidate_page_bitmap(p);
763 }
764
765 tb_invalidated_flag = 1;
766
767 /* remove the TB from the hash list */
768 h = tb_jmp_cache_hash_func(tb->pc);
769 for(env = first_cpu; env != NULL; env = env->next_cpu) {
770 if (env->tb_jmp_cache[h] == tb)
771 env->tb_jmp_cache[h] = NULL;
772 }
773
774 /* suppress this TB from the two jump lists */
775 tb_jmp_remove(tb, 0);
776 tb_jmp_remove(tb, 1);
777
778 /* suppress any remaining jumps to this TB */
779 tb1 = tb->jmp_first;
780 for(;;) {
781 n1 = (long)tb1 & 3;
782 if (n1 == 2)
783 break;
784 tb1 = (TranslationBlock *)((long)tb1 & ~3);
785 tb2 = tb1->jmp_next[n1];
786 tb_reset_jump(tb1, n1);
787 tb1->jmp_next[n1] = NULL;
788 tb1 = tb2;
789 }
790 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
791
792 tb_phys_invalidate_count++;
793 }
794
795 static inline void set_bits(uint8_t *tab, int start, int len)
796 {
797 int end, mask, end1;
798
799 end = start + len;
800 tab += start >> 3;
801 mask = 0xff << (start & 7);
802 if ((start & ~7) == (end & ~7)) {
803 if (start < end) {
804 mask &= ~(0xff << (end & 7));
805 *tab |= mask;
806 }
807 } else {
808 *tab++ |= mask;
809 start = (start + 8) & ~7;
810 end1 = end & ~7;
811 while (start < end1) {
812 *tab++ = 0xff;
813 start += 8;
814 }
815 if (start < end) {
816 mask = ~(0xff << (end & 7));
817 *tab |= mask;
818 }
819 }
820 }
821
822 static void build_page_bitmap(PageDesc *p)
823 {
824 int n, tb_start, tb_end;
825 TranslationBlock *tb;
826
827 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
828 if (!p->code_bitmap)
829 return;
830
831 tb = p->first_tb;
832 while (tb != NULL) {
833 n = (long)tb & 3;
834 tb = (TranslationBlock *)((long)tb & ~3);
835 /* NOTE: this is subtle as a TB may span two physical pages */
836 if (n == 0) {
837 /* NOTE: tb_end may be after the end of the page, but
838 it is not a problem */
839 tb_start = tb->pc & ~TARGET_PAGE_MASK;
840 tb_end = tb_start + tb->size;
841 if (tb_end > TARGET_PAGE_SIZE)
842 tb_end = TARGET_PAGE_SIZE;
843 } else {
844 tb_start = 0;
845 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
846 }
847 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
848 tb = tb->page_next[n];
849 }
850 }
851
852 TranslationBlock *tb_gen_code(CPUState *env,
853 target_ulong pc, target_ulong cs_base,
854 int flags, int cflags)
855 {
856 TranslationBlock *tb;
857 uint8_t *tc_ptr;
858 target_ulong phys_pc, phys_page2, virt_page2;
859 int code_gen_size;
860
861 phys_pc = get_phys_addr_code(env, pc);
862 tb = tb_alloc(pc);
863 if (!tb) {
864 /* flush must be done */
865 tb_flush(env);
866 /* cannot fail at this point */
867 tb = tb_alloc(pc);
868 /* Don't forget to invalidate previous TB info. */
869 tb_invalidated_flag = 1;
870 }
871 tc_ptr = code_gen_ptr;
872 tb->tc_ptr = tc_ptr;
873 tb->cs_base = cs_base;
874 tb->flags = flags;
875 tb->cflags = cflags;
876 cpu_gen_code(env, tb, &code_gen_size);
877 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
878
879 /* check next page if needed */
880 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
881 phys_page2 = -1;
882 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
883 phys_page2 = get_phys_addr_code(env, virt_page2);
884 }
885 tb_link_phys(tb, phys_pc, phys_page2);
886 return tb;
887 }
888
889 /* invalidate all TBs which intersect with the target physical page
890 starting in range [start;end[. NOTE: start and end must refer to
891 the same physical page. 'is_cpu_write_access' should be true if called
892 from a real cpu write access: the virtual CPU will exit the current
893 TB if code is modified inside this TB. */
894 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
895 int is_cpu_write_access)
896 {
897 TranslationBlock *tb, *tb_next, *saved_tb;
898 CPUState *env = cpu_single_env;
899 target_ulong tb_start, tb_end;
900 PageDesc *p;
901 int n;
902 #ifdef TARGET_HAS_PRECISE_SMC
903 int current_tb_not_found = is_cpu_write_access;
904 TranslationBlock *current_tb = NULL;
905 int current_tb_modified = 0;
906 target_ulong current_pc = 0;
907 target_ulong current_cs_base = 0;
908 int current_flags = 0;
909 #endif /* TARGET_HAS_PRECISE_SMC */
910
911 p = page_find(start >> TARGET_PAGE_BITS);
912 if (!p)
913 return;
914 if (!p->code_bitmap &&
915 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
916 is_cpu_write_access) {
917 /* build code bitmap */
918 build_page_bitmap(p);
919 }
920
921 /* we remove all the TBs in the range [start, end[ */
922 /* XXX: see if in some cases it could be faster to invalidate all the code */
923 tb = p->first_tb;
924 while (tb != NULL) {
925 n = (long)tb & 3;
926 tb = (TranslationBlock *)((long)tb & ~3);
927 tb_next = tb->page_next[n];
928 /* NOTE: this is subtle as a TB may span two physical pages */
929 if (n == 0) {
930 /* NOTE: tb_end may be after the end of the page, but
931 it is not a problem */
932 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
933 tb_end = tb_start + tb->size;
934 } else {
935 tb_start = tb->page_addr[1];
936 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
937 }
938 if (!(tb_end <= start || tb_start >= end)) {
939 #ifdef TARGET_HAS_PRECISE_SMC
940 if (current_tb_not_found) {
941 current_tb_not_found = 0;
942 current_tb = NULL;
943 if (env->mem_io_pc) {
944 /* now we have a real cpu fault */
945 current_tb = tb_find_pc(env->mem_io_pc);
946 }
947 }
948 if (current_tb == tb &&
949 (current_tb->cflags & CF_COUNT_MASK) != 1) {
950 /* If we are modifying the current TB, we must stop
951 its execution. We could be more precise by checking
952 that the modification is after the current PC, but it
953 would require a specialized function to partially
954 restore the CPU state */
955
956 current_tb_modified = 1;
957 cpu_restore_state(current_tb, env,
958 env->mem_io_pc, NULL);
959 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
960 &current_flags);
961 }
962 #endif /* TARGET_HAS_PRECISE_SMC */
963 /* we need to do that to handle the case where a signal
964 occurs while doing tb_phys_invalidate() */
965 saved_tb = NULL;
966 if (env) {
967 saved_tb = env->current_tb;
968 env->current_tb = NULL;
969 }
970 tb_phys_invalidate(tb, -1);
971 if (env) {
972 env->current_tb = saved_tb;
973 if (env->interrupt_request && env->current_tb)
974 cpu_interrupt(env, env->interrupt_request);
975 }
976 }
977 tb = tb_next;
978 }
979 #if !defined(CONFIG_USER_ONLY)
980 /* if no code remaining, no need to continue to use slow writes */
981 if (!p->first_tb) {
982 invalidate_page_bitmap(p);
983 if (is_cpu_write_access) {
984 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
985 }
986 }
987 #endif
988 #ifdef TARGET_HAS_PRECISE_SMC
989 if (current_tb_modified) {
990 /* we generate a block containing just the instruction
991 modifying the memory. It will ensure that it cannot modify
992 itself */
993 env->current_tb = NULL;
994 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
995 cpu_resume_from_signal(env, NULL);
996 }
997 #endif
998 }
999
1000 /* len must be <= 8 and start must be a multiple of len */
1001 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1002 {
1003 PageDesc *p;
1004 int offset, b;
1005 #if 0
1006 if (1) {
1007 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1008 cpu_single_env->mem_io_vaddr, len,
1009 cpu_single_env->eip,
1010 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1011 }
1012 #endif
1013 p = page_find(start >> TARGET_PAGE_BITS);
1014 if (!p)
1015 return;
1016 if (p->code_bitmap) {
1017 offset = start & ~TARGET_PAGE_MASK;
1018 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1019 if (b & ((1 << len) - 1))
1020 goto do_invalidate;
1021 } else {
1022 do_invalidate:
1023 tb_invalidate_phys_page_range(start, start + len, 1);
1024 }
1025 }
1026
1027 #if !defined(CONFIG_SOFTMMU)
1028 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1029 unsigned long pc, void *puc)
1030 {
1031 TranslationBlock *tb;
1032 PageDesc *p;
1033 int n;
1034 #ifdef TARGET_HAS_PRECISE_SMC
1035 TranslationBlock *current_tb = NULL;
1036 CPUState *env = cpu_single_env;
1037 int current_tb_modified = 0;
1038 target_ulong current_pc = 0;
1039 target_ulong current_cs_base = 0;
1040 int current_flags = 0;
1041 #endif
1042
1043 addr &= TARGET_PAGE_MASK;
1044 p = page_find(addr >> TARGET_PAGE_BITS);
1045 if (!p)
1046 return;
1047 tb = p->first_tb;
1048 #ifdef TARGET_HAS_PRECISE_SMC
1049 if (tb && pc != 0) {
1050 current_tb = tb_find_pc(pc);
1051 }
1052 #endif
1053 while (tb != NULL) {
1054 n = (long)tb & 3;
1055 tb = (TranslationBlock *)((long)tb & ~3);
1056 #ifdef TARGET_HAS_PRECISE_SMC
1057 if (current_tb == tb &&
1058 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1059 /* If we are modifying the current TB, we must stop
1060 its execution. We could be more precise by checking
1061 that the modification is after the current PC, but it
1062 would require a specialized function to partially
1063 restore the CPU state */
1064
1065 current_tb_modified = 1;
1066 cpu_restore_state(current_tb, env, pc, puc);
1067 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1068 &current_flags);
1069 }
1070 #endif /* TARGET_HAS_PRECISE_SMC */
1071 tb_phys_invalidate(tb, addr);
1072 tb = tb->page_next[n];
1073 }
1074 p->first_tb = NULL;
1075 #ifdef TARGET_HAS_PRECISE_SMC
1076 if (current_tb_modified) {
1077 /* we generate a block containing just the instruction
1078 modifying the memory. It will ensure that it cannot modify
1079 itself */
1080 env->current_tb = NULL;
1081 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1082 cpu_resume_from_signal(env, puc);
1083 }
1084 #endif
1085 }
1086 #endif
1087
1088 /* add the tb in the target page and protect it if necessary */
1089 static inline void tb_alloc_page(TranslationBlock *tb,
1090 unsigned int n, target_ulong page_addr)
1091 {
1092 PageDesc *p;
1093 TranslationBlock *last_first_tb;
1094
1095 tb->page_addr[n] = page_addr;
1096 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1097 tb->page_next[n] = p->first_tb;
1098 last_first_tb = p->first_tb;
1099 p->first_tb = (TranslationBlock *)((long)tb | n);
1100 invalidate_page_bitmap(p);
1101
1102 #if defined(TARGET_HAS_SMC) || 1
1103
1104 #if defined(CONFIG_USER_ONLY)
1105 if (p->flags & PAGE_WRITE) {
1106 target_ulong addr;
1107 PageDesc *p2;
1108 int prot;
1109
1110 /* force the host page as non writable (writes will have a
1111 page fault + mprotect overhead) */
1112 page_addr &= qemu_host_page_mask;
1113 prot = 0;
1114 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1115 addr += TARGET_PAGE_SIZE) {
1116
1117 p2 = page_find (addr >> TARGET_PAGE_BITS);
1118 if (!p2)
1119 continue;
1120 prot |= p2->flags;
1121 p2->flags &= ~PAGE_WRITE;
1122 page_get_flags(addr);
1123 }
1124 mprotect(g2h(page_addr), qemu_host_page_size,
1125 (prot & PAGE_BITS) & ~PAGE_WRITE);
1126 #ifdef DEBUG_TB_INVALIDATE
1127 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1128 page_addr);
1129 #endif
1130 }
1131 #else
1132 /* if some code is already present, then the pages are already
1133 protected. So we handle the case where only the first TB is
1134 allocated in a physical page */
1135 if (!last_first_tb) {
1136 tlb_protect_code(page_addr);
1137 }
1138 #endif
1139
1140 #endif /* TARGET_HAS_SMC */
1141 }
1142
1143 /* Allocate a new translation block. Flush the translation buffer if
1144 too many translation blocks or too much generated code. */
1145 TranslationBlock *tb_alloc(target_ulong pc)
1146 {
1147 TranslationBlock *tb;
1148
1149 if (nb_tbs >= code_gen_max_blocks ||
1150 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1151 return NULL;
1152 tb = &tbs[nb_tbs++];
1153 tb->pc = pc;
1154 tb->cflags = 0;
1155 return tb;
1156 }
1157
1158 void tb_free(TranslationBlock *tb)
1159 {
1160 /* In practice this is mostly used for single use temporary TB
1161 Ignore the hard cases and just back up if this TB happens to
1162 be the last one generated. */
1163 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1164 code_gen_ptr = tb->tc_ptr;
1165 nb_tbs--;
1166 }
1167 }
1168
1169 /* add a new TB and link it to the physical page tables. phys_page2 is
1170 (-1) to indicate that only one page contains the TB. */
1171 void tb_link_phys(TranslationBlock *tb,
1172 target_ulong phys_pc, target_ulong phys_page2)
1173 {
1174 unsigned int h;
1175 TranslationBlock **ptb;
1176
1177 /* Grab the mmap lock to stop another thread invalidating this TB
1178 before we are done. */
1179 mmap_lock();
1180 /* add in the physical hash table */
1181 h = tb_phys_hash_func(phys_pc);
1182 ptb = &tb_phys_hash[h];
1183 tb->phys_hash_next = *ptb;
1184 *ptb = tb;
1185
1186 /* add in the page list */
1187 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1188 if (phys_page2 != -1)
1189 tb_alloc_page(tb, 1, phys_page2);
1190 else
1191 tb->page_addr[1] = -1;
1192
1193 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1194 tb->jmp_next[0] = NULL;
1195 tb->jmp_next[1] = NULL;
1196
1197 /* init original jump addresses */
1198 if (tb->tb_next_offset[0] != 0xffff)
1199 tb_reset_jump(tb, 0);
1200 if (tb->tb_next_offset[1] != 0xffff)
1201 tb_reset_jump(tb, 1);
1202
1203 #ifdef DEBUG_TB_CHECK
1204 tb_page_check();
1205 #endif
1206 mmap_unlock();
1207 }
1208
1209 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1210 tb[1].tc_ptr. Return NULL if not found */
1211 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1212 {
1213 int m_min, m_max, m;
1214 unsigned long v;
1215 TranslationBlock *tb;
1216
1217 if (nb_tbs <= 0)
1218 return NULL;
1219 if (tc_ptr < (unsigned long)code_gen_buffer ||
1220 tc_ptr >= (unsigned long)code_gen_ptr)
1221 return NULL;
1222 /* binary search (cf Knuth) */
1223 m_min = 0;
1224 m_max = nb_tbs - 1;
1225 while (m_min <= m_max) {
1226 m = (m_min + m_max) >> 1;
1227 tb = &tbs[m];
1228 v = (unsigned long)tb->tc_ptr;
1229 if (v == tc_ptr)
1230 return tb;
1231 else if (tc_ptr < v) {
1232 m_max = m - 1;
1233 } else {
1234 m_min = m + 1;
1235 }
1236 }
1237 return &tbs[m_max];
1238 }
1239
1240 static void tb_reset_jump_recursive(TranslationBlock *tb);
1241
1242 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1243 {
1244 TranslationBlock *tb1, *tb_next, **ptb;
1245 unsigned int n1;
1246
1247 tb1 = tb->jmp_next[n];
1248 if (tb1 != NULL) {
1249 /* find head of list */
1250 for(;;) {
1251 n1 = (long)tb1 & 3;
1252 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1253 if (n1 == 2)
1254 break;
1255 tb1 = tb1->jmp_next[n1];
1256 }
1257 /* we are now sure now that tb jumps to tb1 */
1258 tb_next = tb1;
1259
1260 /* remove tb from the jmp_first list */
1261 ptb = &tb_next->jmp_first;
1262 for(;;) {
1263 tb1 = *ptb;
1264 n1 = (long)tb1 & 3;
1265 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1266 if (n1 == n && tb1 == tb)
1267 break;
1268 ptb = &tb1->jmp_next[n1];
1269 }
1270 *ptb = tb->jmp_next[n];
1271 tb->jmp_next[n] = NULL;
1272
1273 /* suppress the jump to next tb in generated code */
1274 tb_reset_jump(tb, n);
1275
1276 /* suppress jumps in the tb on which we could have jumped */
1277 tb_reset_jump_recursive(tb_next);
1278 }
1279 }
1280
1281 static void tb_reset_jump_recursive(TranslationBlock *tb)
1282 {
1283 tb_reset_jump_recursive2(tb, 0);
1284 tb_reset_jump_recursive2(tb, 1);
1285 }
1286
1287 #if defined(TARGET_HAS_ICE)
1288 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1289 {
1290 target_phys_addr_t addr;
1291 target_ulong pd;
1292 ram_addr_t ram_addr;
1293 PhysPageDesc *p;
1294
1295 addr = cpu_get_phys_page_debug(env, pc);
1296 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1297 if (!p) {
1298 pd = IO_MEM_UNASSIGNED;
1299 } else {
1300 pd = p->phys_offset;
1301 }
1302 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1303 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1304 }
1305 #endif
1306
1307 /* Add a watchpoint. */
1308 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1309 int flags, CPUWatchpoint **watchpoint)
1310 {
1311 target_ulong len_mask = ~(len - 1);
1312 CPUWatchpoint *wp;
1313
1314 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1315 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1316 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1317 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1318 return -EINVAL;
1319 }
1320 wp = qemu_malloc(sizeof(*wp));
1321 if (!wp)
1322 return -ENOMEM;
1323
1324 wp->vaddr = addr;
1325 wp->len_mask = len_mask;
1326 wp->flags = flags;
1327
1328 /* keep all GDB-injected watchpoints in front */
1329 if (flags & BP_GDB)
1330 TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1331 else
1332 TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1333
1334 tlb_flush_page(env, addr);
1335
1336 if (watchpoint)
1337 *watchpoint = wp;
1338 return 0;
1339 }
1340
1341 /* Remove a specific watchpoint. */
1342 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1343 int flags)
1344 {
1345 target_ulong len_mask = ~(len - 1);
1346 CPUWatchpoint *wp;
1347
1348 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1349 if (addr == wp->vaddr && len_mask == wp->len_mask
1350 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1351 cpu_watchpoint_remove_by_ref(env, wp);
1352 return 0;
1353 }
1354 }
1355 return -ENOENT;
1356 }
1357
1358 /* Remove a specific watchpoint by reference. */
1359 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1360 {
1361 TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1362
1363 tlb_flush_page(env, watchpoint->vaddr);
1364
1365 qemu_free(watchpoint);
1366 }
1367
1368 /* Remove all matching watchpoints. */
1369 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1370 {
1371 CPUWatchpoint *wp, *next;
1372
1373 TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1374 if (wp->flags & mask)
1375 cpu_watchpoint_remove_by_ref(env, wp);
1376 }
1377 }
1378
1379 /* Add a breakpoint. */
1380 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1381 CPUBreakpoint **breakpoint)
1382 {
1383 #if defined(TARGET_HAS_ICE)
1384 CPUBreakpoint *bp;
1385
1386 bp = qemu_malloc(sizeof(*bp));
1387 if (!bp)
1388 return -ENOMEM;
1389
1390 bp->pc = pc;
1391 bp->flags = flags;
1392
1393 /* keep all GDB-injected breakpoints in front */
1394 if (flags & BP_GDB)
1395 TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1396 else
1397 TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1398
1399 breakpoint_invalidate(env, pc);
1400
1401 if (breakpoint)
1402 *breakpoint = bp;
1403 return 0;
1404 #else
1405 return -ENOSYS;
1406 #endif
1407 }
1408
1409 /* Remove a specific breakpoint. */
1410 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1411 {
1412 #if defined(TARGET_HAS_ICE)
1413 CPUBreakpoint *bp;
1414
1415 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1416 if (bp->pc == pc && bp->flags == flags) {
1417 cpu_breakpoint_remove_by_ref(env, bp);
1418 return 0;
1419 }
1420 }
1421 return -ENOENT;
1422 #else
1423 return -ENOSYS;
1424 #endif
1425 }
1426
1427 /* Remove a specific breakpoint by reference. */
1428 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1429 {
1430 #if defined(TARGET_HAS_ICE)
1431 TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1432
1433 breakpoint_invalidate(env, breakpoint->pc);
1434
1435 qemu_free(breakpoint);
1436 #endif
1437 }
1438
1439 /* Remove all matching breakpoints. */
1440 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1441 {
1442 #if defined(TARGET_HAS_ICE)
1443 CPUBreakpoint *bp, *next;
1444
1445 TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1446 if (bp->flags & mask)
1447 cpu_breakpoint_remove_by_ref(env, bp);
1448 }
1449 #endif
1450 }
1451
1452 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1453 CPU loop after each instruction */
1454 void cpu_single_step(CPUState *env, int enabled)
1455 {
1456 #if defined(TARGET_HAS_ICE)
1457 if (env->singlestep_enabled != enabled) {
1458 env->singlestep_enabled = enabled;
1459 /* must flush all the translated code to avoid inconsistancies */
1460 /* XXX: only flush what is necessary */
1461 tb_flush(env);
1462 }
1463 #endif
1464 }
1465
1466 /* enable or disable low levels log */
1467 void cpu_set_log(int log_flags)
1468 {
1469 loglevel = log_flags;
1470 if (loglevel && !logfile) {
1471 logfile = fopen(logfilename, log_append ? "a" : "w");
1472 if (!logfile) {
1473 perror(logfilename);
1474 _exit(1);
1475 }
1476 #if !defined(CONFIG_SOFTMMU)
1477 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1478 {
1479 static char logfile_buf[4096];
1480 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1481 }
1482 #else
1483 setvbuf(logfile, NULL, _IOLBF, 0);
1484 #endif
1485 log_append = 1;
1486 }
1487 if (!loglevel && logfile) {
1488 fclose(logfile);
1489 logfile = NULL;
1490 }
1491 }
1492
1493 void cpu_set_log_filename(const char *filename)
1494 {
1495 logfilename = strdup(filename);
1496 if (logfile) {
1497 fclose(logfile);
1498 logfile = NULL;
1499 }
1500 cpu_set_log(loglevel);
1501 }
1502
1503 /* mask must never be zero, except for A20 change call */
1504 void cpu_interrupt(CPUState *env, int mask)
1505 {
1506 #if !defined(USE_NPTL)
1507 TranslationBlock *tb;
1508 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1509 #endif
1510 int old_mask;
1511
1512 old_mask = env->interrupt_request;
1513 /* FIXME: This is probably not threadsafe. A different thread could
1514 be in the middle of a read-modify-write operation. */
1515 env->interrupt_request |= mask;
1516 #if defined(USE_NPTL)
1517 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1518 problem and hope the cpu will stop of its own accord. For userspace
1519 emulation this often isn't actually as bad as it sounds. Often
1520 signals are used primarily to interrupt blocking syscalls. */
1521 #else
1522 if (use_icount) {
1523 env->icount_decr.u16.high = 0xffff;
1524 #ifndef CONFIG_USER_ONLY
1525 /* CPU_INTERRUPT_EXIT isn't a real interrupt. It just means
1526 an async event happened and we need to process it. */
1527 if (!can_do_io(env)
1528 && (mask & ~(old_mask | CPU_INTERRUPT_EXIT)) != 0) {
1529 cpu_abort(env, "Raised interrupt while not in I/O function");
1530 }
1531 #endif
1532 } else {
1533 tb = env->current_tb;
1534 /* if the cpu is currently executing code, we must unlink it and
1535 all the potentially executing TB */
1536 if (tb && !testandset(&interrupt_lock)) {
1537 env->current_tb = NULL;
1538 tb_reset_jump_recursive(tb);
1539 resetlock(&interrupt_lock);
1540 }
1541 }
1542 #endif
1543 }
1544
1545 void cpu_reset_interrupt(CPUState *env, int mask)
1546 {
1547 env->interrupt_request &= ~mask;
1548 }
1549
1550 const CPULogItem cpu_log_items[] = {
1551 { CPU_LOG_TB_OUT_ASM, "out_asm",
1552 "show generated host assembly code for each compiled TB" },
1553 { CPU_LOG_TB_IN_ASM, "in_asm",
1554 "show target assembly code for each compiled TB" },
1555 { CPU_LOG_TB_OP, "op",
1556 "show micro ops for each compiled TB" },
1557 { CPU_LOG_TB_OP_OPT, "op_opt",
1558 "show micro ops "
1559 #ifdef TARGET_I386
1560 "before eflags optimization and "
1561 #endif
1562 "after liveness analysis" },
1563 { CPU_LOG_INT, "int",
1564 "show interrupts/exceptions in short format" },
1565 { CPU_LOG_EXEC, "exec",
1566 "show trace before each executed TB (lots of logs)" },
1567 { CPU_LOG_TB_CPU, "cpu",
1568 "show CPU state before block translation" },
1569 #ifdef TARGET_I386
1570 { CPU_LOG_PCALL, "pcall",
1571 "show protected mode far calls/returns/exceptions" },
1572 { CPU_LOG_RESET, "cpu_reset",
1573 "show CPU state before CPU resets" },
1574 #endif
1575 #ifdef DEBUG_IOPORT
1576 { CPU_LOG_IOPORT, "ioport",
1577 "show all i/o ports accesses" },
1578 #endif
1579 { 0, NULL, NULL },
1580 };
1581
1582 static int cmp1(const char *s1, int n, const char *s2)
1583 {
1584 if (strlen(s2) != n)
1585 return 0;
1586 return memcmp(s1, s2, n) == 0;
1587 }
1588
1589 /* takes a comma separated list of log masks. Return 0 if error. */
1590 int cpu_str_to_log_mask(const char *str)
1591 {
1592 const CPULogItem *item;
1593 int mask;
1594 const char *p, *p1;
1595
1596 p = str;
1597 mask = 0;
1598 for(;;) {
1599 p1 = strchr(p, ',');
1600 if (!p1)
1601 p1 = p + strlen(p);
1602 if(cmp1(p,p1-p,"all")) {
1603 for(item = cpu_log_items; item->mask != 0; item++) {
1604 mask |= item->mask;
1605 }
1606 } else {
1607 for(item = cpu_log_items; item->mask != 0; item++) {
1608 if (cmp1(p, p1 - p, item->name))
1609 goto found;
1610 }
1611 return 0;
1612 }
1613 found:
1614 mask |= item->mask;
1615 if (*p1 != ',')
1616 break;
1617 p = p1 + 1;
1618 }
1619 return mask;
1620 }
1621
1622 void cpu_abort(CPUState *env, const char *fmt, ...)
1623 {
1624 va_list ap;
1625 va_list ap2;
1626
1627 va_start(ap, fmt);
1628 va_copy(ap2, ap);
1629 fprintf(stderr, "qemu: fatal: ");
1630 vfprintf(stderr, fmt, ap);
1631 fprintf(stderr, "\n");
1632 #ifdef TARGET_I386
1633 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1634 #else
1635 cpu_dump_state(env, stderr, fprintf, 0);
1636 #endif
1637 if (qemu_log_enabled()) {
1638 qemu_log("qemu: fatal: ");
1639 qemu_log_vprintf(fmt, ap2);
1640 qemu_log("\n");
1641 #ifdef TARGET_I386
1642 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1643 #else
1644 log_cpu_state(env, 0);
1645 #endif
1646 qemu_log_flush();
1647 qemu_log_close();
1648 }
1649 va_end(ap2);
1650 va_end(ap);
1651 abort();
1652 }
1653
1654 CPUState *cpu_copy(CPUState *env)
1655 {
1656 CPUState *new_env = cpu_init(env->cpu_model_str);
1657 CPUState *next_cpu = new_env->next_cpu;
1658 int cpu_index = new_env->cpu_index;
1659 #if defined(TARGET_HAS_ICE)
1660 CPUBreakpoint *bp;
1661 CPUWatchpoint *wp;
1662 #endif
1663
1664 memcpy(new_env, env, sizeof(CPUState));
1665
1666 /* Preserve chaining and index. */
1667 new_env->next_cpu = next_cpu;
1668 new_env->cpu_index = cpu_index;
1669
1670 /* Clone all break/watchpoints.
1671 Note: Once we support ptrace with hw-debug register access, make sure
1672 BP_CPU break/watchpoints are handled correctly on clone. */
1673 TAILQ_INIT(&env->breakpoints);
1674 TAILQ_INIT(&env->watchpoints);
1675 #if defined(TARGET_HAS_ICE)
1676 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1677 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1678 }
1679 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1680 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1681 wp->flags, NULL);
1682 }
1683 #endif
1684
1685 return new_env;
1686 }
1687
1688 #if !defined(CONFIG_USER_ONLY)
1689
1690 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1691 {
1692 unsigned int i;
1693
1694 /* Discard jump cache entries for any tb which might potentially
1695 overlap the flushed page. */
1696 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1697 memset (&env->tb_jmp_cache[i], 0,
1698 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1699
1700 i = tb_jmp_cache_hash_page(addr);
1701 memset (&env->tb_jmp_cache[i], 0,
1702 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1703 }
1704
1705 /* NOTE: if flush_global is true, also flush global entries (not
1706 implemented yet) */
1707 void tlb_flush(CPUState *env, int flush_global)
1708 {
1709 int i;
1710
1711 #if defined(DEBUG_TLB)
1712 printf("tlb_flush:\n");
1713 #endif
1714 /* must reset current TB so that interrupts cannot modify the
1715 links while we are modifying them */
1716 env->current_tb = NULL;
1717
1718 for(i = 0; i < CPU_TLB_SIZE; i++) {
1719 env->tlb_table[0][i].addr_read = -1;
1720 env->tlb_table[0][i].addr_write = -1;
1721 env->tlb_table[0][i].addr_code = -1;
1722 env->tlb_table[1][i].addr_read = -1;
1723 env->tlb_table[1][i].addr_write = -1;
1724 env->tlb_table[1][i].addr_code = -1;
1725 #if (NB_MMU_MODES >= 3)
1726 env->tlb_table[2][i].addr_read = -1;
1727 env->tlb_table[2][i].addr_write = -1;
1728 env->tlb_table[2][i].addr_code = -1;
1729 #if (NB_MMU_MODES == 4)
1730 env->tlb_table[3][i].addr_read = -1;
1731 env->tlb_table[3][i].addr_write = -1;
1732 env->tlb_table[3][i].addr_code = -1;
1733 #endif
1734 #endif
1735 }
1736
1737 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1738
1739 #ifdef USE_KQEMU
1740 if (env->kqemu_enabled) {
1741 kqemu_flush(env, flush_global);
1742 }
1743 #endif
1744 tlb_flush_count++;
1745 }
1746
1747 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1748 {
1749 if (addr == (tlb_entry->addr_read &
1750 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1751 addr == (tlb_entry->addr_write &
1752 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1753 addr == (tlb_entry->addr_code &
1754 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1755 tlb_entry->addr_read = -1;
1756 tlb_entry->addr_write = -1;
1757 tlb_entry->addr_code = -1;
1758 }
1759 }
1760
1761 void tlb_flush_page(CPUState *env, target_ulong addr)
1762 {
1763 int i;
1764
1765 #if defined(DEBUG_TLB)
1766 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1767 #endif
1768 /* must reset current TB so that interrupts cannot modify the
1769 links while we are modifying them */
1770 env->current_tb = NULL;
1771
1772 addr &= TARGET_PAGE_MASK;
1773 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1774 tlb_flush_entry(&env->tlb_table[0][i], addr);
1775 tlb_flush_entry(&env->tlb_table[1][i], addr);
1776 #if (NB_MMU_MODES >= 3)
1777 tlb_flush_entry(&env->tlb_table[2][i], addr);
1778 #if (NB_MMU_MODES == 4)
1779 tlb_flush_entry(&env->tlb_table[3][i], addr);
1780 #endif
1781 #endif
1782
1783 tlb_flush_jmp_cache(env, addr);
1784
1785 #ifdef USE_KQEMU
1786 if (env->kqemu_enabled) {
1787 kqemu_flush_page(env, addr);
1788 }
1789 #endif
1790 }
1791
1792 /* update the TLBs so that writes to code in the virtual page 'addr'
1793 can be detected */
1794 static void tlb_protect_code(ram_addr_t ram_addr)
1795 {
1796 cpu_physical_memory_reset_dirty(ram_addr,
1797 ram_addr + TARGET_PAGE_SIZE,
1798 CODE_DIRTY_FLAG);
1799 }
1800
1801 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1802 tested for self modifying code */
1803 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1804 target_ulong vaddr)
1805 {
1806 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1807 }
1808
1809 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1810 unsigned long start, unsigned long length)
1811 {
1812 unsigned long addr;
1813 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1814 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1815 if ((addr - start) < length) {
1816 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1817 }
1818 }
1819 }
1820
1821 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1822 int dirty_flags)
1823 {
1824 CPUState *env;
1825 unsigned long length, start1;
1826 int i, mask, len;
1827 uint8_t *p;
1828
1829 start &= TARGET_PAGE_MASK;
1830 end = TARGET_PAGE_ALIGN(end);
1831
1832 length = end - start;
1833 if (length == 0)
1834 return;
1835 len = length >> TARGET_PAGE_BITS;
1836 #ifdef USE_KQEMU
1837 /* XXX: should not depend on cpu context */
1838 env = first_cpu;
1839 if (env->kqemu_enabled) {
1840 ram_addr_t addr;
1841 addr = start;
1842 for(i = 0; i < len; i++) {
1843 kqemu_set_notdirty(env, addr);
1844 addr += TARGET_PAGE_SIZE;
1845 }
1846 }
1847 #endif
1848 mask = ~dirty_flags;
1849 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1850 for(i = 0; i < len; i++)
1851 p[i] &= mask;
1852
1853 /* we modify the TLB cache so that the dirty bit will be set again
1854 when accessing the range */
1855 start1 = start + (unsigned long)phys_ram_base;
1856 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1857 for(i = 0; i < CPU_TLB_SIZE; i++)
1858 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1859 for(i = 0; i < CPU_TLB_SIZE; i++)
1860 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1861 #if (NB_MMU_MODES >= 3)
1862 for(i = 0; i < CPU_TLB_SIZE; i++)
1863 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1864 #if (NB_MMU_MODES == 4)
1865 for(i = 0; i < CPU_TLB_SIZE; i++)
1866 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1867 #endif
1868 #endif
1869 }
1870 }
1871
1872 int cpu_physical_memory_set_dirty_tracking(int enable)
1873 {
1874 in_migration = enable;
1875 return 0;
1876 }
1877
1878 int cpu_physical_memory_get_dirty_tracking(void)
1879 {
1880 return in_migration;
1881 }
1882
1883 void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
1884 {
1885 if (kvm_enabled())
1886 kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1887 }
1888
1889 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1890 {
1891 ram_addr_t ram_addr;
1892
1893 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1894 ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1895 tlb_entry->addend - (unsigned long)phys_ram_base;
1896 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1897 tlb_entry->addr_write |= TLB_NOTDIRTY;
1898 }
1899 }
1900 }
1901
1902 /* update the TLB according to the current state of the dirty bits */
1903 void cpu_tlb_update_dirty(CPUState *env)
1904 {
1905 int i;
1906 for(i = 0; i < CPU_TLB_SIZE; i++)
1907 tlb_update_dirty(&env->tlb_table[0][i]);
1908 for(i = 0; i < CPU_TLB_SIZE; i++)
1909 tlb_update_dirty(&env->tlb_table[1][i]);
1910 #if (NB_MMU_MODES >= 3)
1911 for(i = 0; i < CPU_TLB_SIZE; i++)
1912 tlb_update_dirty(&env->tlb_table[2][i]);
1913 #if (NB_MMU_MODES == 4)
1914 for(i = 0; i < CPU_TLB_SIZE; i++)
1915 tlb_update_dirty(&env->tlb_table[3][i]);
1916 #endif
1917 #endif
1918 }
1919
1920 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1921 {
1922 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1923 tlb_entry->addr_write = vaddr;
1924 }
1925
1926 /* update the TLB corresponding to virtual page vaddr
1927 so that it is no longer dirty */
1928 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1929 {
1930 int i;
1931
1932 vaddr &= TARGET_PAGE_MASK;
1933 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1934 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1935 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1936 #if (NB_MMU_MODES >= 3)
1937 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1938 #if (NB_MMU_MODES == 4)
1939 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1940 #endif
1941 #endif
1942 }
1943
1944 /* add a new TLB entry. At most one entry for a given virtual address
1945 is permitted. Return 0 if OK or 2 if the page could not be mapped
1946 (can only happen in non SOFTMMU mode for I/O pages or pages
1947 conflicting with the host address space). */
1948 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1949 target_phys_addr_t paddr, int prot,
1950 int mmu_idx, int is_softmmu)
1951 {
1952 PhysPageDesc *p;
1953 unsigned long pd;
1954 unsigned int index;
1955 target_ulong address;
1956 target_ulong code_address;
1957 target_phys_addr_t addend;
1958 int ret;
1959 CPUTLBEntry *te;
1960 CPUWatchpoint *wp;
1961 target_phys_addr_t iotlb;
1962
1963 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1964 if (!p) {
1965 pd = IO_MEM_UNASSIGNED;
1966 } else {
1967 pd = p->phys_offset;
1968 }
1969 #if defined(DEBUG_TLB)
1970 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1971 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1972 #endif
1973
1974 ret = 0;
1975 address = vaddr;
1976 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1977 /* IO memory case (romd handled later) */
1978 address |= TLB_MMIO;
1979 }
1980 addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
1981 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
1982 /* Normal RAM. */
1983 iotlb = pd & TARGET_PAGE_MASK;
1984 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
1985 iotlb |= IO_MEM_NOTDIRTY;
1986 else
1987 iotlb |= IO_MEM_ROM;
1988 } else {
1989 /* IO handlers are currently passed a phsical address.
1990 It would be nice to pass an offset from the base address
1991 of that region. This would avoid having to special case RAM,
1992 and avoid full address decoding in every device.
1993 We can't use the high bits of pd for this because
1994 IO_MEM_ROMD uses these as a ram address. */
1995 iotlb = (pd & ~TARGET_PAGE_MASK);
1996 if (p) {
1997 iotlb += p->region_offset;
1998 } else {
1999 iotlb += paddr;
2000 }
2001 }
2002
2003 code_address = address;
2004 /* Make accesses to pages with watchpoints go via the
2005 watchpoint trap routines. */
2006 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2007 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2008 iotlb = io_mem_watch + paddr;
2009 /* TODO: The memory case can be optimized by not trapping
2010 reads of pages with a write breakpoint. */
2011 address |= TLB_MMIO;
2012 }
2013 }
2014
2015 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2016 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2017 te = &env->tlb_table[mmu_idx][index];
2018 te->addend = addend - vaddr;
2019 if (prot & PAGE_READ) {
2020 te->addr_read = address;
2021 } else {
2022 te->addr_read = -1;
2023 }
2024
2025 if (prot & PAGE_EXEC) {
2026 te->addr_code = code_address;
2027 } else {
2028 te->addr_code = -1;
2029 }
2030 if (prot & PAGE_WRITE) {
2031 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2032 (pd & IO_MEM_ROMD)) {
2033 /* Write access calls the I/O callback. */
2034 te->addr_write = address | TLB_MMIO;
2035 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2036 !cpu_physical_memory_is_dirty(pd)) {
2037 te->addr_write = address | TLB_NOTDIRTY;
2038 } else {
2039 te->addr_write = address;
2040 }
2041 } else {
2042 te->addr_write = -1;
2043 }
2044 return ret;
2045 }
2046
2047 #else
2048
2049 void tlb_flush(CPUState *env, int flush_global)
2050 {
2051 }
2052
2053 void tlb_flush_page(CPUState *env, target_ulong addr)
2054 {
2055 }
2056
2057 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2058 target_phys_addr_t paddr, int prot,
2059 int mmu_idx, int is_softmmu)
2060 {
2061 return 0;
2062 }
2063
2064 /* dump memory mappings */
2065 void page_dump(FILE *f)
2066 {
2067 unsigned long start, end;
2068 int i, j, prot, prot1;
2069 PageDesc *p;
2070
2071 fprintf(f, "%-8s %-8s %-8s %s\n",
2072 "start", "end", "size", "prot");
2073 start = -1;
2074 end = -1;
2075 prot = 0;
2076 for(i = 0; i <= L1_SIZE; i++) {
2077 if (i < L1_SIZE)
2078 p = l1_map[i];
2079 else
2080 p = NULL;
2081 for(j = 0;j < L2_SIZE; j++) {
2082 if (!p)
2083 prot1 = 0;
2084 else
2085 prot1 = p[j].flags;
2086 if (prot1 != prot) {
2087 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2088 if (start != -1) {
2089 fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2090 start, end, end - start,
2091 prot & PAGE_READ ? 'r' : '-',
2092 prot & PAGE_WRITE ? 'w' : '-',
2093 prot & PAGE_EXEC ? 'x' : '-');
2094 }
2095 if (prot1 != 0)
2096 start = end;
2097 else
2098 start = -1;
2099 prot = prot1;
2100 }
2101 if (!p)
2102 break;
2103 }
2104 }
2105 }
2106
2107 int page_get_flags(target_ulong address)
2108 {
2109 PageDesc *p;
2110
2111 p = page_find(address >> TARGET_PAGE_BITS);
2112 if (!p)
2113 return 0;
2114 return p->flags;
2115 }
2116
2117 /* modify the flags of a page and invalidate the code if
2118 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2119 depending on PAGE_WRITE */
2120 void page_set_flags(target_ulong start, target_ulong end, int flags)
2121 {
2122 PageDesc *p;
2123 target_ulong addr;
2124
2125 /* mmap_lock should already be held. */
2126 start = start & TARGET_PAGE_MASK;
2127 end = TARGET_PAGE_ALIGN(end);
2128 if (flags & PAGE_WRITE)
2129 flags |= PAGE_WRITE_ORG;
2130 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2131 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2132 /* We may be called for host regions that are outside guest
2133 address space. */
2134 if (!p)
2135 return;
2136 /* if the write protection is set, then we invalidate the code
2137 inside */
2138 if (!(p->flags & PAGE_WRITE) &&
2139 (flags & PAGE_WRITE) &&
2140 p->first_tb) {
2141 tb_invalidate_phys_page(addr, 0, NULL);
2142 }
2143 p->flags = flags;
2144 }
2145 }
2146
2147 int page_check_range(target_ulong start, target_ulong len, int flags)
2148 {
2149 PageDesc *p;
2150 target_ulong end;
2151 target_ulong addr;
2152
2153 if (start + len < start)
2154 /* we've wrapped around */
2155 return -1;
2156
2157 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2158 start = start & TARGET_PAGE_MASK;
2159
2160 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2161 p = page_find(addr >> TARGET_PAGE_BITS);
2162 if( !p )
2163 return -1;
2164 if( !(p->flags & PAGE_VALID) )
2165 return -1;
2166
2167 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2168 return -1;
2169 if (flags & PAGE_WRITE) {
2170 if (!(p->flags & PAGE_WRITE_ORG))
2171 return -1;
2172 /* unprotect the page if it was put read-only because it
2173 contains translated code */
2174 if (!(p->flags & PAGE_WRITE)) {
2175 if (!page_unprotect(addr, 0, NULL))
2176 return -1;
2177 }
2178 return 0;
2179 }
2180 }
2181 return 0;
2182 }
2183
2184 /* called from signal handler: invalidate the code and unprotect the
2185 page. Return TRUE if the fault was succesfully handled. */
2186 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2187 {
2188 unsigned int page_index, prot, pindex;
2189 PageDesc *p, *p1;
2190 target_ulong host_start, host_end, addr;
2191
2192 /* Technically this isn't safe inside a signal handler. However we
2193 know this only ever happens in a synchronous SEGV handler, so in
2194 practice it seems to be ok. */
2195 mmap_lock();
2196
2197 host_start = address & qemu_host_page_mask;
2198 page_index = host_start >> TARGET_PAGE_BITS;
2199 p1 = page_find(page_index);
2200 if (!p1) {
2201 mmap_unlock();
2202 return 0;
2203 }
2204 host_end = host_start + qemu_host_page_size;
2205 p = p1;
2206 prot = 0;
2207 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2208 prot |= p->flags;
2209 p++;
2210 }
2211 /* if the page was really writable, then we change its
2212 protection back to writable */
2213 if (prot & PAGE_WRITE_ORG) {
2214 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2215 if (!(p1[pindex].flags & PAGE_WRITE)) {
2216 mprotect((void *)g2h(host_start), qemu_host_page_size,
2217 (prot & PAGE_BITS) | PAGE_WRITE);
2218 p1[pindex].flags |= PAGE_WRITE;
2219 /* and since the content will be modified, we must invalidate
2220 the corresponding translated code. */
2221 tb_invalidate_phys_page(address, pc, puc);
2222 #ifdef DEBUG_TB_CHECK
2223 tb_invalidate_check(address);
2224 #endif
2225 mmap_unlock();
2226 return 1;
2227 }
2228 }
2229 mmap_unlock();
2230 return 0;
2231 }
2232
2233 static inline void tlb_set_dirty(CPUState *env,
2234 unsigned long addr, target_ulong vaddr)
2235 {
2236 }
2237 #endif /* defined(CONFIG_USER_ONLY) */
2238
2239 #if !defined(CONFIG_USER_ONLY)
2240
2241 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2242 ram_addr_t memory, ram_addr_t region_offset);
2243 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2244 ram_addr_t orig_memory, ram_addr_t region_offset);
2245 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2246 need_subpage) \
2247 do { \
2248 if (addr > start_addr) \
2249 start_addr2 = 0; \
2250 else { \
2251 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2252 if (start_addr2 > 0) \
2253 need_subpage = 1; \
2254 } \
2255 \
2256 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2257 end_addr2 = TARGET_PAGE_SIZE - 1; \
2258 else { \
2259 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2260 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2261 need_subpage = 1; \
2262 } \
2263 } while (0)
2264
2265 /* register physical memory. 'size' must be a multiple of the target
2266 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2267 io memory page. The address used when calling the IO function is
2268 the offset from the start of the region, plus region_offset. Both
2269 start_region and regon_offset are rounded down to a page boundary
2270 before calculating this offset. This should not be a problem unless
2271 the low bits of start_addr and region_offset differ. */
2272 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2273 ram_addr_t size,
2274 ram_addr_t phys_offset,
2275 ram_addr_t region_offset)
2276 {
2277 target_phys_addr_t addr, end_addr;
2278 PhysPageDesc *p;
2279 CPUState *env;
2280 ram_addr_t orig_size = size;
2281 void *subpage;
2282
2283 #ifdef USE_KQEMU
2284 /* XXX: should not depend on cpu context */
2285 env = first_cpu;
2286 if (env->kqemu_enabled) {
2287 kqemu_set_phys_mem(start_addr, size, phys_offset);
2288 }
2289 #endif
2290 if (kvm_enabled())
2291 kvm_set_phys_mem(start_addr, size, phys_offset);
2292
2293 region_offset &= TARGET_PAGE_MASK;
2294 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2295 end_addr = start_addr + (target_phys_addr_t)size;
2296 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2297 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2298 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2299 ram_addr_t orig_memory = p->phys_offset;
2300 target_phys_addr_t start_addr2, end_addr2;
2301 int need_subpage = 0;
2302
2303 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2304 need_subpage);
2305 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2306 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2307 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2308 &p->phys_offset, orig_memory,
2309 p->region_offset);
2310 } else {
2311 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2312 >> IO_MEM_SHIFT];
2313 }
2314 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2315 region_offset);
2316 p->region_offset = 0;
2317 } else {
2318 p->phys_offset = phys_offset;
2319 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2320 (phys_offset & IO_MEM_ROMD))
2321 phys_offset += TARGET_PAGE_SIZE;
2322 }
2323 } else {
2324 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2325 p->phys_offset = phys_offset;
2326 p->region_offset = region_offset;
2327 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2328 (phys_offset & IO_MEM_ROMD)) {
2329 phys_offset += TARGET_PAGE_SIZE;
2330 } else {
2331 target_phys_addr_t start_addr2, end_addr2;
2332 int need_subpage = 0;
2333
2334 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2335 end_addr2, need_subpage);
2336
2337 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2338 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2339 &p->phys_offset, IO_MEM_UNASSIGNED,
2340 0);
2341 subpage_register(subpage, start_addr2, end_addr2,
2342 phys_offset, region_offset);
2343 p->region_offset = 0;
2344 }
2345 }
2346 }
2347 region_offset += TARGET_PAGE_SIZE;
2348 }
2349
2350 /* since each CPU stores ram addresses in its TLB cache, we must
2351 reset the modified entries */
2352 /* XXX: slow ! */
2353 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2354 tlb_flush(env, 1);
2355 }
2356 }
2357
2358 /* XXX: temporary until new memory mapping API */
2359 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2360 {
2361 PhysPageDesc *p;
2362
2363 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2364 if (!p)
2365 return IO_MEM_UNASSIGNED;
2366 return p->phys_offset;
2367 }
2368
2369 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2370 {
2371 if (kvm_enabled())
2372 kvm_coalesce_mmio_region(addr, size);
2373 }
2374
2375 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2376 {
2377 if (kvm_enabled())
2378 kvm_uncoalesce_mmio_region(addr, size);
2379 }
2380
2381 /* XXX: better than nothing */
2382 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2383 {
2384 ram_addr_t addr;
2385 if ((phys_ram_alloc_offset + size) > phys_ram_size) {
2386 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2387 (uint64_t)size, (uint64_t)phys_ram_size);
2388 abort();
2389 }
2390 addr = phys_ram_alloc_offset;
2391 phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2392 return addr;
2393 }
2394
2395 void qemu_ram_free(ram_addr_t addr)
2396 {
2397 }
2398
2399 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2400 {
2401 #ifdef DEBUG_UNASSIGNED
2402 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2403 #endif
2404 #if defined(TARGET_SPARC)
2405 do_unassigned_access(addr, 0, 0, 0, 1);
2406 #endif
2407 return 0;
2408 }
2409
2410 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2411 {
2412 #ifdef DEBUG_UNASSIGNED
2413 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2414 #endif
2415 #if defined(TARGET_SPARC)
2416 do_unassigned_access(addr, 0, 0, 0, 2);
2417 #endif
2418 return 0;
2419 }
2420
2421 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2422 {
2423 #ifdef DEBUG_UNASSIGNED
2424 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2425 #endif
2426 #if defined(TARGET_SPARC)
2427 do_unassigned_access(addr, 0, 0, 0, 4);
2428 #endif
2429 return 0;
2430 }
2431
2432 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2433 {
2434 #ifdef DEBUG_UNASSIGNED
2435 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2436 #endif
2437 #if defined(TARGET_SPARC)
2438 do_unassigned_access(addr, 1, 0, 0, 1);
2439 #endif
2440 }
2441
2442 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2443 {
2444 #ifdef DEBUG_UNASSIGNED
2445 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2446 #endif
2447 #if defined(TARGET_SPARC)
2448 do_unassigned_access(addr, 1, 0, 0, 2);
2449 #endif
2450 }
2451
2452 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2453 {
2454 #ifdef DEBUG_UNASSIGNED
2455 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2456 #endif
2457 #if defined(TARGET_SPARC)
2458 do_unassigned_access(addr, 1, 0, 0, 4);
2459 #endif
2460 }
2461
2462 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2463 unassigned_mem_readb,
2464 unassigned_mem_readw,
2465 unassigned_mem_readl,
2466 };
2467
2468 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2469 unassigned_mem_writeb,
2470 unassigned_mem_writew,
2471 unassigned_mem_writel,
2472 };
2473
2474 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2475 uint32_t val)
2476 {
2477 int dirty_flags;
2478 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2479 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2480 #if !defined(CONFIG_USER_ONLY)
2481 tb_invalidate_phys_page_fast(ram_addr, 1);
2482 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2483 #endif
2484 }
2485 stb_p(phys_ram_base + ram_addr, val);
2486 #ifdef USE_KQEMU
2487 if (cpu_single_env->kqemu_enabled &&
2488 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2489 kqemu_modify_page(cpu_single_env, ram_addr);
2490 #endif
2491 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2492 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2493 /* we remove the notdirty callback only if the code has been
2494 flushed */
2495 if (dirty_flags == 0xff)
2496 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2497 }
2498
2499 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2500 uint32_t val)
2501 {
2502 int dirty_flags;
2503 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2504 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2505 #if !defined(CONFIG_USER_ONLY)
2506 tb_invalidate_phys_page_fast(ram_addr, 2);
2507 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2508 #endif
2509 }
2510 stw_p(phys_ram_base + ram_addr, val);
2511 #ifdef USE_KQEMU
2512 if (cpu_single_env->kqemu_enabled &&
2513 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2514 kqemu_modify_page(cpu_single_env, ram_addr);
2515 #endif
2516 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2517 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2518 /* we remove the notdirty callback only if the code has been
2519 flushed */
2520 if (dirty_flags == 0xff)
2521 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2522 }
2523
2524 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2525 uint32_t val)
2526 {
2527 int dirty_flags;
2528 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2529 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2530 #if !defined(CONFIG_USER_ONLY)
2531 tb_invalidate_phys_page_fast(ram_addr, 4);
2532 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2533 #endif
2534 }
2535 stl_p(phys_ram_base + ram_addr, val);
2536 #ifdef USE_KQEMU
2537 if (cpu_single_env->kqemu_enabled &&
2538 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2539 kqemu_modify_page(cpu_single_env, ram_addr);
2540 #endif
2541 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2542 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2543 /* we remove the notdirty callback only if the code has been
2544 flushed */
2545 if (dirty_flags == 0xff)
2546 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2547 }
2548
2549 static CPUReadMemoryFunc *error_mem_read[3] = {
2550 NULL, /* never used */
2551 NULL, /* never used */
2552 NULL, /* never used */
2553 };
2554
2555 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2556 notdirty_mem_writeb,
2557 notdirty_mem_writew,
2558 notdirty_mem_writel,
2559 };
2560
2561 /* Generate a debug exception if a watchpoint has been hit. */
2562 static void check_watchpoint(int offset, int len_mask, int flags)
2563 {
2564 CPUState *env = cpu_single_env;
2565 target_ulong pc, cs_base;
2566 TranslationBlock *tb;
2567 target_ulong vaddr;
2568 CPUWatchpoint *wp;
2569 int cpu_flags;
2570
2571 if (env->watchpoint_hit) {
2572 /* We re-entered the check after replacing the TB. Now raise
2573 * the debug interrupt so that is will trigger after the
2574 * current instruction. */
2575 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2576 return;
2577 }
2578 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2579 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2580 if ((vaddr == (wp->vaddr & len_mask) ||
2581 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2582 wp->flags |= BP_WATCHPOINT_HIT;
2583 if (!env->watchpoint_hit) {
2584 env->watchpoint_hit = wp;
2585 tb = tb_find_pc(env->mem_io_pc);
2586 if (!tb) {
2587 cpu_abort(env, "check_watchpoint: could not find TB for "
2588 "pc=%p", (void *)env->mem_io_pc);
2589 }
2590 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2591 tb_phys_invalidate(tb, -1);
2592 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2593 env->exception_index = EXCP_DEBUG;
2594 } else {
2595 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2596 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2597 }
2598 cpu_resume_from_signal(env, NULL);
2599 }
2600 } else {
2601 wp->flags &= ~BP_WATCHPOINT_HIT;
2602 }
2603 }
2604 }
2605
2606 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2607 so these check for a hit then pass through to the normal out-of-line
2608 phys routines. */
2609 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2610 {
2611 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2612 return ldub_phys(addr);
2613 }
2614
2615 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2616 {
2617 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2618 return lduw_phys(addr);
2619 }
2620
2621 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2622 {
2623 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2624 return ldl_phys(addr);
2625 }
2626
2627 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2628 uint32_t val)
2629 {
2630 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2631 stb_phys(addr, val);
2632 }
2633
2634 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2635 uint32_t val)
2636 {
2637 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2638 stw_phys(addr, val);
2639 }
2640
2641 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2642 uint32_t val)
2643 {
2644 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2645 stl_phys(addr, val);
2646 }
2647
2648 static CPUReadMemoryFunc *watch_mem_read[3] = {
2649 watch_mem_readb,
2650 watch_mem_readw,
2651 watch_mem_readl,
2652 };
2653
2654 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2655 watch_mem_writeb,
2656 watch_mem_writew,
2657 watch_mem_writel,
2658 };
2659
2660 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2661 unsigned int len)
2662 {
2663 uint32_t ret;
2664 unsigned int idx;
2665
2666 idx = SUBPAGE_IDX(addr);
2667 #if defined(DEBUG_SUBPAGE)
2668 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2669 mmio, len, addr, idx);
2670 #endif
2671 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2672 addr + mmio->region_offset[idx][0][len]);
2673
2674 return ret;
2675 }
2676
2677 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2678 uint32_t value, unsigned int len)
2679 {
2680 unsigned int idx;
2681
2682 idx = SUBPAGE_IDX(addr);
2683 #if defined(DEBUG_SUBPAGE)
2684 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2685 mmio, len, addr, idx, value);
2686 #endif
2687 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2688 addr + mmio->region_offset[idx][1][len],
2689 value);
2690 }
2691
2692 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2693 {
2694 #if defined(DEBUG_SUBPAGE)
2695 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2696 #endif
2697
2698 return subpage_readlen(opaque, addr, 0);
2699 }
2700
2701 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2702 uint32_t value)
2703 {
2704 #if defined(DEBUG_SUBPAGE)
2705 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2706 #endif
2707 subpage_writelen(opaque, addr, value, 0);
2708 }
2709
2710 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2711 {
2712 #if defined(DEBUG_SUBPAGE)
2713 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2714 #endif
2715
2716 return subpage_readlen(opaque, addr, 1);
2717 }
2718
2719 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2720 uint32_t value)
2721 {
2722 #if defined(DEBUG_SUBPAGE)
2723 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2724 #endif
2725 subpage_writelen(opaque, addr, value, 1);
2726 }
2727
2728 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2729 {
2730 #if defined(DEBUG_SUBPAGE)
2731 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2732 #endif
2733
2734 return subpage_readlen(opaque, addr, 2);
2735 }
2736
2737 static void subpage_writel (void *opaque,
2738 target_phys_addr_t addr, uint32_t value)
2739 {
2740 #if defined(DEBUG_SUBPAGE)
2741 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2742 #endif
2743 subpage_writelen(opaque, addr, value, 2);
2744 }
2745
2746 static CPUReadMemoryFunc *subpage_read[] = {
2747 &subpage_readb,
2748 &subpage_readw,
2749 &subpage_readl,
2750 };
2751
2752 static CPUWriteMemoryFunc *subpage_write[] = {
2753 &subpage_writeb,
2754 &subpage_writew,
2755 &subpage_writel,
2756 };
2757
2758 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2759 ram_addr_t memory, ram_addr_t region_offset)
2760 {
2761 int idx, eidx;
2762 unsigned int i;
2763
2764 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2765 return -1;
2766 idx = SUBPAGE_IDX(start);
2767 eidx = SUBPAGE_IDX(end);
2768 #if defined(DEBUG_SUBPAGE)
2769 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2770 mmio, start, end, idx, eidx, memory);
2771 #endif
2772 memory >>= IO_MEM_SHIFT;
2773 for (; idx <= eidx; idx++) {
2774 for (i = 0; i < 4; i++) {
2775 if (io_mem_read[memory][i]) {
2776 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2777 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2778 mmio->region_offset[idx][0][i] = region_offset;
2779 }
2780 if (io_mem_write[memory][i]) {
2781 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2782 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2783 mmio->region_offset[idx][1][i] = region_offset;
2784 }
2785 }
2786 }
2787
2788 return 0;
2789 }
2790
2791 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2792 ram_addr_t orig_memory, ram_addr_t region_offset)
2793 {
2794 subpage_t *mmio;
2795 int subpage_memory;
2796
2797 mmio = qemu_mallocz(sizeof(subpage_t));
2798 if (mmio != NULL) {
2799 mmio->base = base;
2800 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2801 #if defined(DEBUG_SUBPAGE)
2802 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2803 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2804 #endif
2805 *phys = subpage_memory | IO_MEM_SUBPAGE;
2806 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
2807 region_offset);
2808 }
2809
2810 return mmio;
2811 }
2812
2813 static void io_mem_init(void)
2814 {
2815 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
2816 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2817 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2818 io_mem_nb = 5;
2819
2820 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2821 watch_mem_write, NULL);
2822 /* alloc dirty bits array */
2823 phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2824 memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2825 }
2826
2827 /* mem_read and mem_write are arrays of functions containing the
2828 function to access byte (index 0), word (index 1) and dword (index
2829 2). Functions can be omitted with a NULL function pointer. The
2830 registered functions may be modified dynamically later.
2831 If io_index is non zero, the corresponding io zone is
2832 modified. If it is zero, a new io zone is allocated. The return
2833 value can be used with cpu_register_physical_memory(). (-1) is
2834 returned if error. */
2835 int cpu_register_io_memory(int io_index,
2836 CPUReadMemoryFunc **mem_read,
2837 CPUWriteMemoryFunc **mem_write,
2838 void *opaque)
2839 {
2840 int i, subwidth = 0;
2841
2842 if (io_index <= 0) {
2843 if (io_mem_nb >= IO_MEM_NB_ENTRIES)
2844 return -1;
2845 io_index = io_mem_nb++;
2846 } else {
2847 if (io_index >= IO_MEM_NB_ENTRIES)
2848 return -1;
2849 }
2850
2851 for(i = 0;i < 3; i++) {
2852 if (!mem_read[i] || !mem_write[i])
2853 subwidth = IO_MEM_SUBWIDTH;
2854 io_mem_read[io_index][i] = mem_read[i];
2855 io_mem_write[io_index][i] = mem_write[i];
2856 }
2857 io_mem_opaque[io_index] = opaque;
2858 return (io_index << IO_MEM_SHIFT) | subwidth;
2859 }
2860
2861 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2862 {
2863 return io_mem_write[io_index >> IO_MEM_SHIFT];
2864 }
2865
2866 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2867 {
2868 return io_mem_read[io_index >> IO_MEM_SHIFT];
2869 }
2870
2871 #endif /* !defined(CONFIG_USER_ONLY) */
2872
2873 /* physical memory access (slow version, mainly for debug) */
2874 #if defined(CONFIG_USER_ONLY)
2875 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2876 int len, int is_write)
2877 {
2878 int l, flags;
2879 target_ulong page;
2880 void * p;
2881
2882 while (len > 0) {
2883 page = addr & TARGET_PAGE_MASK;
2884 l = (page + TARGET_PAGE_SIZE) - addr;
2885 if (l > len)
2886 l = len;
2887 flags = page_get_flags(page);
2888 if (!(flags & PAGE_VALID))
2889 return;
2890 if (is_write) {
2891 if (!(flags & PAGE_WRITE))
2892 return;
2893 /* XXX: this code should not depend on lock_user */
2894 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2895 /* FIXME - should this return an error rather than just fail? */
2896 return;
2897 memcpy(p, buf, l);
2898 unlock_user(p, addr, l);
2899 } else {
2900 if (!(flags & PAGE_READ))
2901 return;
2902 /* XXX: this code should not depend on lock_user */
2903 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2904 /* FIXME - should this return an error rather than just fail? */
2905 return;
2906 memcpy(buf, p, l);
2907 unlock_user(p, addr, 0);
2908 }
2909 len -= l;
2910 buf += l;
2911 addr += l;
2912 }
2913 }
2914
2915 #else
2916 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2917 int len, int is_write)
2918 {
2919 int l, io_index;
2920 uint8_t *ptr;
2921 uint32_t val;
2922 target_phys_addr_t page;
2923 unsigned long pd;
2924 PhysPageDesc *p;
2925
2926 while (len > 0) {
2927 page = addr & TARGET_PAGE_MASK;
2928 l = (page + TARGET_PAGE_SIZE) - addr;
2929 if (l > len)
2930 l = len;
2931 p = phys_page_find(page >> TARGET_PAGE_BITS);
2932 if (!p) {
2933 pd = IO_MEM_UNASSIGNED;
2934 } else {
2935 pd = p->phys_offset;
2936 }
2937
2938 if (is_write) {
2939 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
2940 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2941 if (p)
2942 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
2943 /* XXX: could force cpu_single_env to NULL to avoid
2944 potential bugs */
2945 if (l >= 4 && ((addr & 3) == 0)) {
2946 /* 32 bit write access */
2947 val = ldl_p(buf);
2948 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
2949 l = 4;
2950 } else if (l >= 2 && ((addr & 1) == 0)) {
2951 /* 16 bit write access */
2952 val = lduw_p(buf);
2953 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
2954 l = 2;
2955 } else {
2956 /* 8 bit write access */
2957 val = ldub_p(buf);
2958 io_mem_write[io_index][0](io_mem_opaque[io_index], addr, val);
2959 l = 1;
2960 }
2961 } else {
2962 unsigned long addr1;
2963 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
2964 /* RAM case */
2965 ptr = phys_ram_base + addr1;
2966 memcpy(ptr, buf, l);
2967 if (!cpu_physical_memory_is_dirty(addr1)) {
2968 /* invalidate code */
2969 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
2970 /* set dirty bit */
2971 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
2972 (0xff & ~CODE_DIRTY_FLAG);
2973 }
2974 }
2975 } else {
2976 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
2977 !(pd & IO_MEM_ROMD)) {
2978 /* I/O case */
2979 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2980 if (p)
2981 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
2982 if (l >= 4 && ((addr & 3) == 0)) {
2983 /* 32 bit read access */
2984 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
2985 stl_p(buf, val);
2986 l = 4;
2987 } else if (l >= 2 && ((addr & 1) == 0)) {
2988 /* 16 bit read access */
2989 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
2990 stw_p(buf, val);
2991 l = 2;
2992 } else {
2993 /* 8 bit read access */
2994 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr);
2995 stb_p(buf, val);
2996 l = 1;
2997 }
2998 } else {
2999 /* RAM case */
3000 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3001 (addr & ~TARGET_PAGE_MASK);
3002 memcpy(buf, ptr, l);
3003 }
3004 }
3005 len -= l;
3006 buf += l;
3007 addr += l;
3008 }
3009 }
3010
3011 /* used for ROM loading : can write in RAM and ROM */
3012 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3013 const uint8_t *buf, int len)
3014 {
3015 int l;
3016 uint8_t *ptr;
3017 target_phys_addr_t page;
3018 unsigned long pd;
3019 PhysPageDesc *p;
3020
3021 while (len > 0) {
3022 page = addr & TARGET_PAGE_MASK;
3023 l = (page + TARGET_PAGE_SIZE) - addr;
3024 if (l > len)
3025 l = len;
3026 p = phys_page_find(page >> TARGET_PAGE_BITS);
3027 if (!p) {
3028 pd = IO_MEM_UNASSIGNED;
3029 } else {
3030 pd = p->phys_offset;
3031 }
3032
3033 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3034 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3035 !(pd & IO_MEM_ROMD)) {
3036 /* do nothing */
3037 } else {
3038 unsigned long addr1;
3039 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3040 /* ROM/RAM case */
3041 ptr = phys_ram_base + addr1;
3042 memcpy(ptr, buf, l);
3043 }
3044 len -= l;
3045 buf += l;
3046 addr += l;
3047 }
3048 }
3049
3050 typedef struct {
3051 void *buffer;
3052 target_phys_addr_t addr;
3053 target_phys_addr_t len;
3054 } BounceBuffer;
3055
3056 static BounceBuffer bounce;
3057
3058 typedef struct MapClient {
3059 void *opaque;
3060 void (*callback)(void *opaque);
3061 LIST_ENTRY(MapClient) link;
3062 } MapClient;
3063
3064 static LIST_HEAD(map_client_list, MapClient) map_client_list
3065 = LIST_HEAD_INITIALIZER(map_client_list);
3066
3067 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3068 {
3069 MapClient *client = qemu_malloc(sizeof(*client));
3070
3071 client->opaque = opaque;
3072 client->callback = callback;
3073 LIST_INSERT_HEAD(&map_client_list, client, link);
3074 return client;
3075 }
3076
3077 void cpu_unregister_map_client(void *_client)
3078 {
3079 MapClient *client = (MapClient *)_client;
3080
3081 LIST_REMOVE(client, link);
3082 }
3083
3084 static void cpu_notify_map_clients(void)
3085 {
3086 MapClient *client;
3087
3088 while (!LIST_EMPTY(&map_client_list)) {
3089 client = LIST_FIRST(&map_client_list);
3090 client->callback(client->opaque);
3091 LIST_REMOVE(client, link);
3092 }
3093 }
3094
3095 /* Map a physical memory region into a host virtual address.
3096 * May map a subset of the requested range, given by and returned in *plen.
3097 * May return NULL if resources needed to perform the mapping are exhausted.
3098 * Use only for reads OR writes - not for read-modify-write operations.
3099 * Use cpu_register_map_client() to know when retrying the map operation is
3100 * likely to succeed.
3101 */
3102 void *cpu_physical_memory_map(target_phys_addr_t addr,
3103 target_phys_addr_t *plen,
3104 int is_write)
3105 {
3106 target_phys_addr_t len = *plen;
3107 target_phys_addr_t done = 0;
3108 int l;
3109 uint8_t *ret = NULL;
3110 uint8_t *ptr;
3111 target_phys_addr_t page;
3112 unsigned long pd;
3113 PhysPageDesc *p;
3114 unsigned long addr1;
3115
3116 while (len > 0) {
3117 page = addr & TARGET_PAGE_MASK;
3118 l = (page + TARGET_PAGE_SIZE) - addr;
3119 if (l > len)
3120 l = len;
3121 p = phys_page_find(page >> TARGET_PAGE_BITS);
3122 if (!p) {
3123 pd = IO_MEM_UNASSIGNED;
3124 } else {
3125 pd = p->phys_offset;
3126 }
3127
3128 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3129 if (done || bounce.buffer) {
3130 break;
3131 }
3132 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3133 bounce.addr = addr;
3134 bounce.len = l;
3135 if (!is_write) {
3136 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3137 }
3138 ptr = bounce.buffer;
3139 } else {
3140 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3141 ptr = phys_ram_base + addr1;
3142 }
3143 if (!done) {
3144 ret = ptr;
3145 } else if (ret + done != ptr) {
3146 break;
3147 }
3148
3149 len -= l;
3150 addr += l;
3151 done += l;
3152 }
3153 *plen = done;
3154 return ret;
3155 }
3156
3157 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3158 * Will also mark the memory as dirty if is_write == 1. access_len gives
3159 * the amount of memory that was actually read or written by the caller.
3160 */
3161 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3162 int is_write, target_phys_addr_t access_len)
3163 {
3164 if (buffer != bounce.buffer) {
3165 if (is_write) {
3166 unsigned long addr1 = (uint8_t *)buffer - phys_ram_base;
3167 while (access_len) {
3168 unsigned l;
3169 l = TARGET_PAGE_SIZE;
3170 if (l > access_len)
3171 l = access_len;
3172 if (!cpu_physical_memory_is_dirty(addr1)) {
3173 /* invalidate code */
3174 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3175 /* set dirty bit */
3176 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3177 (0xff & ~CODE_DIRTY_FLAG);
3178 }
3179 addr1 += l;
3180 access_len -= l;
3181 }
3182 }
3183 return;
3184 }
3185 if (is_write) {
3186 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3187 }
3188 qemu_free(bounce.buffer);
3189 bounce.buffer = NULL;
3190 cpu_notify_map_clients();
3191 }
3192
3193 /* warning: addr must be aligned */
3194 uint32_t ldl_phys(target_phys_addr_t addr)
3195 {
3196 int io_index;
3197 uint8_t *ptr;
3198 uint32_t val;
3199 unsigned long pd;
3200 PhysPageDesc *p;
3201
3202 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3203 if (!p) {
3204 pd = IO_MEM_UNASSIGNED;
3205 } else {
3206 pd = p->phys_offset;
3207 }
3208
3209 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3210 !(pd & IO_MEM_ROMD)) {
3211 /* I/O case */
3212 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3213 if (p)
3214 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3215 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3216 } else {
3217 /* RAM case */
3218 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3219 (addr & ~TARGET_PAGE_MASK);
3220 val = ldl_p(ptr);
3221 }
3222 return val;
3223 }
3224
3225 /* warning: addr must be aligned */
3226 uint64_t ldq_phys(target_phys_addr_t addr)
3227 {
3228 int io_index;
3229 uint8_t *ptr;
3230 uint64_t val;
3231 unsigned long pd;
3232 PhysPageDesc *p;
3233
3234 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3235 if (!p) {
3236 pd = IO_MEM_UNASSIGNED;
3237 } else {
3238 pd = p->phys_offset;
3239 }
3240
3241 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3242 !(pd & IO_MEM_ROMD)) {
3243 /* I/O case */
3244 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3245 if (p)
3246 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3247 #ifdef TARGET_WORDS_BIGENDIAN
3248 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3249 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3250 #else
3251 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3252 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3253 #endif
3254 } else {
3255 /* RAM case */
3256 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3257 (addr & ~TARGET_PAGE_MASK);
3258 val = ldq_p(ptr);
3259 }
3260 return val;
3261 }
3262
3263 /* XXX: optimize */
3264 uint32_t ldub_phys(target_phys_addr_t addr)
3265 {
3266 uint8_t val;
3267 cpu_physical_memory_read(addr, &val, 1);
3268 return val;
3269 }
3270
3271 /* XXX: optimize */
3272 uint32_t lduw_phys(target_phys_addr_t addr)
3273 {
3274 uint16_t val;
3275 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3276 return tswap16(val);
3277 }
3278
3279 /* warning: addr must be aligned. The ram page is not masked as dirty
3280 and the code inside is not invalidated. It is useful if the dirty
3281 bits are used to track modified PTEs */
3282 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3283 {
3284 int io_index;
3285 uint8_t *ptr;
3286 unsigned long pd;
3287 PhysPageDesc *p;
3288
3289 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3290 if (!p) {
3291 pd = IO_MEM_UNASSIGNED;
3292 } else {
3293 pd = p->phys_offset;
3294 }
3295
3296 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3297 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3298 if (p)
3299 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3300 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3301 } else {
3302 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3303 ptr = phys_ram_base + addr1;
3304 stl_p(ptr, val);
3305
3306 if (unlikely(in_migration)) {
3307 if (!cpu_physical_memory_is_dirty(addr1)) {
3308 /* invalidate code */
3309 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3310 /* set dirty bit */
3311 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3312 (0xff & ~CODE_DIRTY_FLAG);
3313 }
3314 }
3315 }
3316 }
3317
3318 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3319 {
3320 int io_index;
3321 uint8_t *ptr;
3322 unsigned long pd;
3323 PhysPageDesc *p;
3324
3325 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3326 if (!p) {
3327 pd = IO_MEM_UNASSIGNED;
3328 } else {
3329 pd = p->phys_offset;
3330 }
3331
3332 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3333 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3334 if (p)
3335 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3336 #ifdef TARGET_WORDS_BIGENDIAN
3337 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3338 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3339 #else
3340 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3341 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3342 #endif
3343 } else {
3344 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3345 (addr & ~TARGET_PAGE_MASK);
3346 stq_p(ptr, val);
3347 }
3348 }
3349
3350 /* warning: addr must be aligned */
3351 void stl_phys(target_phys_addr_t addr, uint32_t val)
3352 {
3353 int io_index;
3354 uint8_t *ptr;
3355 unsigned long pd;
3356 PhysPageDesc *p;
3357
3358 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3359 if (!p) {
3360 pd = IO_MEM_UNASSIGNED;
3361 } else {
3362 pd = p->phys_offset;
3363 }
3364
3365 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3366 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3367 if (p)
3368 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3369 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3370 } else {
3371 unsigned long addr1;
3372 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3373 /* RAM case */
3374 ptr = phys_ram_base + addr1;
3375 stl_p(ptr, val);
3376 if (!cpu_physical_memory_is_dirty(addr1)) {
3377 /* invalidate code */
3378 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3379 /* set dirty bit */
3380 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3381 (0xff & ~CODE_DIRTY_FLAG);
3382 }
3383 }
3384 }
3385
3386 /* XXX: optimize */
3387 void stb_phys(target_phys_addr_t addr, uint32_t val)
3388 {
3389 uint8_t v = val;
3390 cpu_physical_memory_write(addr, &v, 1);
3391 }
3392
3393 /* XXX: optimize */
3394 void stw_phys(target_phys_addr_t addr, uint32_t val)
3395 {
3396 uint16_t v = tswap16(val);
3397 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3398 }
3399
3400 /* XXX: optimize */
3401 void stq_phys(target_phys_addr_t addr, uint64_t val)
3402 {
3403 val = tswap64(val);
3404 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3405 }
3406
3407 #endif
3408
3409 /* virtual memory access for debug */
3410 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3411 uint8_t *buf, int len, int is_write)
3412 {
3413 int l;
3414 target_phys_addr_t phys_addr;
3415 target_ulong page;
3416
3417 while (len > 0) {
3418 page = addr & TARGET_PAGE_MASK;
3419 phys_addr = cpu_get_phys_page_debug(env, page);
3420 /* if no physical page mapped, return an error */
3421 if (phys_addr == -1)
3422 return -1;
3423 l = (page + TARGET_PAGE_SIZE) - addr;
3424 if (l > len)
3425 l = len;
3426 cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
3427 buf, l, is_write);
3428 len -= l;
3429 buf += l;
3430 addr += l;
3431 }
3432 return 0;
3433 }
3434
3435 /* in deterministic execution mode, instructions doing device I/Os
3436 must be at the end of the TB */
3437 void cpu_io_recompile(CPUState *env, void *retaddr)
3438 {
3439 TranslationBlock *tb;
3440 uint32_t n, cflags;
3441 target_ulong pc, cs_base;
3442 uint64_t flags;
3443
3444 tb = tb_find_pc((unsigned long)retaddr);
3445 if (!tb) {
3446 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3447 retaddr);
3448 }
3449 n = env->icount_decr.u16.low + tb->icount;
3450 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3451 /* Calculate how many instructions had been executed before the fault
3452 occurred. */
3453 n = n - env->icount_decr.u16.low;
3454 /* Generate a new TB ending on the I/O insn. */
3455 n++;
3456 /* On MIPS and SH, delay slot instructions can only be restarted if
3457 they were already the first instruction in the TB. If this is not
3458 the first instruction in a TB then re-execute the preceding
3459 branch. */
3460 #if defined(TARGET_MIPS)
3461 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3462 env->active_tc.PC -= 4;
3463 env->icount_decr.u16.low++;
3464 env->hflags &= ~MIPS_HFLAG_BMASK;
3465 }
3466 #elif defined(TARGET_SH4)
3467 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3468 && n > 1) {
3469 env->pc -= 2;
3470 env->icount_decr.u16.low++;
3471 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3472 }
3473 #endif
3474 /* This should never happen. */
3475 if (n > CF_COUNT_MASK)
3476 cpu_abort(env, "TB too big during recompile");
3477
3478 cflags = n | CF_LAST_IO;
3479 pc = tb->pc;
3480 cs_base = tb->cs_base;
3481 flags = tb->flags;
3482 tb_phys_invalidate(tb, -1);
3483 /* FIXME: In theory this could raise an exception. In practice
3484 we have already translated the block once so it's probably ok. */
3485 tb_gen_code(env, pc, cs_base, flags, cflags);
3486 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3487 the first in the TB) then we end up generating a whole new TB and
3488 repeating the fault, which is horribly inefficient.
3489 Better would be to execute just this insn uncached, or generate a
3490 second new TB. */
3491 cpu_resume_from_signal(env, NULL);
3492 }
3493
3494 void dump_exec_info(FILE *f,
3495 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3496 {
3497 int i, target_code_size, max_target_code_size;
3498 int direct_jmp_count, direct_jmp2_count, cross_page;
3499 TranslationBlock *tb;
3500
3501 target_code_size = 0;
3502 max_target_code_size = 0;
3503 cross_page = 0;
3504 direct_jmp_count = 0;
3505 direct_jmp2_count = 0;
3506 for(i = 0; i < nb_tbs; i++) {
3507 tb = &tbs[i];
3508 target_code_size += tb->size;
3509 if (tb->size > max_target_code_size)
3510 max_target_code_size = tb->size;
3511 if (tb->page_addr[1] != -1)
3512 cross_page++;
3513 if (tb->tb_next_offset[0] != 0xffff) {
3514 direct_jmp_count++;
3515 if (tb->tb_next_offset[1] != 0xffff) {
3516 direct_jmp2_count++;
3517 }
3518 }
3519 }
3520 /* XXX: avoid using doubles ? */
3521 cpu_fprintf(f, "Translation buffer state:\n");
3522 cpu_fprintf(f, "gen code size %ld/%ld\n",
3523 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3524 cpu_fprintf(f, "TB count %d/%d\n",
3525 nb_tbs, code_gen_max_blocks);
3526 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3527 nb_tbs ? target_code_size / nb_tbs : 0,
3528 max_target_code_size);
3529 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3530 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3531 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3532 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3533 cross_page,
3534 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3535 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3536 direct_jmp_count,
3537 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3538 direct_jmp2_count,
3539 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3540 cpu_fprintf(f, "\nStatistics:\n");
3541 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3542 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3543 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3544 tcg_dump_info(f, cpu_fprintf);
3545 }
3546
3547 #if !defined(CONFIG_USER_ONLY)
3548
3549 #define MMUSUFFIX _cmmu
3550 #define GETPC() NULL
3551 #define env cpu_single_env
3552 #define SOFTMMU_CODE_ACCESS
3553
3554 #define SHIFT 0
3555 #include "softmmu_template.h"
3556
3557 #define SHIFT 1
3558 #include "softmmu_template.h"
3559
3560 #define SHIFT 2
3561 #include "softmmu_template.h"
3562
3563 #define SHIFT 3
3564 #include "softmmu_template.h"
3565
3566 #undef env
3567
3568 #endif