?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
qemu:virtio-net: Check return size on the correct sg list (Alex Williamson)
[qemu/mini2440/sniper_sniper_test.git] / exec.c
blob6010bc70c1a8e9f5c868005dbee6d92046683bb9
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA
19 */
20 #include "config.h"
21 #ifdef _WIN32
22 #define WIN32_LEAN_AND_MEAN
23 #include <windows.h>
24 #else
25 #include <sys/types.h>
26 #include <sys/mman.h>
27 #endif
28 #include <stdlib.h>
29 #include <stdio.h>
30 #include <stdarg.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <unistd.h>
34 #include <inttypes.h>
35
36 #include "cpu.h"
37 #include "exec-all.h"
38 #include "qemu-common.h"
39 #include "tcg.h"
40 #include "hw/hw.h"
41 #include "osdep.h"
42 #include "kvm.h"
43 #if defined(CONFIG_USER_ONLY)
44 #include <qemu.h>
45 #endif
46
47 //#define DEBUG_TB_INVALIDATE
48 //#define DEBUG_FLUSH
49 //#define DEBUG_TLB
50 //#define DEBUG_UNASSIGNED
51
52 /* make various TB consistency checks */
53 //#define DEBUG_TB_CHECK
54 //#define DEBUG_TLB_CHECK
55
56 //#define DEBUG_IOPORT
57 //#define DEBUG_SUBPAGE
58
59 #if !defined(CONFIG_USER_ONLY)
60 /* TB consistency checks only implemented for usermode emulation. */
61 #undef DEBUG_TB_CHECK
62 #endif
63
64 #define SMC_BITMAP_USE_THRESHOLD 10
65
66 #define MMAP_AREA_START 0x00000000
67 #define MMAP_AREA_END 0xa8000000
68
69 #if defined(TARGET_SPARC64)
70 #define TARGET_PHYS_ADDR_SPACE_BITS 41
71 #elif defined(TARGET_SPARC)
72 #define TARGET_PHYS_ADDR_SPACE_BITS 36
73 #elif defined(TARGET_ALPHA)
74 #define TARGET_PHYS_ADDR_SPACE_BITS 42
75 #define TARGET_VIRT_ADDR_SPACE_BITS 42
76 #elif defined(TARGET_PPC64)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 42
78 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
80 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
81 #define TARGET_PHYS_ADDR_SPACE_BITS 36
82 #else
83 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
84 #define TARGET_PHYS_ADDR_SPACE_BITS 32
85 #endif
86
87 static TranslationBlock *tbs;
88 int code_gen_max_blocks;
89 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
90 static int nb_tbs;
91 /* any access to the tbs or the page table must use this lock */
92 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
93
94 #if defined(__arm__) || defined(__sparc_v9__)
95 /* The prologue must be reachable with a direct jump. ARM and Sparc64
96 have limited branch ranges (possibly also PPC) so place it in a
97 section close to code segment. */
98 #define code_gen_section \
99 __attribute__((__section__(".gen_code"))) \
100 __attribute__((aligned (32)))
101 #else
102 #define code_gen_section \
103 __attribute__((aligned (32)))
104 #endif
105
106 uint8_t code_gen_prologue[1024] code_gen_section;
107 static uint8_t *code_gen_buffer;
108 static unsigned long code_gen_buffer_size;
109 /* threshold to flush the translated code buffer */
110 static unsigned long code_gen_buffer_max_size;
111 uint8_t *code_gen_ptr;
112
113 #if !defined(CONFIG_USER_ONLY)
114 ram_addr_t phys_ram_size;
115 int phys_ram_fd;
116 uint8_t *phys_ram_base;
117 uint8_t *phys_ram_dirty;
118 static int in_migration;
119 static ram_addr_t phys_ram_alloc_offset = 0;
120 #endif
121
122 CPUState *first_cpu;
123 /* current CPU in the current thread. It is only valid inside
124 cpu_exec() */
125 CPUState *cpu_single_env;
126 /* 0 = Do not count executed instructions.
127 1 = Precise instruction counting.
128 2 = Adaptive rate instruction counting. */
129 int use_icount = 0;
130 /* Current instruction counter. While executing translated code this may
131 include some instructions that have not yet been executed. */
132 int64_t qemu_icount;
133
134 typedef struct PageDesc {
135 /* list of TBs intersecting this ram page */
136 TranslationBlock *first_tb;
137 /* in order to optimize self modifying code, we count the number
138 of lookups we do to a given page to use a bitmap */
139 unsigned int code_write_count;
140 uint8_t *code_bitmap;
141 #if defined(CONFIG_USER_ONLY)
142 unsigned long flags;
143 #endif
144 } PageDesc;
145
146 typedef struct PhysPageDesc {
147 /* offset in host memory of the page + io_index in the low bits */
148 ram_addr_t phys_offset;
149 ram_addr_t region_offset;
150 } PhysPageDesc;
151
152 #define L2_BITS 10
153 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
154 /* XXX: this is a temporary hack for alpha target.
155 * In the future, this is to be replaced by a multi-level table
156 * to actually be able to handle the complete 64 bits address space.
157 */
158 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
159 #else
160 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
161 #endif
162
163 #define L1_SIZE (1 << L1_BITS)
164 #define L2_SIZE (1 << L2_BITS)
165
166 unsigned long qemu_real_host_page_size;
167 unsigned long qemu_host_page_bits;
168 unsigned long qemu_host_page_size;
169 unsigned long qemu_host_page_mask;
170
171 /* XXX: for system emulation, it could just be an array */
172 static PageDesc *l1_map[L1_SIZE];
173 static PhysPageDesc **l1_phys_map;
174
175 #if !defined(CONFIG_USER_ONLY)
176 static void io_mem_init(void);
177
178 /* io memory support */
179 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
180 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
181 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
182 static char io_mem_used[IO_MEM_NB_ENTRIES];
183 static int io_mem_watch;
184 #endif
185
186 /* log support */
187 static const char *logfilename = "/tmp/qemu.log";
188 FILE *logfile;
189 int loglevel;
190 static int log_append = 0;
191
192 /* statistics */
193 static int tlb_flush_count;
194 static int tb_flush_count;
195 static int tb_phys_invalidate_count;
196
197 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
198 typedef struct subpage_t {
199 target_phys_addr_t base;
200 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
201 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
202 void *opaque[TARGET_PAGE_SIZE][2][4];
203 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
204 } subpage_t;
205
206 #ifdef _WIN32
207 static void map_exec(void *addr, long size)
208 {
209 DWORD old_protect;
210 VirtualProtect(addr, size,
211 PAGE_EXECUTE_READWRITE, &old_protect);
212
213 }
214 #else
215 static void map_exec(void *addr, long size)
216 {
217 unsigned long start, end, page_size;
218
219 page_size = getpagesize();
220 start = (unsigned long)addr;
221 start &= ~(page_size - 1);
222
223 end = (unsigned long)addr + size;
224 end += page_size - 1;
225 end &= ~(page_size - 1);
226
227 mprotect((void *)start, end - start,
228 PROT_READ | PROT_WRITE | PROT_EXEC);
229 }
230 #endif
231
232 static void page_init(void)
233 {
234 /* NOTE: we can always suppose that qemu_host_page_size >=
235 TARGET_PAGE_SIZE */
236 #ifdef _WIN32
237 {
238 SYSTEM_INFO system_info;
239
240 GetSystemInfo(&system_info);
241 qemu_real_host_page_size = system_info.dwPageSize;
242 }
243 #else
244 qemu_real_host_page_size = getpagesize();
245 #endif
246 if (qemu_host_page_size == 0)
247 qemu_host_page_size = qemu_real_host_page_size;
248 if (qemu_host_page_size < TARGET_PAGE_SIZE)
249 qemu_host_page_size = TARGET_PAGE_SIZE;
250 qemu_host_page_bits = 0;
251 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
252 qemu_host_page_bits++;
253 qemu_host_page_mask = ~(qemu_host_page_size - 1);
254 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
255 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
256
257 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
258 {
259 long long startaddr, endaddr;
260 FILE *f;
261 int n;
262
263 mmap_lock();
264 last_brk = (unsigned long)sbrk(0);
265 f = fopen("/proc/self/maps", "r");
266 if (f) {
267 do {
268 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
269 if (n == 2) {
270 startaddr = MIN(startaddr,
271 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
272 endaddr = MIN(endaddr,
273 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
274 page_set_flags(startaddr & TARGET_PAGE_MASK,
275 TARGET_PAGE_ALIGN(endaddr),
276 PAGE_RESERVED);
277 }
278 } while (!feof(f));
279 fclose(f);
280 }
281 mmap_unlock();
282 }
283 #endif
284 }
285
286 static inline PageDesc **page_l1_map(target_ulong index)
287 {
288 #if TARGET_LONG_BITS > 32
289 /* Host memory outside guest VM. For 32-bit targets we have already
290 excluded high addresses. */
291 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
292 return NULL;
293 #endif
294 return &l1_map[index >> L2_BITS];
295 }
296
297 static inline PageDesc *page_find_alloc(target_ulong index)
298 {
299 PageDesc **lp, *p;
300 lp = page_l1_map(index);
301 if (!lp)
302 return NULL;
303
304 p = *lp;
305 if (!p) {
306 /* allocate if not found */
307 #if defined(CONFIG_USER_ONLY)
308 size_t len = sizeof(PageDesc) * L2_SIZE;
309 /* Don't use qemu_malloc because it may recurse. */
310 p = mmap(0, len, PROT_READ | PROT_WRITE,
311 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
312 *lp = p;
313 if (h2g_valid(p)) {
314 unsigned long addr = h2g(p);
315 page_set_flags(addr & TARGET_PAGE_MASK,
316 TARGET_PAGE_ALIGN(addr + len),
317 PAGE_RESERVED);
318 }
319 #else
320 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
321 *lp = p;
322 #endif
323 }
324 return p + (index & (L2_SIZE - 1));
325 }
326
327 static inline PageDesc *page_find(target_ulong index)
328 {
329 PageDesc **lp, *p;
330 lp = page_l1_map(index);
331 if (!lp)
332 return NULL;
333
334 p = *lp;
335 if (!p)
336 return 0;
337 return p + (index & (L2_SIZE - 1));
338 }
339
340 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
341 {
342 void **lp, **p;
343 PhysPageDesc *pd;
344
345 p = (void **)l1_phys_map;
346 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
347
348 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
349 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
350 #endif
351 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
352 p = *lp;
353 if (!p) {
354 /* allocate if not found */
355 if (!alloc)
356 return NULL;
357 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
358 memset(p, 0, sizeof(void *) * L1_SIZE);
359 *lp = p;
360 }
361 #endif
362 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
363 pd = *lp;
364 if (!pd) {
365 int i;
366 /* allocate if not found */
367 if (!alloc)
368 return NULL;
369 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
370 *lp = pd;
371 for (i = 0; i < L2_SIZE; i++) {
372 pd[i].phys_offset = IO_MEM_UNASSIGNED;
373 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
374 }
375 }
376 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
377 }
378
379 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
380 {
381 return phys_page_find_alloc(index, 0);
382 }
383
384 #if !defined(CONFIG_USER_ONLY)
385 static void tlb_protect_code(ram_addr_t ram_addr);
386 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
387 target_ulong vaddr);
388 #define mmap_lock() do { } while(0)
389 #define mmap_unlock() do { } while(0)
390 #endif
391
392 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
393
394 #if defined(CONFIG_USER_ONLY)
395 /* Currently it is not recommanded to allocate big chunks of data in
396 user mode. It will change when a dedicated libc will be used */
397 #define USE_STATIC_CODE_GEN_BUFFER
398 #endif
399
400 #ifdef USE_STATIC_CODE_GEN_BUFFER
401 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
402 #endif
403
404 static void code_gen_alloc(unsigned long tb_size)
405 {
406 #ifdef USE_STATIC_CODE_GEN_BUFFER
407 code_gen_buffer = static_code_gen_buffer;
408 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
409 map_exec(code_gen_buffer, code_gen_buffer_size);
410 #else
411 code_gen_buffer_size = tb_size;
412 if (code_gen_buffer_size == 0) {
413 #if defined(CONFIG_USER_ONLY)
414 /* in user mode, phys_ram_size is not meaningful */
415 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
416 #else
417 /* XXX: needs ajustments */
418 code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
419 #endif
420 }
421 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
422 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
423 /* The code gen buffer location may have constraints depending on
424 the host cpu and OS */
425 #if defined(__linux__)
426 {
427 int flags;
428 void *start = NULL;
429
430 flags = MAP_PRIVATE | MAP_ANONYMOUS;
431 #if defined(__x86_64__)
432 flags |= MAP_32BIT;
433 /* Cannot map more than that */
434 if (code_gen_buffer_size > (800 * 1024 * 1024))
435 code_gen_buffer_size = (800 * 1024 * 1024);
436 #elif defined(__sparc_v9__)
437 // Map the buffer below 2G, so we can use direct calls and branches
438 flags |= MAP_FIXED;
439 start = (void *) 0x60000000UL;
440 if (code_gen_buffer_size > (512 * 1024 * 1024))
441 code_gen_buffer_size = (512 * 1024 * 1024);
442 #elif defined(__arm__)
443 /* Map the buffer below 32M, so we can use direct calls and branches */
444 flags |= MAP_FIXED;
445 start = (void *) 0x01000000UL;
446 if (code_gen_buffer_size > 16 * 1024 * 1024)
447 code_gen_buffer_size = 16 * 1024 * 1024;
448 #endif
449 code_gen_buffer = mmap(start, code_gen_buffer_size,
450 PROT_WRITE | PROT_READ | PROT_EXEC,
451 flags, -1, 0);
452 if (code_gen_buffer == MAP_FAILED) {
453 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
454 exit(1);
455 }
456 }
457 #elif defined(__FreeBSD__) || defined(__DragonFly__)
458 {
459 int flags;
460 void *addr = NULL;
461 flags = MAP_PRIVATE | MAP_ANONYMOUS;
462 #if defined(__x86_64__)
463 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
464 * 0x40000000 is free */
465 flags |= MAP_FIXED;
466 addr = (void *)0x40000000;
467 /* Cannot map more than that */
468 if (code_gen_buffer_size > (800 * 1024 * 1024))
469 code_gen_buffer_size = (800 * 1024 * 1024);
470 #endif
471 code_gen_buffer = mmap(addr, code_gen_buffer_size,
472 PROT_WRITE | PROT_READ | PROT_EXEC,
473 flags, -1, 0);
474 if (code_gen_buffer == MAP_FAILED) {
475 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
476 exit(1);
477 }
478 }
479 #else
480 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
481 map_exec(code_gen_buffer, code_gen_buffer_size);
482 #endif
483 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
484 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
485 code_gen_buffer_max_size = code_gen_buffer_size -
486 code_gen_max_block_size();
487 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
488 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
489 }
490
491 /* Must be called before using the QEMU cpus. 'tb_size' is the size
492 (in bytes) allocated to the translation buffer. Zero means default
493 size. */
494 void cpu_exec_init_all(unsigned long tb_size)
495 {
496 cpu_gen_init();
497 code_gen_alloc(tb_size);
498 code_gen_ptr = code_gen_buffer;
499 page_init();
500 #if !defined(CONFIG_USER_ONLY)
501 io_mem_init();
502 #endif
503 }
504
505 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
506
507 #define CPU_COMMON_SAVE_VERSION 1
508
509 static void cpu_common_save(QEMUFile *f, void *opaque)
510 {
511 CPUState *env = opaque;
512
513 qemu_put_be32s(f, &env->halted);
514 qemu_put_be32s(f, &env->interrupt_request);
515 }
516
517 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
518 {
519 CPUState *env = opaque;
520
521 if (version_id != CPU_COMMON_SAVE_VERSION)
522 return -EINVAL;
523
524 qemu_get_be32s(f, &env->halted);
525 qemu_get_be32s(f, &env->interrupt_request);
526 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
527 version_id is increased. */
528 env->interrupt_request &= ~0x01;
529 tlb_flush(env, 1);
530
531 return 0;
532 }
533 #endif
534
535 void cpu_exec_init(CPUState *env)
536 {
537 CPUState **penv;
538 int cpu_index;
539
540 #if defined(CONFIG_USER_ONLY)
541 cpu_list_lock();
542 #endif
543 env->next_cpu = NULL;
544 penv = &first_cpu;
545 cpu_index = 0;
546 while (*penv != NULL) {
547 penv = (CPUState **)&(*penv)->next_cpu;
548 cpu_index++;
549 }
550 env->cpu_index = cpu_index;
551 TAILQ_INIT(&env->breakpoints);
552 TAILQ_INIT(&env->watchpoints);
553 *penv = env;
554 #if defined(CONFIG_USER_ONLY)
555 cpu_list_unlock();
556 #endif
557 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
558 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
559 cpu_common_save, cpu_common_load, env);
560 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
561 cpu_save, cpu_load, env);
562 #endif
563 }
564
565 static inline void invalidate_page_bitmap(PageDesc *p)
566 {
567 if (p->code_bitmap) {
568 qemu_free(p->code_bitmap);
569 p->code_bitmap = NULL;
570 }
571 p->code_write_count = 0;
572 }
573
574 /* set to NULL all the 'first_tb' fields in all PageDescs */
575 static void page_flush_tb(void)
576 {
577 int i, j;
578 PageDesc *p;
579
580 for(i = 0; i < L1_SIZE; i++) {
581 p = l1_map[i];
582 if (p) {
583 for(j = 0; j < L2_SIZE; j++) {
584 p->first_tb = NULL;
585 invalidate_page_bitmap(p);
586 p++;
587 }
588 }
589 }
590 }
591
592 /* flush all the translation blocks */
593 /* XXX: tb_flush is currently not thread safe */
594 void tb_flush(CPUState *env1)
595 {
596 CPUState *env;
597 #if defined(DEBUG_FLUSH)
598 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
599 (unsigned long)(code_gen_ptr - code_gen_buffer),
600 nb_tbs, nb_tbs > 0 ?
601 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
602 #endif
603 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
604 cpu_abort(env1, "Internal error: code buffer overflow\n");
605
606 nb_tbs = 0;
607
608 for(env = first_cpu; env != NULL; env = env->next_cpu) {
609 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
610 }
611
612 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
613 page_flush_tb();
614
615 code_gen_ptr = code_gen_buffer;
616 /* XXX: flush processor icache at this point if cache flush is
617 expensive */
618 tb_flush_count++;
619 }
620
621 #ifdef DEBUG_TB_CHECK
622
623 static void tb_invalidate_check(target_ulong address)
624 {
625 TranslationBlock *tb;
626 int i;
627 address &= TARGET_PAGE_MASK;
628 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
629 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
630 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
631 address >= tb->pc + tb->size)) {
632 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
633 address, (long)tb->pc, tb->size);
634 }
635 }
636 }
637 }
638
639 /* verify that all the pages have correct rights for code */
640 static void tb_page_check(void)
641 {
642 TranslationBlock *tb;
643 int i, flags1, flags2;
644
645 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
646 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
647 flags1 = page_get_flags(tb->pc);
648 flags2 = page_get_flags(tb->pc + tb->size - 1);
649 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
650 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
651 (long)tb->pc, tb->size, flags1, flags2);
652 }
653 }
654 }
655 }
656
657 static void tb_jmp_check(TranslationBlock *tb)
658 {
659 TranslationBlock *tb1;
660 unsigned int n1;
661
662 /* suppress any remaining jumps to this TB */
663 tb1 = tb->jmp_first;
664 for(;;) {
665 n1 = (long)tb1 & 3;
666 tb1 = (TranslationBlock *)((long)tb1 & ~3);
667 if (n1 == 2)
668 break;
669 tb1 = tb1->jmp_next[n1];
670 }
671 /* check end of list */
672 if (tb1 != tb) {
673 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
674 }
675 }
676
677 #endif
678
679 /* invalidate one TB */
680 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
681 int next_offset)
682 {
683 TranslationBlock *tb1;
684 for(;;) {
685 tb1 = *ptb;
686 if (tb1 == tb) {
687 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
688 break;
689 }
690 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
691 }
692 }
693
694 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
695 {
696 TranslationBlock *tb1;
697 unsigned int n1;
698
699 for(;;) {
700 tb1 = *ptb;
701 n1 = (long)tb1 & 3;
702 tb1 = (TranslationBlock *)((long)tb1 & ~3);
703 if (tb1 == tb) {
704 *ptb = tb1->page_next[n1];
705 break;
706 }
707 ptb = &tb1->page_next[n1];
708 }
709 }
710
711 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
712 {
713 TranslationBlock *tb1, **ptb;
714 unsigned int n1;
715
716 ptb = &tb->jmp_next[n];
717 tb1 = *ptb;
718 if (tb1) {
719 /* find tb(n) in circular list */
720 for(;;) {
721 tb1 = *ptb;
722 n1 = (long)tb1 & 3;
723 tb1 = (TranslationBlock *)((long)tb1 & ~3);
724 if (n1 == n && tb1 == tb)
725 break;
726 if (n1 == 2) {
727 ptb = &tb1->jmp_first;
728 } else {
729 ptb = &tb1->jmp_next[n1];
730 }
731 }
732 /* now we can suppress tb(n) from the list */
733 *ptb = tb->jmp_next[n];
734
735 tb->jmp_next[n] = NULL;
736 }
737 }
738
739 /* reset the jump entry 'n' of a TB so that it is not chained to
740 another TB */
741 static inline void tb_reset_jump(TranslationBlock *tb, int n)
742 {
743 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
744 }
745
746 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
747 {
748 CPUState *env;
749 PageDesc *p;
750 unsigned int h, n1;
751 target_phys_addr_t phys_pc;
752 TranslationBlock *tb1, *tb2;
753
754 /* remove the TB from the hash list */
755 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
756 h = tb_phys_hash_func(phys_pc);
757 tb_remove(&tb_phys_hash[h], tb,
758 offsetof(TranslationBlock, phys_hash_next));
759
760 /* remove the TB from the page list */
761 if (tb->page_addr[0] != page_addr) {
762 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
763 tb_page_remove(&p->first_tb, tb);
764 invalidate_page_bitmap(p);
765 }
766 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
767 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
768 tb_page_remove(&p->first_tb, tb);
769 invalidate_page_bitmap(p);
770 }
771
772 tb_invalidated_flag = 1;
773
774 /* remove the TB from the hash list */
775 h = tb_jmp_cache_hash_func(tb->pc);
776 for(env = first_cpu; env != NULL; env = env->next_cpu) {
777 if (env->tb_jmp_cache[h] == tb)
778 env->tb_jmp_cache[h] = NULL;
779 }
780
781 /* suppress this TB from the two jump lists */
782 tb_jmp_remove(tb, 0);
783 tb_jmp_remove(tb, 1);
784
785 /* suppress any remaining jumps to this TB */
786 tb1 = tb->jmp_first;
787 for(;;) {
788 n1 = (long)tb1 & 3;
789 if (n1 == 2)
790 break;
791 tb1 = (TranslationBlock *)((long)tb1 & ~3);
792 tb2 = tb1->jmp_next[n1];
793 tb_reset_jump(tb1, n1);
794 tb1->jmp_next[n1] = NULL;
795 tb1 = tb2;
796 }
797 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
798
799 tb_phys_invalidate_count++;
800 }
801
802 static inline void set_bits(uint8_t *tab, int start, int len)
803 {
804 int end, mask, end1;
805
806 end = start + len;
807 tab += start >> 3;
808 mask = 0xff << (start & 7);
809 if ((start & ~7) == (end & ~7)) {
810 if (start < end) {
811 mask &= ~(0xff << (end & 7));
812 *tab |= mask;
813 }
814 } else {
815 *tab++ |= mask;
816 start = (start + 8) & ~7;
817 end1 = end & ~7;
818 while (start < end1) {
819 *tab++ = 0xff;
820 start += 8;
821 }
822 if (start < end) {
823 mask = ~(0xff << (end & 7));
824 *tab |= mask;
825 }
826 }
827 }
828
829 static void build_page_bitmap(PageDesc *p)
830 {
831 int n, tb_start, tb_end;
832 TranslationBlock *tb;
833
834 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
835
836 tb = p->first_tb;
837 while (tb != NULL) {
838 n = (long)tb & 3;
839 tb = (TranslationBlock *)((long)tb & ~3);
840 /* NOTE: this is subtle as a TB may span two physical pages */
841 if (n == 0) {
842 /* NOTE: tb_end may be after the end of the page, but
843 it is not a problem */
844 tb_start = tb->pc & ~TARGET_PAGE_MASK;
845 tb_end = tb_start + tb->size;
846 if (tb_end > TARGET_PAGE_SIZE)
847 tb_end = TARGET_PAGE_SIZE;
848 } else {
849 tb_start = 0;
850 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
851 }
852 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
853 tb = tb->page_next[n];
854 }
855 }
856
857 TranslationBlock *tb_gen_code(CPUState *env,
858 target_ulong pc, target_ulong cs_base,
859 int flags, int cflags)
860 {
861 TranslationBlock *tb;
862 uint8_t *tc_ptr;
863 target_ulong phys_pc, phys_page2, virt_page2;
864 int code_gen_size;
865
866 phys_pc = get_phys_addr_code(env, pc);
867 tb = tb_alloc(pc);
868 if (!tb) {
869 /* flush must be done */
870 tb_flush(env);
871 /* cannot fail at this point */
872 tb = tb_alloc(pc);
873 /* Don't forget to invalidate previous TB info. */
874 tb_invalidated_flag = 1;
875 }
876 tc_ptr = code_gen_ptr;
877 tb->tc_ptr = tc_ptr;
878 tb->cs_base = cs_base;
879 tb->flags = flags;
880 tb->cflags = cflags;
881 cpu_gen_code(env, tb, &code_gen_size);
882 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
883
884 /* check next page if needed */
885 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
886 phys_page2 = -1;
887 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
888 phys_page2 = get_phys_addr_code(env, virt_page2);
889 }
890 tb_link_phys(tb, phys_pc, phys_page2);
891 return tb;
892 }
893
894 /* invalidate all TBs which intersect with the target physical page
895 starting in range [start;end[. NOTE: start and end must refer to
896 the same physical page. 'is_cpu_write_access' should be true if called
897 from a real cpu write access: the virtual CPU will exit the current
898 TB if code is modified inside this TB. */
899 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
900 int is_cpu_write_access)
901 {
902 TranslationBlock *tb, *tb_next, *saved_tb;
903 CPUState *env = cpu_single_env;
904 target_ulong tb_start, tb_end;
905 PageDesc *p;
906 int n;
907 #ifdef TARGET_HAS_PRECISE_SMC
908 int current_tb_not_found = is_cpu_write_access;
909 TranslationBlock *current_tb = NULL;
910 int current_tb_modified = 0;
911 target_ulong current_pc = 0;
912 target_ulong current_cs_base = 0;
913 int current_flags = 0;
914 #endif /* TARGET_HAS_PRECISE_SMC */
915
916 p = page_find(start >> TARGET_PAGE_BITS);
917 if (!p)
918 return;
919 if (!p->code_bitmap &&
920 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
921 is_cpu_write_access) {
922 /* build code bitmap */
923 build_page_bitmap(p);
924 }
925
926 /* we remove all the TBs in the range [start, end[ */
927 /* XXX: see if in some cases it could be faster to invalidate all the code */
928 tb = p->first_tb;
929 while (tb != NULL) {
930 n = (long)tb & 3;
931 tb = (TranslationBlock *)((long)tb & ~3);
932 tb_next = tb->page_next[n];
933 /* NOTE: this is subtle as a TB may span two physical pages */
934 if (n == 0) {
935 /* NOTE: tb_end may be after the end of the page, but
936 it is not a problem */
937 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
938 tb_end = tb_start + tb->size;
939 } else {
940 tb_start = tb->page_addr[1];
941 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
942 }
943 if (!(tb_end <= start || tb_start >= end)) {
944 #ifdef TARGET_HAS_PRECISE_SMC
945 if (current_tb_not_found) {
946 current_tb_not_found = 0;
947 current_tb = NULL;
948 if (env->mem_io_pc) {
949 /* now we have a real cpu fault */
950 current_tb = tb_find_pc(env->mem_io_pc);
951 }
952 }
953 if (current_tb == tb &&
954 (current_tb->cflags & CF_COUNT_MASK) != 1) {
955 /* If we are modifying the current TB, we must stop
956 its execution. We could be more precise by checking
957 that the modification is after the current PC, but it
958 would require a specialized function to partially
959 restore the CPU state */
960
961 current_tb_modified = 1;
962 cpu_restore_state(current_tb, env,
963 env->mem_io_pc, NULL);
964 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
965 &current_flags);
966 }
967 #endif /* TARGET_HAS_PRECISE_SMC */
968 /* we need to do that to handle the case where a signal
969 occurs while doing tb_phys_invalidate() */
970 saved_tb = NULL;
971 if (env) {
972 saved_tb = env->current_tb;
973 env->current_tb = NULL;
974 }
975 tb_phys_invalidate(tb, -1);
976 if (env) {
977 env->current_tb = saved_tb;
978 if (env->interrupt_request && env->current_tb)
979 cpu_interrupt(env, env->interrupt_request);
980 }
981 }
982 tb = tb_next;
983 }
984 #if !defined(CONFIG_USER_ONLY)
985 /* if no code remaining, no need to continue to use slow writes */
986 if (!p->first_tb) {
987 invalidate_page_bitmap(p);
988 if (is_cpu_write_access) {
989 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
990 }
991 }
992 #endif
993 #ifdef TARGET_HAS_PRECISE_SMC
994 if (current_tb_modified) {
995 /* we generate a block containing just the instruction
996 modifying the memory. It will ensure that it cannot modify
997 itself */
998 env->current_tb = NULL;
999 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1000 cpu_resume_from_signal(env, NULL);
1001 }
1002 #endif
1003 }
1004
1005 /* len must be <= 8 and start must be a multiple of len */
1006 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1007 {
1008 PageDesc *p;
1009 int offset, b;
1010 #if 0
1011 if (1) {
1012 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1013 cpu_single_env->mem_io_vaddr, len,
1014 cpu_single_env->eip,
1015 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1016 }
1017 #endif
1018 p = page_find(start >> TARGET_PAGE_BITS);
1019 if (!p)
1020 return;
1021 if (p->code_bitmap) {
1022 offset = start & ~TARGET_PAGE_MASK;
1023 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1024 if (b & ((1 << len) - 1))
1025 goto do_invalidate;
1026 } else {
1027 do_invalidate:
1028 tb_invalidate_phys_page_range(start, start + len, 1);
1029 }
1030 }
1031
1032 #if !defined(CONFIG_SOFTMMU)
1033 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1034 unsigned long pc, void *puc)
1035 {
1036 TranslationBlock *tb;
1037 PageDesc *p;
1038 int n;
1039 #ifdef TARGET_HAS_PRECISE_SMC
1040 TranslationBlock *current_tb = NULL;
1041 CPUState *env = cpu_single_env;
1042 int current_tb_modified = 0;
1043 target_ulong current_pc = 0;
1044 target_ulong current_cs_base = 0;
1045 int current_flags = 0;
1046 #endif
1047
1048 addr &= TARGET_PAGE_MASK;
1049 p = page_find(addr >> TARGET_PAGE_BITS);
1050 if (!p)
1051 return;
1052 tb = p->first_tb;
1053 #ifdef TARGET_HAS_PRECISE_SMC
1054 if (tb && pc != 0) {
1055 current_tb = tb_find_pc(pc);
1056 }
1057 #endif
1058 while (tb != NULL) {
1059 n = (long)tb & 3;
1060 tb = (TranslationBlock *)((long)tb & ~3);
1061 #ifdef TARGET_HAS_PRECISE_SMC
1062 if (current_tb == tb &&
1063 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1064 /* If we are modifying the current TB, we must stop
1065 its execution. We could be more precise by checking
1066 that the modification is after the current PC, but it
1067 would require a specialized function to partially
1068 restore the CPU state */
1069
1070 current_tb_modified = 1;
1071 cpu_restore_state(current_tb, env, pc, puc);
1072 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1073 &current_flags);
1074 }
1075 #endif /* TARGET_HAS_PRECISE_SMC */
1076 tb_phys_invalidate(tb, addr);
1077 tb = tb->page_next[n];
1078 }
1079 p->first_tb = NULL;
1080 #ifdef TARGET_HAS_PRECISE_SMC
1081 if (current_tb_modified) {
1082 /* we generate a block containing just the instruction
1083 modifying the memory. It will ensure that it cannot modify
1084 itself */
1085 env->current_tb = NULL;
1086 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1087 cpu_resume_from_signal(env, puc);
1088 }
1089 #endif
1090 }
1091 #endif
1092
1093 /* add the tb in the target page and protect it if necessary */
1094 static inline void tb_alloc_page(TranslationBlock *tb,
1095 unsigned int n, target_ulong page_addr)
1096 {
1097 PageDesc *p;
1098 TranslationBlock *last_first_tb;
1099
1100 tb->page_addr[n] = page_addr;
1101 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1102 tb->page_next[n] = p->first_tb;
1103 last_first_tb = p->first_tb;
1104 p->first_tb = (TranslationBlock *)((long)tb | n);
1105 invalidate_page_bitmap(p);
1106
1107 #if defined(TARGET_HAS_SMC) || 1
1108
1109 #if defined(CONFIG_USER_ONLY)
1110 if (p->flags & PAGE_WRITE) {
1111 target_ulong addr;
1112 PageDesc *p2;
1113 int prot;
1114
1115 /* force the host page as non writable (writes will have a
1116 page fault + mprotect overhead) */
1117 page_addr &= qemu_host_page_mask;
1118 prot = 0;
1119 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1120 addr += TARGET_PAGE_SIZE) {
1121
1122 p2 = page_find (addr >> TARGET_PAGE_BITS);
1123 if (!p2)
1124 continue;
1125 prot |= p2->flags;
1126 p2->flags &= ~PAGE_WRITE;
1127 page_get_flags(addr);
1128 }
1129 mprotect(g2h(page_addr), qemu_host_page_size,
1130 (prot & PAGE_BITS) & ~PAGE_WRITE);
1131 #ifdef DEBUG_TB_INVALIDATE
1132 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1133 page_addr);
1134 #endif
1135 }
1136 #else
1137 /* if some code is already present, then the pages are already
1138 protected. So we handle the case where only the first TB is
1139 allocated in a physical page */
1140 if (!last_first_tb) {
1141 tlb_protect_code(page_addr);
1142 }
1143 #endif
1144
1145 #endif /* TARGET_HAS_SMC */
1146 }
1147
1148 /* Allocate a new translation block. Flush the translation buffer if
1149 too many translation blocks or too much generated code. */
1150 TranslationBlock *tb_alloc(target_ulong pc)
1151 {
1152 TranslationBlock *tb;
1153
1154 if (nb_tbs >= code_gen_max_blocks ||
1155 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1156 return NULL;
1157 tb = &tbs[nb_tbs++];
1158 tb->pc = pc;
1159 tb->cflags = 0;
1160 return tb;
1161 }
1162
1163 void tb_free(TranslationBlock *tb)
1164 {
1165 /* In practice this is mostly used for single use temporary TB
1166 Ignore the hard cases and just back up if this TB happens to
1167 be the last one generated. */
1168 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1169 code_gen_ptr = tb->tc_ptr;
1170 nb_tbs--;
1171 }
1172 }
1173
1174 /* add a new TB and link it to the physical page tables. phys_page2 is
1175 (-1) to indicate that only one page contains the TB. */
1176 void tb_link_phys(TranslationBlock *tb,
1177 target_ulong phys_pc, target_ulong phys_page2)
1178 {
1179 unsigned int h;
1180 TranslationBlock **ptb;
1181
1182 /* Grab the mmap lock to stop another thread invalidating this TB
1183 before we are done. */
1184 mmap_lock();
1185 /* add in the physical hash table */
1186 h = tb_phys_hash_func(phys_pc);
1187 ptb = &tb_phys_hash[h];
1188 tb->phys_hash_next = *ptb;
1189 *ptb = tb;
1190
1191 /* add in the page list */
1192 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1193 if (phys_page2 != -1)
1194 tb_alloc_page(tb, 1, phys_page2);
1195 else
1196 tb->page_addr[1] = -1;
1197
1198 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1199 tb->jmp_next[0] = NULL;
1200 tb->jmp_next[1] = NULL;
1201
1202 /* init original jump addresses */
1203 if (tb->tb_next_offset[0] != 0xffff)
1204 tb_reset_jump(tb, 0);
1205 if (tb->tb_next_offset[1] != 0xffff)
1206 tb_reset_jump(tb, 1);
1207
1208 #ifdef DEBUG_TB_CHECK
1209 tb_page_check();
1210 #endif
1211 mmap_unlock();
1212 }
1213
1214 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1215 tb[1].tc_ptr. Return NULL if not found */
1216 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1217 {
1218 int m_min, m_max, m;
1219 unsigned long v;
1220 TranslationBlock *tb;
1221
1222 if (nb_tbs <= 0)
1223 return NULL;
1224 if (tc_ptr < (unsigned long)code_gen_buffer ||
1225 tc_ptr >= (unsigned long)code_gen_ptr)
1226 return NULL;
1227 /* binary search (cf Knuth) */
1228 m_min = 0;
1229 m_max = nb_tbs - 1;
1230 while (m_min <= m_max) {
1231 m = (m_min + m_max) >> 1;
1232 tb = &tbs[m];
1233 v = (unsigned long)tb->tc_ptr;
1234 if (v == tc_ptr)
1235 return tb;
1236 else if (tc_ptr < v) {
1237 m_max = m - 1;
1238 } else {
1239 m_min = m + 1;
1240 }
1241 }
1242 return &tbs[m_max];
1243 }
1244
1245 static void tb_reset_jump_recursive(TranslationBlock *tb);
1246
1247 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1248 {
1249 TranslationBlock *tb1, *tb_next, **ptb;
1250 unsigned int n1;
1251
1252 tb1 = tb->jmp_next[n];
1253 if (tb1 != NULL) {
1254 /* find head of list */
1255 for(;;) {
1256 n1 = (long)tb1 & 3;
1257 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1258 if (n1 == 2)
1259 break;
1260 tb1 = tb1->jmp_next[n1];
1261 }
1262 /* we are now sure now that tb jumps to tb1 */
1263 tb_next = tb1;
1264
1265 /* remove tb from the jmp_first list */
1266 ptb = &tb_next->jmp_first;
1267 for(;;) {
1268 tb1 = *ptb;
1269 n1 = (long)tb1 & 3;
1270 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1271 if (n1 == n && tb1 == tb)
1272 break;
1273 ptb = &tb1->jmp_next[n1];
1274 }
1275 *ptb = tb->jmp_next[n];
1276 tb->jmp_next[n] = NULL;
1277
1278 /* suppress the jump to next tb in generated code */
1279 tb_reset_jump(tb, n);
1280
1281 /* suppress jumps in the tb on which we could have jumped */
1282 tb_reset_jump_recursive(tb_next);
1283 }
1284 }
1285
1286 static void tb_reset_jump_recursive(TranslationBlock *tb)
1287 {
1288 tb_reset_jump_recursive2(tb, 0);
1289 tb_reset_jump_recursive2(tb, 1);
1290 }
1291
1292 #if defined(TARGET_HAS_ICE)
1293 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1294 {
1295 target_phys_addr_t addr;
1296 target_ulong pd;
1297 ram_addr_t ram_addr;
1298 PhysPageDesc *p;
1299
1300 addr = cpu_get_phys_page_debug(env, pc);
1301 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1302 if (!p) {
1303 pd = IO_MEM_UNASSIGNED;
1304 } else {
1305 pd = p->phys_offset;
1306 }
1307 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1308 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1309 }
1310 #endif
1311
1312 /* Add a watchpoint. */
1313 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1314 int flags, CPUWatchpoint **watchpoint)
1315 {
1316 target_ulong len_mask = ~(len - 1);
1317 CPUWatchpoint *wp;
1318
1319 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1320 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1321 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1322 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1323 return -EINVAL;
1324 }
1325 wp = qemu_malloc(sizeof(*wp));
1326
1327 wp->vaddr = addr;
1328 wp->len_mask = len_mask;
1329 wp->flags = flags;
1330
1331 /* keep all GDB-injected watchpoints in front */
1332 if (flags & BP_GDB)
1333 TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1334 else
1335 TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1336
1337 tlb_flush_page(env, addr);
1338
1339 if (watchpoint)
1340 *watchpoint = wp;
1341 return 0;
1342 }
1343
1344 /* Remove a specific watchpoint. */
1345 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1346 int flags)
1347 {
1348 target_ulong len_mask = ~(len - 1);
1349 CPUWatchpoint *wp;
1350
1351 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1352 if (addr == wp->vaddr && len_mask == wp->len_mask
1353 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1354 cpu_watchpoint_remove_by_ref(env, wp);
1355 return 0;
1356 }
1357 }
1358 return -ENOENT;
1359 }
1360
1361 /* Remove a specific watchpoint by reference. */
1362 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1363 {
1364 TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1365
1366 tlb_flush_page(env, watchpoint->vaddr);
1367
1368 qemu_free(watchpoint);
1369 }
1370
1371 /* Remove all matching watchpoints. */
1372 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1373 {
1374 CPUWatchpoint *wp, *next;
1375
1376 TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1377 if (wp->flags & mask)
1378 cpu_watchpoint_remove_by_ref(env, wp);
1379 }
1380 }
1381
1382 /* Add a breakpoint. */
1383 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1384 CPUBreakpoint **breakpoint)
1385 {
1386 #if defined(TARGET_HAS_ICE)
1387 CPUBreakpoint *bp;
1388
1389 bp = qemu_malloc(sizeof(*bp));
1390
1391 bp->pc = pc;
1392 bp->flags = flags;
1393
1394 /* keep all GDB-injected breakpoints in front */
1395 if (flags & BP_GDB)
1396 TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1397 else
1398 TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1399
1400 breakpoint_invalidate(env, pc);
1401
1402 if (breakpoint)
1403 *breakpoint = bp;
1404 return 0;
1405 #else
1406 return -ENOSYS;
1407 #endif
1408 }
1409
1410 /* Remove a specific breakpoint. */
1411 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1412 {
1413 #if defined(TARGET_HAS_ICE)
1414 CPUBreakpoint *bp;
1415
1416 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1417 if (bp->pc == pc && bp->flags == flags) {
1418 cpu_breakpoint_remove_by_ref(env, bp);
1419 return 0;
1420 }
1421 }
1422 return -ENOENT;
1423 #else
1424 return -ENOSYS;
1425 #endif
1426 }
1427
1428 /* Remove a specific breakpoint by reference. */
1429 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1430 {
1431 #if defined(TARGET_HAS_ICE)
1432 TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1433
1434 breakpoint_invalidate(env, breakpoint->pc);
1435
1436 qemu_free(breakpoint);
1437 #endif
1438 }
1439
1440 /* Remove all matching breakpoints. */
1441 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1442 {
1443 #if defined(TARGET_HAS_ICE)
1444 CPUBreakpoint *bp, *next;
1445
1446 TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1447 if (bp->flags & mask)
1448 cpu_breakpoint_remove_by_ref(env, bp);
1449 }
1450 #endif
1451 }
1452
1453 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1454 CPU loop after each instruction */
1455 void cpu_single_step(CPUState *env, int enabled)
1456 {
1457 #if defined(TARGET_HAS_ICE)
1458 if (env->singlestep_enabled != enabled) {
1459 env->singlestep_enabled = enabled;
1460 if (kvm_enabled())
1461 kvm_update_guest_debug(env, 0);
1462 else {
1463 /* must flush all the translated code to avoid inconsistancies */
1464 /* XXX: only flush what is necessary */
1465 tb_flush(env);
1466 }
1467 }
1468 #endif
1469 }
1470
1471 /* enable or disable low levels log */
1472 void cpu_set_log(int log_flags)
1473 {
1474 loglevel = log_flags;
1475 if (loglevel && !logfile) {
1476 logfile = fopen(logfilename, log_append ? "a" : "w");
1477 if (!logfile) {
1478 perror(logfilename);
1479 _exit(1);
1480 }
1481 #if !defined(CONFIG_SOFTMMU)
1482 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1483 {
1484 static char logfile_buf[4096];
1485 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1486 }
1487 #else
1488 setvbuf(logfile, NULL, _IOLBF, 0);
1489 #endif
1490 log_append = 1;
1491 }
1492 if (!loglevel && logfile) {
1493 fclose(logfile);
1494 logfile = NULL;
1495 }
1496 }
1497
1498 void cpu_set_log_filename(const char *filename)
1499 {
1500 logfilename = strdup(filename);
1501 if (logfile) {
1502 fclose(logfile);
1503 logfile = NULL;
1504 }
1505 cpu_set_log(loglevel);
1506 }
1507
1508 static void cpu_unlink_tb(CPUState *env)
1509 {
1510 #if defined(USE_NPTL)
1511 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1512 problem and hope the cpu will stop of its own accord. For userspace
1513 emulation this often isn't actually as bad as it sounds. Often
1514 signals are used primarily to interrupt blocking syscalls. */
1515 #else
1516 TranslationBlock *tb;
1517 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1518
1519 tb = env->current_tb;
1520 /* if the cpu is currently executing code, we must unlink it and
1521 all the potentially executing TB */
1522 if (tb && !testandset(&interrupt_lock)) {
1523 env->current_tb = NULL;
1524 tb_reset_jump_recursive(tb);
1525 resetlock(&interrupt_lock);
1526 }
1527 #endif
1528 }
1529
1530 /* mask must never be zero, except for A20 change call */
1531 void cpu_interrupt(CPUState *env, int mask)
1532 {
1533 int old_mask;
1534
1535 old_mask = env->interrupt_request;
1536 env->interrupt_request |= mask;
1537
1538 if (use_icount) {
1539 env->icount_decr.u16.high = 0xffff;
1540 #ifndef CONFIG_USER_ONLY
1541 if (!can_do_io(env)
1542 && (mask & ~old_mask) != 0) {
1543 cpu_abort(env, "Raised interrupt while not in I/O function");
1544 }
1545 #endif
1546 } else {
1547 cpu_unlink_tb(env);
1548 }
1549 }
1550
1551 void cpu_reset_interrupt(CPUState *env, int mask)
1552 {
1553 env->interrupt_request &= ~mask;
1554 }
1555
1556 void cpu_exit(CPUState *env)
1557 {
1558 env->exit_request = 1;
1559 cpu_unlink_tb(env);
1560 }
1561
1562 const CPULogItem cpu_log_items[] = {
1563 { CPU_LOG_TB_OUT_ASM, "out_asm",
1564 "show generated host assembly code for each compiled TB" },
1565 { CPU_LOG_TB_IN_ASM, "in_asm",
1566 "show target assembly code for each compiled TB" },
1567 { CPU_LOG_TB_OP, "op",
1568 "show micro ops for each compiled TB" },
1569 { CPU_LOG_TB_OP_OPT, "op_opt",
1570 "show micro ops "
1571 #ifdef TARGET_I386
1572 "before eflags optimization and "
1573 #endif
1574 "after liveness analysis" },
1575 { CPU_LOG_INT, "int",
1576 "show interrupts/exceptions in short format" },
1577 { CPU_LOG_EXEC, "exec",
1578 "show trace before each executed TB (lots of logs)" },
1579 { CPU_LOG_TB_CPU, "cpu",
1580 "show CPU state before block translation" },
1581 #ifdef TARGET_I386
1582 { CPU_LOG_PCALL, "pcall",
1583 "show protected mode far calls/returns/exceptions" },
1584 { CPU_LOG_RESET, "cpu_reset",
1585 "show CPU state before CPU resets" },
1586 #endif
1587 #ifdef DEBUG_IOPORT
1588 { CPU_LOG_IOPORT, "ioport",
1589 "show all i/o ports accesses" },
1590 #endif
1591 { 0, NULL, NULL },
1592 };
1593
1594 static int cmp1(const char *s1, int n, const char *s2)
1595 {
1596 if (strlen(s2) != n)
1597 return 0;
1598 return memcmp(s1, s2, n) == 0;
1599 }
1600
1601 /* takes a comma separated list of log masks. Return 0 if error. */
1602 int cpu_str_to_log_mask(const char *str)
1603 {
1604 const CPULogItem *item;
1605 int mask;
1606 const char *p, *p1;
1607
1608 p = str;
1609 mask = 0;
1610 for(;;) {
1611 p1 = strchr(p, ',');
1612 if (!p1)
1613 p1 = p + strlen(p);
1614 if(cmp1(p,p1-p,"all")) {
1615 for(item = cpu_log_items; item->mask != 0; item++) {
1616 mask |= item->mask;
1617 }
1618 } else {
1619 for(item = cpu_log_items; item->mask != 0; item++) {
1620 if (cmp1(p, p1 - p, item->name))
1621 goto found;
1622 }
1623 return 0;
1624 }
1625 found:
1626 mask |= item->mask;
1627 if (*p1 != ',')
1628 break;
1629 p = p1 + 1;
1630 }
1631 return mask;
1632 }
1633
1634 void cpu_abort(CPUState *env, const char *fmt, ...)
1635 {
1636 va_list ap;
1637 va_list ap2;
1638
1639 va_start(ap, fmt);
1640 va_copy(ap2, ap);
1641 fprintf(stderr, "qemu: fatal: ");
1642 vfprintf(stderr, fmt, ap);
1643 fprintf(stderr, "\n");
1644 #ifdef TARGET_I386
1645 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1646 #else
1647 cpu_dump_state(env, stderr, fprintf, 0);
1648 #endif
1649 if (qemu_log_enabled()) {
1650 qemu_log("qemu: fatal: ");
1651 qemu_log_vprintf(fmt, ap2);
1652 qemu_log("\n");
1653 #ifdef TARGET_I386
1654 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1655 #else
1656 log_cpu_state(env, 0);
1657 #endif
1658 qemu_log_flush();
1659 qemu_log_close();
1660 }
1661 va_end(ap2);
1662 va_end(ap);
1663 abort();
1664 }
1665
1666 CPUState *cpu_copy(CPUState *env)
1667 {
1668 CPUState *new_env = cpu_init(env->cpu_model_str);
1669 CPUState *next_cpu = new_env->next_cpu;
1670 int cpu_index = new_env->cpu_index;
1671 #if defined(TARGET_HAS_ICE)
1672 CPUBreakpoint *bp;
1673 CPUWatchpoint *wp;
1674 #endif
1675
1676 memcpy(new_env, env, sizeof(CPUState));
1677
1678 /* Preserve chaining and index. */
1679 new_env->next_cpu = next_cpu;
1680 new_env->cpu_index = cpu_index;
1681
1682 /* Clone all break/watchpoints.
1683 Note: Once we support ptrace with hw-debug register access, make sure
1684 BP_CPU break/watchpoints are handled correctly on clone. */
1685 TAILQ_INIT(&env->breakpoints);
1686 TAILQ_INIT(&env->watchpoints);
1687 #if defined(TARGET_HAS_ICE)
1688 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1689 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1690 }
1691 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1692 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1693 wp->flags, NULL);
1694 }
1695 #endif
1696
1697 return new_env;
1698 }
1699
1700 #if !defined(CONFIG_USER_ONLY)
1701
1702 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1703 {
1704 unsigned int i;
1705
1706 /* Discard jump cache entries for any tb which might potentially
1707 overlap the flushed page. */
1708 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1709 memset (&env->tb_jmp_cache[i], 0,
1710 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1711
1712 i = tb_jmp_cache_hash_page(addr);
1713 memset (&env->tb_jmp_cache[i], 0,
1714 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1715 }
1716
1717 /* NOTE: if flush_global is true, also flush global entries (not
1718 implemented yet) */
1719 void tlb_flush(CPUState *env, int flush_global)
1720 {
1721 int i;
1722
1723 #if defined(DEBUG_TLB)
1724 printf("tlb_flush:\n");
1725 #endif
1726 /* must reset current TB so that interrupts cannot modify the
1727 links while we are modifying them */
1728 env->current_tb = NULL;
1729
1730 for(i = 0; i < CPU_TLB_SIZE; i++) {
1731 env->tlb_table[0][i].addr_read = -1;
1732 env->tlb_table[0][i].addr_write = -1;
1733 env->tlb_table[0][i].addr_code = -1;
1734 env->tlb_table[1][i].addr_read = -1;
1735 env->tlb_table[1][i].addr_write = -1;
1736 env->tlb_table[1][i].addr_code = -1;
1737 #if (NB_MMU_MODES >= 3)
1738 env->tlb_table[2][i].addr_read = -1;
1739 env->tlb_table[2][i].addr_write = -1;
1740 env->tlb_table[2][i].addr_code = -1;
1741 #if (NB_MMU_MODES == 4)
1742 env->tlb_table[3][i].addr_read = -1;
1743 env->tlb_table[3][i].addr_write = -1;
1744 env->tlb_table[3][i].addr_code = -1;
1745 #endif
1746 #endif
1747 }
1748
1749 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1750
1751 #ifdef USE_KQEMU
1752 if (env->kqemu_enabled) {
1753 kqemu_flush(env, flush_global);
1754 }
1755 #endif
1756 tlb_flush_count++;
1757 }
1758
1759 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1760 {
1761 if (addr == (tlb_entry->addr_read &
1762 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1763 addr == (tlb_entry->addr_write &
1764 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1765 addr == (tlb_entry->addr_code &
1766 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1767 tlb_entry->addr_read = -1;
1768 tlb_entry->addr_write = -1;
1769 tlb_entry->addr_code = -1;
1770 }
1771 }
1772
1773 void tlb_flush_page(CPUState *env, target_ulong addr)
1774 {
1775 int i;
1776
1777 #if defined(DEBUG_TLB)
1778 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1779 #endif
1780 /* must reset current TB so that interrupts cannot modify the
1781 links while we are modifying them */
1782 env->current_tb = NULL;
1783
1784 addr &= TARGET_PAGE_MASK;
1785 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1786 tlb_flush_entry(&env->tlb_table[0][i], addr);
1787 tlb_flush_entry(&env->tlb_table[1][i], addr);
1788 #if (NB_MMU_MODES >= 3)
1789 tlb_flush_entry(&env->tlb_table[2][i], addr);
1790 #if (NB_MMU_MODES == 4)
1791 tlb_flush_entry(&env->tlb_table[3][i], addr);
1792 #endif
1793 #endif
1794
1795 tlb_flush_jmp_cache(env, addr);
1796
1797 #ifdef USE_KQEMU
1798 if (env->kqemu_enabled) {
1799 kqemu_flush_page(env, addr);
1800 }
1801 #endif
1802 }
1803
1804 /* update the TLBs so that writes to code in the virtual page 'addr'
1805 can be detected */
1806 static void tlb_protect_code(ram_addr_t ram_addr)
1807 {
1808 cpu_physical_memory_reset_dirty(ram_addr,
1809 ram_addr + TARGET_PAGE_SIZE,
1810 CODE_DIRTY_FLAG);
1811 }
1812
1813 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1814 tested for self modifying code */
1815 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1816 target_ulong vaddr)
1817 {
1818 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1819 }
1820
1821 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1822 unsigned long start, unsigned long length)
1823 {
1824 unsigned long addr;
1825 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1826 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1827 if ((addr - start) < length) {
1828 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1829 }
1830 }
1831 }
1832
1833 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1834 int dirty_flags)
1835 {
1836 CPUState *env;
1837 unsigned long length, start1;
1838 int i, mask, len;
1839 uint8_t *p;
1840
1841 start &= TARGET_PAGE_MASK;
1842 end = TARGET_PAGE_ALIGN(end);
1843
1844 length = end - start;
1845 if (length == 0)
1846 return;
1847 len = length >> TARGET_PAGE_BITS;
1848 #ifdef USE_KQEMU
1849 /* XXX: should not depend on cpu context */
1850 env = first_cpu;
1851 if (env->kqemu_enabled) {
1852 ram_addr_t addr;
1853 addr = start;
1854 for(i = 0; i < len; i++) {
1855 kqemu_set_notdirty(env, addr);
1856 addr += TARGET_PAGE_SIZE;
1857 }
1858 }
1859 #endif
1860 mask = ~dirty_flags;
1861 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1862 for(i = 0; i < len; i++)
1863 p[i] &= mask;
1864
1865 /* we modify the TLB cache so that the dirty bit will be set again
1866 when accessing the range */
1867 start1 = start + (unsigned long)phys_ram_base;
1868 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1869 for(i = 0; i < CPU_TLB_SIZE; i++)
1870 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1871 for(i = 0; i < CPU_TLB_SIZE; i++)
1872 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1873 #if (NB_MMU_MODES >= 3)
1874 for(i = 0; i < CPU_TLB_SIZE; i++)
1875 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1876 #if (NB_MMU_MODES == 4)
1877 for(i = 0; i < CPU_TLB_SIZE; i++)
1878 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1879 #endif
1880 #endif
1881 }
1882 }
1883
1884 int cpu_physical_memory_set_dirty_tracking(int enable)
1885 {
1886 in_migration = enable;
1887 return 0;
1888 }
1889
1890 int cpu_physical_memory_get_dirty_tracking(void)
1891 {
1892 return in_migration;
1893 }
1894
1895 void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
1896 {
1897 if (kvm_enabled())
1898 kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1899 }
1900
1901 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1902 {
1903 ram_addr_t ram_addr;
1904
1905 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1906 ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1907 tlb_entry->addend - (unsigned long)phys_ram_base;
1908 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1909 tlb_entry->addr_write |= TLB_NOTDIRTY;
1910 }
1911 }
1912 }
1913
1914 /* update the TLB according to the current state of the dirty bits */
1915 void cpu_tlb_update_dirty(CPUState *env)
1916 {
1917 int i;
1918 for(i = 0; i < CPU_TLB_SIZE; i++)
1919 tlb_update_dirty(&env->tlb_table[0][i]);
1920 for(i = 0; i < CPU_TLB_SIZE; i++)
1921 tlb_update_dirty(&env->tlb_table[1][i]);
1922 #if (NB_MMU_MODES >= 3)
1923 for(i = 0; i < CPU_TLB_SIZE; i++)
1924 tlb_update_dirty(&env->tlb_table[2][i]);
1925 #if (NB_MMU_MODES == 4)
1926 for(i = 0; i < CPU_TLB_SIZE; i++)
1927 tlb_update_dirty(&env->tlb_table[3][i]);
1928 #endif
1929 #endif
1930 }
1931
1932 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1933 {
1934 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1935 tlb_entry->addr_write = vaddr;
1936 }
1937
1938 /* update the TLB corresponding to virtual page vaddr
1939 so that it is no longer dirty */
1940 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1941 {
1942 int i;
1943
1944 vaddr &= TARGET_PAGE_MASK;
1945 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1946 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1947 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1948 #if (NB_MMU_MODES >= 3)
1949 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1950 #if (NB_MMU_MODES == 4)
1951 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1952 #endif
1953 #endif
1954 }
1955
1956 /* add a new TLB entry. At most one entry for a given virtual address
1957 is permitted. Return 0 if OK or 2 if the page could not be mapped
1958 (can only happen in non SOFTMMU mode for I/O pages or pages
1959 conflicting with the host address space). */
1960 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1961 target_phys_addr_t paddr, int prot,
1962 int mmu_idx, int is_softmmu)
1963 {
1964 PhysPageDesc *p;
1965 unsigned long pd;
1966 unsigned int index;
1967 target_ulong address;
1968 target_ulong code_address;
1969 target_phys_addr_t addend;
1970 int ret;
1971 CPUTLBEntry *te;
1972 CPUWatchpoint *wp;
1973 target_phys_addr_t iotlb;
1974
1975 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1976 if (!p) {
1977 pd = IO_MEM_UNASSIGNED;
1978 } else {
1979 pd = p->phys_offset;
1980 }
1981 #if defined(DEBUG_TLB)
1982 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1983 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1984 #endif
1985
1986 ret = 0;
1987 address = vaddr;
1988 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1989 /* IO memory case (romd handled later) */
1990 address |= TLB_MMIO;
1991 }
1992 addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
1993 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
1994 /* Normal RAM. */
1995 iotlb = pd & TARGET_PAGE_MASK;
1996 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
1997 iotlb |= IO_MEM_NOTDIRTY;
1998 else
1999 iotlb |= IO_MEM_ROM;
2000 } else {
2001 /* IO handlers are currently passed a phsical address.
2002 It would be nice to pass an offset from the base address
2003 of that region. This would avoid having to special case RAM,
2004 and avoid full address decoding in every device.
2005 We can't use the high bits of pd for this because
2006 IO_MEM_ROMD uses these as a ram address. */
2007 iotlb = (pd & ~TARGET_PAGE_MASK);
2008 if (p) {
2009 iotlb += p->region_offset;
2010 } else {
2011 iotlb += paddr;
2012 }
2013 }
2014
2015 code_address = address;
2016 /* Make accesses to pages with watchpoints go via the
2017 watchpoint trap routines. */
2018 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2019 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2020 iotlb = io_mem_watch + paddr;
2021 /* TODO: The memory case can be optimized by not trapping
2022 reads of pages with a write breakpoint. */
2023 address |= TLB_MMIO;
2024 }
2025 }
2026
2027 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2028 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2029 te = &env->tlb_table[mmu_idx][index];
2030 te->addend = addend - vaddr;
2031 if (prot & PAGE_READ) {
2032 te->addr_read = address;
2033 } else {
2034 te->addr_read = -1;
2035 }
2036
2037 if (prot & PAGE_EXEC) {
2038 te->addr_code = code_address;
2039 } else {
2040 te->addr_code = -1;
2041 }
2042 if (prot & PAGE_WRITE) {
2043 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2044 (pd & IO_MEM_ROMD)) {
2045 /* Write access calls the I/O callback. */
2046 te->addr_write = address | TLB_MMIO;
2047 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2048 !cpu_physical_memory_is_dirty(pd)) {
2049 te->addr_write = address | TLB_NOTDIRTY;
2050 } else {
2051 te->addr_write = address;
2052 }
2053 } else {
2054 te->addr_write = -1;
2055 }
2056 return ret;
2057 }
2058
2059 #else
2060
2061 void tlb_flush(CPUState *env, int flush_global)
2062 {
2063 }
2064
2065 void tlb_flush_page(CPUState *env, target_ulong addr)
2066 {
2067 }
2068
2069 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2070 target_phys_addr_t paddr, int prot,
2071 int mmu_idx, int is_softmmu)
2072 {
2073 return 0;
2074 }
2075
2076 /* dump memory mappings */
2077 void page_dump(FILE *f)
2078 {
2079 unsigned long start, end;
2080 int i, j, prot, prot1;
2081 PageDesc *p;
2082
2083 fprintf(f, "%-8s %-8s %-8s %s\n",
2084 "start", "end", "size", "prot");
2085 start = -1;
2086 end = -1;
2087 prot = 0;
2088 for(i = 0; i <= L1_SIZE; i++) {
2089 if (i < L1_SIZE)
2090 p = l1_map[i];
2091 else
2092 p = NULL;
2093 for(j = 0;j < L2_SIZE; j++) {
2094 if (!p)
2095 prot1 = 0;
2096 else
2097 prot1 = p[j].flags;
2098 if (prot1 != prot) {
2099 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2100 if (start != -1) {
2101 fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2102 start, end, end - start,
2103 prot & PAGE_READ ? 'r' : '-',
2104 prot & PAGE_WRITE ? 'w' : '-',
2105 prot & PAGE_EXEC ? 'x' : '-');
2106 }
2107 if (prot1 != 0)
2108 start = end;
2109 else
2110 start = -1;
2111 prot = prot1;
2112 }
2113 if (!p)
2114 break;
2115 }
2116 }
2117 }
2118
2119 int page_get_flags(target_ulong address)
2120 {
2121 PageDesc *p;
2122
2123 p = page_find(address >> TARGET_PAGE_BITS);
2124 if (!p)
2125 return 0;
2126 return p->flags;
2127 }
2128
2129 /* modify the flags of a page and invalidate the code if
2130 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2131 depending on PAGE_WRITE */
2132 void page_set_flags(target_ulong start, target_ulong end, int flags)
2133 {
2134 PageDesc *p;
2135 target_ulong addr;
2136
2137 /* mmap_lock should already be held. */
2138 start = start & TARGET_PAGE_MASK;
2139 end = TARGET_PAGE_ALIGN(end);
2140 if (flags & PAGE_WRITE)
2141 flags |= PAGE_WRITE_ORG;
2142 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2143 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2144 /* We may be called for host regions that are outside guest
2145 address space. */
2146 if (!p)
2147 return;
2148 /* if the write protection is set, then we invalidate the code
2149 inside */
2150 if (!(p->flags & PAGE_WRITE) &&
2151 (flags & PAGE_WRITE) &&
2152 p->first_tb) {
2153 tb_invalidate_phys_page(addr, 0, NULL);
2154 }
2155 p->flags = flags;
2156 }
2157 }
2158
2159 int page_check_range(target_ulong start, target_ulong len, int flags)
2160 {
2161 PageDesc *p;
2162 target_ulong end;
2163 target_ulong addr;
2164
2165 if (start + len < start)
2166 /* we've wrapped around */
2167 return -1;
2168
2169 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2170 start = start & TARGET_PAGE_MASK;
2171
2172 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2173 p = page_find(addr >> TARGET_PAGE_BITS);
2174 if( !p )
2175 return -1;
2176 if( !(p->flags & PAGE_VALID) )
2177 return -1;
2178
2179 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2180 return -1;
2181 if (flags & PAGE_WRITE) {
2182 if (!(p->flags & PAGE_WRITE_ORG))
2183 return -1;
2184 /* unprotect the page if it was put read-only because it
2185 contains translated code */
2186 if (!(p->flags & PAGE_WRITE)) {
2187 if (!page_unprotect(addr, 0, NULL))
2188 return -1;
2189 }
2190 return 0;
2191 }
2192 }
2193 return 0;
2194 }
2195
2196 /* called from signal handler: invalidate the code and unprotect the
2197 page. Return TRUE if the fault was succesfully handled. */
2198 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2199 {
2200 unsigned int page_index, prot, pindex;
2201 PageDesc *p, *p1;
2202 target_ulong host_start, host_end, addr;
2203
2204 /* Technically this isn't safe inside a signal handler. However we
2205 know this only ever happens in a synchronous SEGV handler, so in
2206 practice it seems to be ok. */
2207 mmap_lock();
2208
2209 host_start = address & qemu_host_page_mask;
2210 page_index = host_start >> TARGET_PAGE_BITS;
2211 p1 = page_find(page_index);
2212 if (!p1) {
2213 mmap_unlock();
2214 return 0;
2215 }
2216 host_end = host_start + qemu_host_page_size;
2217 p = p1;
2218 prot = 0;
2219 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2220 prot |= p->flags;
2221 p++;
2222 }
2223 /* if the page was really writable, then we change its
2224 protection back to writable */
2225 if (prot & PAGE_WRITE_ORG) {
2226 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2227 if (!(p1[pindex].flags & PAGE_WRITE)) {
2228 mprotect((void *)g2h(host_start), qemu_host_page_size,
2229 (prot & PAGE_BITS) | PAGE_WRITE);
2230 p1[pindex].flags |= PAGE_WRITE;
2231 /* and since the content will be modified, we must invalidate
2232 the corresponding translated code. */
2233 tb_invalidate_phys_page(address, pc, puc);
2234 #ifdef DEBUG_TB_CHECK
2235 tb_invalidate_check(address);
2236 #endif
2237 mmap_unlock();
2238 return 1;
2239 }
2240 }
2241 mmap_unlock();
2242 return 0;
2243 }
2244
2245 static inline void tlb_set_dirty(CPUState *env,
2246 unsigned long addr, target_ulong vaddr)
2247 {
2248 }
2249 #endif /* defined(CONFIG_USER_ONLY) */
2250
2251 #if !defined(CONFIG_USER_ONLY)
2252
2253 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2254 ram_addr_t memory, ram_addr_t region_offset);
2255 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2256 ram_addr_t orig_memory, ram_addr_t region_offset);
2257 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2258 need_subpage) \
2259 do { \
2260 if (addr > start_addr) \
2261 start_addr2 = 0; \
2262 else { \
2263 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2264 if (start_addr2 > 0) \
2265 need_subpage = 1; \
2266 } \
2267 \
2268 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2269 end_addr2 = TARGET_PAGE_SIZE - 1; \
2270 else { \
2271 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2272 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2273 need_subpage = 1; \
2274 } \
2275 } while (0)
2276
2277 /* register physical memory. 'size' must be a multiple of the target
2278 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2279 io memory page. The address used when calling the IO function is
2280 the offset from the start of the region, plus region_offset. Both
2281 start_region and regon_offset are rounded down to a page boundary
2282 before calculating this offset. This should not be a problem unless
2283 the low bits of start_addr and region_offset differ. */
2284 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2285 ram_addr_t size,
2286 ram_addr_t phys_offset,
2287 ram_addr_t region_offset)
2288 {
2289 target_phys_addr_t addr, end_addr;
2290 PhysPageDesc *p;
2291 CPUState *env;
2292 ram_addr_t orig_size = size;
2293 void *subpage;
2294
2295 #ifdef USE_KQEMU
2296 /* XXX: should not depend on cpu context */
2297 env = first_cpu;
2298 if (env->kqemu_enabled) {
2299 kqemu_set_phys_mem(start_addr, size, phys_offset);
2300 }
2301 #endif
2302 if (kvm_enabled())
2303 kvm_set_phys_mem(start_addr, size, phys_offset);
2304
2305 if (phys_offset == IO_MEM_UNASSIGNED) {
2306 region_offset = start_addr;
2307 }
2308 region_offset &= TARGET_PAGE_MASK;
2309 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2310 end_addr = start_addr + (target_phys_addr_t)size;
2311 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2312 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2313 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2314 ram_addr_t orig_memory = p->phys_offset;
2315 target_phys_addr_t start_addr2, end_addr2;
2316 int need_subpage = 0;
2317
2318 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2319 need_subpage);
2320 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2321 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2322 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2323 &p->phys_offset, orig_memory,
2324 p->region_offset);
2325 } else {
2326 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2327 >> IO_MEM_SHIFT];
2328 }
2329 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2330 region_offset);
2331 p->region_offset = 0;
2332 } else {
2333 p->phys_offset = phys_offset;
2334 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2335 (phys_offset & IO_MEM_ROMD))
2336 phys_offset += TARGET_PAGE_SIZE;
2337 }
2338 } else {
2339 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2340 p->phys_offset = phys_offset;
2341 p->region_offset = region_offset;
2342 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2343 (phys_offset & IO_MEM_ROMD)) {
2344 phys_offset += TARGET_PAGE_SIZE;
2345 } else {
2346 target_phys_addr_t start_addr2, end_addr2;
2347 int need_subpage = 0;
2348
2349 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2350 end_addr2, need_subpage);
2351
2352 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2353 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2354 &p->phys_offset, IO_MEM_UNASSIGNED,
2355 addr & TARGET_PAGE_MASK);
2356 subpage_register(subpage, start_addr2, end_addr2,
2357 phys_offset, region_offset);
2358 p->region_offset = 0;
2359 }
2360 }
2361 }
2362 region_offset += TARGET_PAGE_SIZE;
2363 }
2364
2365 /* since each CPU stores ram addresses in its TLB cache, we must
2366 reset the modified entries */
2367 /* XXX: slow ! */
2368 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2369 tlb_flush(env, 1);
2370 }
2371 }
2372
2373 /* XXX: temporary until new memory mapping API */
2374 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2375 {
2376 PhysPageDesc *p;
2377
2378 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2379 if (!p)
2380 return IO_MEM_UNASSIGNED;
2381 return p->phys_offset;
2382 }
2383
2384 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2385 {
2386 if (kvm_enabled())
2387 kvm_coalesce_mmio_region(addr, size);
2388 }
2389
2390 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2391 {
2392 if (kvm_enabled())
2393 kvm_uncoalesce_mmio_region(addr, size);
2394 }
2395
2396 /* XXX: better than nothing */
2397 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2398 {
2399 ram_addr_t addr;
2400 if ((phys_ram_alloc_offset + size) > phys_ram_size) {
2401 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2402 (uint64_t)size, (uint64_t)phys_ram_size);
2403 abort();
2404 }
2405 addr = phys_ram_alloc_offset;
2406 phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2407 return addr;
2408 }
2409
2410 void qemu_ram_free(ram_addr_t addr)
2411 {
2412 }
2413
2414 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2415 {
2416 #ifdef DEBUG_UNASSIGNED
2417 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2418 #endif
2419 #if defined(TARGET_SPARC)
2420 do_unassigned_access(addr, 0, 0, 0, 1);
2421 #endif
2422 return 0;
2423 }
2424
2425 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2426 {
2427 #ifdef DEBUG_UNASSIGNED
2428 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2429 #endif
2430 #if defined(TARGET_SPARC)
2431 do_unassigned_access(addr, 0, 0, 0, 2);
2432 #endif
2433 return 0;
2434 }
2435
2436 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2437 {
2438 #ifdef DEBUG_UNASSIGNED
2439 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2440 #endif
2441 #if defined(TARGET_SPARC)
2442 do_unassigned_access(addr, 0, 0, 0, 4);
2443 #endif
2444 return 0;
2445 }
2446
2447 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2448 {
2449 #ifdef DEBUG_UNASSIGNED
2450 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2451 #endif
2452 #if defined(TARGET_SPARC)
2453 do_unassigned_access(addr, 1, 0, 0, 1);
2454 #endif
2455 }
2456
2457 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2458 {
2459 #ifdef DEBUG_UNASSIGNED
2460 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2461 #endif
2462 #if defined(TARGET_SPARC)
2463 do_unassigned_access(addr, 1, 0, 0, 2);
2464 #endif
2465 }
2466
2467 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2468 {
2469 #ifdef DEBUG_UNASSIGNED
2470 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2471 #endif
2472 #if defined(TARGET_SPARC)
2473 do_unassigned_access(addr, 1, 0, 0, 4);
2474 #endif
2475 }
2476
2477 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2478 unassigned_mem_readb,
2479 unassigned_mem_readw,
2480 unassigned_mem_readl,
2481 };
2482
2483 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2484 unassigned_mem_writeb,
2485 unassigned_mem_writew,
2486 unassigned_mem_writel,
2487 };
2488
2489 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2490 uint32_t val)
2491 {
2492 int dirty_flags;
2493 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2494 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2495 #if !defined(CONFIG_USER_ONLY)
2496 tb_invalidate_phys_page_fast(ram_addr, 1);
2497 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2498 #endif
2499 }
2500 stb_p(phys_ram_base + ram_addr, val);
2501 #ifdef USE_KQEMU
2502 if (cpu_single_env->kqemu_enabled &&
2503 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2504 kqemu_modify_page(cpu_single_env, ram_addr);
2505 #endif
2506 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2507 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2508 /* we remove the notdirty callback only if the code has been
2509 flushed */
2510 if (dirty_flags == 0xff)
2511 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2512 }
2513
2514 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2515 uint32_t val)
2516 {
2517 int dirty_flags;
2518 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2519 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2520 #if !defined(CONFIG_USER_ONLY)
2521 tb_invalidate_phys_page_fast(ram_addr, 2);
2522 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2523 #endif
2524 }
2525 stw_p(phys_ram_base + ram_addr, val);
2526 #ifdef USE_KQEMU
2527 if (cpu_single_env->kqemu_enabled &&
2528 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2529 kqemu_modify_page(cpu_single_env, ram_addr);
2530 #endif
2531 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2532 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2533 /* we remove the notdirty callback only if the code has been
2534 flushed */
2535 if (dirty_flags == 0xff)
2536 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2537 }
2538
2539 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2540 uint32_t val)
2541 {
2542 int dirty_flags;
2543 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2544 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2545 #if !defined(CONFIG_USER_ONLY)
2546 tb_invalidate_phys_page_fast(ram_addr, 4);
2547 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2548 #endif
2549 }
2550 stl_p(phys_ram_base + ram_addr, val);
2551 #ifdef USE_KQEMU
2552 if (cpu_single_env->kqemu_enabled &&
2553 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2554 kqemu_modify_page(cpu_single_env, ram_addr);
2555 #endif
2556 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2557 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2558 /* we remove the notdirty callback only if the code has been
2559 flushed */
2560 if (dirty_flags == 0xff)
2561 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2562 }
2563
2564 static CPUReadMemoryFunc *error_mem_read[3] = {
2565 NULL, /* never used */
2566 NULL, /* never used */
2567 NULL, /* never used */
2568 };
2569
2570 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2571 notdirty_mem_writeb,
2572 notdirty_mem_writew,
2573 notdirty_mem_writel,
2574 };
2575
2576 /* Generate a debug exception if a watchpoint has been hit. */
2577 static void check_watchpoint(int offset, int len_mask, int flags)
2578 {
2579 CPUState *env = cpu_single_env;
2580 target_ulong pc, cs_base;
2581 TranslationBlock *tb;
2582 target_ulong vaddr;
2583 CPUWatchpoint *wp;
2584 int cpu_flags;
2585
2586 if (env->watchpoint_hit) {
2587 /* We re-entered the check after replacing the TB. Now raise
2588 * the debug interrupt so that is will trigger after the
2589 * current instruction. */
2590 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2591 return;
2592 }
2593 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2594 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2595 if ((vaddr == (wp->vaddr & len_mask) ||
2596 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2597 wp->flags |= BP_WATCHPOINT_HIT;
2598 if (!env->watchpoint_hit) {
2599 env->watchpoint_hit = wp;
2600 tb = tb_find_pc(env->mem_io_pc);
2601 if (!tb) {
2602 cpu_abort(env, "check_watchpoint: could not find TB for "
2603 "pc=%p", (void *)env->mem_io_pc);
2604 }
2605 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2606 tb_phys_invalidate(tb, -1);
2607 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2608 env->exception_index = EXCP_DEBUG;
2609 } else {
2610 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2611 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2612 }
2613 cpu_resume_from_signal(env, NULL);
2614 }
2615 } else {
2616 wp->flags &= ~BP_WATCHPOINT_HIT;
2617 }
2618 }
2619 }
2620
2621 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2622 so these check for a hit then pass through to the normal out-of-line
2623 phys routines. */
2624 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2625 {
2626 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2627 return ldub_phys(addr);
2628 }
2629
2630 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2631 {
2632 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2633 return lduw_phys(addr);
2634 }
2635
2636 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2637 {
2638 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2639 return ldl_phys(addr);
2640 }
2641
2642 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2643 uint32_t val)
2644 {
2645 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2646 stb_phys(addr, val);
2647 }
2648
2649 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2650 uint32_t val)
2651 {
2652 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2653 stw_phys(addr, val);
2654 }
2655
2656 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2657 uint32_t val)
2658 {
2659 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2660 stl_phys(addr, val);
2661 }
2662
2663 static CPUReadMemoryFunc *watch_mem_read[3] = {
2664 watch_mem_readb,
2665 watch_mem_readw,
2666 watch_mem_readl,
2667 };
2668
2669 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2670 watch_mem_writeb,
2671 watch_mem_writew,
2672 watch_mem_writel,
2673 };
2674
2675 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2676 unsigned int len)
2677 {
2678 uint32_t ret;
2679 unsigned int idx;
2680
2681 idx = SUBPAGE_IDX(addr);
2682 #if defined(DEBUG_SUBPAGE)
2683 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2684 mmio, len, addr, idx);
2685 #endif
2686 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2687 addr + mmio->region_offset[idx][0][len]);
2688
2689 return ret;
2690 }
2691
2692 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2693 uint32_t value, unsigned int len)
2694 {
2695 unsigned int idx;
2696
2697 idx = SUBPAGE_IDX(addr);
2698 #if defined(DEBUG_SUBPAGE)
2699 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2700 mmio, len, addr, idx, value);
2701 #endif
2702 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2703 addr + mmio->region_offset[idx][1][len],
2704 value);
2705 }
2706
2707 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2708 {
2709 #if defined(DEBUG_SUBPAGE)
2710 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2711 #endif
2712
2713 return subpage_readlen(opaque, addr, 0);
2714 }
2715
2716 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2717 uint32_t value)
2718 {
2719 #if defined(DEBUG_SUBPAGE)
2720 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2721 #endif
2722 subpage_writelen(opaque, addr, value, 0);
2723 }
2724
2725 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2726 {
2727 #if defined(DEBUG_SUBPAGE)
2728 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2729 #endif
2730
2731 return subpage_readlen(opaque, addr, 1);
2732 }
2733
2734 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2735 uint32_t value)
2736 {
2737 #if defined(DEBUG_SUBPAGE)
2738 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2739 #endif
2740 subpage_writelen(opaque, addr, value, 1);
2741 }
2742
2743 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2744 {
2745 #if defined(DEBUG_SUBPAGE)
2746 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2747 #endif
2748
2749 return subpage_readlen(opaque, addr, 2);
2750 }
2751
2752 static void subpage_writel (void *opaque,
2753 target_phys_addr_t addr, uint32_t value)
2754 {
2755 #if defined(DEBUG_SUBPAGE)
2756 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2757 #endif
2758 subpage_writelen(opaque, addr, value, 2);
2759 }
2760
2761 static CPUReadMemoryFunc *subpage_read[] = {
2762 &subpage_readb,
2763 &subpage_readw,
2764 &subpage_readl,
2765 };
2766
2767 static CPUWriteMemoryFunc *subpage_write[] = {
2768 &subpage_writeb,
2769 &subpage_writew,
2770 &subpage_writel,
2771 };
2772
2773 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2774 ram_addr_t memory, ram_addr_t region_offset)
2775 {
2776 int idx, eidx;
2777 unsigned int i;
2778
2779 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2780 return -1;
2781 idx = SUBPAGE_IDX(start);
2782 eidx = SUBPAGE_IDX(end);
2783 #if defined(DEBUG_SUBPAGE)
2784 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2785 mmio, start, end, idx, eidx, memory);
2786 #endif
2787 memory >>= IO_MEM_SHIFT;
2788 for (; idx <= eidx; idx++) {
2789 for (i = 0; i < 4; i++) {
2790 if (io_mem_read[memory][i]) {
2791 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2792 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2793 mmio->region_offset[idx][0][i] = region_offset;
2794 }
2795 if (io_mem_write[memory][i]) {
2796 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2797 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2798 mmio->region_offset[idx][1][i] = region_offset;
2799 }
2800 }
2801 }
2802
2803 return 0;
2804 }
2805
2806 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2807 ram_addr_t orig_memory, ram_addr_t region_offset)
2808 {
2809 subpage_t *mmio;
2810 int subpage_memory;
2811
2812 mmio = qemu_mallocz(sizeof(subpage_t));
2813
2814 mmio->base = base;
2815 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2816 #if defined(DEBUG_SUBPAGE)
2817 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2818 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2819 #endif
2820 *phys = subpage_memory | IO_MEM_SUBPAGE;
2821 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
2822 region_offset);
2823
2824 return mmio;
2825 }
2826
2827 static int get_free_io_mem_idx(void)
2828 {
2829 int i;
2830
2831 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
2832 if (!io_mem_used[i]) {
2833 io_mem_used[i] = 1;
2834 return i;
2835 }
2836
2837 return -1;
2838 }
2839
2840 static void io_mem_init(void)
2841 {
2842 int i;
2843
2844 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
2845 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2846 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2847 for (i=0; i<5; i++)
2848 io_mem_used[i] = 1;
2849
2850 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2851 watch_mem_write, NULL);
2852 /* alloc dirty bits array */
2853 phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2854 memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2855 }
2856
2857 /* mem_read and mem_write are arrays of functions containing the
2858 function to access byte (index 0), word (index 1) and dword (index
2859 2). Functions can be omitted with a NULL function pointer. The
2860 registered functions may be modified dynamically later.
2861 If io_index is non zero, the corresponding io zone is
2862 modified. If it is zero, a new io zone is allocated. The return
2863 value can be used with cpu_register_physical_memory(). (-1) is
2864 returned if error. */
2865 int cpu_register_io_memory(int io_index,
2866 CPUReadMemoryFunc **mem_read,
2867 CPUWriteMemoryFunc **mem_write,
2868 void *opaque)
2869 {
2870 int i, subwidth = 0;
2871
2872 if (io_index <= 0) {
2873 io_index = get_free_io_mem_idx();
2874 if (io_index == -1)
2875 return io_index;
2876 } else {
2877 if (io_index >= IO_MEM_NB_ENTRIES)
2878 return -1;
2879 }
2880
2881 for(i = 0;i < 3; i++) {
2882 if (!mem_read[i] || !mem_write[i])
2883 subwidth = IO_MEM_SUBWIDTH;
2884 io_mem_read[io_index][i] = mem_read[i];
2885 io_mem_write[io_index][i] = mem_write[i];
2886 }
2887 io_mem_opaque[io_index] = opaque;
2888 return (io_index << IO_MEM_SHIFT) | subwidth;
2889 }
2890
2891 void cpu_unregister_io_memory(int io_table_address)
2892 {
2893 int i;
2894 int io_index = io_table_address >> IO_MEM_SHIFT;
2895
2896 for (i=0;i < 3; i++) {
2897 io_mem_read[io_index][i] = unassigned_mem_read[i];
2898 io_mem_write[io_index][i] = unassigned_mem_write[i];
2899 }
2900 io_mem_opaque[io_index] = NULL;
2901 io_mem_used[io_index] = 0;
2902 }
2903
2904 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2905 {
2906 return io_mem_write[io_index >> IO_MEM_SHIFT];
2907 }
2908
2909 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2910 {
2911 return io_mem_read[io_index >> IO_MEM_SHIFT];
2912 }
2913
2914 #endif /* !defined(CONFIG_USER_ONLY) */
2915
2916 /* physical memory access (slow version, mainly for debug) */
2917 #if defined(CONFIG_USER_ONLY)
2918 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2919 int len, int is_write)
2920 {
2921 int l, flags;
2922 target_ulong page;
2923 void * p;
2924
2925 while (len > 0) {
2926 page = addr & TARGET_PAGE_MASK;
2927 l = (page + TARGET_PAGE_SIZE) - addr;
2928 if (l > len)
2929 l = len;
2930 flags = page_get_flags(page);
2931 if (!(flags & PAGE_VALID))
2932 return;
2933 if (is_write) {
2934 if (!(flags & PAGE_WRITE))
2935 return;
2936 /* XXX: this code should not depend on lock_user */
2937 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2938 /* FIXME - should this return an error rather than just fail? */
2939 return;
2940 memcpy(p, buf, l);
2941 unlock_user(p, addr, l);
2942 } else {
2943 if (!(flags & PAGE_READ))
2944 return;
2945 /* XXX: this code should not depend on lock_user */
2946 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2947 /* FIXME - should this return an error rather than just fail? */
2948 return;
2949 memcpy(buf, p, l);
2950 unlock_user(p, addr, 0);
2951 }
2952 len -= l;
2953 buf += l;
2954 addr += l;
2955 }
2956 }
2957
2958 #else
2959 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2960 int len, int is_write)
2961 {
2962 int l, io_index;
2963 uint8_t *ptr;
2964 uint32_t val;
2965 target_phys_addr_t page;
2966 unsigned long pd;
2967 PhysPageDesc *p;
2968
2969 while (len > 0) {
2970 page = addr & TARGET_PAGE_MASK;
2971 l = (page + TARGET_PAGE_SIZE) - addr;
2972 if (l > len)
2973 l = len;
2974 p = phys_page_find(page >> TARGET_PAGE_BITS);
2975 if (!p) {
2976 pd = IO_MEM_UNASSIGNED;
2977 } else {
2978 pd = p->phys_offset;
2979 }
2980
2981 if (is_write) {
2982 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
2983 target_phys_addr_t addr1 = addr;
2984 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2985 if (p)
2986 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
2987 /* XXX: could force cpu_single_env to NULL to avoid
2988 potential bugs */
2989 if (l >= 4 && ((addr1 & 3) == 0)) {
2990 /* 32 bit write access */
2991 val = ldl_p(buf);
2992 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
2993 l = 4;
2994 } else if (l >= 2 && ((addr1 & 1) == 0)) {
2995 /* 16 bit write access */
2996 val = lduw_p(buf);
2997 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
2998 l = 2;
2999 } else {
3000 /* 8 bit write access */
3001 val = ldub_p(buf);
3002 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3003 l = 1;
3004 }
3005 } else {
3006 unsigned long addr1;
3007 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3008 /* RAM case */
3009 ptr = phys_ram_base + addr1;
3010 memcpy(ptr, buf, l);
3011 if (!cpu_physical_memory_is_dirty(addr1)) {
3012 /* invalidate code */
3013 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3014 /* set dirty bit */
3015 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3016 (0xff & ~CODE_DIRTY_FLAG);
3017 }
3018 }
3019 } else {
3020 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3021 !(pd & IO_MEM_ROMD)) {
3022 target_phys_addr_t addr1 = addr;
3023 /* I/O case */
3024 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3025 if (p)
3026 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3027 if (l >= 4 && ((addr1 & 3) == 0)) {
3028 /* 32 bit read access */
3029 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3030 stl_p(buf, val);
3031 l = 4;
3032 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3033 /* 16 bit read access */
3034 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3035 stw_p(buf, val);
3036 l = 2;
3037 } else {
3038 /* 8 bit read access */
3039 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3040 stb_p(buf, val);
3041 l = 1;
3042 }
3043 } else {
3044 /* RAM case */
3045 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3046 (addr & ~TARGET_PAGE_MASK);
3047 memcpy(buf, ptr, l);
3048 }
3049 }
3050 len -= l;
3051 buf += l;
3052 addr += l;
3053 }
3054 }
3055
3056 /* used for ROM loading : can write in RAM and ROM */
3057 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3058 const uint8_t *buf, int len)
3059 {
3060 int l;
3061 uint8_t *ptr;
3062 target_phys_addr_t page;
3063 unsigned long pd;
3064 PhysPageDesc *p;
3065
3066 while (len > 0) {
3067 page = addr & TARGET_PAGE_MASK;
3068 l = (page + TARGET_PAGE_SIZE) - addr;
3069 if (l > len)
3070 l = len;
3071 p = phys_page_find(page >> TARGET_PAGE_BITS);
3072 if (!p) {
3073 pd = IO_MEM_UNASSIGNED;
3074 } else {
3075 pd = p->phys_offset;
3076 }
3077
3078 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3079 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3080 !(pd & IO_MEM_ROMD)) {
3081 /* do nothing */
3082 } else {
3083 unsigned long addr1;
3084 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3085 /* ROM/RAM case */
3086 ptr = phys_ram_base + addr1;
3087 memcpy(ptr, buf, l);
3088 }
3089 len -= l;
3090 buf += l;
3091 addr += l;
3092 }
3093 }
3094
3095 typedef struct {
3096 void *buffer;
3097 target_phys_addr_t addr;
3098 target_phys_addr_t len;
3099 } BounceBuffer;
3100
3101 static BounceBuffer bounce;
3102
3103 typedef struct MapClient {
3104 void *opaque;
3105 void (*callback)(void *opaque);
3106 LIST_ENTRY(MapClient) link;
3107 } MapClient;
3108
3109 static LIST_HEAD(map_client_list, MapClient) map_client_list
3110 = LIST_HEAD_INITIALIZER(map_client_list);
3111
3112 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3113 {
3114 MapClient *client = qemu_malloc(sizeof(*client));
3115
3116 client->opaque = opaque;
3117 client->callback = callback;
3118 LIST_INSERT_HEAD(&map_client_list, client, link);
3119 return client;
3120 }
3121
3122 void cpu_unregister_map_client(void *_client)
3123 {
3124 MapClient *client = (MapClient *)_client;
3125
3126 LIST_REMOVE(client, link);
3127 }
3128
3129 static void cpu_notify_map_clients(void)
3130 {
3131 MapClient *client;
3132
3133 while (!LIST_EMPTY(&map_client_list)) {
3134 client = LIST_FIRST(&map_client_list);
3135 client->callback(client->opaque);
3136 LIST_REMOVE(client, link);
3137 }
3138 }
3139
3140 /* Map a physical memory region into a host virtual address.
3141 * May map a subset of the requested range, given by and returned in *plen.
3142 * May return NULL if resources needed to perform the mapping are exhausted.
3143 * Use only for reads OR writes - not for read-modify-write operations.
3144 * Use cpu_register_map_client() to know when retrying the map operation is
3145 * likely to succeed.
3146 */
3147 void *cpu_physical_memory_map(target_phys_addr_t addr,
3148 target_phys_addr_t *plen,
3149 int is_write)
3150 {
3151 target_phys_addr_t len = *plen;
3152 target_phys_addr_t done = 0;
3153 int l;
3154 uint8_t *ret = NULL;
3155 uint8_t *ptr;
3156 target_phys_addr_t page;
3157 unsigned long pd;
3158 PhysPageDesc *p;
3159 unsigned long addr1;
3160
3161 while (len > 0) {
3162 page = addr & TARGET_PAGE_MASK;
3163 l = (page + TARGET_PAGE_SIZE) - addr;
3164 if (l > len)
3165 l = len;
3166 p = phys_page_find(page >> TARGET_PAGE_BITS);
3167 if (!p) {
3168 pd = IO_MEM_UNASSIGNED;
3169 } else {
3170 pd = p->phys_offset;
3171 }
3172
3173 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3174 if (done || bounce.buffer) {
3175 break;
3176 }
3177 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3178 bounce.addr = addr;
3179 bounce.len = l;
3180 if (!is_write) {
3181 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3182 }
3183 ptr = bounce.buffer;
3184 } else {
3185 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3186 ptr = phys_ram_base + addr1;
3187 }
3188 if (!done) {
3189 ret = ptr;
3190 } else if (ret + done != ptr) {
3191 break;
3192 }
3193
3194 len -= l;
3195 addr += l;
3196 done += l;
3197 }
3198 *plen = done;
3199 return ret;
3200 }
3201
3202 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3203 * Will also mark the memory as dirty if is_write == 1. access_len gives
3204 * the amount of memory that was actually read or written by the caller.
3205 */
3206 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3207 int is_write, target_phys_addr_t access_len)
3208 {
3209 if (buffer != bounce.buffer) {
3210 if (is_write) {
3211 unsigned long addr1 = (uint8_t *)buffer - phys_ram_base;
3212 while (access_len) {
3213 unsigned l;
3214 l = TARGET_PAGE_SIZE;
3215 if (l > access_len)
3216 l = access_len;
3217 if (!cpu_physical_memory_is_dirty(addr1)) {
3218 /* invalidate code */
3219 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3220 /* set dirty bit */
3221 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3222 (0xff & ~CODE_DIRTY_FLAG);
3223 }
3224 addr1 += l;
3225 access_len -= l;
3226 }
3227 }
3228 return;
3229 }
3230 if (is_write) {
3231 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3232 }
3233 qemu_free(bounce.buffer);
3234 bounce.buffer = NULL;
3235 cpu_notify_map_clients();
3236 }
3237
3238 /* warning: addr must be aligned */
3239 uint32_t ldl_phys(target_phys_addr_t addr)
3240 {
3241 int io_index;
3242 uint8_t *ptr;
3243 uint32_t val;
3244 unsigned long pd;
3245 PhysPageDesc *p;
3246
3247 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3248 if (!p) {
3249 pd = IO_MEM_UNASSIGNED;
3250 } else {
3251 pd = p->phys_offset;
3252 }
3253
3254 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3255 !(pd & IO_MEM_ROMD)) {
3256 /* I/O case */
3257 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3258 if (p)
3259 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3260 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3261 } else {
3262 /* RAM case */
3263 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3264 (addr & ~TARGET_PAGE_MASK);
3265 val = ldl_p(ptr);
3266 }
3267 return val;
3268 }
3269
3270 /* warning: addr must be aligned */
3271 uint64_t ldq_phys(target_phys_addr_t addr)
3272 {
3273 int io_index;
3274 uint8_t *ptr;
3275 uint64_t val;
3276 unsigned long pd;
3277 PhysPageDesc *p;
3278
3279 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3280 if (!p) {
3281 pd = IO_MEM_UNASSIGNED;
3282 } else {
3283 pd = p->phys_offset;
3284 }
3285
3286 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3287 !(pd & IO_MEM_ROMD)) {
3288 /* I/O case */
3289 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3290 if (p)
3291 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3292 #ifdef TARGET_WORDS_BIGENDIAN
3293 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3294 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3295 #else
3296 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3297 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3298 #endif
3299 } else {
3300 /* RAM case */
3301 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3302 (addr & ~TARGET_PAGE_MASK);
3303 val = ldq_p(ptr);
3304 }
3305 return val;
3306 }
3307
3308 /* XXX: optimize */
3309 uint32_t ldub_phys(target_phys_addr_t addr)
3310 {
3311 uint8_t val;
3312 cpu_physical_memory_read(addr, &val, 1);
3313 return val;
3314 }
3315
3316 /* XXX: optimize */
3317 uint32_t lduw_phys(target_phys_addr_t addr)
3318 {
3319 uint16_t val;
3320 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3321 return tswap16(val);
3322 }
3323
3324 /* warning: addr must be aligned. The ram page is not masked as dirty
3325 and the code inside is not invalidated. It is useful if the dirty
3326 bits are used to track modified PTEs */
3327 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3328 {
3329 int io_index;
3330 uint8_t *ptr;
3331 unsigned long pd;
3332 PhysPageDesc *p;
3333
3334 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3335 if (!p) {
3336 pd = IO_MEM_UNASSIGNED;
3337 } else {
3338 pd = p->phys_offset;
3339 }
3340
3341 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3342 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3343 if (p)
3344 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3345 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3346 } else {
3347 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3348 ptr = phys_ram_base + addr1;
3349 stl_p(ptr, val);
3350
3351 if (unlikely(in_migration)) {
3352 if (!cpu_physical_memory_is_dirty(addr1)) {
3353 /* invalidate code */
3354 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3355 /* set dirty bit */
3356 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3357 (0xff & ~CODE_DIRTY_FLAG);
3358 }
3359 }
3360 }
3361 }
3362
3363 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3364 {
3365 int io_index;
3366 uint8_t *ptr;
3367 unsigned long pd;
3368 PhysPageDesc *p;
3369
3370 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3371 if (!p) {
3372 pd = IO_MEM_UNASSIGNED;
3373 } else {
3374 pd = p->phys_offset;
3375 }
3376
3377 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3378 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3379 if (p)
3380 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3381 #ifdef TARGET_WORDS_BIGENDIAN
3382 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3383 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3384 #else
3385 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3386 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3387 #endif
3388 } else {
3389 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3390 (addr & ~TARGET_PAGE_MASK);
3391 stq_p(ptr, val);
3392 }
3393 }
3394
3395 /* warning: addr must be aligned */
3396 void stl_phys(target_phys_addr_t addr, uint32_t val)
3397 {
3398 int io_index;
3399 uint8_t *ptr;
3400 unsigned long pd;
3401 PhysPageDesc *p;
3402
3403 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3404 if (!p) {
3405 pd = IO_MEM_UNASSIGNED;
3406 } else {
3407 pd = p->phys_offset;
3408 }
3409
3410 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3411 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3412 if (p)
3413 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3414 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3415 } else {
3416 unsigned long addr1;
3417 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3418 /* RAM case */
3419 ptr = phys_ram_base + addr1;
3420 stl_p(ptr, val);
3421 if (!cpu_physical_memory_is_dirty(addr1)) {
3422 /* invalidate code */
3423 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3424 /* set dirty bit */
3425 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3426 (0xff & ~CODE_DIRTY_FLAG);
3427 }
3428 }
3429 }
3430
3431 /* XXX: optimize */
3432 void stb_phys(target_phys_addr_t addr, uint32_t val)
3433 {
3434 uint8_t v = val;
3435 cpu_physical_memory_write(addr, &v, 1);
3436 }
3437
3438 /* XXX: optimize */
3439 void stw_phys(target_phys_addr_t addr, uint32_t val)
3440 {
3441 uint16_t v = tswap16(val);
3442 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3443 }
3444
3445 /* XXX: optimize */
3446 void stq_phys(target_phys_addr_t addr, uint64_t val)
3447 {
3448 val = tswap64(val);
3449 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3450 }
3451
3452 #endif
3453
3454 /* virtual memory access for debug */
3455 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3456 uint8_t *buf, int len, int is_write)
3457 {
3458 int l;
3459 target_phys_addr_t phys_addr;
3460 target_ulong page;
3461
3462 while (len > 0) {
3463 page = addr & TARGET_PAGE_MASK;
3464 phys_addr = cpu_get_phys_page_debug(env, page);
3465 /* if no physical page mapped, return an error */
3466 if (phys_addr == -1)
3467 return -1;
3468 l = (page + TARGET_PAGE_SIZE) - addr;
3469 if (l > len)
3470 l = len;
3471 cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
3472 buf, l, is_write);
3473 len -= l;
3474 buf += l;
3475 addr += l;
3476 }
3477 return 0;
3478 }
3479
3480 /* in deterministic execution mode, instructions doing device I/Os
3481 must be at the end of the TB */
3482 void cpu_io_recompile(CPUState *env, void *retaddr)
3483 {
3484 TranslationBlock *tb;
3485 uint32_t n, cflags;
3486 target_ulong pc, cs_base;
3487 uint64_t flags;
3488
3489 tb = tb_find_pc((unsigned long)retaddr);
3490 if (!tb) {
3491 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3492 retaddr);
3493 }
3494 n = env->icount_decr.u16.low + tb->icount;
3495 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3496 /* Calculate how many instructions had been executed before the fault
3497 occurred. */
3498 n = n - env->icount_decr.u16.low;
3499 /* Generate a new TB ending on the I/O insn. */
3500 n++;
3501 /* On MIPS and SH, delay slot instructions can only be restarted if
3502 they were already the first instruction in the TB. If this is not
3503 the first instruction in a TB then re-execute the preceding
3504 branch. */
3505 #if defined(TARGET_MIPS)
3506 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3507 env->active_tc.PC -= 4;
3508 env->icount_decr.u16.low++;
3509 env->hflags &= ~MIPS_HFLAG_BMASK;
3510 }
3511 #elif defined(TARGET_SH4)
3512 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3513 && n > 1) {
3514 env->pc -= 2;
3515 env->icount_decr.u16.low++;
3516 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3517 }
3518 #endif
3519 /* This should never happen. */
3520 if (n > CF_COUNT_MASK)
3521 cpu_abort(env, "TB too big during recompile");
3522
3523 cflags = n | CF_LAST_IO;
3524 pc = tb->pc;
3525 cs_base = tb->cs_base;
3526 flags = tb->flags;
3527 tb_phys_invalidate(tb, -1);
3528 /* FIXME: In theory this could raise an exception. In practice
3529 we have already translated the block once so it's probably ok. */
3530 tb_gen_code(env, pc, cs_base, flags, cflags);
3531 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3532 the first in the TB) then we end up generating a whole new TB and
3533 repeating the fault, which is horribly inefficient.
3534 Better would be to execute just this insn uncached, or generate a
3535 second new TB. */
3536 cpu_resume_from_signal(env, NULL);
3537 }
3538
3539 void dump_exec_info(FILE *f,
3540 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3541 {
3542 int i, target_code_size, max_target_code_size;
3543 int direct_jmp_count, direct_jmp2_count, cross_page;
3544 TranslationBlock *tb;
3545
3546 target_code_size = 0;
3547 max_target_code_size = 0;
3548 cross_page = 0;
3549 direct_jmp_count = 0;
3550 direct_jmp2_count = 0;
3551 for(i = 0; i < nb_tbs; i++) {
3552 tb = &tbs[i];
3553 target_code_size += tb->size;
3554 if (tb->size > max_target_code_size)
3555 max_target_code_size = tb->size;
3556 if (tb->page_addr[1] != -1)
3557 cross_page++;
3558 if (tb->tb_next_offset[0] != 0xffff) {
3559 direct_jmp_count++;
3560 if (tb->tb_next_offset[1] != 0xffff) {
3561 direct_jmp2_count++;
3562 }
3563 }
3564 }
3565 /* XXX: avoid using doubles ? */
3566 cpu_fprintf(f, "Translation buffer state:\n");
3567 cpu_fprintf(f, "gen code size %ld/%ld\n",
3568 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3569 cpu_fprintf(f, "TB count %d/%d\n",
3570 nb_tbs, code_gen_max_blocks);
3571 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3572 nb_tbs ? target_code_size / nb_tbs : 0,
3573 max_target_code_size);
3574 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3575 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3576 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3577 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3578 cross_page,
3579 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3580 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3581 direct_jmp_count,
3582 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3583 direct_jmp2_count,
3584 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3585 cpu_fprintf(f, "\nStatistics:\n");
3586 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3587 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3588 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3589 tcg_dump_info(f, cpu_fprintf);
3590 }
3591
3592 #if !defined(CONFIG_USER_ONLY)
3593
3594 #define MMUSUFFIX _cmmu
3595 #define GETPC() NULL
3596 #define env cpu_single_env
3597 #define SOFTMMU_CODE_ACCESS
3598
3599 #define SHIFT 0
3600 #include "softmmu_template.h"
3601
3602 #define SHIFT 1
3603 #include "softmmu_template.h"
3604
3605 #define SHIFT 2
3606 #include "softmmu_template.h"
3607
3608 #define SHIFT 3
3609 #include "softmmu_template.h"
3610
3611 #undef env
3612
3613 #endif