?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Disable the vnc CopyRect encoding
[qemu-kvm/fedora.git] / exec.c
blob22f75e54ad2ee1634400622b39c211e0dac12edb
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA
19 */
20 #include "config.h"
21 #ifdef _WIN32
22 #define WIN32_LEAN_AND_MEAN
23 #include <windows.h>
24 #else
25 #include <sys/types.h>
26 #include <sys/mman.h>
27 #endif
28 #include <stdlib.h>
29 #include <stdio.h>
30 #include <stdarg.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <unistd.h>
34 #include <inttypes.h>
35
36 #include "cpu.h"
37 #include "exec-all.h"
38 #include "qemu-common.h"
39
40 #if !defined(TARGET_IA64)
41 #include "tcg.h"
42 #endif
43 #include "qemu-kvm.h"
44
45 #include "hw/hw.h"
46 #include "osdep.h"
47 #include "kvm.h"
48 #if defined(CONFIG_USER_ONLY)
49 #include <qemu.h>
50 #endif
51
52 //#define DEBUG_TB_INVALIDATE
53 //#define DEBUG_FLUSH
54 //#define DEBUG_TLB
55 //#define DEBUG_UNASSIGNED
56
57 /* make various TB consistency checks */
58 //#define DEBUG_TB_CHECK
59 //#define DEBUG_TLB_CHECK
60
61 //#define DEBUG_IOPORT
62 //#define DEBUG_SUBPAGE
63
64 #if !defined(CONFIG_USER_ONLY)
65 /* TB consistency checks only implemented for usermode emulation. */
66 #undef DEBUG_TB_CHECK
67 #endif
68
69 #define SMC_BITMAP_USE_THRESHOLD 10
70
71 #define MMAP_AREA_START 0x00000000
72 #define MMAP_AREA_END 0xa8000000
73
74 #if defined(TARGET_SPARC64)
75 #define TARGET_PHYS_ADDR_SPACE_BITS 41
76 #elif defined(TARGET_SPARC)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 36
78 #elif defined(TARGET_ALPHA)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
80 #define TARGET_VIRT_ADDR_SPACE_BITS 42
81 #elif defined(TARGET_PPC64)
82 #define TARGET_PHYS_ADDR_SPACE_BITS 42
83 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
84 #define TARGET_PHYS_ADDR_SPACE_BITS 42
85 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
86 #define TARGET_PHYS_ADDR_SPACE_BITS 36
87 #elif defined(TARGET_IA64)
88 #define TARGET_PHYS_ADDR_SPACE_BITS 36
89 #else
90 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
91 #define TARGET_PHYS_ADDR_SPACE_BITS 32
92 #endif
93
94 static TranslationBlock *tbs;
95 int code_gen_max_blocks;
96 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
97 static int nb_tbs;
98 /* any access to the tbs or the page table must use this lock */
99 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
100
101 #if defined(__arm__) || defined(__sparc_v9__)
102 /* The prologue must be reachable with a direct jump. ARM and Sparc64
103 have limited branch ranges (possibly also PPC) so place it in a
104 section close to code segment. */
105 #define code_gen_section \
106 __attribute__((__section__(".gen_code"))) \
107 __attribute__((aligned (32)))
108 #else
109 #define code_gen_section \
110 __attribute__((aligned (32)))
111 #endif
112
113 uint8_t code_gen_prologue[1024] code_gen_section;
114 static uint8_t *code_gen_buffer;
115 static unsigned long code_gen_buffer_size;
116 /* threshold to flush the translated code buffer */
117 static unsigned long code_gen_buffer_max_size;
118 uint8_t *code_gen_ptr;
119
120 #if !defined(CONFIG_USER_ONLY)
121 ram_addr_t phys_ram_size;
122 int phys_ram_fd;
123 uint8_t *phys_ram_base;
124 uint8_t *phys_ram_dirty;
125 uint8_t *bios_mem;
126 static int in_migration;
127 static ram_addr_t phys_ram_alloc_offset = 0;
128 #endif
129
130 CPUState *first_cpu;
131 /* current CPU in the current thread. It is only valid inside
132 cpu_exec() */
133 CPUState *cpu_single_env;
134 /* 0 = Do not count executed instructions.
135 1 = Precise instruction counting.
136 2 = Adaptive rate instruction counting. */
137 int use_icount = 0;
138 /* Current instruction counter. While executing translated code this may
139 include some instructions that have not yet been executed. */
140 int64_t qemu_icount;
141
142 typedef struct PageDesc {
143 /* list of TBs intersecting this ram page */
144 TranslationBlock *first_tb;
145 /* in order to optimize self modifying code, we count the number
146 of lookups we do to a given page to use a bitmap */
147 unsigned int code_write_count;
148 uint8_t *code_bitmap;
149 #if defined(CONFIG_USER_ONLY)
150 unsigned long flags;
151 #endif
152 } PageDesc;
153
154 typedef struct PhysPageDesc {
155 /* offset in host memory of the page + io_index in the low bits */
156 ram_addr_t phys_offset;
157 ram_addr_t region_offset;
158 } PhysPageDesc;
159
160 #define L2_BITS 10
161 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
162 /* XXX: this is a temporary hack for alpha target.
163 * In the future, this is to be replaced by a multi-level table
164 * to actually be able to handle the complete 64 bits address space.
165 */
166 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
167 #else
168 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
169 #endif
170
171 #define L1_SIZE (1 << L1_BITS)
172 #define L2_SIZE (1 << L2_BITS)
173
174 unsigned long qemu_real_host_page_size;
175 unsigned long qemu_host_page_bits;
176 unsigned long qemu_host_page_size;
177 unsigned long qemu_host_page_mask;
178
179 /* XXX: for system emulation, it could just be an array */
180 static PageDesc *l1_map[L1_SIZE];
181 static PhysPageDesc **l1_phys_map;
182
183 #if !defined(CONFIG_USER_ONLY)
184 static void io_mem_init(void);
185
186 /* io memory support */
187 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
188 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
189 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
190 char io_mem_used[IO_MEM_NB_ENTRIES];
191 static int io_mem_watch;
192 #endif
193
194 /* log support */
195 static const char *logfilename = "/tmp/qemu.log";
196 FILE *logfile;
197 int loglevel;
198 static int log_append = 0;
199
200 /* statistics */
201 static int tlb_flush_count;
202 static int tb_flush_count;
203 static int tb_phys_invalidate_count;
204
205 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
206 typedef struct subpage_t {
207 target_phys_addr_t base;
208 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
209 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
210 void *opaque[TARGET_PAGE_SIZE][2][4];
211 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
212 } subpage_t;
213
214 #ifdef _WIN32
215 static void map_exec(void *addr, long size)
216 {
217 DWORD old_protect;
218 VirtualProtect(addr, size,
219 PAGE_EXECUTE_READWRITE, &old_protect);
220
221 }
222 #else
223 static void map_exec(void *addr, long size)
224 {
225 unsigned long start, end, page_size;
226
227 page_size = getpagesize();
228 start = (unsigned long)addr;
229 start &= ~(page_size - 1);
230
231 end = (unsigned long)addr + size;
232 end += page_size - 1;
233 end &= ~(page_size - 1);
234
235 mprotect((void *)start, end - start,
236 PROT_READ | PROT_WRITE | PROT_EXEC);
237 }
238 #endif
239
240 static void page_init(void)
241 {
242 /* NOTE: we can always suppose that qemu_host_page_size >=
243 TARGET_PAGE_SIZE */
244 #ifdef _WIN32
245 {
246 SYSTEM_INFO system_info;
247
248 GetSystemInfo(&system_info);
249 qemu_real_host_page_size = system_info.dwPageSize;
250 }
251 #else
252 qemu_real_host_page_size = getpagesize();
253 #endif
254 if (qemu_host_page_size == 0)
255 qemu_host_page_size = qemu_real_host_page_size;
256 if (qemu_host_page_size < TARGET_PAGE_SIZE)
257 qemu_host_page_size = TARGET_PAGE_SIZE;
258 qemu_host_page_bits = 0;
259 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
260 qemu_host_page_bits++;
261 qemu_host_page_mask = ~(qemu_host_page_size - 1);
262 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
263 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
264
265 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
266 {
267 long long startaddr, endaddr;
268 FILE *f;
269 int n;
270
271 mmap_lock();
272 last_brk = (unsigned long)sbrk(0);
273 f = fopen("/proc/self/maps", "r");
274 if (f) {
275 do {
276 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
277 if (n == 2) {
278 startaddr = MIN(startaddr,
279 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
280 endaddr = MIN(endaddr,
281 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
282 page_set_flags(startaddr & TARGET_PAGE_MASK,
283 TARGET_PAGE_ALIGN(endaddr),
284 PAGE_RESERVED);
285 }
286 } while (!feof(f));
287 fclose(f);
288 }
289 mmap_unlock();
290 }
291 #endif
292 }
293
294 static inline PageDesc **page_l1_map(target_ulong index)
295 {
296 #if TARGET_LONG_BITS > 32
297 /* Host memory outside guest VM. For 32-bit targets we have already
298 excluded high addresses. */
299 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
300 return NULL;
301 #endif
302 return &l1_map[index >> L2_BITS];
303 }
304
305 static inline PageDesc *page_find_alloc(target_ulong index)
306 {
307 PageDesc **lp, *p;
308 lp = page_l1_map(index);
309 if (!lp)
310 return NULL;
311
312 p = *lp;
313 if (!p) {
314 /* allocate if not found */
315 #if defined(CONFIG_USER_ONLY)
316 size_t len = sizeof(PageDesc) * L2_SIZE;
317 /* Don't use qemu_malloc because it may recurse. */
318 p = mmap(0, len, PROT_READ | PROT_WRITE,
319 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
320 *lp = p;
321 if (h2g_valid(p)) {
322 unsigned long addr = h2g(p);
323 page_set_flags(addr & TARGET_PAGE_MASK,
324 TARGET_PAGE_ALIGN(addr + len),
325 PAGE_RESERVED);
326 }
327 #else
328 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
329 *lp = p;
330 #endif
331 }
332 return p + (index & (L2_SIZE - 1));
333 }
334
335 static inline PageDesc *page_find(target_ulong index)
336 {
337 PageDesc **lp, *p;
338 lp = page_l1_map(index);
339 if (!lp)
340 return NULL;
341
342 p = *lp;
343 if (!p)
344 return 0;
345 return p + (index & (L2_SIZE - 1));
346 }
347
348 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
349 {
350 void **lp, **p;
351 PhysPageDesc *pd;
352
353 p = (void **)l1_phys_map;
354 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
355
356 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
357 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
358 #endif
359 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
360 p = *lp;
361 if (!p) {
362 /* allocate if not found */
363 if (!alloc)
364 return NULL;
365 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
366 memset(p, 0, sizeof(void *) * L1_SIZE);
367 *lp = p;
368 }
369 #endif
370 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
371 pd = *lp;
372 if (!pd) {
373 int i;
374 /* allocate if not found */
375 if (!alloc)
376 return NULL;
377 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
378 *lp = pd;
379 for (i = 0; i < L2_SIZE; i++) {
380 pd[i].phys_offset = IO_MEM_UNASSIGNED;
381 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
382 }
383 }
384 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
385 }
386
387 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
388 {
389 return phys_page_find_alloc(index, 0);
390 }
391
392 #if !defined(CONFIG_USER_ONLY)
393 static void tlb_protect_code(ram_addr_t ram_addr);
394 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
395 target_ulong vaddr);
396 #define mmap_lock() do { } while(0)
397 #define mmap_unlock() do { } while(0)
398 #endif
399
400 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
401
402 #if defined(CONFIG_USER_ONLY)
403 /* Currently it is not recommanded to allocate big chunks of data in
404 user mode. It will change when a dedicated libc will be used */
405 #define USE_STATIC_CODE_GEN_BUFFER
406 #endif
407
408 #ifdef USE_STATIC_CODE_GEN_BUFFER
409 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
410 #endif
411
412 static void code_gen_alloc(unsigned long tb_size)
413 {
414 if (kvm_enabled())
415 return;
416
417 #ifdef USE_STATIC_CODE_GEN_BUFFER
418 code_gen_buffer = static_code_gen_buffer;
419 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
420 map_exec(code_gen_buffer, code_gen_buffer_size);
421 #else
422 code_gen_buffer_size = tb_size;
423 if (code_gen_buffer_size == 0) {
424 #if defined(CONFIG_USER_ONLY)
425 /* in user mode, phys_ram_size is not meaningful */
426 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
427 #else
428 /* XXX: needs ajustments */
429 code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
430 #endif
431 }
432 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
433 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
434 /* The code gen buffer location may have constraints depending on
435 the host cpu and OS */
436 #if defined(__linux__)
437 {
438 int flags;
439 void *start = NULL;
440
441 flags = MAP_PRIVATE | MAP_ANONYMOUS;
442 #if defined(__x86_64__)
443 flags |= MAP_32BIT;
444 /* Cannot map more than that */
445 if (code_gen_buffer_size > (800 * 1024 * 1024))
446 code_gen_buffer_size = (800 * 1024 * 1024);
447 #elif defined(__sparc_v9__)
448 // Map the buffer below 2G, so we can use direct calls and branches
449 flags |= MAP_FIXED;
450 start = (void *) 0x60000000UL;
451 if (code_gen_buffer_size > (512 * 1024 * 1024))
452 code_gen_buffer_size = (512 * 1024 * 1024);
453 #elif defined(__arm__)
454 /* Map the buffer below 32M, so we can use direct calls and branches */
455 flags |= MAP_FIXED;
456 start = (void *) 0x01000000UL;
457 if (code_gen_buffer_size > 16 * 1024 * 1024)
458 code_gen_buffer_size = 16 * 1024 * 1024;
459 #endif
460 code_gen_buffer = mmap(start, code_gen_buffer_size,
461 PROT_WRITE | PROT_READ | PROT_EXEC,
462 flags, -1, 0);
463 if (code_gen_buffer == MAP_FAILED) {
464 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
465 exit(1);
466 }
467 }
468 #elif defined(__FreeBSD__)
469 {
470 int flags;
471 void *addr = NULL;
472 flags = MAP_PRIVATE | MAP_ANONYMOUS;
473 #if defined(__x86_64__)
474 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
475 * 0x40000000 is free */
476 flags |= MAP_FIXED;
477 addr = (void *)0x40000000;
478 /* Cannot map more than that */
479 if (code_gen_buffer_size > (800 * 1024 * 1024))
480 code_gen_buffer_size = (800 * 1024 * 1024);
481 #endif
482 code_gen_buffer = mmap(addr, code_gen_buffer_size,
483 PROT_WRITE | PROT_READ | PROT_EXEC,
484 flags, -1, 0);
485 if (code_gen_buffer == MAP_FAILED) {
486 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
487 exit(1);
488 }
489 }
490 #else
491 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
492 map_exec(code_gen_buffer, code_gen_buffer_size);
493 #endif
494 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
495 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
496 code_gen_buffer_max_size = code_gen_buffer_size -
497 code_gen_max_block_size();
498 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
499 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
500 }
501
502 /* Must be called before using the QEMU cpus. 'tb_size' is the size
503 (in bytes) allocated to the translation buffer. Zero means default
504 size. */
505 void cpu_exec_init_all(unsigned long tb_size)
506 {
507 cpu_gen_init();
508 code_gen_alloc(tb_size);
509 code_gen_ptr = code_gen_buffer;
510 page_init();
511 #if !defined(CONFIG_USER_ONLY)
512 io_mem_init();
513 #endif
514 }
515
516 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
517
518 #define CPU_COMMON_SAVE_VERSION 1
519
520 static void cpu_common_save(QEMUFile *f, void *opaque)
521 {
522 CPUState *env = opaque;
523
524 qemu_put_be32s(f, &env->halted);
525 qemu_put_be32s(f, &env->interrupt_request);
526 }
527
528 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
529 {
530 CPUState *env = opaque;
531
532 if (version_id != CPU_COMMON_SAVE_VERSION)
533 return -EINVAL;
534
535 qemu_get_be32s(f, &env->halted);
536 qemu_get_be32s(f, &env->interrupt_request);
537 env->interrupt_request &= ~CPU_INTERRUPT_EXIT;
538 tlb_flush(env, 1);
539
540 return 0;
541 }
542 #endif
543
544 void cpu_exec_init(CPUState *env)
545 {
546 CPUState **penv;
547 int cpu_index;
548
549 env->next_cpu = NULL;
550 penv = &first_cpu;
551 cpu_index = 0;
552 while (*penv != NULL) {
553 penv = (CPUState **)&(*penv)->next_cpu;
554 cpu_index++;
555 }
556 env->cpu_index = cpu_index;
557 TAILQ_INIT(&env->breakpoints);
558 TAILQ_INIT(&env->watchpoints);
559 #ifdef __WIN32
560 env->thread_id = GetCurrentProcessId();
561 #else
562 env->thread_id = getpid();
563 #endif
564 *penv = env;
565 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
566 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
567 cpu_common_save, cpu_common_load, env);
568 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
569 cpu_save, cpu_load, env);
570 #endif
571 }
572
573 static inline void invalidate_page_bitmap(PageDesc *p)
574 {
575 if (p->code_bitmap) {
576 qemu_free(p->code_bitmap);
577 p->code_bitmap = NULL;
578 }
579 p->code_write_count = 0;
580 }
581
582 /* set to NULL all the 'first_tb' fields in all PageDescs */
583 static void page_flush_tb(void)
584 {
585 int i, j;
586 PageDesc *p;
587
588 for(i = 0; i < L1_SIZE; i++) {
589 p = l1_map[i];
590 if (p) {
591 for(j = 0; j < L2_SIZE; j++) {
592 p->first_tb = NULL;
593 invalidate_page_bitmap(p);
594 p++;
595 }
596 }
597 }
598 }
599
600 /* flush all the translation blocks */
601 /* XXX: tb_flush is currently not thread safe */
602 void tb_flush(CPUState *env1)
603 {
604 CPUState *env;
605 #if defined(DEBUG_FLUSH)
606 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
607 (unsigned long)(code_gen_ptr - code_gen_buffer),
608 nb_tbs, nb_tbs > 0 ?
609 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
610 #endif
611 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
612 cpu_abort(env1, "Internal error: code buffer overflow\n");
613
614 nb_tbs = 0;
615
616 for(env = first_cpu; env != NULL; env = env->next_cpu) {
617 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
618 }
619
620 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
621 page_flush_tb();
622
623 code_gen_ptr = code_gen_buffer;
624 /* XXX: flush processor icache at this point if cache flush is
625 expensive */
626 tb_flush_count++;
627 }
628
629 #ifdef DEBUG_TB_CHECK
630
631 static void tb_invalidate_check(target_ulong address)
632 {
633 TranslationBlock *tb;
634 int i;
635 address &= TARGET_PAGE_MASK;
636 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
637 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
638 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
639 address >= tb->pc + tb->size)) {
640 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
641 address, (long)tb->pc, tb->size);
642 }
643 }
644 }
645 }
646
647 /* verify that all the pages have correct rights for code */
648 static void tb_page_check(void)
649 {
650 TranslationBlock *tb;
651 int i, flags1, flags2;
652
653 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
654 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
655 flags1 = page_get_flags(tb->pc);
656 flags2 = page_get_flags(tb->pc + tb->size - 1);
657 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
658 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
659 (long)tb->pc, tb->size, flags1, flags2);
660 }
661 }
662 }
663 }
664
665 static void tb_jmp_check(TranslationBlock *tb)
666 {
667 TranslationBlock *tb1;
668 unsigned int n1;
669
670 /* suppress any remaining jumps to this TB */
671 tb1 = tb->jmp_first;
672 for(;;) {
673 n1 = (long)tb1 & 3;
674 tb1 = (TranslationBlock *)((long)tb1 & ~3);
675 if (n1 == 2)
676 break;
677 tb1 = tb1->jmp_next[n1];
678 }
679 /* check end of list */
680 if (tb1 != tb) {
681 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
682 }
683 }
684
685 #endif
686
687 /* invalidate one TB */
688 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
689 int next_offset)
690 {
691 TranslationBlock *tb1;
692 for(;;) {
693 tb1 = *ptb;
694 if (tb1 == tb) {
695 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
696 break;
697 }
698 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
699 }
700 }
701
702 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
703 {
704 TranslationBlock *tb1;
705 unsigned int n1;
706
707 for(;;) {
708 tb1 = *ptb;
709 n1 = (long)tb1 & 3;
710 tb1 = (TranslationBlock *)((long)tb1 & ~3);
711 if (tb1 == tb) {
712 *ptb = tb1->page_next[n1];
713 break;
714 }
715 ptb = &tb1->page_next[n1];
716 }
717 }
718
719 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
720 {
721 TranslationBlock *tb1, **ptb;
722 unsigned int n1;
723
724 ptb = &tb->jmp_next[n];
725 tb1 = *ptb;
726 if (tb1) {
727 /* find tb(n) in circular list */
728 for(;;) {
729 tb1 = *ptb;
730 n1 = (long)tb1 & 3;
731 tb1 = (TranslationBlock *)((long)tb1 & ~3);
732 if (n1 == n && tb1 == tb)
733 break;
734 if (n1 == 2) {
735 ptb = &tb1->jmp_first;
736 } else {
737 ptb = &tb1->jmp_next[n1];
738 }
739 }
740 /* now we can suppress tb(n) from the list */
741 *ptb = tb->jmp_next[n];
742
743 tb->jmp_next[n] = NULL;
744 }
745 }
746
747 /* reset the jump entry 'n' of a TB so that it is not chained to
748 another TB */
749 static inline void tb_reset_jump(TranslationBlock *tb, int n)
750 {
751 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
752 }
753
754 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
755 {
756 CPUState *env;
757 PageDesc *p;
758 unsigned int h, n1;
759 target_phys_addr_t phys_pc;
760 TranslationBlock *tb1, *tb2;
761
762 /* remove the TB from the hash list */
763 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
764 h = tb_phys_hash_func(phys_pc);
765 tb_remove(&tb_phys_hash[h], tb,
766 offsetof(TranslationBlock, phys_hash_next));
767
768 /* remove the TB from the page list */
769 if (tb->page_addr[0] != page_addr) {
770 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
771 tb_page_remove(&p->first_tb, tb);
772 invalidate_page_bitmap(p);
773 }
774 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
775 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
776 tb_page_remove(&p->first_tb, tb);
777 invalidate_page_bitmap(p);
778 }
779
780 tb_invalidated_flag = 1;
781
782 /* remove the TB from the hash list */
783 h = tb_jmp_cache_hash_func(tb->pc);
784 for(env = first_cpu; env != NULL; env = env->next_cpu) {
785 if (env->tb_jmp_cache[h] == tb)
786 env->tb_jmp_cache[h] = NULL;
787 }
788
789 /* suppress this TB from the two jump lists */
790 tb_jmp_remove(tb, 0);
791 tb_jmp_remove(tb, 1);
792
793 /* suppress any remaining jumps to this TB */
794 tb1 = tb->jmp_first;
795 for(;;) {
796 n1 = (long)tb1 & 3;
797 if (n1 == 2)
798 break;
799 tb1 = (TranslationBlock *)((long)tb1 & ~3);
800 tb2 = tb1->jmp_next[n1];
801 tb_reset_jump(tb1, n1);
802 tb1->jmp_next[n1] = NULL;
803 tb1 = tb2;
804 }
805 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
806
807 tb_phys_invalidate_count++;
808 }
809
810 static inline void set_bits(uint8_t *tab, int start, int len)
811 {
812 int end, mask, end1;
813
814 end = start + len;
815 tab += start >> 3;
816 mask = 0xff << (start & 7);
817 if ((start & ~7) == (end & ~7)) {
818 if (start < end) {
819 mask &= ~(0xff << (end & 7));
820 *tab |= mask;
821 }
822 } else {
823 *tab++ |= mask;
824 start = (start + 8) & ~7;
825 end1 = end & ~7;
826 while (start < end1) {
827 *tab++ = 0xff;
828 start += 8;
829 }
830 if (start < end) {
831 mask = ~(0xff << (end & 7));
832 *tab |= mask;
833 }
834 }
835 }
836
837 static void build_page_bitmap(PageDesc *p)
838 {
839 int n, tb_start, tb_end;
840 TranslationBlock *tb;
841
842 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
843
844 tb = p->first_tb;
845 while (tb != NULL) {
846 n = (long)tb & 3;
847 tb = (TranslationBlock *)((long)tb & ~3);
848 /* NOTE: this is subtle as a TB may span two physical pages */
849 if (n == 0) {
850 /* NOTE: tb_end may be after the end of the page, but
851 it is not a problem */
852 tb_start = tb->pc & ~TARGET_PAGE_MASK;
853 tb_end = tb_start + tb->size;
854 if (tb_end > TARGET_PAGE_SIZE)
855 tb_end = TARGET_PAGE_SIZE;
856 } else {
857 tb_start = 0;
858 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
859 }
860 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
861 tb = tb->page_next[n];
862 }
863 }
864
865 TranslationBlock *tb_gen_code(CPUState *env,
866 target_ulong pc, target_ulong cs_base,
867 int flags, int cflags)
868 {
869 TranslationBlock *tb;
870 uint8_t *tc_ptr;
871 target_ulong phys_pc, phys_page2, virt_page2;
872 int code_gen_size;
873
874 phys_pc = get_phys_addr_code(env, pc);
875 tb = tb_alloc(pc);
876 if (!tb) {
877 /* flush must be done */
878 tb_flush(env);
879 /* cannot fail at this point */
880 tb = tb_alloc(pc);
881 /* Don't forget to invalidate previous TB info. */
882 tb_invalidated_flag = 1;
883 }
884 tc_ptr = code_gen_ptr;
885 tb->tc_ptr = tc_ptr;
886 tb->cs_base = cs_base;
887 tb->flags = flags;
888 tb->cflags = cflags;
889 cpu_gen_code(env, tb, &code_gen_size);
890 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
891
892 /* check next page if needed */
893 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
894 phys_page2 = -1;
895 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
896 phys_page2 = get_phys_addr_code(env, virt_page2);
897 }
898 tb_link_phys(tb, phys_pc, phys_page2);
899 return tb;
900 }
901
902 /* invalidate all TBs which intersect with the target physical page
903 starting in range [start;end[. NOTE: start and end must refer to
904 the same physical page. 'is_cpu_write_access' should be true if called
905 from a real cpu write access: the virtual CPU will exit the current
906 TB if code is modified inside this TB. */
907 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
908 int is_cpu_write_access)
909 {
910 TranslationBlock *tb, *tb_next, *saved_tb;
911 CPUState *env = cpu_single_env;
912 target_ulong tb_start, tb_end;
913 PageDesc *p;
914 int n;
915 #ifdef TARGET_HAS_PRECISE_SMC
916 int current_tb_not_found = is_cpu_write_access;
917 TranslationBlock *current_tb = NULL;
918 int current_tb_modified = 0;
919 target_ulong current_pc = 0;
920 target_ulong current_cs_base = 0;
921 int current_flags = 0;
922 #endif /* TARGET_HAS_PRECISE_SMC */
923
924 p = page_find(start >> TARGET_PAGE_BITS);
925 if (!p)
926 return;
927 if (!p->code_bitmap &&
928 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
929 is_cpu_write_access) {
930 /* build code bitmap */
931 build_page_bitmap(p);
932 }
933
934 /* we remove all the TBs in the range [start, end[ */
935 /* XXX: see if in some cases it could be faster to invalidate all the code */
936 tb = p->first_tb;
937 while (tb != NULL) {
938 n = (long)tb & 3;
939 tb = (TranslationBlock *)((long)tb & ~3);
940 tb_next = tb->page_next[n];
941 /* NOTE: this is subtle as a TB may span two physical pages */
942 if (n == 0) {
943 /* NOTE: tb_end may be after the end of the page, but
944 it is not a problem */
945 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
946 tb_end = tb_start + tb->size;
947 } else {
948 tb_start = tb->page_addr[1];
949 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
950 }
951 if (!(tb_end <= start || tb_start >= end)) {
952 #ifdef TARGET_HAS_PRECISE_SMC
953 if (current_tb_not_found) {
954 current_tb_not_found = 0;
955 current_tb = NULL;
956 if (env->mem_io_pc) {
957 /* now we have a real cpu fault */
958 current_tb = tb_find_pc(env->mem_io_pc);
959 }
960 }
961 if (current_tb == tb &&
962 (current_tb->cflags & CF_COUNT_MASK) != 1) {
963 /* If we are modifying the current TB, we must stop
964 its execution. We could be more precise by checking
965 that the modification is after the current PC, but it
966 would require a specialized function to partially
967 restore the CPU state */
968
969 current_tb_modified = 1;
970 cpu_restore_state(current_tb, env,
971 env->mem_io_pc, NULL);
972 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
973 &current_flags);
974 }
975 #endif /* TARGET_HAS_PRECISE_SMC */
976 /* we need to do that to handle the case where a signal
977 occurs while doing tb_phys_invalidate() */
978 saved_tb = NULL;
979 if (env) {
980 saved_tb = env->current_tb;
981 env->current_tb = NULL;
982 }
983 tb_phys_invalidate(tb, -1);
984 if (env) {
985 env->current_tb = saved_tb;
986 if (env->interrupt_request && env->current_tb)
987 cpu_interrupt(env, env->interrupt_request);
988 }
989 }
990 tb = tb_next;
991 }
992 #if !defined(CONFIG_USER_ONLY)
993 /* if no code remaining, no need to continue to use slow writes */
994 if (!p->first_tb) {
995 invalidate_page_bitmap(p);
996 if (is_cpu_write_access) {
997 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
998 }
999 }
1000 #endif
1001 #ifdef TARGET_HAS_PRECISE_SMC
1002 if (current_tb_modified) {
1003 /* we generate a block containing just the instruction
1004 modifying the memory. It will ensure that it cannot modify
1005 itself */
1006 env->current_tb = NULL;
1007 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1008 cpu_resume_from_signal(env, NULL);
1009 }
1010 #endif
1011 }
1012
1013 /* len must be <= 8 and start must be a multiple of len */
1014 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1015 {
1016 PageDesc *p;
1017 int offset, b;
1018 #if 0
1019 if (1) {
1020 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1021 cpu_single_env->mem_io_vaddr, len,
1022 cpu_single_env->eip,
1023 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1024 }
1025 #endif
1026 p = page_find(start >> TARGET_PAGE_BITS);
1027 if (!p)
1028 return;
1029 if (p->code_bitmap) {
1030 offset = start & ~TARGET_PAGE_MASK;
1031 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1032 if (b & ((1 << len) - 1))
1033 goto do_invalidate;
1034 } else {
1035 do_invalidate:
1036 tb_invalidate_phys_page_range(start, start + len, 1);
1037 }
1038 }
1039
1040 #if !defined(CONFIG_SOFTMMU)
1041 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1042 unsigned long pc, void *puc)
1043 {
1044 TranslationBlock *tb;
1045 PageDesc *p;
1046 int n;
1047 #ifdef TARGET_HAS_PRECISE_SMC
1048 TranslationBlock *current_tb = NULL;
1049 CPUState *env = cpu_single_env;
1050 int current_tb_modified = 0;
1051 target_ulong current_pc = 0;
1052 target_ulong current_cs_base = 0;
1053 int current_flags = 0;
1054 #endif
1055
1056 addr &= TARGET_PAGE_MASK;
1057 p = page_find(addr >> TARGET_PAGE_BITS);
1058 if (!p)
1059 return;
1060 tb = p->first_tb;
1061 #ifdef TARGET_HAS_PRECISE_SMC
1062 if (tb && pc != 0) {
1063 current_tb = tb_find_pc(pc);
1064 }
1065 #endif
1066 while (tb != NULL) {
1067 n = (long)tb & 3;
1068 tb = (TranslationBlock *)((long)tb & ~3);
1069 #ifdef TARGET_HAS_PRECISE_SMC
1070 if (current_tb == tb &&
1071 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1072 /* If we are modifying the current TB, we must stop
1073 its execution. We could be more precise by checking
1074 that the modification is after the current PC, but it
1075 would require a specialized function to partially
1076 restore the CPU state */
1077
1078 current_tb_modified = 1;
1079 cpu_restore_state(current_tb, env, pc, puc);
1080 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1081 &current_flags);
1082 }
1083 #endif /* TARGET_HAS_PRECISE_SMC */
1084 tb_phys_invalidate(tb, addr);
1085 tb = tb->page_next[n];
1086 }
1087 p->first_tb = NULL;
1088 #ifdef TARGET_HAS_PRECISE_SMC
1089 if (current_tb_modified) {
1090 /* we generate a block containing just the instruction
1091 modifying the memory. It will ensure that it cannot modify
1092 itself */
1093 env->current_tb = NULL;
1094 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1095 cpu_resume_from_signal(env, puc);
1096 }
1097 #endif
1098 }
1099 #endif
1100
1101 /* add the tb in the target page and protect it if necessary */
1102 static inline void tb_alloc_page(TranslationBlock *tb,
1103 unsigned int n, target_ulong page_addr)
1104 {
1105 PageDesc *p;
1106 TranslationBlock *last_first_tb;
1107
1108 tb->page_addr[n] = page_addr;
1109 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1110 tb->page_next[n] = p->first_tb;
1111 last_first_tb = p->first_tb;
1112 p->first_tb = (TranslationBlock *)((long)tb | n);
1113 invalidate_page_bitmap(p);
1114
1115 #if defined(TARGET_HAS_SMC) || 1
1116
1117 #if defined(CONFIG_USER_ONLY)
1118 if (p->flags & PAGE_WRITE) {
1119 target_ulong addr;
1120 PageDesc *p2;
1121 int prot;
1122
1123 /* force the host page as non writable (writes will have a
1124 page fault + mprotect overhead) */
1125 page_addr &= qemu_host_page_mask;
1126 prot = 0;
1127 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1128 addr += TARGET_PAGE_SIZE) {
1129
1130 p2 = page_find (addr >> TARGET_PAGE_BITS);
1131 if (!p2)
1132 continue;
1133 prot |= p2->flags;
1134 p2->flags &= ~PAGE_WRITE;
1135 page_get_flags(addr);
1136 }
1137 mprotect(g2h(page_addr), qemu_host_page_size,
1138 (prot & PAGE_BITS) & ~PAGE_WRITE);
1139 #ifdef DEBUG_TB_INVALIDATE
1140 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1141 page_addr);
1142 #endif
1143 }
1144 #else
1145 /* if some code is already present, then the pages are already
1146 protected. So we handle the case where only the first TB is
1147 allocated in a physical page */
1148 if (!last_first_tb) {
1149 tlb_protect_code(page_addr);
1150 }
1151 #endif
1152
1153 #endif /* TARGET_HAS_SMC */
1154 }
1155
1156 /* Allocate a new translation block. Flush the translation buffer if
1157 too many translation blocks or too much generated code. */
1158 TranslationBlock *tb_alloc(target_ulong pc)
1159 {
1160 TranslationBlock *tb;
1161
1162 if (nb_tbs >= code_gen_max_blocks ||
1163 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1164 return NULL;
1165 tb = &tbs[nb_tbs++];
1166 tb->pc = pc;
1167 tb->cflags = 0;
1168 return tb;
1169 }
1170
1171 void tb_free(TranslationBlock *tb)
1172 {
1173 /* In practice this is mostly used for single use temporary TB
1174 Ignore the hard cases and just back up if this TB happens to
1175 be the last one generated. */
1176 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1177 code_gen_ptr = tb->tc_ptr;
1178 nb_tbs--;
1179 }
1180 }
1181
1182 /* add a new TB and link it to the physical page tables. phys_page2 is
1183 (-1) to indicate that only one page contains the TB. */
1184 void tb_link_phys(TranslationBlock *tb,
1185 target_ulong phys_pc, target_ulong phys_page2)
1186 {
1187 unsigned int h;
1188 TranslationBlock **ptb;
1189
1190 /* Grab the mmap lock to stop another thread invalidating this TB
1191 before we are done. */
1192 mmap_lock();
1193 /* add in the physical hash table */
1194 h = tb_phys_hash_func(phys_pc);
1195 ptb = &tb_phys_hash[h];
1196 tb->phys_hash_next = *ptb;
1197 *ptb = tb;
1198
1199 /* add in the page list */
1200 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1201 if (phys_page2 != -1)
1202 tb_alloc_page(tb, 1, phys_page2);
1203 else
1204 tb->page_addr[1] = -1;
1205
1206 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1207 tb->jmp_next[0] = NULL;
1208 tb->jmp_next[1] = NULL;
1209
1210 /* init original jump addresses */
1211 if (tb->tb_next_offset[0] != 0xffff)
1212 tb_reset_jump(tb, 0);
1213 if (tb->tb_next_offset[1] != 0xffff)
1214 tb_reset_jump(tb, 1);
1215
1216 #ifdef DEBUG_TB_CHECK
1217 tb_page_check();
1218 #endif
1219 mmap_unlock();
1220 }
1221
1222 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1223 tb[1].tc_ptr. Return NULL if not found */
1224 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1225 {
1226 int m_min, m_max, m;
1227 unsigned long v;
1228 TranslationBlock *tb;
1229
1230 if (nb_tbs <= 0)
1231 return NULL;
1232 if (tc_ptr < (unsigned long)code_gen_buffer ||
1233 tc_ptr >= (unsigned long)code_gen_ptr)
1234 return NULL;
1235 /* binary search (cf Knuth) */
1236 m_min = 0;
1237 m_max = nb_tbs - 1;
1238 while (m_min <= m_max) {
1239 m = (m_min + m_max) >> 1;
1240 tb = &tbs[m];
1241 v = (unsigned long)tb->tc_ptr;
1242 if (v == tc_ptr)
1243 return tb;
1244 else if (tc_ptr < v) {
1245 m_max = m - 1;
1246 } else {
1247 m_min = m + 1;
1248 }
1249 }
1250 return &tbs[m_max];
1251 }
1252
1253 static void tb_reset_jump_recursive(TranslationBlock *tb);
1254
1255 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1256 {
1257 TranslationBlock *tb1, *tb_next, **ptb;
1258 unsigned int n1;
1259
1260 tb1 = tb->jmp_next[n];
1261 if (tb1 != NULL) {
1262 /* find head of list */
1263 for(;;) {
1264 n1 = (long)tb1 & 3;
1265 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1266 if (n1 == 2)
1267 break;
1268 tb1 = tb1->jmp_next[n1];
1269 }
1270 /* we are now sure now that tb jumps to tb1 */
1271 tb_next = tb1;
1272
1273 /* remove tb from the jmp_first list */
1274 ptb = &tb_next->jmp_first;
1275 for(;;) {
1276 tb1 = *ptb;
1277 n1 = (long)tb1 & 3;
1278 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1279 if (n1 == n && tb1 == tb)
1280 break;
1281 ptb = &tb1->jmp_next[n1];
1282 }
1283 *ptb = tb->jmp_next[n];
1284 tb->jmp_next[n] = NULL;
1285
1286 /* suppress the jump to next tb in generated code */
1287 tb_reset_jump(tb, n);
1288
1289 /* suppress jumps in the tb on which we could have jumped */
1290 tb_reset_jump_recursive(tb_next);
1291 }
1292 }
1293
1294 static void tb_reset_jump_recursive(TranslationBlock *tb)
1295 {
1296 tb_reset_jump_recursive2(tb, 0);
1297 tb_reset_jump_recursive2(tb, 1);
1298 }
1299
1300 #if defined(TARGET_HAS_ICE)
1301 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1302 {
1303 target_phys_addr_t addr;
1304 target_ulong pd;
1305 ram_addr_t ram_addr;
1306 PhysPageDesc *p;
1307
1308 addr = cpu_get_phys_page_debug(env, pc);
1309 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1310 if (!p) {
1311 pd = IO_MEM_UNASSIGNED;
1312 } else {
1313 pd = p->phys_offset;
1314 }
1315 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1316 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1317 }
1318 #endif
1319
1320 /* Add a watchpoint. */
1321 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1322 int flags, CPUWatchpoint **watchpoint)
1323 {
1324 target_ulong len_mask = ~(len - 1);
1325 CPUWatchpoint *wp;
1326
1327 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1328 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1329 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1330 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1331 return -EINVAL;
1332 }
1333 wp = qemu_malloc(sizeof(*wp));
1334
1335 wp->vaddr = addr;
1336 wp->len_mask = len_mask;
1337 wp->flags = flags;
1338
1339 /* keep all GDB-injected watchpoints in front */
1340 if (flags & BP_GDB)
1341 TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1342 else
1343 TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1344
1345 tlb_flush_page(env, addr);
1346
1347 if (watchpoint)
1348 *watchpoint = wp;
1349 return 0;
1350 }
1351
1352 /* Remove a specific watchpoint. */
1353 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1354 int flags)
1355 {
1356 target_ulong len_mask = ~(len - 1);
1357 CPUWatchpoint *wp;
1358
1359 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1360 if (addr == wp->vaddr && len_mask == wp->len_mask
1361 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1362 cpu_watchpoint_remove_by_ref(env, wp);
1363 return 0;
1364 }
1365 }
1366 return -ENOENT;
1367 }
1368
1369 /* Remove a specific watchpoint by reference. */
1370 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1371 {
1372 TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1373
1374 tlb_flush_page(env, watchpoint->vaddr);
1375
1376 qemu_free(watchpoint);
1377 }
1378
1379 /* Remove all matching watchpoints. */
1380 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1381 {
1382 CPUWatchpoint *wp, *next;
1383
1384 TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1385 if (wp->flags & mask)
1386 cpu_watchpoint_remove_by_ref(env, wp);
1387 }
1388 }
1389
1390 /* Add a breakpoint. */
1391 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1392 CPUBreakpoint **breakpoint)
1393 {
1394 #if defined(TARGET_HAS_ICE)
1395 CPUBreakpoint *bp;
1396
1397 bp = qemu_malloc(sizeof(*bp));
1398
1399 bp->pc = pc;
1400 bp->flags = flags;
1401
1402 /* keep all GDB-injected breakpoints in front */
1403 if (flags & BP_GDB)
1404 TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1405 else
1406 TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1407
1408 breakpoint_invalidate(env, pc);
1409
1410 if (breakpoint)
1411 *breakpoint = bp;
1412 return 0;
1413 #else
1414 return -ENOSYS;
1415 #endif
1416 }
1417
1418 /* Remove a specific breakpoint. */
1419 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1420 {
1421 #if defined(TARGET_HAS_ICE)
1422 CPUBreakpoint *bp;
1423
1424 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1425 if (bp->pc == pc && bp->flags == flags) {
1426 cpu_breakpoint_remove_by_ref(env, bp);
1427 return 0;
1428 }
1429 }
1430 return -ENOENT;
1431 #else
1432 return -ENOSYS;
1433 #endif
1434 }
1435
1436 /* Remove a specific breakpoint by reference. */
1437 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1438 {
1439 #if defined(TARGET_HAS_ICE)
1440 TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1441
1442 breakpoint_invalidate(env, breakpoint->pc);
1443
1444 qemu_free(breakpoint);
1445 #endif
1446 }
1447
1448 /* Remove all matching breakpoints. */
1449 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1450 {
1451 #if defined(TARGET_HAS_ICE)
1452 CPUBreakpoint *bp, *next;
1453
1454 TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1455 if (bp->flags & mask)
1456 cpu_breakpoint_remove_by_ref(env, bp);
1457 }
1458 #endif
1459 }
1460
1461 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1462 CPU loop after each instruction */
1463 void cpu_single_step(CPUState *env, int enabled)
1464 {
1465 #if defined(TARGET_HAS_ICE)
1466 if (env->singlestep_enabled != enabled) {
1467 env->singlestep_enabled = enabled;
1468 if (kvm_enabled())
1469 kvm_update_guest_debug(env, 0);
1470 else {
1471 /* must flush all the translated code to avoid inconsistancies */
1472 /* XXX: only flush what is necessary */
1473 tb_flush(env);
1474 }
1475 }
1476 #endif
1477 }
1478
1479 /* enable or disable low levels log */
1480 void cpu_set_log(int log_flags)
1481 {
1482 loglevel = log_flags;
1483 if (loglevel && !logfile) {
1484 logfile = fopen(logfilename, log_append ? "a" : "w");
1485 if (!logfile) {
1486 perror(logfilename);
1487 _exit(1);
1488 }
1489 #if !defined(CONFIG_SOFTMMU)
1490 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1491 {
1492 static char logfile_buf[4096];
1493 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1494 }
1495 #else
1496 setvbuf(logfile, NULL, _IOLBF, 0);
1497 #endif
1498 log_append = 1;
1499 }
1500 if (!loglevel && logfile) {
1501 fclose(logfile);
1502 logfile = NULL;
1503 }
1504 }
1505
1506 void cpu_set_log_filename(const char *filename)
1507 {
1508 logfilename = strdup(filename);
1509 if (logfile) {
1510 fclose(logfile);
1511 logfile = NULL;
1512 }
1513 cpu_set_log(loglevel);
1514 }
1515
1516 /* mask must never be zero, except for A20 change call */
1517 void cpu_interrupt(CPUState *env, int mask)
1518 {
1519 #if !defined(USE_NPTL)
1520 TranslationBlock *tb;
1521 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1522 #endif
1523 int old_mask;
1524
1525 if (mask & CPU_INTERRUPT_EXIT) {
1526 env->exit_request = 1;
1527 mask &= ~CPU_INTERRUPT_EXIT;
1528 }
1529
1530 old_mask = env->interrupt_request;
1531 env->interrupt_request |= mask;
1532 if (kvm_enabled() && !qemu_kvm_irqchip_in_kernel())
1533 kvm_update_interrupt_request(env);
1534 #if defined(USE_NPTL)
1535 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1536 problem and hope the cpu will stop of its own accord. For userspace
1537 emulation this often isn't actually as bad as it sounds. Often
1538 signals are used primarily to interrupt blocking syscalls. */
1539 #else
1540 if (use_icount) {
1541 env->icount_decr.u16.high = 0xffff;
1542 #ifndef CONFIG_USER_ONLY
1543 if (!can_do_io(env)
1544 && (mask & ~old_mask) != 0) {
1545 cpu_abort(env, "Raised interrupt while not in I/O function");
1546 }
1547 #endif
1548 } else {
1549 tb = env->current_tb;
1550 /* if the cpu is currently executing code, we must unlink it and
1551 all the potentially executing TB */
1552 if (tb && !testandset(&interrupt_lock)) {
1553 env->current_tb = NULL;
1554 tb_reset_jump_recursive(tb);
1555 resetlock(&interrupt_lock);
1556 }
1557 }
1558 #endif
1559 }
1560
1561 void cpu_reset_interrupt(CPUState *env, int mask)
1562 {
1563 env->interrupt_request &= ~mask;
1564 }
1565
1566 const CPULogItem cpu_log_items[] = {
1567 { CPU_LOG_TB_OUT_ASM, "out_asm",
1568 "show generated host assembly code for each compiled TB" },
1569 { CPU_LOG_TB_IN_ASM, "in_asm",
1570 "show target assembly code for each compiled TB" },
1571 { CPU_LOG_TB_OP, "op",
1572 "show micro ops for each compiled TB" },
1573 { CPU_LOG_TB_OP_OPT, "op_opt",
1574 "show micro ops "
1575 #ifdef TARGET_I386
1576 "before eflags optimization and "
1577 #endif
1578 "after liveness analysis" },
1579 { CPU_LOG_INT, "int",
1580 "show interrupts/exceptions in short format" },
1581 { CPU_LOG_EXEC, "exec",
1582 "show trace before each executed TB (lots of logs)" },
1583 { CPU_LOG_TB_CPU, "cpu",
1584 "show CPU state before block translation" },
1585 #ifdef TARGET_I386
1586 { CPU_LOG_PCALL, "pcall",
1587 "show protected mode far calls/returns/exceptions" },
1588 { CPU_LOG_RESET, "cpu_reset",
1589 "show CPU state before CPU resets" },
1590 #endif
1591 #ifdef DEBUG_IOPORT
1592 { CPU_LOG_IOPORT, "ioport",
1593 "show all i/o ports accesses" },
1594 #endif
1595 { 0, NULL, NULL },
1596 };
1597
1598 static int cmp1(const char *s1, int n, const char *s2)
1599 {
1600 if (strlen(s2) != n)
1601 return 0;
1602 return memcmp(s1, s2, n) == 0;
1603 }
1604
1605 /* takes a comma separated list of log masks. Return 0 if error. */
1606 int cpu_str_to_log_mask(const char *str)
1607 {
1608 const CPULogItem *item;
1609 int mask;
1610 const char *p, *p1;
1611
1612 p = str;
1613 mask = 0;
1614 for(;;) {
1615 p1 = strchr(p, ',');
1616 if (!p1)
1617 p1 = p + strlen(p);
1618 if(cmp1(p,p1-p,"all")) {
1619 for(item = cpu_log_items; item->mask != 0; item++) {
1620 mask |= item->mask;
1621 }
1622 } else {
1623 for(item = cpu_log_items; item->mask != 0; item++) {
1624 if (cmp1(p, p1 - p, item->name))
1625 goto found;
1626 }
1627 return 0;
1628 }
1629 found:
1630 mask |= item->mask;
1631 if (*p1 != ',')
1632 break;
1633 p = p1 + 1;
1634 }
1635 return mask;
1636 }
1637
1638 void cpu_abort(CPUState *env, const char *fmt, ...)
1639 {
1640 va_list ap;
1641 va_list ap2;
1642
1643 va_start(ap, fmt);
1644 va_copy(ap2, ap);
1645 fprintf(stderr, "qemu: fatal: ");
1646 vfprintf(stderr, fmt, ap);
1647 fprintf(stderr, "\n");
1648 #ifdef TARGET_I386
1649 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1650 #else
1651 cpu_dump_state(env, stderr, fprintf, 0);
1652 #endif
1653 if (qemu_log_enabled()) {
1654 qemu_log("qemu: fatal: ");
1655 qemu_log_vprintf(fmt, ap2);
1656 qemu_log("\n");
1657 #ifdef TARGET_I386
1658 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1659 #else
1660 log_cpu_state(env, 0);
1661 #endif
1662 qemu_log_flush();
1663 qemu_log_close();
1664 }
1665 va_end(ap2);
1666 va_end(ap);
1667 abort();
1668 }
1669
1670 CPUState *cpu_copy(CPUState *env)
1671 {
1672 CPUState *new_env = cpu_init(env->cpu_model_str);
1673 CPUState *next_cpu = new_env->next_cpu;
1674 int cpu_index = new_env->cpu_index;
1675 #if defined(TARGET_HAS_ICE)
1676 CPUBreakpoint *bp;
1677 CPUWatchpoint *wp;
1678 #endif
1679
1680 memcpy(new_env, env, sizeof(CPUState));
1681
1682 /* Preserve chaining and index. */
1683 new_env->next_cpu = next_cpu;
1684 new_env->cpu_index = cpu_index;
1685
1686 /* Clone all break/watchpoints.
1687 Note: Once we support ptrace with hw-debug register access, make sure
1688 BP_CPU break/watchpoints are handled correctly on clone. */
1689 TAILQ_INIT(&env->breakpoints);
1690 TAILQ_INIT(&env->watchpoints);
1691 #if defined(TARGET_HAS_ICE)
1692 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1693 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1694 }
1695 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1696 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1697 wp->flags, NULL);
1698 }
1699 #endif
1700
1701 return new_env;
1702 }
1703
1704 #if !defined(CONFIG_USER_ONLY)
1705
1706 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1707 {
1708 unsigned int i;
1709
1710 /* Discard jump cache entries for any tb which might potentially
1711 overlap the flushed page. */
1712 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1713 memset (&env->tb_jmp_cache[i], 0,
1714 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1715
1716 i = tb_jmp_cache_hash_page(addr);
1717 memset (&env->tb_jmp_cache[i], 0,
1718 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1719 }
1720
1721 /* NOTE: if flush_global is true, also flush global entries (not
1722 implemented yet) */
1723 void tlb_flush(CPUState *env, int flush_global)
1724 {
1725 int i;
1726
1727 #if defined(DEBUG_TLB)
1728 printf("tlb_flush:\n");
1729 #endif
1730 /* must reset current TB so that interrupts cannot modify the
1731 links while we are modifying them */
1732 env->current_tb = NULL;
1733
1734 for(i = 0; i < CPU_TLB_SIZE; i++) {
1735 env->tlb_table[0][i].addr_read = -1;
1736 env->tlb_table[0][i].addr_write = -1;
1737 env->tlb_table[0][i].addr_code = -1;
1738 env->tlb_table[1][i].addr_read = -1;
1739 env->tlb_table[1][i].addr_write = -1;
1740 env->tlb_table[1][i].addr_code = -1;
1741 #if (NB_MMU_MODES >= 3)
1742 env->tlb_table[2][i].addr_read = -1;
1743 env->tlb_table[2][i].addr_write = -1;
1744 env->tlb_table[2][i].addr_code = -1;
1745 #if (NB_MMU_MODES == 4)
1746 env->tlb_table[3][i].addr_read = -1;
1747 env->tlb_table[3][i].addr_write = -1;
1748 env->tlb_table[3][i].addr_code = -1;
1749 #endif
1750 #endif
1751 }
1752
1753 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1754
1755 #ifdef USE_KQEMU
1756 if (env->kqemu_enabled) {
1757 kqemu_flush(env, flush_global);
1758 }
1759 #endif
1760 tlb_flush_count++;
1761 }
1762
1763 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1764 {
1765 if (addr == (tlb_entry->addr_read &
1766 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1767 addr == (tlb_entry->addr_write &
1768 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1769 addr == (tlb_entry->addr_code &
1770 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1771 tlb_entry->addr_read = -1;
1772 tlb_entry->addr_write = -1;
1773 tlb_entry->addr_code = -1;
1774 }
1775 }
1776
1777 void tlb_flush_page(CPUState *env, target_ulong addr)
1778 {
1779 int i;
1780
1781 #if defined(DEBUG_TLB)
1782 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1783 #endif
1784 /* must reset current TB so that interrupts cannot modify the
1785 links while we are modifying them */
1786 env->current_tb = NULL;
1787
1788 addr &= TARGET_PAGE_MASK;
1789 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1790 tlb_flush_entry(&env->tlb_table[0][i], addr);
1791 tlb_flush_entry(&env->tlb_table[1][i], addr);
1792 #if (NB_MMU_MODES >= 3)
1793 tlb_flush_entry(&env->tlb_table[2][i], addr);
1794 #if (NB_MMU_MODES == 4)
1795 tlb_flush_entry(&env->tlb_table[3][i], addr);
1796 #endif
1797 #endif
1798
1799 tlb_flush_jmp_cache(env, addr);
1800
1801 #ifdef USE_KQEMU
1802 if (env->kqemu_enabled) {
1803 kqemu_flush_page(env, addr);
1804 }
1805 #endif
1806 }
1807
1808 /* update the TLBs so that writes to code in the virtual page 'addr'
1809 can be detected */
1810 static void tlb_protect_code(ram_addr_t ram_addr)
1811 {
1812 cpu_physical_memory_reset_dirty(ram_addr,
1813 ram_addr + TARGET_PAGE_SIZE,
1814 CODE_DIRTY_FLAG);
1815 }
1816
1817 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1818 tested for self modifying code */
1819 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1820 target_ulong vaddr)
1821 {
1822 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1823 }
1824
1825 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1826 unsigned long start, unsigned long length)
1827 {
1828 unsigned long addr;
1829 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1830 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1831 if ((addr - start) < length) {
1832 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1833 }
1834 }
1835 }
1836
1837 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1838 int dirty_flags)
1839 {
1840 CPUState *env;
1841 unsigned long length, start1;
1842 int i, mask, len;
1843 uint8_t *p;
1844
1845 start &= TARGET_PAGE_MASK;
1846 end = TARGET_PAGE_ALIGN(end);
1847
1848 length = end - start;
1849 if (length == 0)
1850 return;
1851 len = length >> TARGET_PAGE_BITS;
1852 #ifdef USE_KQEMU
1853 /* XXX: should not depend on cpu context */
1854 env = first_cpu;
1855 if (env->kqemu_enabled) {
1856 ram_addr_t addr;
1857 addr = start;
1858 for(i = 0; i < len; i++) {
1859 kqemu_set_notdirty(env, addr);
1860 addr += TARGET_PAGE_SIZE;
1861 }
1862 }
1863 #endif
1864 mask = ~dirty_flags;
1865 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1866 for(i = 0; i < len; i++)
1867 p[i] &= mask;
1868
1869 /* we modify the TLB cache so that the dirty bit will be set again
1870 when accessing the range */
1871 start1 = start + (unsigned long)phys_ram_base;
1872 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1873 for(i = 0; i < CPU_TLB_SIZE; i++)
1874 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1875 for(i = 0; i < CPU_TLB_SIZE; i++)
1876 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1877 #if (NB_MMU_MODES >= 3)
1878 for(i = 0; i < CPU_TLB_SIZE; i++)
1879 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1880 #if (NB_MMU_MODES == 4)
1881 for(i = 0; i < CPU_TLB_SIZE; i++)
1882 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1883 #endif
1884 #endif
1885 }
1886 }
1887
1888 int cpu_physical_memory_set_dirty_tracking(int enable)
1889 {
1890 int r=0;
1891
1892 if (kvm_enabled())
1893 r = kvm_physical_memory_set_dirty_tracking(enable);
1894 in_migration = enable;
1895 return r;
1896 }
1897
1898 int cpu_physical_memory_get_dirty_tracking(void)
1899 {
1900 return in_migration;
1901 }
1902
1903 void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
1904 {
1905 if (kvm_enabled())
1906 kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1907 }
1908
1909 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1910 {
1911 ram_addr_t ram_addr;
1912
1913 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1914 ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1915 tlb_entry->addend - (unsigned long)phys_ram_base;
1916 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1917 tlb_entry->addr_write |= TLB_NOTDIRTY;
1918 }
1919 }
1920 }
1921
1922 /* update the TLB according to the current state of the dirty bits */
1923 void cpu_tlb_update_dirty(CPUState *env)
1924 {
1925 int i;
1926 for(i = 0; i < CPU_TLB_SIZE; i++)
1927 tlb_update_dirty(&env->tlb_table[0][i]);
1928 for(i = 0; i < CPU_TLB_SIZE; i++)
1929 tlb_update_dirty(&env->tlb_table[1][i]);
1930 #if (NB_MMU_MODES >= 3)
1931 for(i = 0; i < CPU_TLB_SIZE; i++)
1932 tlb_update_dirty(&env->tlb_table[2][i]);
1933 #if (NB_MMU_MODES == 4)
1934 for(i = 0; i < CPU_TLB_SIZE; i++)
1935 tlb_update_dirty(&env->tlb_table[3][i]);
1936 #endif
1937 #endif
1938 }
1939
1940 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1941 {
1942 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1943 tlb_entry->addr_write = vaddr;
1944 }
1945
1946 /* update the TLB corresponding to virtual page vaddr
1947 so that it is no longer dirty */
1948 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1949 {
1950 int i;
1951
1952 vaddr &= TARGET_PAGE_MASK;
1953 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1954 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1955 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1956 #if (NB_MMU_MODES >= 3)
1957 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1958 #if (NB_MMU_MODES == 4)
1959 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1960 #endif
1961 #endif
1962 }
1963
1964 /* add a new TLB entry. At most one entry for a given virtual address
1965 is permitted. Return 0 if OK or 2 if the page could not be mapped
1966 (can only happen in non SOFTMMU mode for I/O pages or pages
1967 conflicting with the host address space). */
1968 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1969 target_phys_addr_t paddr, int prot,
1970 int mmu_idx, int is_softmmu)
1971 {
1972 PhysPageDesc *p;
1973 unsigned long pd;
1974 unsigned int index;
1975 target_ulong address;
1976 target_ulong code_address;
1977 target_phys_addr_t addend;
1978 int ret;
1979 CPUTLBEntry *te;
1980 CPUWatchpoint *wp;
1981 target_phys_addr_t iotlb;
1982
1983 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1984 if (!p) {
1985 pd = IO_MEM_UNASSIGNED;
1986 } else {
1987 pd = p->phys_offset;
1988 }
1989 #if defined(DEBUG_TLB)
1990 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1991 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1992 #endif
1993
1994 ret = 0;
1995 address = vaddr;
1996 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1997 /* IO memory case (romd handled later) */
1998 address |= TLB_MMIO;
1999 }
2000 addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
2001 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2002 /* Normal RAM. */
2003 iotlb = pd & TARGET_PAGE_MASK;
2004 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2005 iotlb |= IO_MEM_NOTDIRTY;
2006 else
2007 iotlb |= IO_MEM_ROM;
2008 } else {
2009 /* IO handlers are currently passed a phsical address.
2010 It would be nice to pass an offset from the base address
2011 of that region. This would avoid having to special case RAM,
2012 and avoid full address decoding in every device.
2013 We can't use the high bits of pd for this because
2014 IO_MEM_ROMD uses these as a ram address. */
2015 iotlb = (pd & ~TARGET_PAGE_MASK);
2016 if (p) {
2017 iotlb += p->region_offset;
2018 } else {
2019 iotlb += paddr;
2020 }
2021 }
2022
2023 code_address = address;
2024 /* Make accesses to pages with watchpoints go via the
2025 watchpoint trap routines. */
2026 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2027 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2028 iotlb = io_mem_watch + paddr;
2029 /* TODO: The memory case can be optimized by not trapping
2030 reads of pages with a write breakpoint. */
2031 address |= TLB_MMIO;
2032 }
2033 }
2034
2035 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2036 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2037 te = &env->tlb_table[mmu_idx][index];
2038 te->addend = addend - vaddr;
2039 if (prot & PAGE_READ) {
2040 te->addr_read = address;
2041 } else {
2042 te->addr_read = -1;
2043 }
2044
2045 if (prot & PAGE_EXEC) {
2046 te->addr_code = code_address;
2047 } else {
2048 te->addr_code = -1;
2049 }
2050 if (prot & PAGE_WRITE) {
2051 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2052 (pd & IO_MEM_ROMD)) {
2053 /* Write access calls the I/O callback. */
2054 te->addr_write = address | TLB_MMIO;
2055 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2056 !cpu_physical_memory_is_dirty(pd)) {
2057 te->addr_write = address | TLB_NOTDIRTY;
2058 } else {
2059 te->addr_write = address;
2060 }
2061 } else {
2062 te->addr_write = -1;
2063 }
2064 return ret;
2065 }
2066
2067 #else
2068
2069 void tlb_flush(CPUState *env, int flush_global)
2070 {
2071 }
2072
2073 void tlb_flush_page(CPUState *env, target_ulong addr)
2074 {
2075 }
2076
2077 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2078 target_phys_addr_t paddr, int prot,
2079 int mmu_idx, int is_softmmu)
2080 {
2081 return 0;
2082 }
2083
2084 /* dump memory mappings */
2085 void page_dump(FILE *f)
2086 {
2087 unsigned long start, end;
2088 int i, j, prot, prot1;
2089 PageDesc *p;
2090
2091 fprintf(f, "%-8s %-8s %-8s %s\n",
2092 "start", "end", "size", "prot");
2093 start = -1;
2094 end = -1;
2095 prot = 0;
2096 for(i = 0; i <= L1_SIZE; i++) {
2097 if (i < L1_SIZE)
2098 p = l1_map[i];
2099 else
2100 p = NULL;
2101 for(j = 0;j < L2_SIZE; j++) {
2102 if (!p)
2103 prot1 = 0;
2104 else
2105 prot1 = p[j].flags;
2106 if (prot1 != prot) {
2107 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2108 if (start != -1) {
2109 fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2110 start, end, end - start,
2111 prot & PAGE_READ ? 'r' : '-',
2112 prot & PAGE_WRITE ? 'w' : '-',
2113 prot & PAGE_EXEC ? 'x' : '-');
2114 }
2115 if (prot1 != 0)
2116 start = end;
2117 else
2118 start = -1;
2119 prot = prot1;
2120 }
2121 if (!p)
2122 break;
2123 }
2124 }
2125 }
2126
2127 int page_get_flags(target_ulong address)
2128 {
2129 PageDesc *p;
2130
2131 p = page_find(address >> TARGET_PAGE_BITS);
2132 if (!p)
2133 return 0;
2134 return p->flags;
2135 }
2136
2137 /* modify the flags of a page and invalidate the code if
2138 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2139 depending on PAGE_WRITE */
2140 void page_set_flags(target_ulong start, target_ulong end, int flags)
2141 {
2142 PageDesc *p;
2143 target_ulong addr;
2144
2145 /* mmap_lock should already be held. */
2146 start = start & TARGET_PAGE_MASK;
2147 end = TARGET_PAGE_ALIGN(end);
2148 if (flags & PAGE_WRITE)
2149 flags |= PAGE_WRITE_ORG;
2150 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2151 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2152 /* We may be called for host regions that are outside guest
2153 address space. */
2154 if (!p)
2155 return;
2156 /* if the write protection is set, then we invalidate the code
2157 inside */
2158 if (!(p->flags & PAGE_WRITE) &&
2159 (flags & PAGE_WRITE) &&
2160 p->first_tb) {
2161 tb_invalidate_phys_page(addr, 0, NULL);
2162 }
2163 p->flags = flags;
2164 }
2165 }
2166
2167 int page_check_range(target_ulong start, target_ulong len, int flags)
2168 {
2169 PageDesc *p;
2170 target_ulong end;
2171 target_ulong addr;
2172
2173 if (start + len < start)
2174 /* we've wrapped around */
2175 return -1;
2176
2177 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2178 start = start & TARGET_PAGE_MASK;
2179
2180 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2181 p = page_find(addr >> TARGET_PAGE_BITS);
2182 if( !p )
2183 return -1;
2184 if( !(p->flags & PAGE_VALID) )
2185 return -1;
2186
2187 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2188 return -1;
2189 if (flags & PAGE_WRITE) {
2190 if (!(p->flags & PAGE_WRITE_ORG))
2191 return -1;
2192 /* unprotect the page if it was put read-only because it
2193 contains translated code */
2194 if (!(p->flags & PAGE_WRITE)) {
2195 if (!page_unprotect(addr, 0, NULL))
2196 return -1;
2197 }
2198 return 0;
2199 }
2200 }
2201 return 0;
2202 }
2203
2204 /* called from signal handler: invalidate the code and unprotect the
2205 page. Return TRUE if the fault was succesfully handled. */
2206 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2207 {
2208 unsigned int page_index, prot, pindex;
2209 PageDesc *p, *p1;
2210 target_ulong host_start, host_end, addr;
2211
2212 /* Technically this isn't safe inside a signal handler. However we
2213 know this only ever happens in a synchronous SEGV handler, so in
2214 practice it seems to be ok. */
2215 mmap_lock();
2216
2217 host_start = address & qemu_host_page_mask;
2218 page_index = host_start >> TARGET_PAGE_BITS;
2219 p1 = page_find(page_index);
2220 if (!p1) {
2221 mmap_unlock();
2222 return 0;
2223 }
2224 host_end = host_start + qemu_host_page_size;
2225 p = p1;
2226 prot = 0;
2227 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2228 prot |= p->flags;
2229 p++;
2230 }
2231 /* if the page was really writable, then we change its
2232 protection back to writable */
2233 if (prot & PAGE_WRITE_ORG) {
2234 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2235 if (!(p1[pindex].flags & PAGE_WRITE)) {
2236 mprotect((void *)g2h(host_start), qemu_host_page_size,
2237 (prot & PAGE_BITS) | PAGE_WRITE);
2238 p1[pindex].flags |= PAGE_WRITE;
2239 /* and since the content will be modified, we must invalidate
2240 the corresponding translated code. */
2241 tb_invalidate_phys_page(address, pc, puc);
2242 #ifdef DEBUG_TB_CHECK
2243 tb_invalidate_check(address);
2244 #endif
2245 mmap_unlock();
2246 return 1;
2247 }
2248 }
2249 mmap_unlock();
2250 return 0;
2251 }
2252
2253 static inline void tlb_set_dirty(CPUState *env,
2254 unsigned long addr, target_ulong vaddr)
2255 {
2256 }
2257 #endif /* defined(CONFIG_USER_ONLY) */
2258
2259 #if !defined(CONFIG_USER_ONLY)
2260
2261 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2262 ram_addr_t memory, ram_addr_t region_offset);
2263 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2264 ram_addr_t orig_memory, ram_addr_t region_offset);
2265 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2266 need_subpage) \
2267 do { \
2268 if (addr > start_addr) \
2269 start_addr2 = 0; \
2270 else { \
2271 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2272 if (start_addr2 > 0) \
2273 need_subpage = 1; \
2274 } \
2275 \
2276 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2277 end_addr2 = TARGET_PAGE_SIZE - 1; \
2278 else { \
2279 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2280 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2281 need_subpage = 1; \
2282 } \
2283 } while (0)
2284
2285 /* register physical memory. 'size' must be a multiple of the target
2286 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2287 io memory page. The address used when calling the IO function is
2288 the offset from the start of the region, plus region_offset. Both
2289 start_region and regon_offset are rounded down to a page boundary
2290 before calculating this offset. This should not be a problem unless
2291 the low bits of start_addr and region_offset differ. */
2292 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2293 ram_addr_t size,
2294 ram_addr_t phys_offset,
2295 ram_addr_t region_offset)
2296 {
2297 target_phys_addr_t addr, end_addr;
2298 PhysPageDesc *p;
2299 CPUState *env;
2300 ram_addr_t orig_size = size;
2301 void *subpage;
2302
2303 #ifdef USE_KQEMU
2304 /* XXX: should not depend on cpu context */
2305 env = first_cpu;
2306 if (env->kqemu_enabled) {
2307 kqemu_set_phys_mem(start_addr, size, phys_offset);
2308 }
2309 #endif
2310 if (kvm_enabled())
2311 kvm_set_phys_mem(start_addr, size, phys_offset);
2312
2313 if (phys_offset == IO_MEM_UNASSIGNED) {
2314 region_offset = start_addr;
2315 }
2316 region_offset &= TARGET_PAGE_MASK;
2317 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2318 end_addr = start_addr + (target_phys_addr_t)size;
2319 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2320 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2321 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2322 ram_addr_t orig_memory = p->phys_offset;
2323 target_phys_addr_t start_addr2, end_addr2;
2324 int need_subpage = 0;
2325
2326 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2327 need_subpage);
2328 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2329 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2330 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2331 &p->phys_offset, orig_memory,
2332 p->region_offset);
2333 } else {
2334 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2335 >> IO_MEM_SHIFT];
2336 }
2337 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2338 region_offset);
2339 p->region_offset = 0;
2340 } else {
2341 p->phys_offset = phys_offset;
2342 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2343 (phys_offset & IO_MEM_ROMD))
2344 phys_offset += TARGET_PAGE_SIZE;
2345 }
2346 } else {
2347 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2348 p->phys_offset = phys_offset;
2349 p->region_offset = region_offset;
2350 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2351 (phys_offset & IO_MEM_ROMD)) {
2352 phys_offset += TARGET_PAGE_SIZE;
2353 } else {
2354 target_phys_addr_t start_addr2, end_addr2;
2355 int need_subpage = 0;
2356
2357 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2358 end_addr2, need_subpage);
2359
2360 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2361 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2362 &p->phys_offset, IO_MEM_UNASSIGNED,
2363 addr & TARGET_PAGE_MASK);
2364 subpage_register(subpage, start_addr2, end_addr2,
2365 phys_offset, region_offset);
2366 p->region_offset = 0;
2367 }
2368 }
2369 }
2370 region_offset += TARGET_PAGE_SIZE;
2371 }
2372
2373 /* since each CPU stores ram addresses in its TLB cache, we must
2374 reset the modified entries */
2375 /* XXX: slow ! */
2376 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2377 tlb_flush(env, 1);
2378 }
2379 }
2380
2381 /* XXX: temporary until new memory mapping API */
2382 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2383 {
2384 PhysPageDesc *p;
2385
2386 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2387 if (!p)
2388 return IO_MEM_UNASSIGNED;
2389 return p->phys_offset;
2390 }
2391
2392 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2393 {
2394 if (kvm_enabled())
2395 kvm_coalesce_mmio_region(addr, size);
2396 }
2397
2398 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2399 {
2400 if (kvm_enabled())
2401 kvm_uncoalesce_mmio_region(addr, size);
2402 }
2403
2404 /* XXX: better than nothing */
2405 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2406 {
2407 ram_addr_t addr;
2408 if ((phys_ram_alloc_offset + size) > phys_ram_size) {
2409 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2410 (uint64_t)size, (uint64_t)phys_ram_size);
2411 abort();
2412 }
2413 addr = phys_ram_alloc_offset;
2414 phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2415
2416 if (kvm_enabled())
2417 kvm_setup_guest_memory(phys_ram_base + addr, size);
2418
2419 return addr;
2420 }
2421
2422 void qemu_ram_free(ram_addr_t addr)
2423 {
2424 }
2425
2426 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2427 {
2428 #ifdef DEBUG_UNASSIGNED
2429 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2430 #endif
2431 #if defined(TARGET_SPARC)
2432 do_unassigned_access(addr, 0, 0, 0, 1);
2433 #endif
2434 return 0;
2435 }
2436
2437 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2438 {
2439 #ifdef DEBUG_UNASSIGNED
2440 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2441 #endif
2442 #if defined(TARGET_SPARC)
2443 do_unassigned_access(addr, 0, 0, 0, 2);
2444 #endif
2445 return 0;
2446 }
2447
2448 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2449 {
2450 #ifdef DEBUG_UNASSIGNED
2451 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2452 #endif
2453 #if defined(TARGET_SPARC)
2454 do_unassigned_access(addr, 0, 0, 0, 4);
2455 #endif
2456 return 0;
2457 }
2458
2459 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2460 {
2461 #ifdef DEBUG_UNASSIGNED
2462 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2463 #endif
2464 #if defined(TARGET_SPARC)
2465 do_unassigned_access(addr, 1, 0, 0, 1);
2466 #endif
2467 }
2468
2469 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2470 {
2471 #ifdef DEBUG_UNASSIGNED
2472 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2473 #endif
2474 #if defined(TARGET_SPARC)
2475 do_unassigned_access(addr, 1, 0, 0, 2);
2476 #endif
2477 }
2478
2479 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2480 {
2481 #ifdef DEBUG_UNASSIGNED
2482 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2483 #endif
2484 #if defined(TARGET_SPARC)
2485 do_unassigned_access(addr, 1, 0, 0, 4);
2486 #endif
2487 }
2488
2489 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2490 unassigned_mem_readb,
2491 unassigned_mem_readw,
2492 unassigned_mem_readl,
2493 };
2494
2495 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2496 unassigned_mem_writeb,
2497 unassigned_mem_writew,
2498 unassigned_mem_writel,
2499 };
2500
2501 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2502 uint32_t val)
2503 {
2504 int dirty_flags;
2505 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2506 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2507 #if !defined(CONFIG_USER_ONLY)
2508 tb_invalidate_phys_page_fast(ram_addr, 1);
2509 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2510 #endif
2511 }
2512 stb_p(phys_ram_base + ram_addr, val);
2513 #ifdef USE_KQEMU
2514 if (cpu_single_env->kqemu_enabled &&
2515 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2516 kqemu_modify_page(cpu_single_env, ram_addr);
2517 #endif
2518 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2519 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2520 /* we remove the notdirty callback only if the code has been
2521 flushed */
2522 if (dirty_flags == 0xff)
2523 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2524 }
2525
2526 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2527 uint32_t val)
2528 {
2529 int dirty_flags;
2530 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2531 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2532 #if !defined(CONFIG_USER_ONLY)
2533 tb_invalidate_phys_page_fast(ram_addr, 2);
2534 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2535 #endif
2536 }
2537 stw_p(phys_ram_base + ram_addr, val);
2538 #ifdef USE_KQEMU
2539 if (cpu_single_env->kqemu_enabled &&
2540 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2541 kqemu_modify_page(cpu_single_env, ram_addr);
2542 #endif
2543 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2544 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2545 /* we remove the notdirty callback only if the code has been
2546 flushed */
2547 if (dirty_flags == 0xff)
2548 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2549 }
2550
2551 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2552 uint32_t val)
2553 {
2554 int dirty_flags;
2555 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2556 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2557 #if !defined(CONFIG_USER_ONLY)
2558 tb_invalidate_phys_page_fast(ram_addr, 4);
2559 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2560 #endif
2561 }
2562 stl_p(phys_ram_base + ram_addr, val);
2563 #ifdef USE_KQEMU
2564 if (cpu_single_env->kqemu_enabled &&
2565 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2566 kqemu_modify_page(cpu_single_env, ram_addr);
2567 #endif
2568 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2569 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2570 /* we remove the notdirty callback only if the code has been
2571 flushed */
2572 if (dirty_flags == 0xff)
2573 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2574 }
2575
2576 static CPUReadMemoryFunc *error_mem_read[3] = {
2577 NULL, /* never used */
2578 NULL, /* never used */
2579 NULL, /* never used */
2580 };
2581
2582 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2583 notdirty_mem_writeb,
2584 notdirty_mem_writew,
2585 notdirty_mem_writel,
2586 };
2587
2588 /* Generate a debug exception if a watchpoint has been hit. */
2589 static void check_watchpoint(int offset, int len_mask, int flags)
2590 {
2591 CPUState *env = cpu_single_env;
2592 target_ulong pc, cs_base;
2593 TranslationBlock *tb;
2594 target_ulong vaddr;
2595 CPUWatchpoint *wp;
2596 int cpu_flags;
2597
2598 if (env->watchpoint_hit) {
2599 /* We re-entered the check after replacing the TB. Now raise
2600 * the debug interrupt so that is will trigger after the
2601 * current instruction. */
2602 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2603 return;
2604 }
2605 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2606 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2607 if ((vaddr == (wp->vaddr & len_mask) ||
2608 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2609 wp->flags |= BP_WATCHPOINT_HIT;
2610 if (!env->watchpoint_hit) {
2611 env->watchpoint_hit = wp;
2612 tb = tb_find_pc(env->mem_io_pc);
2613 if (!tb) {
2614 cpu_abort(env, "check_watchpoint: could not find TB for "
2615 "pc=%p", (void *)env->mem_io_pc);
2616 }
2617 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2618 tb_phys_invalidate(tb, -1);
2619 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2620 env->exception_index = EXCP_DEBUG;
2621 } else {
2622 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2623 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2624 }
2625 cpu_resume_from_signal(env, NULL);
2626 }
2627 } else {
2628 wp->flags &= ~BP_WATCHPOINT_HIT;
2629 }
2630 }
2631 }
2632
2633 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2634 so these check for a hit then pass through to the normal out-of-line
2635 phys routines. */
2636 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2637 {
2638 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2639 return ldub_phys(addr);
2640 }
2641
2642 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2643 {
2644 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2645 return lduw_phys(addr);
2646 }
2647
2648 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2649 {
2650 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2651 return ldl_phys(addr);
2652 }
2653
2654 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2655 uint32_t val)
2656 {
2657 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2658 stb_phys(addr, val);
2659 }
2660
2661 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2662 uint32_t val)
2663 {
2664 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2665 stw_phys(addr, val);
2666 }
2667
2668 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2669 uint32_t val)
2670 {
2671 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2672 stl_phys(addr, val);
2673 }
2674
2675 static CPUReadMemoryFunc *watch_mem_read[3] = {
2676 watch_mem_readb,
2677 watch_mem_readw,
2678 watch_mem_readl,
2679 };
2680
2681 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2682 watch_mem_writeb,
2683 watch_mem_writew,
2684 watch_mem_writel,
2685 };
2686
2687 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2688 unsigned int len)
2689 {
2690 uint32_t ret;
2691 unsigned int idx;
2692
2693 idx = SUBPAGE_IDX(addr);
2694 #if defined(DEBUG_SUBPAGE)
2695 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2696 mmio, len, addr, idx);
2697 #endif
2698 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2699 addr + mmio->region_offset[idx][0][len]);
2700
2701 return ret;
2702 }
2703
2704 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2705 uint32_t value, unsigned int len)
2706 {
2707 unsigned int idx;
2708
2709 idx = SUBPAGE_IDX(addr);
2710 #if defined(DEBUG_SUBPAGE)
2711 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2712 mmio, len, addr, idx, value);
2713 #endif
2714 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2715 addr + mmio->region_offset[idx][1][len],
2716 value);
2717 }
2718
2719 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2720 {
2721 #if defined(DEBUG_SUBPAGE)
2722 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2723 #endif
2724
2725 return subpage_readlen(opaque, addr, 0);
2726 }
2727
2728 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2729 uint32_t value)
2730 {
2731 #if defined(DEBUG_SUBPAGE)
2732 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2733 #endif
2734 subpage_writelen(opaque, addr, value, 0);
2735 }
2736
2737 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2738 {
2739 #if defined(DEBUG_SUBPAGE)
2740 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2741 #endif
2742
2743 return subpage_readlen(opaque, addr, 1);
2744 }
2745
2746 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2747 uint32_t value)
2748 {
2749 #if defined(DEBUG_SUBPAGE)
2750 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2751 #endif
2752 subpage_writelen(opaque, addr, value, 1);
2753 }
2754
2755 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2756 {
2757 #if defined(DEBUG_SUBPAGE)
2758 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2759 #endif
2760
2761 return subpage_readlen(opaque, addr, 2);
2762 }
2763
2764 static void subpage_writel (void *opaque,
2765 target_phys_addr_t addr, uint32_t value)
2766 {
2767 #if defined(DEBUG_SUBPAGE)
2768 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2769 #endif
2770 subpage_writelen(opaque, addr, value, 2);
2771 }
2772
2773 static CPUReadMemoryFunc *subpage_read[] = {
2774 &subpage_readb,
2775 &subpage_readw,
2776 &subpage_readl,
2777 };
2778
2779 static CPUWriteMemoryFunc *subpage_write[] = {
2780 &subpage_writeb,
2781 &subpage_writew,
2782 &subpage_writel,
2783 };
2784
2785 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2786 ram_addr_t memory, ram_addr_t region_offset)
2787 {
2788 int idx, eidx;
2789 unsigned int i;
2790
2791 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2792 return -1;
2793 idx = SUBPAGE_IDX(start);
2794 eidx = SUBPAGE_IDX(end);
2795 #if defined(DEBUG_SUBPAGE)
2796 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2797 mmio, start, end, idx, eidx, memory);
2798 #endif
2799 memory >>= IO_MEM_SHIFT;
2800 for (; idx <= eidx; idx++) {
2801 for (i = 0; i < 4; i++) {
2802 if (io_mem_read[memory][i]) {
2803 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2804 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2805 mmio->region_offset[idx][0][i] = region_offset;
2806 }
2807 if (io_mem_write[memory][i]) {
2808 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2809 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2810 mmio->region_offset[idx][1][i] = region_offset;
2811 }
2812 }
2813 }
2814
2815 return 0;
2816 }
2817
2818 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2819 ram_addr_t orig_memory, ram_addr_t region_offset)
2820 {
2821 subpage_t *mmio;
2822 int subpage_memory;
2823
2824 mmio = qemu_mallocz(sizeof(subpage_t));
2825
2826 mmio->base = base;
2827 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2828 #if defined(DEBUG_SUBPAGE)
2829 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2830 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2831 #endif
2832 *phys = subpage_memory | IO_MEM_SUBPAGE;
2833 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
2834 region_offset);
2835
2836 return mmio;
2837 }
2838
2839 static int get_free_io_mem_idx(void)
2840 {
2841 int i;
2842
2843 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
2844 if (!io_mem_used[i]) {
2845 io_mem_used[i] = 1;
2846 return i;
2847 }
2848
2849 return -1;
2850 }
2851
2852 static void io_mem_init(void)
2853 {
2854 int i;
2855
2856 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
2857 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2858 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2859 for (i=0; i<5; i++)
2860 io_mem_used[i] = 1;
2861
2862 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2863 watch_mem_write, NULL);
2864 /* alloc dirty bits array */
2865 phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2866 memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2867 }
2868
2869 /* mem_read and mem_write are arrays of functions containing the
2870 function to access byte (index 0), word (index 1) and dword (index
2871 2). Functions can be omitted with a NULL function pointer. The
2872 registered functions may be modified dynamically later.
2873 If io_index is non zero, the corresponding io zone is
2874 modified. If it is zero, a new io zone is allocated. The return
2875 value can be used with cpu_register_physical_memory(). (-1) is
2876 returned if error. */
2877 int cpu_register_io_memory(int io_index,
2878 CPUReadMemoryFunc **mem_read,
2879 CPUWriteMemoryFunc **mem_write,
2880 void *opaque)
2881 {
2882 int i, subwidth = 0;
2883
2884 if (io_index <= 0) {
2885 io_index = get_free_io_mem_idx();
2886 if (io_index == -1)
2887 return io_index;
2888 } else {
2889 if (io_index >= IO_MEM_NB_ENTRIES)
2890 return -1;
2891 }
2892
2893 for(i = 0;i < 3; i++) {
2894 if (!mem_read[i] || !mem_write[i])
2895 subwidth = IO_MEM_SUBWIDTH;
2896 io_mem_read[io_index][i] = mem_read[i];
2897 io_mem_write[io_index][i] = mem_write[i];
2898 }
2899 io_mem_opaque[io_index] = opaque;
2900 return (io_index << IO_MEM_SHIFT) | subwidth;
2901 }
2902
2903 void cpu_unregister_io_memory(int io_table_address)
2904 {
2905 int i;
2906 int io_index = io_table_address >> IO_MEM_SHIFT;
2907
2908 for (i=0;i < 3; i++) {
2909 io_mem_read[io_index][i] = unassigned_mem_read[i];
2910 io_mem_write[io_index][i] = unassigned_mem_write[i];
2911 }
2912 io_mem_opaque[io_index] = NULL;
2913 io_mem_used[io_index] = 0;
2914 }
2915
2916 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2917 {
2918 return io_mem_write[io_index >> IO_MEM_SHIFT];
2919 }
2920
2921 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2922 {
2923 return io_mem_read[io_index >> IO_MEM_SHIFT];
2924 }
2925
2926 #endif /* !defined(CONFIG_USER_ONLY) */
2927
2928 /* physical memory access (slow version, mainly for debug) */
2929 #if defined(CONFIG_USER_ONLY)
2930 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2931 int len, int is_write)
2932 {
2933 int l, flags;
2934 target_ulong page;
2935 void * p;
2936
2937 while (len > 0) {
2938 page = addr & TARGET_PAGE_MASK;
2939 l = (page + TARGET_PAGE_SIZE) - addr;
2940 if (l > len)
2941 l = len;
2942 flags = page_get_flags(page);
2943 if (!(flags & PAGE_VALID))
2944 return;
2945 if (is_write) {
2946 if (!(flags & PAGE_WRITE))
2947 return;
2948 /* XXX: this code should not depend on lock_user */
2949 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2950 /* FIXME - should this return an error rather than just fail? */
2951 return;
2952 memcpy(p, buf, l);
2953 unlock_user(p, addr, l);
2954 } else {
2955 if (!(flags & PAGE_READ))
2956 return;
2957 /* XXX: this code should not depend on lock_user */
2958 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2959 /* FIXME - should this return an error rather than just fail? */
2960 return;
2961 memcpy(buf, p, l);
2962 unlock_user(p, addr, 0);
2963 }
2964 len -= l;
2965 buf += l;
2966 addr += l;
2967 }
2968 }
2969
2970 #else
2971 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2972 int len, int is_write)
2973 {
2974 int l, io_index;
2975 uint8_t *ptr;
2976 uint32_t val;
2977 target_phys_addr_t page;
2978 unsigned long pd;
2979 PhysPageDesc *p;
2980
2981 while (len > 0) {
2982 page = addr & TARGET_PAGE_MASK;
2983 l = (page + TARGET_PAGE_SIZE) - addr;
2984 if (l > len)
2985 l = len;
2986 p = phys_page_find(page >> TARGET_PAGE_BITS);
2987 if (!p) {
2988 pd = IO_MEM_UNASSIGNED;
2989 } else {
2990 pd = p->phys_offset;
2991 }
2992
2993 if (is_write) {
2994 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
2995 target_phys_addr_t addr1 = addr;
2996 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2997 if (p)
2998 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
2999 /* XXX: could force cpu_single_env to NULL to avoid
3000 potential bugs */
3001 if (l >= 4 && ((addr1 & 3) == 0)) {
3002 /* 32 bit write access */
3003 val = ldl_p(buf);
3004 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3005 l = 4;
3006 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3007 /* 16 bit write access */
3008 val = lduw_p(buf);
3009 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3010 l = 2;
3011 } else {
3012 /* 8 bit write access */
3013 val = ldub_p(buf);
3014 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3015 l = 1;
3016 }
3017 } else {
3018 unsigned long addr1;
3019 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3020 /* RAM case */
3021 ptr = phys_ram_base + addr1;
3022 memcpy(ptr, buf, l);
3023 if (!cpu_physical_memory_is_dirty(addr1)) {
3024 /* invalidate code */
3025 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3026 /* set dirty bit */
3027 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3028 (0xff & ~CODE_DIRTY_FLAG);
3029 }
3030 /* qemu doesn't execute guest code directly, but kvm does
3031 therefore fluch instruction caches */
3032 if (kvm_enabled())
3033 flush_icache_range((unsigned long)ptr,
3034 ((unsigned long)ptr)+l);
3035 }
3036 } else {
3037 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3038 !(pd & IO_MEM_ROMD)) {
3039 target_phys_addr_t addr1 = addr;
3040 /* I/O case */
3041 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3042 if (p)
3043 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3044 if (l >= 4 && ((addr1 & 3) == 0)) {
3045 /* 32 bit read access */
3046 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3047 stl_p(buf, val);
3048 l = 4;
3049 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3050 /* 16 bit read access */
3051 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3052 stw_p(buf, val);
3053 l = 2;
3054 } else {
3055 /* 8 bit read access */
3056 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3057 stb_p(buf, val);
3058 l = 1;
3059 }
3060 } else {
3061 /* RAM case */
3062 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3063 (addr & ~TARGET_PAGE_MASK);
3064 memcpy(buf, ptr, l);
3065 }
3066 }
3067 len -= l;
3068 buf += l;
3069 addr += l;
3070 }
3071 }
3072
3073 /* used for ROM loading : can write in RAM and ROM */
3074 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3075 const uint8_t *buf, int len)
3076 {
3077 int l;
3078 uint8_t *ptr;
3079 target_phys_addr_t page;
3080 unsigned long pd;
3081 PhysPageDesc *p;
3082
3083 while (len > 0) {
3084 page = addr & TARGET_PAGE_MASK;
3085 l = (page + TARGET_PAGE_SIZE) - addr;
3086 if (l > len)
3087 l = len;
3088 p = phys_page_find(page >> TARGET_PAGE_BITS);
3089 if (!p) {
3090 pd = IO_MEM_UNASSIGNED;
3091 } else {
3092 pd = p->phys_offset;
3093 }
3094
3095 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3096 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3097 !(pd & IO_MEM_ROMD)) {
3098 /* do nothing */
3099 } else {
3100 unsigned long addr1;
3101 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3102 /* ROM/RAM case */
3103 ptr = phys_ram_base + addr1;
3104 memcpy(ptr, buf, l);
3105 }
3106 len -= l;
3107 buf += l;
3108 addr += l;
3109 }
3110 }
3111
3112 typedef struct {
3113 void *buffer;
3114 target_phys_addr_t addr;
3115 target_phys_addr_t len;
3116 } BounceBuffer;
3117
3118 static BounceBuffer bounce;
3119
3120 typedef struct MapClient {
3121 void *opaque;
3122 void (*callback)(void *opaque);
3123 LIST_ENTRY(MapClient) link;
3124 } MapClient;
3125
3126 static LIST_HEAD(map_client_list, MapClient) map_client_list
3127 = LIST_HEAD_INITIALIZER(map_client_list);
3128
3129 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3130 {
3131 MapClient *client = qemu_malloc(sizeof(*client));
3132
3133 client->opaque = opaque;
3134 client->callback = callback;
3135 LIST_INSERT_HEAD(&map_client_list, client, link);
3136 return client;
3137 }
3138
3139 void cpu_unregister_map_client(void *_client)
3140 {
3141 MapClient *client = (MapClient *)_client;
3142
3143 LIST_REMOVE(client, link);
3144 qemu_free(client);
3145 }
3146
3147 static void cpu_notify_map_clients(void)
3148 {
3149 MapClient *client;
3150
3151 while (!LIST_EMPTY(&map_client_list)) {
3152 client = LIST_FIRST(&map_client_list);
3153 client->callback(client->opaque);
3154 cpu_unregister_map_client(client);
3155 }
3156 }
3157
3158 /* Map a physical memory region into a host virtual address.
3159 * May map a subset of the requested range, given by and returned in *plen.
3160 * May return NULL if resources needed to perform the mapping are exhausted.
3161 * Use only for reads OR writes - not for read-modify-write operations.
3162 * Use cpu_register_map_client() to know when retrying the map operation is
3163 * likely to succeed.
3164 */
3165 void *cpu_physical_memory_map(target_phys_addr_t addr,
3166 target_phys_addr_t *plen,
3167 int is_write)
3168 {
3169 target_phys_addr_t len = *plen;
3170 target_phys_addr_t done = 0;
3171 int l;
3172 uint8_t *ret = NULL;
3173 uint8_t *ptr;
3174 target_phys_addr_t page;
3175 unsigned long pd;
3176 PhysPageDesc *p;
3177 unsigned long addr1;
3178
3179 while (len > 0) {
3180 page = addr & TARGET_PAGE_MASK;
3181 l = (page + TARGET_PAGE_SIZE) - addr;
3182 if (l > len)
3183 l = len;
3184 p = phys_page_find(page >> TARGET_PAGE_BITS);
3185 if (!p) {
3186 pd = IO_MEM_UNASSIGNED;
3187 } else {
3188 pd = p->phys_offset;
3189 }
3190
3191 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3192 if (done || bounce.buffer) {
3193 break;
3194 }
3195 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3196 bounce.addr = addr;
3197 bounce.len = l;
3198 if (!is_write) {
3199 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3200 }
3201 ptr = bounce.buffer;
3202 } else {
3203 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3204 ptr = phys_ram_base + addr1;
3205 }
3206 if (!done) {
3207 ret = ptr;
3208 } else if (ret + done != ptr) {
3209 break;
3210 }
3211
3212 len -= l;
3213 addr += l;
3214 done += l;
3215 }
3216 *plen = done;
3217 return ret;
3218 }
3219
3220 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3221 * Will also mark the memory as dirty if is_write == 1. access_len gives
3222 * the amount of memory that was actually read or written by the caller.
3223 */
3224 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3225 int is_write, target_phys_addr_t access_len)
3226 {
3227 if (buffer != bounce.buffer) {
3228 if (is_write) {
3229 unsigned long addr1 = (uint8_t *)buffer - phys_ram_base;
3230 while (access_len) {
3231 unsigned l;
3232 l = TARGET_PAGE_SIZE;
3233 if (l > access_len)
3234 l = access_len;
3235 if (!cpu_physical_memory_is_dirty(addr1)) {
3236 /* invalidate code */
3237 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3238 /* set dirty bit */
3239 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3240 (0xff & ~CODE_DIRTY_FLAG);
3241 }
3242 addr1 += l;
3243 access_len -= l;
3244 }
3245 }
3246 return;
3247 }
3248 if (is_write) {
3249 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3250 }
3251 qemu_free(bounce.buffer);
3252 bounce.buffer = NULL;
3253 cpu_notify_map_clients();
3254 }
3255
3256 /* warning: addr must be aligned */
3257 uint32_t ldl_phys(target_phys_addr_t addr)
3258 {
3259 int io_index;
3260 uint8_t *ptr;
3261 uint32_t val;
3262 unsigned long pd;
3263 PhysPageDesc *p;
3264
3265 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3266 if (!p) {
3267 pd = IO_MEM_UNASSIGNED;
3268 } else {
3269 pd = p->phys_offset;
3270 }
3271
3272 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3273 !(pd & IO_MEM_ROMD)) {
3274 /* I/O case */
3275 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3276 if (p)
3277 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3278 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3279 } else {
3280 /* RAM case */
3281 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3282 (addr & ~TARGET_PAGE_MASK);
3283 val = ldl_p(ptr);
3284 }
3285 return val;
3286 }
3287
3288 /* warning: addr must be aligned */
3289 uint64_t ldq_phys(target_phys_addr_t addr)
3290 {
3291 int io_index;
3292 uint8_t *ptr;
3293 uint64_t val;
3294 unsigned long pd;
3295 PhysPageDesc *p;
3296
3297 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3298 if (!p) {
3299 pd = IO_MEM_UNASSIGNED;
3300 } else {
3301 pd = p->phys_offset;
3302 }
3303
3304 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3305 !(pd & IO_MEM_ROMD)) {
3306 /* I/O case */
3307 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3308 if (p)
3309 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3310 #ifdef TARGET_WORDS_BIGENDIAN
3311 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3312 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3313 #else
3314 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3315 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3316 #endif
3317 } else {
3318 /* RAM case */
3319 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3320 (addr & ~TARGET_PAGE_MASK);
3321 val = ldq_p(ptr);
3322 }
3323 return val;
3324 }
3325
3326 /* XXX: optimize */
3327 uint32_t ldub_phys(target_phys_addr_t addr)
3328 {
3329 uint8_t val;
3330 cpu_physical_memory_read(addr, &val, 1);
3331 return val;
3332 }
3333
3334 /* XXX: optimize */
3335 uint32_t lduw_phys(target_phys_addr_t addr)
3336 {
3337 uint16_t val;
3338 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3339 return tswap16(val);
3340 }
3341
3342 #ifdef __GNUC__
3343 #define likely(x) __builtin_expect(!!(x), 1)
3344 #define unlikely(x) __builtin_expect(!!(x), 0)
3345 #else
3346 #define likely(x) x
3347 #define unlikely(x) x
3348 #endif
3349
3350 /* warning: addr must be aligned. The ram page is not masked as dirty
3351 and the code inside is not invalidated. It is useful if the dirty
3352 bits are used to track modified PTEs */
3353 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3354 {
3355 int io_index;
3356 uint8_t *ptr;
3357 unsigned long pd;
3358 PhysPageDesc *p;
3359
3360 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3361 if (!p) {
3362 pd = IO_MEM_UNASSIGNED;
3363 } else {
3364 pd = p->phys_offset;
3365 }
3366
3367 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3368 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3369 if (p)
3370 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3371 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3372 } else {
3373 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3374 ptr = phys_ram_base + addr1;
3375 stl_p(ptr, val);
3376
3377 if (unlikely(in_migration)) {
3378 if (!cpu_physical_memory_is_dirty(addr1)) {
3379 /* invalidate code */
3380 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3381 /* set dirty bit */
3382 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3383 (0xff & ~CODE_DIRTY_FLAG);
3384 }
3385 }
3386 }
3387 }
3388
3389 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3390 {
3391 int io_index;
3392 uint8_t *ptr;
3393 unsigned long pd;
3394 PhysPageDesc *p;
3395
3396 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3397 if (!p) {
3398 pd = IO_MEM_UNASSIGNED;
3399 } else {
3400 pd = p->phys_offset;
3401 }
3402
3403 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3404 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3405 if (p)
3406 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3407 #ifdef TARGET_WORDS_BIGENDIAN
3408 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3409 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3410 #else
3411 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3412 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3413 #endif
3414 } else {
3415 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3416 (addr & ~TARGET_PAGE_MASK);
3417 stq_p(ptr, val);
3418 }
3419 }
3420
3421 /* warning: addr must be aligned */
3422 void stl_phys(target_phys_addr_t addr, uint32_t val)
3423 {
3424 int io_index;
3425 uint8_t *ptr;
3426 unsigned long pd;
3427 PhysPageDesc *p;
3428
3429 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3430 if (!p) {
3431 pd = IO_MEM_UNASSIGNED;
3432 } else {
3433 pd = p->phys_offset;
3434 }
3435
3436 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3437 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3438 if (p)
3439 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3440 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3441 } else {
3442 unsigned long addr1;
3443 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3444 /* RAM case */
3445 ptr = phys_ram_base + addr1;
3446 stl_p(ptr, val);
3447 if (!cpu_physical_memory_is_dirty(addr1)) {
3448 /* invalidate code */
3449 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3450 /* set dirty bit */
3451 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3452 (0xff & ~CODE_DIRTY_FLAG);
3453 }
3454 }
3455 }
3456
3457 /* XXX: optimize */
3458 void stb_phys(target_phys_addr_t addr, uint32_t val)
3459 {
3460 uint8_t v = val;
3461 cpu_physical_memory_write(addr, &v, 1);
3462 }
3463
3464 /* XXX: optimize */
3465 void stw_phys(target_phys_addr_t addr, uint32_t val)
3466 {
3467 uint16_t v = tswap16(val);
3468 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3469 }
3470
3471 /* XXX: optimize */
3472 void stq_phys(target_phys_addr_t addr, uint64_t val)
3473 {
3474 val = tswap64(val);
3475 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3476 }
3477
3478 #endif
3479
3480 /* virtual memory access for debug */
3481 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3482 uint8_t *buf, int len, int is_write)
3483 {
3484 int l;
3485 target_phys_addr_t phys_addr;
3486 target_ulong page;
3487
3488 while (len > 0) {
3489 page = addr & TARGET_PAGE_MASK;
3490 phys_addr = cpu_get_phys_page_debug(env, page);
3491 /* if no physical page mapped, return an error */
3492 if (phys_addr == -1)
3493 return -1;
3494 l = (page + TARGET_PAGE_SIZE) - addr;
3495 if (l > len)
3496 l = len;
3497 cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
3498 buf, l, is_write);
3499 len -= l;
3500 buf += l;
3501 addr += l;
3502 }
3503 return 0;
3504 }
3505
3506 /* in deterministic execution mode, instructions doing device I/Os
3507 must be at the end of the TB */
3508 void cpu_io_recompile(CPUState *env, void *retaddr)
3509 {
3510 TranslationBlock *tb;
3511 uint32_t n, cflags;
3512 target_ulong pc, cs_base;
3513 uint64_t flags;
3514
3515 tb = tb_find_pc((unsigned long)retaddr);
3516 if (!tb) {
3517 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3518 retaddr);
3519 }
3520 n = env->icount_decr.u16.low + tb->icount;
3521 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3522 /* Calculate how many instructions had been executed before the fault
3523 occurred. */
3524 n = n - env->icount_decr.u16.low;
3525 /* Generate a new TB ending on the I/O insn. */
3526 n++;
3527 /* On MIPS and SH, delay slot instructions can only be restarted if
3528 they were already the first instruction in the TB. If this is not
3529 the first instruction in a TB then re-execute the preceding
3530 branch. */
3531 #if defined(TARGET_MIPS)
3532 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3533 env->active_tc.PC -= 4;
3534 env->icount_decr.u16.low++;
3535 env->hflags &= ~MIPS_HFLAG_BMASK;
3536 }
3537 #elif defined(TARGET_SH4)
3538 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3539 && n > 1) {
3540 env->pc -= 2;
3541 env->icount_decr.u16.low++;
3542 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3543 }
3544 #endif
3545 /* This should never happen. */
3546 if (n > CF_COUNT_MASK)
3547 cpu_abort(env, "TB too big during recompile");
3548
3549 cflags = n | CF_LAST_IO;
3550 pc = tb->pc;
3551 cs_base = tb->cs_base;
3552 flags = tb->flags;
3553 tb_phys_invalidate(tb, -1);
3554 /* FIXME: In theory this could raise an exception. In practice
3555 we have already translated the block once so it's probably ok. */
3556 tb_gen_code(env, pc, cs_base, flags, cflags);
3557 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3558 the first in the TB) then we end up generating a whole new TB and
3559 repeating the fault, which is horribly inefficient.
3560 Better would be to execute just this insn uncached, or generate a
3561 second new TB. */
3562 cpu_resume_from_signal(env, NULL);
3563 }
3564
3565 void dump_exec_info(FILE *f,
3566 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3567 {
3568 int i, target_code_size, max_target_code_size;
3569 int direct_jmp_count, direct_jmp2_count, cross_page;
3570 TranslationBlock *tb;
3571
3572 target_code_size = 0;
3573 max_target_code_size = 0;
3574 cross_page = 0;
3575 direct_jmp_count = 0;
3576 direct_jmp2_count = 0;
3577 for(i = 0; i < nb_tbs; i++) {
3578 tb = &tbs[i];
3579 target_code_size += tb->size;
3580 if (tb->size > max_target_code_size)
3581 max_target_code_size = tb->size;
3582 if (tb->page_addr[1] != -1)
3583 cross_page++;
3584 if (tb->tb_next_offset[0] != 0xffff) {
3585 direct_jmp_count++;
3586 if (tb->tb_next_offset[1] != 0xffff) {
3587 direct_jmp2_count++;
3588 }
3589 }
3590 }
3591 /* XXX: avoid using doubles ? */
3592 cpu_fprintf(f, "Translation buffer state:\n");
3593 cpu_fprintf(f, "gen code size %ld/%ld\n",
3594 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3595 cpu_fprintf(f, "TB count %d/%d\n",
3596 nb_tbs, code_gen_max_blocks);
3597 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3598 nb_tbs ? target_code_size / nb_tbs : 0,
3599 max_target_code_size);
3600 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3601 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3602 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3603 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3604 cross_page,
3605 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3606 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3607 direct_jmp_count,
3608 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3609 direct_jmp2_count,
3610 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3611 cpu_fprintf(f, "\nStatistics:\n");
3612 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3613 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3614 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3615 tcg_dump_info(f, cpu_fprintf);
3616 }
3617
3618 #if !defined(CONFIG_USER_ONLY)
3619
3620 #define MMUSUFFIX _cmmu
3621 #define GETPC() NULL
3622 #define env cpu_single_env
3623 #define SOFTMMU_CODE_ACCESS
3624
3625 #define SHIFT 0
3626 #include "softmmu_template.h"
3627
3628 #define SHIFT 1
3629 #include "softmmu_template.h"
3630
3631 #define SHIFT 2
3632 #include "softmmu_template.h"
3633
3634 #define SHIFT 3
3635 #include "softmmu_template.h"
3636
3637 #undef env
3638
3639 #endif
Patches shipped by Fedora