?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
kvm: bios: Reset stack pointer to zero on S3 resume
[qemu-kvm/fedora.git] / exec.c
blobf3646fe7639cbeff246c4f932d923b030b3a5b70
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20 #include "config.h"
21 #ifdef _WIN32
22 #define WIN32_LEAN_AND_MEAN
23 #include <windows.h>
24 #else
25 #include <sys/types.h>
26 #include <sys/mman.h>
27 #endif
28 #include <stdlib.h>
29 #include <stdio.h>
30 #include <stdarg.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <unistd.h>
34 #include <inttypes.h>
35
36 #include "cpu.h"
37 #include "exec-all.h"
38 #include "qemu-common.h"
39
40 #if !defined(TARGET_IA64)
41 #include "tcg.h"
42 #endif
43 #include "qemu-kvm.h"
44
45 #include "hw/hw.h"
46 #include "osdep.h"
47 #include "kvm.h"
48 #if defined(CONFIG_USER_ONLY)
49 #include <qemu.h>
50 #endif
51
52 //#define DEBUG_TB_INVALIDATE
53 //#define DEBUG_FLUSH
54 //#define DEBUG_TLB
55 //#define DEBUG_UNASSIGNED
56
57 /* make various TB consistency checks */
58 //#define DEBUG_TB_CHECK
59 //#define DEBUG_TLB_CHECK
60
61 //#define DEBUG_IOPORT
62 //#define DEBUG_SUBPAGE
63
64 #if !defined(CONFIG_USER_ONLY)
65 /* TB consistency checks only implemented for usermode emulation. */
66 #undef DEBUG_TB_CHECK
67 #endif
68
69 #define SMC_BITMAP_USE_THRESHOLD 10
70
71 #define MMAP_AREA_START 0x00000000
72 #define MMAP_AREA_END 0xa8000000
73
74 #if defined(TARGET_SPARC64)
75 #define TARGET_PHYS_ADDR_SPACE_BITS 41
76 #elif defined(TARGET_SPARC)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 36
78 #elif defined(TARGET_ALPHA)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
80 #define TARGET_VIRT_ADDR_SPACE_BITS 42
81 #elif defined(TARGET_PPC64)
82 #define TARGET_PHYS_ADDR_SPACE_BITS 42
83 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
84 #define TARGET_PHYS_ADDR_SPACE_BITS 42
85 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
86 #define TARGET_PHYS_ADDR_SPACE_BITS 36
87 #elif defined(TARGET_IA64)
88 #define TARGET_PHYS_ADDR_SPACE_BITS 36
89 #else
90 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
91 #define TARGET_PHYS_ADDR_SPACE_BITS 32
92 #endif
93
94 static TranslationBlock *tbs;
95 int code_gen_max_blocks;
96 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
97 static int nb_tbs;
98 /* any access to the tbs or the page table must use this lock */
99 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
100
101 #if defined(__arm__) || defined(__sparc_v9__)
102 /* The prologue must be reachable with a direct jump. ARM and Sparc64
103 have limited branch ranges (possibly also PPC) so place it in a
104 section close to code segment. */
105 #define code_gen_section \
106 __attribute__((__section__(".gen_code"))) \
107 __attribute__((aligned (32)))
108 #else
109 #define code_gen_section \
110 __attribute__((aligned (32)))
111 #endif
112
113 uint8_t code_gen_prologue[1024] code_gen_section;
114 static uint8_t *code_gen_buffer;
115 static unsigned long code_gen_buffer_size;
116 /* threshold to flush the translated code buffer */
117 static unsigned long code_gen_buffer_max_size;
118 uint8_t *code_gen_ptr;
119
120 #if !defined(CONFIG_USER_ONLY)
121 ram_addr_t phys_ram_size;
122 int phys_ram_fd;
123 uint8_t *phys_ram_base;
124 uint8_t *phys_ram_dirty;
125 uint8_t *bios_mem;
126 static int in_migration;
127 static ram_addr_t phys_ram_alloc_offset = 0;
128 #endif
129
130 CPUState *first_cpu;
131 /* current CPU in the current thread. It is only valid inside
132 cpu_exec() */
133 CPUState *cpu_single_env;
134 /* 0 = Do not count executed instructions.
135 1 = Precise instruction counting.
136 2 = Adaptive rate instruction counting. */
137 int use_icount = 0;
138 /* Current instruction counter. While executing translated code this may
139 include some instructions that have not yet been executed. */
140 int64_t qemu_icount;
141
142 typedef struct PageDesc {
143 /* list of TBs intersecting this ram page */
144 TranslationBlock *first_tb;
145 /* in order to optimize self modifying code, we count the number
146 of lookups we do to a given page to use a bitmap */
147 unsigned int code_write_count;
148 uint8_t *code_bitmap;
149 #if defined(CONFIG_USER_ONLY)
150 unsigned long flags;
151 #endif
152 } PageDesc;
153
154 typedef struct PhysPageDesc {
155 /* offset in host memory of the page + io_index in the low bits */
156 ram_addr_t phys_offset;
157 ram_addr_t region_offset;
158 } PhysPageDesc;
159
160 #define L2_BITS 10
161 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
162 /* XXX: this is a temporary hack for alpha target.
163 * In the future, this is to be replaced by a multi-level table
164 * to actually be able to handle the complete 64 bits address space.
165 */
166 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
167 #else
168 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
169 #endif
170
171 #define L1_SIZE (1 << L1_BITS)
172 #define L2_SIZE (1 << L2_BITS)
173
174 unsigned long qemu_real_host_page_size;
175 unsigned long qemu_host_page_bits;
176 unsigned long qemu_host_page_size;
177 unsigned long qemu_host_page_mask;
178
179 /* XXX: for system emulation, it could just be an array */
180 static PageDesc *l1_map[L1_SIZE];
181 static PhysPageDesc **l1_phys_map;
182
183 #if !defined(CONFIG_USER_ONLY)
184 static void io_mem_init(void);
185
186 /* io memory support */
187 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
188 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
189 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
190 char io_mem_used[IO_MEM_NB_ENTRIES];
191 static int io_mem_watch;
192 #endif
193
194 /* log support */
195 static const char *logfilename = "/tmp/qemu.log";
196 FILE *logfile;
197 int loglevel;
198 static int log_append = 0;
199
200 /* statistics */
201 static int tlb_flush_count;
202 static int tb_flush_count;
203 static int tb_phys_invalidate_count;
204
205 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
206 typedef struct subpage_t {
207 target_phys_addr_t base;
208 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
209 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
210 void *opaque[TARGET_PAGE_SIZE][2][4];
211 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
212 } subpage_t;
213
214 #ifdef _WIN32
215 static void map_exec(void *addr, long size)
216 {
217 DWORD old_protect;
218 VirtualProtect(addr, size,
219 PAGE_EXECUTE_READWRITE, &old_protect);
220
221 }
222 #else
223 static void map_exec(void *addr, long size)
224 {
225 unsigned long start, end, page_size;
226
227 page_size = getpagesize();
228 start = (unsigned long)addr;
229 start &= ~(page_size - 1);
230
231 end = (unsigned long)addr + size;
232 end += page_size - 1;
233 end &= ~(page_size - 1);
234
235 mprotect((void *)start, end - start,
236 PROT_READ | PROT_WRITE | PROT_EXEC);
237 }
238 #endif
239
240 static void page_init(void)
241 {
242 /* NOTE: we can always suppose that qemu_host_page_size >=
243 TARGET_PAGE_SIZE */
244 #ifdef _WIN32
245 {
246 SYSTEM_INFO system_info;
247
248 GetSystemInfo(&system_info);
249 qemu_real_host_page_size = system_info.dwPageSize;
250 }
251 #else
252 qemu_real_host_page_size = getpagesize();
253 #endif
254 if (qemu_host_page_size == 0)
255 qemu_host_page_size = qemu_real_host_page_size;
256 if (qemu_host_page_size < TARGET_PAGE_SIZE)
257 qemu_host_page_size = TARGET_PAGE_SIZE;
258 qemu_host_page_bits = 0;
259 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
260 qemu_host_page_bits++;
261 qemu_host_page_mask = ~(qemu_host_page_size - 1);
262 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
263 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
264
265 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
266 {
267 long long startaddr, endaddr;
268 FILE *f;
269 int n;
270
271 mmap_lock();
272 last_brk = (unsigned long)sbrk(0);
273 f = fopen("/proc/self/maps", "r");
274 if (f) {
275 do {
276 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
277 if (n == 2) {
278 startaddr = MIN(startaddr,
279 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
280 endaddr = MIN(endaddr,
281 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
282 page_set_flags(startaddr & TARGET_PAGE_MASK,
283 TARGET_PAGE_ALIGN(endaddr),
284 PAGE_RESERVED);
285 }
286 } while (!feof(f));
287 fclose(f);
288 }
289 mmap_unlock();
290 }
291 #endif
292 }
293
294 static inline PageDesc **page_l1_map(target_ulong index)
295 {
296 #if TARGET_LONG_BITS > 32
297 /* Host memory outside guest VM. For 32-bit targets we have already
298 excluded high addresses. */
299 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
300 return NULL;
301 #endif
302 return &l1_map[index >> L2_BITS];
303 }
304
305 static inline PageDesc *page_find_alloc(target_ulong index)
306 {
307 PageDesc **lp, *p;
308 lp = page_l1_map(index);
309 if (!lp)
310 return NULL;
311
312 p = *lp;
313 if (!p) {
314 /* allocate if not found */
315 #if defined(CONFIG_USER_ONLY)
316 size_t len = sizeof(PageDesc) * L2_SIZE;
317 /* Don't use qemu_malloc because it may recurse. */
318 p = mmap(0, len, PROT_READ | PROT_WRITE,
319 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
320 *lp = p;
321 if (h2g_valid(p)) {
322 unsigned long addr = h2g(p);
323 page_set_flags(addr & TARGET_PAGE_MASK,
324 TARGET_PAGE_ALIGN(addr + len),
325 PAGE_RESERVED);
326 }
327 #else
328 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
329 *lp = p;
330 #endif
331 }
332 return p + (index & (L2_SIZE - 1));
333 }
334
335 static inline PageDesc *page_find(target_ulong index)
336 {
337 PageDesc **lp, *p;
338 lp = page_l1_map(index);
339 if (!lp)
340 return NULL;
341
342 p = *lp;
343 if (!p)
344 return 0;
345 return p + (index & (L2_SIZE - 1));
346 }
347
348 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
349 {
350 void **lp, **p;
351 PhysPageDesc *pd;
352
353 p = (void **)l1_phys_map;
354 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
355
356 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
357 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
358 #endif
359 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
360 p = *lp;
361 if (!p) {
362 /* allocate if not found */
363 if (!alloc)
364 return NULL;
365 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
366 memset(p, 0, sizeof(void *) * L1_SIZE);
367 *lp = p;
368 }
369 #endif
370 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
371 pd = *lp;
372 if (!pd) {
373 int i;
374 /* allocate if not found */
375 if (!alloc)
376 return NULL;
377 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
378 *lp = pd;
379 for (i = 0; i < L2_SIZE; i++)
380 pd[i].phys_offset = IO_MEM_UNASSIGNED;
381 }
382 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
383 }
384
385 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
386 {
387 return phys_page_find_alloc(index, 0);
388 }
389
390 #if !defined(CONFIG_USER_ONLY)
391 static void tlb_protect_code(ram_addr_t ram_addr);
392 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
393 target_ulong vaddr);
394 #define mmap_lock() do { } while(0)
395 #define mmap_unlock() do { } while(0)
396 #endif
397
398 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
399
400 #if defined(CONFIG_USER_ONLY)
401 /* Currently it is not recommanded to allocate big chunks of data in
402 user mode. It will change when a dedicated libc will be used */
403 #define USE_STATIC_CODE_GEN_BUFFER
404 #endif
405
406 #ifdef USE_STATIC_CODE_GEN_BUFFER
407 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
408 #endif
409
410 static void code_gen_alloc(unsigned long tb_size)
411 {
412 if (kvm_enabled())
413 return;
414
415 #ifdef USE_STATIC_CODE_GEN_BUFFER
416 code_gen_buffer = static_code_gen_buffer;
417 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
418 map_exec(code_gen_buffer, code_gen_buffer_size);
419 #else
420 code_gen_buffer_size = tb_size;
421 if (code_gen_buffer_size == 0) {
422 #if defined(CONFIG_USER_ONLY)
423 /* in user mode, phys_ram_size is not meaningful */
424 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
425 #else
426 /* XXX: needs ajustments */
427 code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
428 #endif
429 }
430 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
431 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
432 /* The code gen buffer location may have constraints depending on
433 the host cpu and OS */
434 #if defined(__linux__)
435 {
436 int flags;
437 void *start = NULL;
438
439 flags = MAP_PRIVATE | MAP_ANONYMOUS;
440 #if defined(__x86_64__)
441 flags |= MAP_32BIT;
442 /* Cannot map more than that */
443 if (code_gen_buffer_size > (800 * 1024 * 1024))
444 code_gen_buffer_size = (800 * 1024 * 1024);
445 #elif defined(__sparc_v9__)
446 // Map the buffer below 2G, so we can use direct calls and branches
447 flags |= MAP_FIXED;
448 start = (void *) 0x60000000UL;
449 if (code_gen_buffer_size > (512 * 1024 * 1024))
450 code_gen_buffer_size = (512 * 1024 * 1024);
451 #elif defined(__arm__)
452 /* Map the buffer below 32M, so we can use direct calls and branches */
453 flags |= MAP_FIXED;
454 start = (void *) 0x01000000UL;
455 if (code_gen_buffer_size > 16 * 1024 * 1024)
456 code_gen_buffer_size = 16 * 1024 * 1024;
457 #endif
458 code_gen_buffer = mmap(start, code_gen_buffer_size,
459 PROT_WRITE | PROT_READ | PROT_EXEC,
460 flags, -1, 0);
461 if (code_gen_buffer == MAP_FAILED) {
462 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
463 exit(1);
464 }
465 }
466 #elif defined(__FreeBSD__)
467 {
468 int flags;
469 void *addr = NULL;
470 flags = MAP_PRIVATE | MAP_ANONYMOUS;
471 #if defined(__x86_64__)
472 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
473 * 0x40000000 is free */
474 flags |= MAP_FIXED;
475 addr = (void *)0x40000000;
476 /* Cannot map more than that */
477 if (code_gen_buffer_size > (800 * 1024 * 1024))
478 code_gen_buffer_size = (800 * 1024 * 1024);
479 #endif
480 code_gen_buffer = mmap(addr, code_gen_buffer_size,
481 PROT_WRITE | PROT_READ | PROT_EXEC,
482 flags, -1, 0);
483 if (code_gen_buffer == MAP_FAILED) {
484 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
485 exit(1);
486 }
487 }
488 #else
489 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
490 if (!code_gen_buffer) {
491 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
492 exit(1);
493 }
494 map_exec(code_gen_buffer, code_gen_buffer_size);
495 #endif
496 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
497 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
498 code_gen_buffer_max_size = code_gen_buffer_size -
499 code_gen_max_block_size();
500 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
501 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
502 }
503
504 /* Must be called before using the QEMU cpus. 'tb_size' is the size
505 (in bytes) allocated to the translation buffer. Zero means default
506 size. */
507 void cpu_exec_init_all(unsigned long tb_size)
508 {
509 cpu_gen_init();
510 code_gen_alloc(tb_size);
511 code_gen_ptr = code_gen_buffer;
512 page_init();
513 #if !defined(CONFIG_USER_ONLY)
514 io_mem_init();
515 #endif
516 }
517
518 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
519
520 #define CPU_COMMON_SAVE_VERSION 1
521
522 static void cpu_common_save(QEMUFile *f, void *opaque)
523 {
524 CPUState *env = opaque;
525
526 qemu_put_be32s(f, &env->halted);
527 qemu_put_be32s(f, &env->interrupt_request);
528 }
529
530 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
531 {
532 CPUState *env = opaque;
533
534 if (version_id != CPU_COMMON_SAVE_VERSION)
535 return -EINVAL;
536
537 qemu_get_be32s(f, &env->halted);
538 qemu_get_be32s(f, &env->interrupt_request);
539 tlb_flush(env, 1);
540
541 return 0;
542 }
543 #endif
544
545 void cpu_exec_init(CPUState *env)
546 {
547 CPUState **penv;
548 int cpu_index;
549
550 env->next_cpu = NULL;
551 penv = &first_cpu;
552 cpu_index = 0;
553 while (*penv != NULL) {
554 penv = (CPUState **)&(*penv)->next_cpu;
555 cpu_index++;
556 }
557 env->cpu_index = cpu_index;
558 TAILQ_INIT(&env->breakpoints);
559 TAILQ_INIT(&env->watchpoints);
560 #ifdef __WIN32
561 env->thread_id = GetCurrentProcessId();
562 #else
563 env->thread_id = getpid();
564 #endif
565 *penv = env;
566 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
567 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
568 cpu_common_save, cpu_common_load, env);
569 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
570 cpu_save, cpu_load, env);
571 #endif
572 }
573
574 static inline void invalidate_page_bitmap(PageDesc *p)
575 {
576 if (p->code_bitmap) {
577 qemu_free(p->code_bitmap);
578 p->code_bitmap = NULL;
579 }
580 p->code_write_count = 0;
581 }
582
583 /* set to NULL all the 'first_tb' fields in all PageDescs */
584 static void page_flush_tb(void)
585 {
586 int i, j;
587 PageDesc *p;
588
589 for(i = 0; i < L1_SIZE; i++) {
590 p = l1_map[i];
591 if (p) {
592 for(j = 0; j < L2_SIZE; j++) {
593 p->first_tb = NULL;
594 invalidate_page_bitmap(p);
595 p++;
596 }
597 }
598 }
599 }
600
601 /* flush all the translation blocks */
602 /* XXX: tb_flush is currently not thread safe */
603 void tb_flush(CPUState *env1)
604 {
605 CPUState *env;
606 #if defined(DEBUG_FLUSH)
607 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
608 (unsigned long)(code_gen_ptr - code_gen_buffer),
609 nb_tbs, nb_tbs > 0 ?
610 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
611 #endif
612 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
613 cpu_abort(env1, "Internal error: code buffer overflow\n");
614
615 nb_tbs = 0;
616
617 for(env = first_cpu; env != NULL; env = env->next_cpu) {
618 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
619 }
620
621 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
622 page_flush_tb();
623
624 code_gen_ptr = code_gen_buffer;
625 /* XXX: flush processor icache at this point if cache flush is
626 expensive */
627 tb_flush_count++;
628 }
629
630 #ifdef DEBUG_TB_CHECK
631
632 static void tb_invalidate_check(target_ulong address)
633 {
634 TranslationBlock *tb;
635 int i;
636 address &= TARGET_PAGE_MASK;
637 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
638 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
639 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
640 address >= tb->pc + tb->size)) {
641 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
642 address, (long)tb->pc, tb->size);
643 }
644 }
645 }
646 }
647
648 /* verify that all the pages have correct rights for code */
649 static void tb_page_check(void)
650 {
651 TranslationBlock *tb;
652 int i, flags1, flags2;
653
654 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
655 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
656 flags1 = page_get_flags(tb->pc);
657 flags2 = page_get_flags(tb->pc + tb->size - 1);
658 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
659 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
660 (long)tb->pc, tb->size, flags1, flags2);
661 }
662 }
663 }
664 }
665
666 static void tb_jmp_check(TranslationBlock *tb)
667 {
668 TranslationBlock *tb1;
669 unsigned int n1;
670
671 /* suppress any remaining jumps to this TB */
672 tb1 = tb->jmp_first;
673 for(;;) {
674 n1 = (long)tb1 & 3;
675 tb1 = (TranslationBlock *)((long)tb1 & ~3);
676 if (n1 == 2)
677 break;
678 tb1 = tb1->jmp_next[n1];
679 }
680 /* check end of list */
681 if (tb1 != tb) {
682 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
683 }
684 }
685
686 #endif
687
688 /* invalidate one TB */
689 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
690 int next_offset)
691 {
692 TranslationBlock *tb1;
693 for(;;) {
694 tb1 = *ptb;
695 if (tb1 == tb) {
696 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
697 break;
698 }
699 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
700 }
701 }
702
703 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
704 {
705 TranslationBlock *tb1;
706 unsigned int n1;
707
708 for(;;) {
709 tb1 = *ptb;
710 n1 = (long)tb1 & 3;
711 tb1 = (TranslationBlock *)((long)tb1 & ~3);
712 if (tb1 == tb) {
713 *ptb = tb1->page_next[n1];
714 break;
715 }
716 ptb = &tb1->page_next[n1];
717 }
718 }
719
720 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
721 {
722 TranslationBlock *tb1, **ptb;
723 unsigned int n1;
724
725 ptb = &tb->jmp_next[n];
726 tb1 = *ptb;
727 if (tb1) {
728 /* find tb(n) in circular list */
729 for(;;) {
730 tb1 = *ptb;
731 n1 = (long)tb1 & 3;
732 tb1 = (TranslationBlock *)((long)tb1 & ~3);
733 if (n1 == n && tb1 == tb)
734 break;
735 if (n1 == 2) {
736 ptb = &tb1->jmp_first;
737 } else {
738 ptb = &tb1->jmp_next[n1];
739 }
740 }
741 /* now we can suppress tb(n) from the list */
742 *ptb = tb->jmp_next[n];
743
744 tb->jmp_next[n] = NULL;
745 }
746 }
747
748 /* reset the jump entry 'n' of a TB so that it is not chained to
749 another TB */
750 static inline void tb_reset_jump(TranslationBlock *tb, int n)
751 {
752 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
753 }
754
755 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
756 {
757 CPUState *env;
758 PageDesc *p;
759 unsigned int h, n1;
760 target_phys_addr_t phys_pc;
761 TranslationBlock *tb1, *tb2;
762
763 /* remove the TB from the hash list */
764 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
765 h = tb_phys_hash_func(phys_pc);
766 tb_remove(&tb_phys_hash[h], tb,
767 offsetof(TranslationBlock, phys_hash_next));
768
769 /* remove the TB from the page list */
770 if (tb->page_addr[0] != page_addr) {
771 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
772 tb_page_remove(&p->first_tb, tb);
773 invalidate_page_bitmap(p);
774 }
775 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
776 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
777 tb_page_remove(&p->first_tb, tb);
778 invalidate_page_bitmap(p);
779 }
780
781 tb_invalidated_flag = 1;
782
783 /* remove the TB from the hash list */
784 h = tb_jmp_cache_hash_func(tb->pc);
785 for(env = first_cpu; env != NULL; env = env->next_cpu) {
786 if (env->tb_jmp_cache[h] == tb)
787 env->tb_jmp_cache[h] = NULL;
788 }
789
790 /* suppress this TB from the two jump lists */
791 tb_jmp_remove(tb, 0);
792 tb_jmp_remove(tb, 1);
793
794 /* suppress any remaining jumps to this TB */
795 tb1 = tb->jmp_first;
796 for(;;) {
797 n1 = (long)tb1 & 3;
798 if (n1 == 2)
799 break;
800 tb1 = (TranslationBlock *)((long)tb1 & ~3);
801 tb2 = tb1->jmp_next[n1];
802 tb_reset_jump(tb1, n1);
803 tb1->jmp_next[n1] = NULL;
804 tb1 = tb2;
805 }
806 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
807
808 tb_phys_invalidate_count++;
809 }
810
811 static inline void set_bits(uint8_t *tab, int start, int len)
812 {
813 int end, mask, end1;
814
815 end = start + len;
816 tab += start >> 3;
817 mask = 0xff << (start & 7);
818 if ((start & ~7) == (end & ~7)) {
819 if (start < end) {
820 mask &= ~(0xff << (end & 7));
821 *tab |= mask;
822 }
823 } else {
824 *tab++ |= mask;
825 start = (start + 8) & ~7;
826 end1 = end & ~7;
827 while (start < end1) {
828 *tab++ = 0xff;
829 start += 8;
830 }
831 if (start < end) {
832 mask = ~(0xff << (end & 7));
833 *tab |= mask;
834 }
835 }
836 }
837
838 static void build_page_bitmap(PageDesc *p)
839 {
840 int n, tb_start, tb_end;
841 TranslationBlock *tb;
842
843 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
844 if (!p->code_bitmap)
845 return;
846
847 tb = p->first_tb;
848 while (tb != NULL) {
849 n = (long)tb & 3;
850 tb = (TranslationBlock *)((long)tb & ~3);
851 /* NOTE: this is subtle as a TB may span two physical pages */
852 if (n == 0) {
853 /* NOTE: tb_end may be after the end of the page, but
854 it is not a problem */
855 tb_start = tb->pc & ~TARGET_PAGE_MASK;
856 tb_end = tb_start + tb->size;
857 if (tb_end > TARGET_PAGE_SIZE)
858 tb_end = TARGET_PAGE_SIZE;
859 } else {
860 tb_start = 0;
861 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
862 }
863 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
864 tb = tb->page_next[n];
865 }
866 }
867
868 TranslationBlock *tb_gen_code(CPUState *env,
869 target_ulong pc, target_ulong cs_base,
870 int flags, int cflags)
871 {
872 TranslationBlock *tb;
873 uint8_t *tc_ptr;
874 target_ulong phys_pc, phys_page2, virt_page2;
875 int code_gen_size;
876
877 phys_pc = get_phys_addr_code(env, pc);
878 tb = tb_alloc(pc);
879 if (!tb) {
880 /* flush must be done */
881 tb_flush(env);
882 /* cannot fail at this point */
883 tb = tb_alloc(pc);
884 /* Don't forget to invalidate previous TB info. */
885 tb_invalidated_flag = 1;
886 }
887 tc_ptr = code_gen_ptr;
888 tb->tc_ptr = tc_ptr;
889 tb->cs_base = cs_base;
890 tb->flags = flags;
891 tb->cflags = cflags;
892 cpu_gen_code(env, tb, &code_gen_size);
893 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
894
895 /* check next page if needed */
896 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
897 phys_page2 = -1;
898 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
899 phys_page2 = get_phys_addr_code(env, virt_page2);
900 }
901 tb_link_phys(tb, phys_pc, phys_page2);
902 return tb;
903 }
904
905 /* invalidate all TBs which intersect with the target physical page
906 starting in range [start;end[. NOTE: start and end must refer to
907 the same physical page. 'is_cpu_write_access' should be true if called
908 from a real cpu write access: the virtual CPU will exit the current
909 TB if code is modified inside this TB. */
910 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
911 int is_cpu_write_access)
912 {
913 TranslationBlock *tb, *tb_next, *saved_tb;
914 CPUState *env = cpu_single_env;
915 target_ulong tb_start, tb_end;
916 PageDesc *p;
917 int n;
918 #ifdef TARGET_HAS_PRECISE_SMC
919 int current_tb_not_found = is_cpu_write_access;
920 TranslationBlock *current_tb = NULL;
921 int current_tb_modified = 0;
922 target_ulong current_pc = 0;
923 target_ulong current_cs_base = 0;
924 int current_flags = 0;
925 #endif /* TARGET_HAS_PRECISE_SMC */
926
927 p = page_find(start >> TARGET_PAGE_BITS);
928 if (!p)
929 return;
930 if (!p->code_bitmap &&
931 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
932 is_cpu_write_access) {
933 /* build code bitmap */
934 build_page_bitmap(p);
935 }
936
937 /* we remove all the TBs in the range [start, end[ */
938 /* XXX: see if in some cases it could be faster to invalidate all the code */
939 tb = p->first_tb;
940 while (tb != NULL) {
941 n = (long)tb & 3;
942 tb = (TranslationBlock *)((long)tb & ~3);
943 tb_next = tb->page_next[n];
944 /* NOTE: this is subtle as a TB may span two physical pages */
945 if (n == 0) {
946 /* NOTE: tb_end may be after the end of the page, but
947 it is not a problem */
948 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
949 tb_end = tb_start + tb->size;
950 } else {
951 tb_start = tb->page_addr[1];
952 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
953 }
954 if (!(tb_end <= start || tb_start >= end)) {
955 #ifdef TARGET_HAS_PRECISE_SMC
956 if (current_tb_not_found) {
957 current_tb_not_found = 0;
958 current_tb = NULL;
959 if (env->mem_io_pc) {
960 /* now we have a real cpu fault */
961 current_tb = tb_find_pc(env->mem_io_pc);
962 }
963 }
964 if (current_tb == tb &&
965 (current_tb->cflags & CF_COUNT_MASK) != 1) {
966 /* If we are modifying the current TB, we must stop
967 its execution. We could be more precise by checking
968 that the modification is after the current PC, but it
969 would require a specialized function to partially
970 restore the CPU state */
971
972 current_tb_modified = 1;
973 cpu_restore_state(current_tb, env,
974 env->mem_io_pc, NULL);
975 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
976 &current_flags);
977 }
978 #endif /* TARGET_HAS_PRECISE_SMC */
979 /* we need to do that to handle the case where a signal
980 occurs while doing tb_phys_invalidate() */
981 saved_tb = NULL;
982 if (env) {
983 saved_tb = env->current_tb;
984 env->current_tb = NULL;
985 }
986 tb_phys_invalidate(tb, -1);
987 if (env) {
988 env->current_tb = saved_tb;
989 if (env->interrupt_request && env->current_tb)
990 cpu_interrupt(env, env->interrupt_request);
991 }
992 }
993 tb = tb_next;
994 }
995 #if !defined(CONFIG_USER_ONLY)
996 /* if no code remaining, no need to continue to use slow writes */
997 if (!p->first_tb) {
998 invalidate_page_bitmap(p);
999 if (is_cpu_write_access) {
1000 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1001 }
1002 }
1003 #endif
1004 #ifdef TARGET_HAS_PRECISE_SMC
1005 if (current_tb_modified) {
1006 /* we generate a block containing just the instruction
1007 modifying the memory. It will ensure that it cannot modify
1008 itself */
1009 env->current_tb = NULL;
1010 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1011 cpu_resume_from_signal(env, NULL);
1012 }
1013 #endif
1014 }
1015
1016 /* len must be <= 8 and start must be a multiple of len */
1017 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1018 {
1019 PageDesc *p;
1020 int offset, b;
1021 #if 0
1022 if (1) {
1023 if (loglevel) {
1024 fprintf(logfile, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1025 cpu_single_env->mem_io_vaddr, len,
1026 cpu_single_env->eip,
1027 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1028 }
1029 }
1030 #endif
1031 p = page_find(start >> TARGET_PAGE_BITS);
1032 if (!p)
1033 return;
1034 if (p->code_bitmap) {
1035 offset = start & ~TARGET_PAGE_MASK;
1036 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1037 if (b & ((1 << len) - 1))
1038 goto do_invalidate;
1039 } else {
1040 do_invalidate:
1041 tb_invalidate_phys_page_range(start, start + len, 1);
1042 }
1043 }
1044
1045 #if !defined(CONFIG_SOFTMMU)
1046 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1047 unsigned long pc, void *puc)
1048 {
1049 TranslationBlock *tb;
1050 PageDesc *p;
1051 int n;
1052 #ifdef TARGET_HAS_PRECISE_SMC
1053 TranslationBlock *current_tb = NULL;
1054 CPUState *env = cpu_single_env;
1055 int current_tb_modified = 0;
1056 target_ulong current_pc = 0;
1057 target_ulong current_cs_base = 0;
1058 int current_flags = 0;
1059 #endif
1060
1061 addr &= TARGET_PAGE_MASK;
1062 p = page_find(addr >> TARGET_PAGE_BITS);
1063 if (!p)
1064 return;
1065 tb = p->first_tb;
1066 #ifdef TARGET_HAS_PRECISE_SMC
1067 if (tb && pc != 0) {
1068 current_tb = tb_find_pc(pc);
1069 }
1070 #endif
1071 while (tb != NULL) {
1072 n = (long)tb & 3;
1073 tb = (TranslationBlock *)((long)tb & ~3);
1074 #ifdef TARGET_HAS_PRECISE_SMC
1075 if (current_tb == tb &&
1076 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1077 /* If we are modifying the current TB, we must stop
1078 its execution. We could be more precise by checking
1079 that the modification is after the current PC, but it
1080 would require a specialized function to partially
1081 restore the CPU state */
1082
1083 current_tb_modified = 1;
1084 cpu_restore_state(current_tb, env, pc, puc);
1085 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1086 &current_flags);
1087 }
1088 #endif /* TARGET_HAS_PRECISE_SMC */
1089 tb_phys_invalidate(tb, addr);
1090 tb = tb->page_next[n];
1091 }
1092 p->first_tb = NULL;
1093 #ifdef TARGET_HAS_PRECISE_SMC
1094 if (current_tb_modified) {
1095 /* we generate a block containing just the instruction
1096 modifying the memory. It will ensure that it cannot modify
1097 itself */
1098 env->current_tb = NULL;
1099 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1100 cpu_resume_from_signal(env, puc);
1101 }
1102 #endif
1103 }
1104 #endif
1105
1106 /* add the tb in the target page and protect it if necessary */
1107 static inline void tb_alloc_page(TranslationBlock *tb,
1108 unsigned int n, target_ulong page_addr)
1109 {
1110 PageDesc *p;
1111 TranslationBlock *last_first_tb;
1112
1113 tb->page_addr[n] = page_addr;
1114 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1115 tb->page_next[n] = p->first_tb;
1116 last_first_tb = p->first_tb;
1117 p->first_tb = (TranslationBlock *)((long)tb | n);
1118 invalidate_page_bitmap(p);
1119
1120 #if defined(TARGET_HAS_SMC) || 1
1121
1122 #if defined(CONFIG_USER_ONLY)
1123 if (p->flags & PAGE_WRITE) {
1124 target_ulong addr;
1125 PageDesc *p2;
1126 int prot;
1127
1128 /* force the host page as non writable (writes will have a
1129 page fault + mprotect overhead) */
1130 page_addr &= qemu_host_page_mask;
1131 prot = 0;
1132 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1133 addr += TARGET_PAGE_SIZE) {
1134
1135 p2 = page_find (addr >> TARGET_PAGE_BITS);
1136 if (!p2)
1137 continue;
1138 prot |= p2->flags;
1139 p2->flags &= ~PAGE_WRITE;
1140 page_get_flags(addr);
1141 }
1142 mprotect(g2h(page_addr), qemu_host_page_size,
1143 (prot & PAGE_BITS) & ~PAGE_WRITE);
1144 #ifdef DEBUG_TB_INVALIDATE
1145 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1146 page_addr);
1147 #endif
1148 }
1149 #else
1150 /* if some code is already present, then the pages are already
1151 protected. So we handle the case where only the first TB is
1152 allocated in a physical page */
1153 if (!last_first_tb) {
1154 tlb_protect_code(page_addr);
1155 }
1156 #endif
1157
1158 #endif /* TARGET_HAS_SMC */
1159 }
1160
1161 /* Allocate a new translation block. Flush the translation buffer if
1162 too many translation blocks or too much generated code. */
1163 TranslationBlock *tb_alloc(target_ulong pc)
1164 {
1165 TranslationBlock *tb;
1166
1167 if (nb_tbs >= code_gen_max_blocks ||
1168 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1169 return NULL;
1170 tb = &tbs[nb_tbs++];
1171 tb->pc = pc;
1172 tb->cflags = 0;
1173 return tb;
1174 }
1175
1176 void tb_free(TranslationBlock *tb)
1177 {
1178 /* In practice this is mostly used for single use temporary TB
1179 Ignore the hard cases and just back up if this TB happens to
1180 be the last one generated. */
1181 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1182 code_gen_ptr = tb->tc_ptr;
1183 nb_tbs--;
1184 }
1185 }
1186
1187 /* add a new TB and link it to the physical page tables. phys_page2 is
1188 (-1) to indicate that only one page contains the TB. */
1189 void tb_link_phys(TranslationBlock *tb,
1190 target_ulong phys_pc, target_ulong phys_page2)
1191 {
1192 unsigned int h;
1193 TranslationBlock **ptb;
1194
1195 /* Grab the mmap lock to stop another thread invalidating this TB
1196 before we are done. */
1197 mmap_lock();
1198 /* add in the physical hash table */
1199 h = tb_phys_hash_func(phys_pc);
1200 ptb = &tb_phys_hash[h];
1201 tb->phys_hash_next = *ptb;
1202 *ptb = tb;
1203
1204 /* add in the page list */
1205 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1206 if (phys_page2 != -1)
1207 tb_alloc_page(tb, 1, phys_page2);
1208 else
1209 tb->page_addr[1] = -1;
1210
1211 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1212 tb->jmp_next[0] = NULL;
1213 tb->jmp_next[1] = NULL;
1214
1215 /* init original jump addresses */
1216 if (tb->tb_next_offset[0] != 0xffff)
1217 tb_reset_jump(tb, 0);
1218 if (tb->tb_next_offset[1] != 0xffff)
1219 tb_reset_jump(tb, 1);
1220
1221 #ifdef DEBUG_TB_CHECK
1222 tb_page_check();
1223 #endif
1224 mmap_unlock();
1225 }
1226
1227 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1228 tb[1].tc_ptr. Return NULL if not found */
1229 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1230 {
1231 int m_min, m_max, m;
1232 unsigned long v;
1233 TranslationBlock *tb;
1234
1235 if (nb_tbs <= 0)
1236 return NULL;
1237 if (tc_ptr < (unsigned long)code_gen_buffer ||
1238 tc_ptr >= (unsigned long)code_gen_ptr)
1239 return NULL;
1240 /* binary search (cf Knuth) */
1241 m_min = 0;
1242 m_max = nb_tbs - 1;
1243 while (m_min <= m_max) {
1244 m = (m_min + m_max) >> 1;
1245 tb = &tbs[m];
1246 v = (unsigned long)tb->tc_ptr;
1247 if (v == tc_ptr)
1248 return tb;
1249 else if (tc_ptr < v) {
1250 m_max = m - 1;
1251 } else {
1252 m_min = m + 1;
1253 }
1254 }
1255 return &tbs[m_max];
1256 }
1257
1258 static void tb_reset_jump_recursive(TranslationBlock *tb);
1259
1260 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1261 {
1262 TranslationBlock *tb1, *tb_next, **ptb;
1263 unsigned int n1;
1264
1265 tb1 = tb->jmp_next[n];
1266 if (tb1 != NULL) {
1267 /* find head of list */
1268 for(;;) {
1269 n1 = (long)tb1 & 3;
1270 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1271 if (n1 == 2)
1272 break;
1273 tb1 = tb1->jmp_next[n1];
1274 }
1275 /* we are now sure now that tb jumps to tb1 */
1276 tb_next = tb1;
1277
1278 /* remove tb from the jmp_first list */
1279 ptb = &tb_next->jmp_first;
1280 for(;;) {
1281 tb1 = *ptb;
1282 n1 = (long)tb1 & 3;
1283 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1284 if (n1 == n && tb1 == tb)
1285 break;
1286 ptb = &tb1->jmp_next[n1];
1287 }
1288 *ptb = tb->jmp_next[n];
1289 tb->jmp_next[n] = NULL;
1290
1291 /* suppress the jump to next tb in generated code */
1292 tb_reset_jump(tb, n);
1293
1294 /* suppress jumps in the tb on which we could have jumped */
1295 tb_reset_jump_recursive(tb_next);
1296 }
1297 }
1298
1299 static void tb_reset_jump_recursive(TranslationBlock *tb)
1300 {
1301 tb_reset_jump_recursive2(tb, 0);
1302 tb_reset_jump_recursive2(tb, 1);
1303 }
1304
1305 #if defined(TARGET_HAS_ICE)
1306 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1307 {
1308 target_phys_addr_t addr;
1309 target_ulong pd;
1310 ram_addr_t ram_addr;
1311 PhysPageDesc *p;
1312
1313 addr = cpu_get_phys_page_debug(env, pc);
1314 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1315 if (!p) {
1316 pd = IO_MEM_UNASSIGNED;
1317 } else {
1318 pd = p->phys_offset;
1319 }
1320 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1321 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1322 }
1323 #endif
1324
1325 /* Add a watchpoint. */
1326 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1327 int flags, CPUWatchpoint **watchpoint)
1328 {
1329 target_ulong len_mask = ~(len - 1);
1330 CPUWatchpoint *wp;
1331
1332 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1333 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1334 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1335 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1336 return -EINVAL;
1337 }
1338 wp = qemu_malloc(sizeof(*wp));
1339 if (!wp)
1340 return -ENOMEM;
1341
1342 wp->vaddr = addr;
1343 wp->len_mask = len_mask;
1344 wp->flags = flags;
1345
1346 /* keep all GDB-injected watchpoints in front */
1347 if (flags & BP_GDB)
1348 TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1349 else
1350 TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1351
1352 tlb_flush_page(env, addr);
1353
1354 if (watchpoint)
1355 *watchpoint = wp;
1356 return 0;
1357 }
1358
1359 /* Remove a specific watchpoint. */
1360 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1361 int flags)
1362 {
1363 target_ulong len_mask = ~(len - 1);
1364 CPUWatchpoint *wp;
1365
1366 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1367 if (addr == wp->vaddr && len_mask == wp->len_mask
1368 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1369 cpu_watchpoint_remove_by_ref(env, wp);
1370 return 0;
1371 }
1372 }
1373 return -ENOENT;
1374 }
1375
1376 /* Remove a specific watchpoint by reference. */
1377 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1378 {
1379 TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1380
1381 tlb_flush_page(env, watchpoint->vaddr);
1382
1383 qemu_free(watchpoint);
1384 }
1385
1386 /* Remove all matching watchpoints. */
1387 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1388 {
1389 CPUWatchpoint *wp, *next;
1390
1391 TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1392 if (wp->flags & mask)
1393 cpu_watchpoint_remove_by_ref(env, wp);
1394 }
1395 }
1396
1397 /* Add a breakpoint. */
1398 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1399 CPUBreakpoint **breakpoint)
1400 {
1401 #if defined(TARGET_HAS_ICE)
1402 CPUBreakpoint *bp;
1403
1404 bp = qemu_malloc(sizeof(*bp));
1405 if (!bp)
1406 return -ENOMEM;
1407
1408 bp->pc = pc;
1409 bp->flags = flags;
1410
1411 /* keep all GDB-injected breakpoints in front */
1412 if (flags & BP_GDB)
1413 TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1414 else
1415 TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1416
1417 breakpoint_invalidate(env, pc);
1418
1419 if (breakpoint)
1420 *breakpoint = bp;
1421 return 0;
1422 #else
1423 return -ENOSYS;
1424 #endif
1425 }
1426
1427 /* Remove a specific breakpoint. */
1428 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1429 {
1430 #if defined(TARGET_HAS_ICE)
1431 CPUBreakpoint *bp;
1432
1433 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1434 if (bp->pc == pc && bp->flags == flags) {
1435 cpu_breakpoint_remove_by_ref(env, bp);
1436 return 0;
1437 }
1438 }
1439 return -ENOENT;
1440 #else
1441 return -ENOSYS;
1442 #endif
1443 }
1444
1445 /* Remove a specific breakpoint by reference. */
1446 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1447 {
1448 #if defined(TARGET_HAS_ICE)
1449 TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1450
1451 breakpoint_invalidate(env, breakpoint->pc);
1452
1453 qemu_free(breakpoint);
1454 #endif
1455 }
1456
1457 /* Remove all matching breakpoints. */
1458 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1459 {
1460 #if defined(TARGET_HAS_ICE)
1461 CPUBreakpoint *bp, *next;
1462
1463 TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1464 if (bp->flags & mask)
1465 cpu_breakpoint_remove_by_ref(env, bp);
1466 }
1467 #endif
1468 }
1469
1470 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1471 CPU loop after each instruction */
1472 void cpu_single_step(CPUState *env, int enabled)
1473 {
1474 #if defined(TARGET_HAS_ICE)
1475 if (env->singlestep_enabled != enabled) {
1476 env->singlestep_enabled = enabled;
1477 if (kvm_enabled())
1478 kvm_update_guest_debug(env, 0);
1479 else {
1480 /* must flush all the translated code to avoid inconsistancies */
1481 /* XXX: only flush what is necessary */
1482 tb_flush(env);
1483 }
1484 }
1485 #endif
1486 }
1487
1488 /* enable or disable low levels log */
1489 void cpu_set_log(int log_flags)
1490 {
1491 loglevel = log_flags;
1492 if (loglevel && !logfile) {
1493 logfile = fopen(logfilename, log_append ? "a" : "w");
1494 if (!logfile) {
1495 perror(logfilename);
1496 _exit(1);
1497 }
1498 #if !defined(CONFIG_SOFTMMU)
1499 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1500 {
1501 static char logfile_buf[4096];
1502 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1503 }
1504 #else
1505 setvbuf(logfile, NULL, _IOLBF, 0);
1506 #endif
1507 log_append = 1;
1508 }
1509 if (!loglevel && logfile) {
1510 fclose(logfile);
1511 logfile = NULL;
1512 }
1513 }
1514
1515 void cpu_set_log_filename(const char *filename)
1516 {
1517 logfilename = strdup(filename);
1518 if (logfile) {
1519 fclose(logfile);
1520 logfile = NULL;
1521 }
1522 cpu_set_log(loglevel);
1523 }
1524
1525 /* mask must never be zero, except for A20 change call */
1526 void cpu_interrupt(CPUState *env, int mask)
1527 {
1528 #if !defined(USE_NPTL)
1529 TranslationBlock *tb;
1530 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1531 #endif
1532 int old_mask;
1533
1534 old_mask = env->interrupt_request;
1535 /* FIXME: This is probably not threadsafe. A different thread could
1536 be in the middle of a read-modify-write operation. */
1537 env->interrupt_request |= mask;
1538 if (kvm_enabled() && !qemu_kvm_irqchip_in_kernel())
1539 kvm_update_interrupt_request(env);
1540 #if defined(USE_NPTL)
1541 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1542 problem and hope the cpu will stop of its own accord. For userspace
1543 emulation this often isn't actually as bad as it sounds. Often
1544 signals are used primarily to interrupt blocking syscalls. */
1545 #else
1546 if (use_icount) {
1547 env->icount_decr.u16.high = 0xffff;
1548 #ifndef CONFIG_USER_ONLY
1549 /* CPU_INTERRUPT_EXIT isn't a real interrupt. It just means
1550 an async event happened and we need to process it. */
1551 if (!can_do_io(env)
1552 && (mask & ~(old_mask | CPU_INTERRUPT_EXIT)) != 0) {
1553 cpu_abort(env, "Raised interrupt while not in I/O function");
1554 }
1555 #endif
1556 } else {
1557 tb = env->current_tb;
1558 /* if the cpu is currently executing code, we must unlink it and
1559 all the potentially executing TB */
1560 if (tb && !testandset(&interrupt_lock)) {
1561 env->current_tb = NULL;
1562 tb_reset_jump_recursive(tb);
1563 resetlock(&interrupt_lock);
1564 }
1565 }
1566 #endif
1567 }
1568
1569 void cpu_reset_interrupt(CPUState *env, int mask)
1570 {
1571 env->interrupt_request &= ~mask;
1572 }
1573
1574 const CPULogItem cpu_log_items[] = {
1575 { CPU_LOG_TB_OUT_ASM, "out_asm",
1576 "show generated host assembly code for each compiled TB" },
1577 { CPU_LOG_TB_IN_ASM, "in_asm",
1578 "show target assembly code for each compiled TB" },
1579 { CPU_LOG_TB_OP, "op",
1580 "show micro ops for each compiled TB" },
1581 { CPU_LOG_TB_OP_OPT, "op_opt",
1582 "show micro ops "
1583 #ifdef TARGET_I386
1584 "before eflags optimization and "
1585 #endif
1586 "after liveness analysis" },
1587 { CPU_LOG_INT, "int",
1588 "show interrupts/exceptions in short format" },
1589 { CPU_LOG_EXEC, "exec",
1590 "show trace before each executed TB (lots of logs)" },
1591 { CPU_LOG_TB_CPU, "cpu",
1592 "show CPU state before block translation" },
1593 #ifdef TARGET_I386
1594 { CPU_LOG_PCALL, "pcall",
1595 "show protected mode far calls/returns/exceptions" },
1596 #endif
1597 #ifdef DEBUG_IOPORT
1598 { CPU_LOG_IOPORT, "ioport",
1599 "show all i/o ports accesses" },
1600 #endif
1601 { 0, NULL, NULL },
1602 };
1603
1604 static int cmp1(const char *s1, int n, const char *s2)
1605 {
1606 if (strlen(s2) != n)
1607 return 0;
1608 return memcmp(s1, s2, n) == 0;
1609 }
1610
1611 /* takes a comma separated list of log masks. Return 0 if error. */
1612 int cpu_str_to_log_mask(const char *str)
1613 {
1614 const CPULogItem *item;
1615 int mask;
1616 const char *p, *p1;
1617
1618 p = str;
1619 mask = 0;
1620 for(;;) {
1621 p1 = strchr(p, ',');
1622 if (!p1)
1623 p1 = p + strlen(p);
1624 if(cmp1(p,p1-p,"all")) {
1625 for(item = cpu_log_items; item->mask != 0; item++) {
1626 mask |= item->mask;
1627 }
1628 } else {
1629 for(item = cpu_log_items; item->mask != 0; item++) {
1630 if (cmp1(p, p1 - p, item->name))
1631 goto found;
1632 }
1633 return 0;
1634 }
1635 found:
1636 mask |= item->mask;
1637 if (*p1 != ',')
1638 break;
1639 p = p1 + 1;
1640 }
1641 return mask;
1642 }
1643
1644 void cpu_abort(CPUState *env, const char *fmt, ...)
1645 {
1646 va_list ap;
1647 va_list ap2;
1648
1649 va_start(ap, fmt);
1650 va_copy(ap2, ap);
1651 fprintf(stderr, "qemu: fatal: ");
1652 vfprintf(stderr, fmt, ap);
1653 fprintf(stderr, "\n");
1654 #ifdef TARGET_I386
1655 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1656 #else
1657 cpu_dump_state(env, stderr, fprintf, 0);
1658 #endif
1659 if (logfile) {
1660 fprintf(logfile, "qemu: fatal: ");
1661 vfprintf(logfile, fmt, ap2);
1662 fprintf(logfile, "\n");
1663 #ifdef TARGET_I386
1664 cpu_dump_state(env, logfile, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1665 #else
1666 cpu_dump_state(env, logfile, fprintf, 0);
1667 #endif
1668 fflush(logfile);
1669 fclose(logfile);
1670 }
1671 va_end(ap2);
1672 va_end(ap);
1673 abort();
1674 }
1675
1676 CPUState *cpu_copy(CPUState *env)
1677 {
1678 CPUState *new_env = cpu_init(env->cpu_model_str);
1679 /* preserve chaining and index */
1680 CPUState *next_cpu = new_env->next_cpu;
1681 int cpu_index = new_env->cpu_index;
1682 memcpy(new_env, env, sizeof(CPUState));
1683 new_env->next_cpu = next_cpu;
1684 new_env->cpu_index = cpu_index;
1685 return new_env;
1686 }
1687
1688 #if !defined(CONFIG_USER_ONLY)
1689
1690 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1691 {
1692 unsigned int i;
1693
1694 /* Discard jump cache entries for any tb which might potentially
1695 overlap the flushed page. */
1696 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1697 memset (&env->tb_jmp_cache[i], 0,
1698 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1699
1700 i = tb_jmp_cache_hash_page(addr);
1701 memset (&env->tb_jmp_cache[i], 0,
1702 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1703 }
1704
1705 /* NOTE: if flush_global is true, also flush global entries (not
1706 implemented yet) */
1707 void tlb_flush(CPUState *env, int flush_global)
1708 {
1709 int i;
1710
1711 #if defined(DEBUG_TLB)
1712 printf("tlb_flush:\n");
1713 #endif
1714 /* must reset current TB so that interrupts cannot modify the
1715 links while we are modifying them */
1716 env->current_tb = NULL;
1717
1718 for(i = 0; i < CPU_TLB_SIZE; i++) {
1719 env->tlb_table[0][i].addr_read = -1;
1720 env->tlb_table[0][i].addr_write = -1;
1721 env->tlb_table[0][i].addr_code = -1;
1722 env->tlb_table[1][i].addr_read = -1;
1723 env->tlb_table[1][i].addr_write = -1;
1724 env->tlb_table[1][i].addr_code = -1;
1725 #if (NB_MMU_MODES >= 3)
1726 env->tlb_table[2][i].addr_read = -1;
1727 env->tlb_table[2][i].addr_write = -1;
1728 env->tlb_table[2][i].addr_code = -1;
1729 #if (NB_MMU_MODES == 4)
1730 env->tlb_table[3][i].addr_read = -1;
1731 env->tlb_table[3][i].addr_write = -1;
1732 env->tlb_table[3][i].addr_code = -1;
1733 #endif
1734 #endif
1735 }
1736
1737 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1738
1739 #ifdef USE_KQEMU
1740 if (env->kqemu_enabled) {
1741 kqemu_flush(env, flush_global);
1742 }
1743 #endif
1744 tlb_flush_count++;
1745 }
1746
1747 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1748 {
1749 if (addr == (tlb_entry->addr_read &
1750 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1751 addr == (tlb_entry->addr_write &
1752 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1753 addr == (tlb_entry->addr_code &
1754 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1755 tlb_entry->addr_read = -1;
1756 tlb_entry->addr_write = -1;
1757 tlb_entry->addr_code = -1;
1758 }
1759 }
1760
1761 void tlb_flush_page(CPUState *env, target_ulong addr)
1762 {
1763 int i;
1764
1765 #if defined(DEBUG_TLB)
1766 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1767 #endif
1768 /* must reset current TB so that interrupts cannot modify the
1769 links while we are modifying them */
1770 env->current_tb = NULL;
1771
1772 addr &= TARGET_PAGE_MASK;
1773 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1774 tlb_flush_entry(&env->tlb_table[0][i], addr);
1775 tlb_flush_entry(&env->tlb_table[1][i], addr);
1776 #if (NB_MMU_MODES >= 3)
1777 tlb_flush_entry(&env->tlb_table[2][i], addr);
1778 #if (NB_MMU_MODES == 4)
1779 tlb_flush_entry(&env->tlb_table[3][i], addr);
1780 #endif
1781 #endif
1782
1783 tlb_flush_jmp_cache(env, addr);
1784
1785 #ifdef USE_KQEMU
1786 if (env->kqemu_enabled) {
1787 kqemu_flush_page(env, addr);
1788 }
1789 #endif
1790 }
1791
1792 /* update the TLBs so that writes to code in the virtual page 'addr'
1793 can be detected */
1794 static void tlb_protect_code(ram_addr_t ram_addr)
1795 {
1796 cpu_physical_memory_reset_dirty(ram_addr,
1797 ram_addr + TARGET_PAGE_SIZE,
1798 CODE_DIRTY_FLAG);
1799 }
1800
1801 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1802 tested for self modifying code */
1803 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1804 target_ulong vaddr)
1805 {
1806 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1807 }
1808
1809 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1810 unsigned long start, unsigned long length)
1811 {
1812 unsigned long addr;
1813 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1814 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1815 if ((addr - start) < length) {
1816 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1817 }
1818 }
1819 }
1820
1821 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1822 int dirty_flags)
1823 {
1824 CPUState *env;
1825 unsigned long length, start1;
1826 int i, mask, len;
1827 uint8_t *p;
1828
1829 start &= TARGET_PAGE_MASK;
1830 end = TARGET_PAGE_ALIGN(end);
1831
1832 length = end - start;
1833 if (length == 0)
1834 return;
1835 len = length >> TARGET_PAGE_BITS;
1836 #ifdef USE_KQEMU
1837 /* XXX: should not depend on cpu context */
1838 env = first_cpu;
1839 if (env->kqemu_enabled) {
1840 ram_addr_t addr;
1841 addr = start;
1842 for(i = 0; i < len; i++) {
1843 kqemu_set_notdirty(env, addr);
1844 addr += TARGET_PAGE_SIZE;
1845 }
1846 }
1847 #endif
1848 mask = ~dirty_flags;
1849 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1850 for(i = 0; i < len; i++)
1851 p[i] &= mask;
1852
1853 /* we modify the TLB cache so that the dirty bit will be set again
1854 when accessing the range */
1855 start1 = start + (unsigned long)phys_ram_base;
1856 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1857 for(i = 0; i < CPU_TLB_SIZE; i++)
1858 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1859 for(i = 0; i < CPU_TLB_SIZE; i++)
1860 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1861 #if (NB_MMU_MODES >= 3)
1862 for(i = 0; i < CPU_TLB_SIZE; i++)
1863 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1864 #if (NB_MMU_MODES == 4)
1865 for(i = 0; i < CPU_TLB_SIZE; i++)
1866 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1867 #endif
1868 #endif
1869 }
1870 }
1871
1872 int cpu_physical_memory_set_dirty_tracking(int enable)
1873 {
1874 int r=0;
1875
1876 if (kvm_enabled())
1877 r = kvm_physical_memory_set_dirty_tracking(enable);
1878 in_migration = enable;
1879 return r;
1880 }
1881
1882 int cpu_physical_memory_get_dirty_tracking(void)
1883 {
1884 return in_migration;
1885 }
1886
1887 void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
1888 {
1889 if (kvm_enabled())
1890 kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1891 }
1892
1893 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1894 {
1895 ram_addr_t ram_addr;
1896
1897 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1898 ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1899 tlb_entry->addend - (unsigned long)phys_ram_base;
1900 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1901 tlb_entry->addr_write |= TLB_NOTDIRTY;
1902 }
1903 }
1904 }
1905
1906 /* update the TLB according to the current state of the dirty bits */
1907 void cpu_tlb_update_dirty(CPUState *env)
1908 {
1909 int i;
1910 for(i = 0; i < CPU_TLB_SIZE; i++)
1911 tlb_update_dirty(&env->tlb_table[0][i]);
1912 for(i = 0; i < CPU_TLB_SIZE; i++)
1913 tlb_update_dirty(&env->tlb_table[1][i]);
1914 #if (NB_MMU_MODES >= 3)
1915 for(i = 0; i < CPU_TLB_SIZE; i++)
1916 tlb_update_dirty(&env->tlb_table[2][i]);
1917 #if (NB_MMU_MODES == 4)
1918 for(i = 0; i < CPU_TLB_SIZE; i++)
1919 tlb_update_dirty(&env->tlb_table[3][i]);
1920 #endif
1921 #endif
1922 }
1923
1924 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1925 {
1926 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1927 tlb_entry->addr_write = vaddr;
1928 }
1929
1930 /* update the TLB corresponding to virtual page vaddr
1931 so that it is no longer dirty */
1932 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1933 {
1934 int i;
1935
1936 vaddr &= TARGET_PAGE_MASK;
1937 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1938 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1939 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1940 #if (NB_MMU_MODES >= 3)
1941 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1942 #if (NB_MMU_MODES == 4)
1943 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1944 #endif
1945 #endif
1946 }
1947
1948 /* add a new TLB entry. At most one entry for a given virtual address
1949 is permitted. Return 0 if OK or 2 if the page could not be mapped
1950 (can only happen in non SOFTMMU mode for I/O pages or pages
1951 conflicting with the host address space). */
1952 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1953 target_phys_addr_t paddr, int prot,
1954 int mmu_idx, int is_softmmu)
1955 {
1956 PhysPageDesc *p;
1957 unsigned long pd;
1958 unsigned int index;
1959 target_ulong address;
1960 target_ulong code_address;
1961 target_phys_addr_t addend;
1962 int ret;
1963 CPUTLBEntry *te;
1964 CPUWatchpoint *wp;
1965 target_phys_addr_t iotlb;
1966
1967 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1968 if (!p) {
1969 pd = IO_MEM_UNASSIGNED;
1970 } else {
1971 pd = p->phys_offset;
1972 }
1973 #if defined(DEBUG_TLB)
1974 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1975 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1976 #endif
1977
1978 ret = 0;
1979 address = vaddr;
1980 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1981 /* IO memory case (romd handled later) */
1982 address |= TLB_MMIO;
1983 }
1984 addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
1985 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
1986 /* Normal RAM. */
1987 iotlb = pd & TARGET_PAGE_MASK;
1988 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
1989 iotlb |= IO_MEM_NOTDIRTY;
1990 else
1991 iotlb |= IO_MEM_ROM;
1992 } else {
1993 /* IO handlers are currently passed a phsical address.
1994 It would be nice to pass an offset from the base address
1995 of that region. This would avoid having to special case RAM,
1996 and avoid full address decoding in every device.
1997 We can't use the high bits of pd for this because
1998 IO_MEM_ROMD uses these as a ram address. */
1999 iotlb = (pd & ~TARGET_PAGE_MASK);
2000 if (p) {
2001 iotlb += p->region_offset;
2002 } else {
2003 iotlb += paddr;
2004 }
2005 }
2006
2007 code_address = address;
2008 /* Make accesses to pages with watchpoints go via the
2009 watchpoint trap routines. */
2010 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2011 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2012 iotlb = io_mem_watch + paddr;
2013 /* TODO: The memory case can be optimized by not trapping
2014 reads of pages with a write breakpoint. */
2015 address |= TLB_MMIO;
2016 }
2017 }
2018
2019 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2020 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2021 te = &env->tlb_table[mmu_idx][index];
2022 te->addend = addend - vaddr;
2023 if (prot & PAGE_READ) {
2024 te->addr_read = address;
2025 } else {
2026 te->addr_read = -1;
2027 }
2028
2029 if (prot & PAGE_EXEC) {
2030 te->addr_code = code_address;
2031 } else {
2032 te->addr_code = -1;
2033 }
2034 if (prot & PAGE_WRITE) {
2035 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2036 (pd & IO_MEM_ROMD)) {
2037 /* Write access calls the I/O callback. */
2038 te->addr_write = address | TLB_MMIO;
2039 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2040 !cpu_physical_memory_is_dirty(pd)) {
2041 te->addr_write = address | TLB_NOTDIRTY;
2042 } else {
2043 te->addr_write = address;
2044 }
2045 } else {
2046 te->addr_write = -1;
2047 }
2048 return ret;
2049 }
2050
2051 #else
2052
2053 void tlb_flush(CPUState *env, int flush_global)
2054 {
2055 }
2056
2057 void tlb_flush_page(CPUState *env, target_ulong addr)
2058 {
2059 }
2060
2061 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2062 target_phys_addr_t paddr, int prot,
2063 int mmu_idx, int is_softmmu)
2064 {
2065 return 0;
2066 }
2067
2068 /* dump memory mappings */
2069 void page_dump(FILE *f)
2070 {
2071 unsigned long start, end;
2072 int i, j, prot, prot1;
2073 PageDesc *p;
2074
2075 fprintf(f, "%-8s %-8s %-8s %s\n",
2076 "start", "end", "size", "prot");
2077 start = -1;
2078 end = -1;
2079 prot = 0;
2080 for(i = 0; i <= L1_SIZE; i++) {
2081 if (i < L1_SIZE)
2082 p = l1_map[i];
2083 else
2084 p = NULL;
2085 for(j = 0;j < L2_SIZE; j++) {
2086 if (!p)
2087 prot1 = 0;
2088 else
2089 prot1 = p[j].flags;
2090 if (prot1 != prot) {
2091 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2092 if (start != -1) {
2093 fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2094 start, end, end - start,
2095 prot & PAGE_READ ? 'r' : '-',
2096 prot & PAGE_WRITE ? 'w' : '-',
2097 prot & PAGE_EXEC ? 'x' : '-');
2098 }
2099 if (prot1 != 0)
2100 start = end;
2101 else
2102 start = -1;
2103 prot = prot1;
2104 }
2105 if (!p)
2106 break;
2107 }
2108 }
2109 }
2110
2111 int page_get_flags(target_ulong address)
2112 {
2113 PageDesc *p;
2114
2115 p = page_find(address >> TARGET_PAGE_BITS);
2116 if (!p)
2117 return 0;
2118 return p->flags;
2119 }
2120
2121 /* modify the flags of a page and invalidate the code if
2122 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2123 depending on PAGE_WRITE */
2124 void page_set_flags(target_ulong start, target_ulong end, int flags)
2125 {
2126 PageDesc *p;
2127 target_ulong addr;
2128
2129 /* mmap_lock should already be held. */
2130 start = start & TARGET_PAGE_MASK;
2131 end = TARGET_PAGE_ALIGN(end);
2132 if (flags & PAGE_WRITE)
2133 flags |= PAGE_WRITE_ORG;
2134 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2135 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2136 /* We may be called for host regions that are outside guest
2137 address space. */
2138 if (!p)
2139 return;
2140 /* if the write protection is set, then we invalidate the code
2141 inside */
2142 if (!(p->flags & PAGE_WRITE) &&
2143 (flags & PAGE_WRITE) &&
2144 p->first_tb) {
2145 tb_invalidate_phys_page(addr, 0, NULL);
2146 }
2147 p->flags = flags;
2148 }
2149 }
2150
2151 int page_check_range(target_ulong start, target_ulong len, int flags)
2152 {
2153 PageDesc *p;
2154 target_ulong end;
2155 target_ulong addr;
2156
2157 if (start + len < start)
2158 /* we've wrapped around */
2159 return -1;
2160
2161 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2162 start = start & TARGET_PAGE_MASK;
2163
2164 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2165 p = page_find(addr >> TARGET_PAGE_BITS);
2166 if( !p )
2167 return -1;
2168 if( !(p->flags & PAGE_VALID) )
2169 return -1;
2170
2171 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2172 return -1;
2173 if (flags & PAGE_WRITE) {
2174 if (!(p->flags & PAGE_WRITE_ORG))
2175 return -1;
2176 /* unprotect the page if it was put read-only because it
2177 contains translated code */
2178 if (!(p->flags & PAGE_WRITE)) {
2179 if (!page_unprotect(addr, 0, NULL))
2180 return -1;
2181 }
2182 return 0;
2183 }
2184 }
2185 return 0;
2186 }
2187
2188 /* called from signal handler: invalidate the code and unprotect the
2189 page. Return TRUE if the fault was succesfully handled. */
2190 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2191 {
2192 unsigned int page_index, prot, pindex;
2193 PageDesc *p, *p1;
2194 target_ulong host_start, host_end, addr;
2195
2196 /* Technically this isn't safe inside a signal handler. However we
2197 know this only ever happens in a synchronous SEGV handler, so in
2198 practice it seems to be ok. */
2199 mmap_lock();
2200
2201 host_start = address & qemu_host_page_mask;
2202 page_index = host_start >> TARGET_PAGE_BITS;
2203 p1 = page_find(page_index);
2204 if (!p1) {
2205 mmap_unlock();
2206 return 0;
2207 }
2208 host_end = host_start + qemu_host_page_size;
2209 p = p1;
2210 prot = 0;
2211 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2212 prot |= p->flags;
2213 p++;
2214 }
2215 /* if the page was really writable, then we change its
2216 protection back to writable */
2217 if (prot & PAGE_WRITE_ORG) {
2218 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2219 if (!(p1[pindex].flags & PAGE_WRITE)) {
2220 mprotect((void *)g2h(host_start), qemu_host_page_size,
2221 (prot & PAGE_BITS) | PAGE_WRITE);
2222 p1[pindex].flags |= PAGE_WRITE;
2223 /* and since the content will be modified, we must invalidate
2224 the corresponding translated code. */
2225 tb_invalidate_phys_page(address, pc, puc);
2226 #ifdef DEBUG_TB_CHECK
2227 tb_invalidate_check(address);
2228 #endif
2229 mmap_unlock();
2230 return 1;
2231 }
2232 }
2233 mmap_unlock();
2234 return 0;
2235 }
2236
2237 static inline void tlb_set_dirty(CPUState *env,
2238 unsigned long addr, target_ulong vaddr)
2239 {
2240 }
2241 #endif /* defined(CONFIG_USER_ONLY) */
2242
2243 #if !defined(CONFIG_USER_ONLY)
2244
2245 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2246 ram_addr_t memory, ram_addr_t region_offset);
2247 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2248 ram_addr_t orig_memory, ram_addr_t region_offset);
2249 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2250 need_subpage) \
2251 do { \
2252 if (addr > start_addr) \
2253 start_addr2 = 0; \
2254 else { \
2255 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2256 if (start_addr2 > 0) \
2257 need_subpage = 1; \
2258 } \
2259 \
2260 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2261 end_addr2 = TARGET_PAGE_SIZE - 1; \
2262 else { \
2263 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2264 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2265 need_subpage = 1; \
2266 } \
2267 } while (0)
2268
2269 /* register physical memory. 'size' must be a multiple of the target
2270 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2271 io memory page. The address used when calling the IO function is
2272 the offset from the start of the region, plus region_offset. Both
2273 start_region and regon_offset are rounded down to a page boundary
2274 before calculating this offset. This should not be a problem unless
2275 the low bits of start_addr and region_offset differ. */
2276 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2277 ram_addr_t size,
2278 ram_addr_t phys_offset,
2279 ram_addr_t region_offset)
2280 {
2281 target_phys_addr_t addr, end_addr;
2282 PhysPageDesc *p;
2283 CPUState *env;
2284 ram_addr_t orig_size = size;
2285 void *subpage;
2286
2287 #ifdef USE_KQEMU
2288 /* XXX: should not depend on cpu context */
2289 env = first_cpu;
2290 if (env->kqemu_enabled) {
2291 kqemu_set_phys_mem(start_addr, size, phys_offset);
2292 }
2293 #endif
2294 if (kvm_enabled())
2295 kvm_set_phys_mem(start_addr, size, phys_offset);
2296
2297 region_offset &= TARGET_PAGE_MASK;
2298 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2299 end_addr = start_addr + (target_phys_addr_t)size;
2300 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2301 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2302 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2303 ram_addr_t orig_memory = p->phys_offset;
2304 target_phys_addr_t start_addr2, end_addr2;
2305 int need_subpage = 0;
2306
2307 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2308 need_subpage);
2309 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2310 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2311 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2312 &p->phys_offset, orig_memory,
2313 p->region_offset);
2314 } else {
2315 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2316 >> IO_MEM_SHIFT];
2317 }
2318 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2319 region_offset);
2320 p->region_offset = 0;
2321 } else {
2322 p->phys_offset = phys_offset;
2323 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2324 (phys_offset & IO_MEM_ROMD))
2325 phys_offset += TARGET_PAGE_SIZE;
2326 }
2327 } else {
2328 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2329 p->phys_offset = phys_offset;
2330 p->region_offset = region_offset;
2331 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2332 (phys_offset & IO_MEM_ROMD)) {
2333 phys_offset += TARGET_PAGE_SIZE;
2334 } else {
2335 target_phys_addr_t start_addr2, end_addr2;
2336 int need_subpage = 0;
2337
2338 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2339 end_addr2, need_subpage);
2340
2341 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2342 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2343 &p->phys_offset, IO_MEM_UNASSIGNED,
2344 0);
2345 subpage_register(subpage, start_addr2, end_addr2,
2346 phys_offset, region_offset);
2347 p->region_offset = 0;
2348 }
2349 }
2350 }
2351 region_offset += TARGET_PAGE_SIZE;
2352 }
2353
2354 /* since each CPU stores ram addresses in its TLB cache, we must
2355 reset the modified entries */
2356 /* XXX: slow ! */
2357 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2358 tlb_flush(env, 1);
2359 }
2360 }
2361
2362 /* XXX: temporary until new memory mapping API */
2363 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2364 {
2365 PhysPageDesc *p;
2366
2367 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2368 if (!p)
2369 return IO_MEM_UNASSIGNED;
2370 return p->phys_offset;
2371 }
2372
2373 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2374 {
2375 if (kvm_enabled())
2376 kvm_coalesce_mmio_region(addr, size);
2377 }
2378
2379 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2380 {
2381 if (kvm_enabled())
2382 kvm_uncoalesce_mmio_region(addr, size);
2383 }
2384
2385 /* XXX: better than nothing */
2386 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2387 {
2388 ram_addr_t addr;
2389 if ((phys_ram_alloc_offset + size) > phys_ram_size) {
2390 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2391 (uint64_t)size, (uint64_t)phys_ram_size);
2392 abort();
2393 }
2394 addr = phys_ram_alloc_offset;
2395 phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2396 return addr;
2397 }
2398
2399 void qemu_ram_free(ram_addr_t addr)
2400 {
2401 }
2402
2403 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2404 {
2405 #ifdef DEBUG_UNASSIGNED
2406 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2407 #endif
2408 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2409 do_unassigned_access(addr, 0, 0, 0, 1);
2410 #endif
2411 return 0;
2412 }
2413
2414 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2415 {
2416 #ifdef DEBUG_UNASSIGNED
2417 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2418 #endif
2419 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2420 do_unassigned_access(addr, 0, 0, 0, 2);
2421 #endif
2422 return 0;
2423 }
2424
2425 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2426 {
2427 #ifdef DEBUG_UNASSIGNED
2428 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2429 #endif
2430 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2431 do_unassigned_access(addr, 0, 0, 0, 4);
2432 #endif
2433 return 0;
2434 }
2435
2436 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2437 {
2438 #ifdef DEBUG_UNASSIGNED
2439 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2440 #endif
2441 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2442 do_unassigned_access(addr, 1, 0, 0, 1);
2443 #endif
2444 }
2445
2446 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2447 {
2448 #ifdef DEBUG_UNASSIGNED
2449 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2450 #endif
2451 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2452 do_unassigned_access(addr, 1, 0, 0, 2);
2453 #endif
2454 }
2455
2456 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2457 {
2458 #ifdef DEBUG_UNASSIGNED
2459 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2460 #endif
2461 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2462 do_unassigned_access(addr, 1, 0, 0, 4);
2463 #endif
2464 }
2465
2466 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2467 unassigned_mem_readb,
2468 unassigned_mem_readw,
2469 unassigned_mem_readl,
2470 };
2471
2472 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2473 unassigned_mem_writeb,
2474 unassigned_mem_writew,
2475 unassigned_mem_writel,
2476 };
2477
2478 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2479 uint32_t val)
2480 {
2481 int dirty_flags;
2482 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2483 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2484 #if !defined(CONFIG_USER_ONLY)
2485 tb_invalidate_phys_page_fast(ram_addr, 1);
2486 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2487 #endif
2488 }
2489 stb_p(phys_ram_base + ram_addr, val);
2490 #ifdef USE_KQEMU
2491 if (cpu_single_env->kqemu_enabled &&
2492 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2493 kqemu_modify_page(cpu_single_env, ram_addr);
2494 #endif
2495 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2496 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2497 /* we remove the notdirty callback only if the code has been
2498 flushed */
2499 if (dirty_flags == 0xff)
2500 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2501 }
2502
2503 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2504 uint32_t val)
2505 {
2506 int dirty_flags;
2507 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2508 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2509 #if !defined(CONFIG_USER_ONLY)
2510 tb_invalidate_phys_page_fast(ram_addr, 2);
2511 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2512 #endif
2513 }
2514 stw_p(phys_ram_base + ram_addr, val);
2515 #ifdef USE_KQEMU
2516 if (cpu_single_env->kqemu_enabled &&
2517 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2518 kqemu_modify_page(cpu_single_env, ram_addr);
2519 #endif
2520 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2521 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2522 /* we remove the notdirty callback only if the code has been
2523 flushed */
2524 if (dirty_flags == 0xff)
2525 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2526 }
2527
2528 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2529 uint32_t val)
2530 {
2531 int dirty_flags;
2532 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2533 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2534 #if !defined(CONFIG_USER_ONLY)
2535 tb_invalidate_phys_page_fast(ram_addr, 4);
2536 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2537 #endif
2538 }
2539 stl_p(phys_ram_base + ram_addr, val);
2540 #ifdef USE_KQEMU
2541 if (cpu_single_env->kqemu_enabled &&
2542 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2543 kqemu_modify_page(cpu_single_env, ram_addr);
2544 #endif
2545 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2546 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2547 /* we remove the notdirty callback only if the code has been
2548 flushed */
2549 if (dirty_flags == 0xff)
2550 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2551 }
2552
2553 static CPUReadMemoryFunc *error_mem_read[3] = {
2554 NULL, /* never used */
2555 NULL, /* never used */
2556 NULL, /* never used */
2557 };
2558
2559 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2560 notdirty_mem_writeb,
2561 notdirty_mem_writew,
2562 notdirty_mem_writel,
2563 };
2564
2565 /* Generate a debug exception if a watchpoint has been hit. */
2566 static void check_watchpoint(int offset, int len_mask, int flags)
2567 {
2568 CPUState *env = cpu_single_env;
2569 target_ulong pc, cs_base;
2570 TranslationBlock *tb;
2571 target_ulong vaddr;
2572 CPUWatchpoint *wp;
2573 int cpu_flags;
2574
2575 if (env->watchpoint_hit) {
2576 /* We re-entered the check after replacing the TB. Now raise
2577 * the debug interrupt so that is will trigger after the
2578 * current instruction. */
2579 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2580 return;
2581 }
2582 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2583 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2584 if ((vaddr == (wp->vaddr & len_mask) ||
2585 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2586 wp->flags |= BP_WATCHPOINT_HIT;
2587 if (!env->watchpoint_hit) {
2588 env->watchpoint_hit = wp;
2589 tb = tb_find_pc(env->mem_io_pc);
2590 if (!tb) {
2591 cpu_abort(env, "check_watchpoint: could not find TB for "
2592 "pc=%p", (void *)env->mem_io_pc);
2593 }
2594 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2595 tb_phys_invalidate(tb, -1);
2596 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2597 env->exception_index = EXCP_DEBUG;
2598 } else {
2599 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2600 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2601 }
2602 cpu_resume_from_signal(env, NULL);
2603 }
2604 } else {
2605 wp->flags &= ~BP_WATCHPOINT_HIT;
2606 }
2607 }
2608 }
2609
2610 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2611 so these check for a hit then pass through to the normal out-of-line
2612 phys routines. */
2613 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2614 {
2615 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2616 return ldub_phys(addr);
2617 }
2618
2619 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2620 {
2621 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2622 return lduw_phys(addr);
2623 }
2624
2625 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2626 {
2627 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2628 return ldl_phys(addr);
2629 }
2630
2631 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2632 uint32_t val)
2633 {
2634 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2635 stb_phys(addr, val);
2636 }
2637
2638 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2639 uint32_t val)
2640 {
2641 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2642 stw_phys(addr, val);
2643 }
2644
2645 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2646 uint32_t val)
2647 {
2648 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2649 stl_phys(addr, val);
2650 }
2651
2652 static CPUReadMemoryFunc *watch_mem_read[3] = {
2653 watch_mem_readb,
2654 watch_mem_readw,
2655 watch_mem_readl,
2656 };
2657
2658 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2659 watch_mem_writeb,
2660 watch_mem_writew,
2661 watch_mem_writel,
2662 };
2663
2664 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2665 unsigned int len)
2666 {
2667 uint32_t ret;
2668 unsigned int idx;
2669
2670 idx = SUBPAGE_IDX(addr);
2671 #if defined(DEBUG_SUBPAGE)
2672 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2673 mmio, len, addr, idx);
2674 #endif
2675 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2676 addr + mmio->region_offset[idx][0][len]);
2677
2678 return ret;
2679 }
2680
2681 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2682 uint32_t value, unsigned int len)
2683 {
2684 unsigned int idx;
2685
2686 idx = SUBPAGE_IDX(addr);
2687 #if defined(DEBUG_SUBPAGE)
2688 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2689 mmio, len, addr, idx, value);
2690 #endif
2691 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2692 addr + mmio->region_offset[idx][1][len],
2693 value);
2694 }
2695
2696 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2697 {
2698 #if defined(DEBUG_SUBPAGE)
2699 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2700 #endif
2701
2702 return subpage_readlen(opaque, addr, 0);
2703 }
2704
2705 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2706 uint32_t value)
2707 {
2708 #if defined(DEBUG_SUBPAGE)
2709 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2710 #endif
2711 subpage_writelen(opaque, addr, value, 0);
2712 }
2713
2714 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2715 {
2716 #if defined(DEBUG_SUBPAGE)
2717 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2718 #endif
2719
2720 return subpage_readlen(opaque, addr, 1);
2721 }
2722
2723 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2724 uint32_t value)
2725 {
2726 #if defined(DEBUG_SUBPAGE)
2727 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2728 #endif
2729 subpage_writelen(opaque, addr, value, 1);
2730 }
2731
2732 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2733 {
2734 #if defined(DEBUG_SUBPAGE)
2735 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2736 #endif
2737
2738 return subpage_readlen(opaque, addr, 2);
2739 }
2740
2741 static void subpage_writel (void *opaque,
2742 target_phys_addr_t addr, uint32_t value)
2743 {
2744 #if defined(DEBUG_SUBPAGE)
2745 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2746 #endif
2747 subpage_writelen(opaque, addr, value, 2);
2748 }
2749
2750 static CPUReadMemoryFunc *subpage_read[] = {
2751 &subpage_readb,
2752 &subpage_readw,
2753 &subpage_readl,
2754 };
2755
2756 static CPUWriteMemoryFunc *subpage_write[] = {
2757 &subpage_writeb,
2758 &subpage_writew,
2759 &subpage_writel,
2760 };
2761
2762 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2763 ram_addr_t memory, ram_addr_t region_offset)
2764 {
2765 int idx, eidx;
2766 unsigned int i;
2767
2768 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2769 return -1;
2770 idx = SUBPAGE_IDX(start);
2771 eidx = SUBPAGE_IDX(end);
2772 #if defined(DEBUG_SUBPAGE)
2773 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2774 mmio, start, end, idx, eidx, memory);
2775 #endif
2776 memory >>= IO_MEM_SHIFT;
2777 for (; idx <= eidx; idx++) {
2778 for (i = 0; i < 4; i++) {
2779 if (io_mem_read[memory][i]) {
2780 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2781 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2782 mmio->region_offset[idx][0][i] = region_offset;
2783 }
2784 if (io_mem_write[memory][i]) {
2785 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2786 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2787 mmio->region_offset[idx][1][i] = region_offset;
2788 }
2789 }
2790 }
2791
2792 return 0;
2793 }
2794
2795 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2796 ram_addr_t orig_memory, ram_addr_t region_offset)
2797 {
2798 subpage_t *mmio;
2799 int subpage_memory;
2800
2801 mmio = qemu_mallocz(sizeof(subpage_t));
2802 if (mmio != NULL) {
2803 mmio->base = base;
2804 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2805 #if defined(DEBUG_SUBPAGE)
2806 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2807 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2808 #endif
2809 *phys = subpage_memory | IO_MEM_SUBPAGE;
2810 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
2811 region_offset);
2812 }
2813
2814 return mmio;
2815 }
2816
2817 static int get_free_io_mem_idx(void)
2818 {
2819 int i;
2820
2821 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
2822 if (!io_mem_used[i]) {
2823 io_mem_used[i] = 1;
2824 return i;
2825 }
2826
2827 return -1;
2828 }
2829
2830 static void io_mem_init(void)
2831 {
2832 int i;
2833
2834 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
2835 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2836 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2837 for (i=0; i<5; i++)
2838 io_mem_used[i] = 1;
2839
2840 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2841 watch_mem_write, NULL);
2842 /* alloc dirty bits array */
2843 phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2844 memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2845 }
2846
2847 /* mem_read and mem_write are arrays of functions containing the
2848 function to access byte (index 0), word (index 1) and dword (index
2849 2). Functions can be omitted with a NULL function pointer. The
2850 registered functions may be modified dynamically later.
2851 If io_index is non zero, the corresponding io zone is
2852 modified. If it is zero, a new io zone is allocated. The return
2853 value can be used with cpu_register_physical_memory(). (-1) is
2854 returned if error. */
2855 int cpu_register_io_memory(int io_index,
2856 CPUReadMemoryFunc **mem_read,
2857 CPUWriteMemoryFunc **mem_write,
2858 void *opaque)
2859 {
2860 int i, subwidth = 0;
2861
2862 if (io_index <= 0) {
2863 io_index = get_free_io_mem_idx();
2864 if (io_index == -1)
2865 return io_index;
2866 } else {
2867 if (io_index >= IO_MEM_NB_ENTRIES)
2868 return -1;
2869 }
2870
2871 for(i = 0;i < 3; i++) {
2872 if (!mem_read[i] || !mem_write[i])
2873 subwidth = IO_MEM_SUBWIDTH;
2874 io_mem_read[io_index][i] = mem_read[i];
2875 io_mem_write[io_index][i] = mem_write[i];
2876 }
2877 io_mem_opaque[io_index] = opaque;
2878 return (io_index << IO_MEM_SHIFT) | subwidth;
2879 }
2880
2881 void cpu_unregister_io_memory(int io_table_address)
2882 {
2883 int i;
2884 int io_index = io_table_address >> IO_MEM_SHIFT;
2885
2886 for (i=0;i < 3; i++) {
2887 io_mem_read[io_index][i] = unassigned_mem_read[i];
2888 io_mem_write[io_index][i] = unassigned_mem_write[i];
2889 }
2890 io_mem_opaque[io_index] = NULL;
2891 io_mem_used[io_index] = 0;
2892 }
2893
2894 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2895 {
2896 return io_mem_write[io_index >> IO_MEM_SHIFT];
2897 }
2898
2899 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2900 {
2901 return io_mem_read[io_index >> IO_MEM_SHIFT];
2902 }
2903
2904 #endif /* !defined(CONFIG_USER_ONLY) */
2905
2906 /* physical memory access (slow version, mainly for debug) */
2907 #if defined(CONFIG_USER_ONLY)
2908 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2909 int len, int is_write)
2910 {
2911 int l, flags;
2912 target_ulong page;
2913 void * p;
2914
2915 while (len > 0) {
2916 page = addr & TARGET_PAGE_MASK;
2917 l = (page + TARGET_PAGE_SIZE) - addr;
2918 if (l > len)
2919 l = len;
2920 flags = page_get_flags(page);
2921 if (!(flags & PAGE_VALID))
2922 return;
2923 if (is_write) {
2924 if (!(flags & PAGE_WRITE))
2925 return;
2926 /* XXX: this code should not depend on lock_user */
2927 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2928 /* FIXME - should this return an error rather than just fail? */
2929 return;
2930 memcpy(p, buf, l);
2931 unlock_user(p, addr, l);
2932 } else {
2933 if (!(flags & PAGE_READ))
2934 return;
2935 /* XXX: this code should not depend on lock_user */
2936 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2937 /* FIXME - should this return an error rather than just fail? */
2938 return;
2939 memcpy(buf, p, l);
2940 unlock_user(p, addr, 0);
2941 }
2942 len -= l;
2943 buf += l;
2944 addr += l;
2945 }
2946 }
2947
2948 #else
2949 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2950 int len, int is_write)
2951 {
2952 int l, io_index;
2953 uint8_t *ptr;
2954 uint32_t val;
2955 target_phys_addr_t page;
2956 unsigned long pd;
2957 PhysPageDesc *p;
2958
2959 while (len > 0) {
2960 page = addr & TARGET_PAGE_MASK;
2961 l = (page + TARGET_PAGE_SIZE) - addr;
2962 if (l > len)
2963 l = len;
2964 p = phys_page_find(page >> TARGET_PAGE_BITS);
2965 if (!p) {
2966 pd = IO_MEM_UNASSIGNED;
2967 } else {
2968 pd = p->phys_offset;
2969 }
2970
2971 if (is_write) {
2972 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
2973 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2974 if (p)
2975 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
2976 /* XXX: could force cpu_single_env to NULL to avoid
2977 potential bugs */
2978 if (l >= 4 && ((addr & 3) == 0)) {
2979 /* 32 bit write access */
2980 val = ldl_p(buf);
2981 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
2982 l = 4;
2983 } else if (l >= 2 && ((addr & 1) == 0)) {
2984 /* 16 bit write access */
2985 val = lduw_p(buf);
2986 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
2987 l = 2;
2988 } else {
2989 /* 8 bit write access */
2990 val = ldub_p(buf);
2991 io_mem_write[io_index][0](io_mem_opaque[io_index], addr, val);
2992 l = 1;
2993 }
2994 } else {
2995 unsigned long addr1;
2996 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
2997 /* RAM case */
2998 ptr = phys_ram_base + addr1;
2999 memcpy(ptr, buf, l);
3000 if (!cpu_physical_memory_is_dirty(addr1)) {
3001 /* invalidate code */
3002 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3003 /* set dirty bit */
3004 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3005 (0xff & ~CODE_DIRTY_FLAG);
3006 }
3007 /* qemu doesn't execute guest code directly, but kvm does
3008 therefore fluch instruction caches */
3009 if (kvm_enabled())
3010 flush_icache_range((unsigned long)ptr,
3011 ((unsigned long)ptr)+l);
3012 }
3013 } else {
3014 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3015 !(pd & IO_MEM_ROMD)) {
3016 /* I/O case */
3017 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3018 if (p)
3019 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3020 if (l >= 4 && ((addr & 3) == 0)) {
3021 /* 32 bit read access */
3022 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3023 stl_p(buf, val);
3024 l = 4;
3025 } else if (l >= 2 && ((addr & 1) == 0)) {
3026 /* 16 bit read access */
3027 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
3028 stw_p(buf, val);
3029 l = 2;
3030 } else {
3031 /* 8 bit read access */
3032 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr);
3033 stb_p(buf, val);
3034 l = 1;
3035 }
3036 } else {
3037 /* RAM case */
3038 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3039 (addr & ~TARGET_PAGE_MASK);
3040 memcpy(buf, ptr, l);
3041 }
3042 }
3043 len -= l;
3044 buf += l;
3045 addr += l;
3046 }
3047 }
3048
3049 /* used for ROM loading : can write in RAM and ROM */
3050 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3051 const uint8_t *buf, int len)
3052 {
3053 int l;
3054 uint8_t *ptr;
3055 target_phys_addr_t page;
3056 unsigned long pd;
3057 PhysPageDesc *p;
3058
3059 while (len > 0) {
3060 page = addr & TARGET_PAGE_MASK;
3061 l = (page + TARGET_PAGE_SIZE) - addr;
3062 if (l > len)
3063 l = len;
3064 p = phys_page_find(page >> TARGET_PAGE_BITS);
3065 if (!p) {
3066 pd = IO_MEM_UNASSIGNED;
3067 } else {
3068 pd = p->phys_offset;
3069 }
3070
3071 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3072 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3073 !(pd & IO_MEM_ROMD)) {
3074 /* do nothing */
3075 } else {
3076 unsigned long addr1;
3077 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3078 /* ROM/RAM case */
3079 ptr = phys_ram_base + addr1;
3080 memcpy(ptr, buf, l);
3081 }
3082 len -= l;
3083 buf += l;
3084 addr += l;
3085 }
3086 }
3087
3088
3089 /* warning: addr must be aligned */
3090 uint32_t ldl_phys(target_phys_addr_t addr)
3091 {
3092 int io_index;
3093 uint8_t *ptr;
3094 uint32_t val;
3095 unsigned long pd;
3096 PhysPageDesc *p;
3097
3098 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3099 if (!p) {
3100 pd = IO_MEM_UNASSIGNED;
3101 } else {
3102 pd = p->phys_offset;
3103 }
3104
3105 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3106 !(pd & IO_MEM_ROMD)) {
3107 /* I/O case */
3108 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3109 if (p)
3110 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3111 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3112 } else {
3113 /* RAM case */
3114 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3115 (addr & ~TARGET_PAGE_MASK);
3116 val = ldl_p(ptr);
3117 }
3118 return val;
3119 }
3120
3121 /* warning: addr must be aligned */
3122 uint64_t ldq_phys(target_phys_addr_t addr)
3123 {
3124 int io_index;
3125 uint8_t *ptr;
3126 uint64_t val;
3127 unsigned long pd;
3128 PhysPageDesc *p;
3129
3130 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3131 if (!p) {
3132 pd = IO_MEM_UNASSIGNED;
3133 } else {
3134 pd = p->phys_offset;
3135 }
3136
3137 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3138 !(pd & IO_MEM_ROMD)) {
3139 /* I/O case */
3140 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3141 if (p)
3142 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3143 #ifdef TARGET_WORDS_BIGENDIAN
3144 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3145 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3146 #else
3147 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3148 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3149 #endif
3150 } else {
3151 /* RAM case */
3152 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3153 (addr & ~TARGET_PAGE_MASK);
3154 val = ldq_p(ptr);
3155 }
3156 return val;
3157 }
3158
3159 /* XXX: optimize */
3160 uint32_t ldub_phys(target_phys_addr_t addr)
3161 {
3162 uint8_t val;
3163 cpu_physical_memory_read(addr, &val, 1);
3164 return val;
3165 }
3166
3167 /* XXX: optimize */
3168 uint32_t lduw_phys(target_phys_addr_t addr)
3169 {
3170 uint16_t val;
3171 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3172 return tswap16(val);
3173 }
3174
3175 #ifdef __GNUC__
3176 #define likely(x) __builtin_expect(!!(x), 1)
3177 #define unlikely(x) __builtin_expect(!!(x), 0)
3178 #else
3179 #define likely(x) x
3180 #define unlikely(x) x
3181 #endif
3182
3183 /* warning: addr must be aligned. The ram page is not masked as dirty
3184 and the code inside is not invalidated. It is useful if the dirty
3185 bits are used to track modified PTEs */
3186 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3187 {
3188 int io_index;
3189 uint8_t *ptr;
3190 unsigned long pd;
3191 PhysPageDesc *p;
3192
3193 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3194 if (!p) {
3195 pd = IO_MEM_UNASSIGNED;
3196 } else {
3197 pd = p->phys_offset;
3198 }
3199
3200 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3201 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3202 if (p)
3203 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3204 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3205 } else {
3206 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3207 ptr = phys_ram_base + addr1;
3208 stl_p(ptr, val);
3209
3210 if (unlikely(in_migration)) {
3211 if (!cpu_physical_memory_is_dirty(addr1)) {
3212 /* invalidate code */
3213 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3214 /* set dirty bit */
3215 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3216 (0xff & ~CODE_DIRTY_FLAG);
3217 }
3218 }
3219 }
3220 }
3221
3222 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3223 {
3224 int io_index;
3225 uint8_t *ptr;
3226 unsigned long pd;
3227 PhysPageDesc *p;
3228
3229 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3230 if (!p) {
3231 pd = IO_MEM_UNASSIGNED;
3232 } else {
3233 pd = p->phys_offset;
3234 }
3235
3236 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3237 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3238 if (p)
3239 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3240 #ifdef TARGET_WORDS_BIGENDIAN
3241 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3242 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3243 #else
3244 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3245 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3246 #endif
3247 } else {
3248 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3249 (addr & ~TARGET_PAGE_MASK);
3250 stq_p(ptr, val);
3251 }
3252 }
3253
3254 /* warning: addr must be aligned */
3255 void stl_phys(target_phys_addr_t addr, uint32_t val)
3256 {
3257 int io_index;
3258 uint8_t *ptr;
3259 unsigned long pd;
3260 PhysPageDesc *p;
3261
3262 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3263 if (!p) {
3264 pd = IO_MEM_UNASSIGNED;
3265 } else {
3266 pd = p->phys_offset;
3267 }
3268
3269 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3270 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3271 if (p)
3272 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3273 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3274 } else {
3275 unsigned long addr1;
3276 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3277 /* RAM case */
3278 ptr = phys_ram_base + addr1;
3279 stl_p(ptr, val);
3280 if (!cpu_physical_memory_is_dirty(addr1)) {
3281 /* invalidate code */
3282 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3283 /* set dirty bit */
3284 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3285 (0xff & ~CODE_DIRTY_FLAG);
3286 }
3287 }
3288 }
3289
3290 /* XXX: optimize */
3291 void stb_phys(target_phys_addr_t addr, uint32_t val)
3292 {
3293 uint8_t v = val;
3294 cpu_physical_memory_write(addr, &v, 1);
3295 }
3296
3297 /* XXX: optimize */
3298 void stw_phys(target_phys_addr_t addr, uint32_t val)
3299 {
3300 uint16_t v = tswap16(val);
3301 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3302 }
3303
3304 /* XXX: optimize */
3305 void stq_phys(target_phys_addr_t addr, uint64_t val)
3306 {
3307 val = tswap64(val);
3308 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3309 }
3310
3311 #endif
3312
3313 /* virtual memory access for debug */
3314 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3315 uint8_t *buf, int len, int is_write)
3316 {
3317 int l;
3318 target_phys_addr_t phys_addr;
3319 target_ulong page;
3320
3321 while (len > 0) {
3322 page = addr & TARGET_PAGE_MASK;
3323 phys_addr = cpu_get_phys_page_debug(env, page);
3324 /* if no physical page mapped, return an error */
3325 if (phys_addr == -1)
3326 return -1;
3327 l = (page + TARGET_PAGE_SIZE) - addr;
3328 if (l > len)
3329 l = len;
3330 cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
3331 buf, l, is_write);
3332 len -= l;
3333 buf += l;
3334 addr += l;
3335 }
3336 return 0;
3337 }
3338
3339 /* in deterministic execution mode, instructions doing device I/Os
3340 must be at the end of the TB */
3341 void cpu_io_recompile(CPUState *env, void *retaddr)
3342 {
3343 TranslationBlock *tb;
3344 uint32_t n, cflags;
3345 target_ulong pc, cs_base;
3346 uint64_t flags;
3347
3348 tb = tb_find_pc((unsigned long)retaddr);
3349 if (!tb) {
3350 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3351 retaddr);
3352 }
3353 n = env->icount_decr.u16.low + tb->icount;
3354 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3355 /* Calculate how many instructions had been executed before the fault
3356 occurred. */
3357 n = n - env->icount_decr.u16.low;
3358 /* Generate a new TB ending on the I/O insn. */
3359 n++;
3360 /* On MIPS and SH, delay slot instructions can only be restarted if
3361 they were already the first instruction in the TB. If this is not
3362 the first instruction in a TB then re-execute the preceding
3363 branch. */
3364 #if defined(TARGET_MIPS)
3365 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3366 env->active_tc.PC -= 4;
3367 env->icount_decr.u16.low++;
3368 env->hflags &= ~MIPS_HFLAG_BMASK;
3369 }
3370 #elif defined(TARGET_SH4)
3371 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3372 && n > 1) {
3373 env->pc -= 2;
3374 env->icount_decr.u16.low++;
3375 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3376 }
3377 #endif
3378 /* This should never happen. */
3379 if (n > CF_COUNT_MASK)
3380 cpu_abort(env, "TB too big during recompile");
3381
3382 cflags = n | CF_LAST_IO;
3383 pc = tb->pc;
3384 cs_base = tb->cs_base;
3385 flags = tb->flags;
3386 tb_phys_invalidate(tb, -1);
3387 /* FIXME: In theory this could raise an exception. In practice
3388 we have already translated the block once so it's probably ok. */
3389 tb_gen_code(env, pc, cs_base, flags, cflags);
3390 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3391 the first in the TB) then we end up generating a whole new TB and
3392 repeating the fault, which is horribly inefficient.
3393 Better would be to execute just this insn uncached, or generate a
3394 second new TB. */
3395 cpu_resume_from_signal(env, NULL);
3396 }
3397
3398 void dump_exec_info(FILE *f,
3399 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3400 {
3401 int i, target_code_size, max_target_code_size;
3402 int direct_jmp_count, direct_jmp2_count, cross_page;
3403 TranslationBlock *tb;
3404
3405 target_code_size = 0;
3406 max_target_code_size = 0;
3407 cross_page = 0;
3408 direct_jmp_count = 0;
3409 direct_jmp2_count = 0;
3410 for(i = 0; i < nb_tbs; i++) {
3411 tb = &tbs[i];
3412 target_code_size += tb->size;
3413 if (tb->size > max_target_code_size)
3414 max_target_code_size = tb->size;
3415 if (tb->page_addr[1] != -1)
3416 cross_page++;
3417 if (tb->tb_next_offset[0] != 0xffff) {
3418 direct_jmp_count++;
3419 if (tb->tb_next_offset[1] != 0xffff) {
3420 direct_jmp2_count++;
3421 }
3422 }
3423 }
3424 /* XXX: avoid using doubles ? */
3425 cpu_fprintf(f, "Translation buffer state:\n");
3426 cpu_fprintf(f, "gen code size %ld/%ld\n",
3427 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3428 cpu_fprintf(f, "TB count %d/%d\n",
3429 nb_tbs, code_gen_max_blocks);
3430 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3431 nb_tbs ? target_code_size / nb_tbs : 0,
3432 max_target_code_size);
3433 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3434 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3435 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3436 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3437 cross_page,
3438 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3439 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3440 direct_jmp_count,
3441 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3442 direct_jmp2_count,
3443 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3444 cpu_fprintf(f, "\nStatistics:\n");
3445 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3446 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3447 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3448 tcg_dump_info(f, cpu_fprintf);
3449 }
3450
3451 #if !defined(CONFIG_USER_ONLY)
3452
3453 #define MMUSUFFIX _cmmu
3454 #define GETPC() NULL
3455 #define env cpu_single_env
3456 #define SOFTMMU_CODE_ACCESS
3457
3458 #define SHIFT 0
3459 #include "softmmu_template.h"
3460
3461 #define SHIFT 1
3462 #include "softmmu_template.h"
3463
3464 #define SHIFT 2
3465 #include "softmmu_template.h"
3466
3467 #define SHIFT 3
3468 #include "softmmu_template.h"
3469
3470 #undef env
3471
3472 #endif
Patches shipped by Fedora