kqemu.c

   1 /*
   2  *  KQEMU support
   3  *
   4  *  Copyright (c) 2005-2008 Fabrice Bellard
   5  *  Copyright (c) 2011 Stefan Weil
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20 #include "qemu/osdep.h"
  21 #include "qemu-common.h"
  22 #ifdef _WIN32
  23 #include <windows.h>
  24 #include <winioctl.h>
  25 #else
  26 #include <sys/mman.h>
  27 #include <sys/ioctl.h>
  28 #endif
  29 #ifdef CONFIG_SOLARIS
  30 #include <sys/ioccom.h>
  31 #endif
  32
  33 #include "cpu.h"
  34 #include "exec-all.h"
  35
  36 #ifdef CONFIG_KQEMU
  37
  38 #define DEBUG
  39 //#define PROFILE
  40
  41
  42 #ifdef DEBUG
  43 #  define LOG_INT(...) qemu_log_mask(CPU_LOG_INT, ## __VA_ARGS__)
  44 #  define LOG_INT_STATE(env) log_cpu_state_mask(CPU_LOG_INT, (env), 0)
  45 #else
  46 #  define LOG_INT(...) do { } while (0)
  47 #  define LOG_INT_STATE(env) do { } while (0)
  48 #endif
  49
  50 #include "kqemu.h"
  51
  52 #ifdef _WIN32
  53 #define KQEMU_DEVICE "\\\\.\\kqemu"
  54 #else
  55 #define KQEMU_DEVICE "/dev/kqemu"
  56 #endif
  57
  58 static void qpi_init(void);
  59
  60 #ifdef _WIN32
  61 #define KQEMU_INVALID_FD INVALID_HANDLE_VALUE
  62 HANDLE kqemu_fd = KQEMU_INVALID_FD;
  63 #define kqemu_closefd(x) CloseHandle(x)
  64 #else
  65 #define KQEMU_INVALID_FD -1
  66 int kqemu_fd = KQEMU_INVALID_FD;
  67 #define kqemu_closefd(x) close(x)
  68 #endif
  69
  70 /* 0 = not allowed
  71    1 = user kqemu
  72    2 = kernel kqemu
  73 */
  74 int kqemu_allowed = 0;
  75 uint64_t *pages_to_flush;
  76 unsigned int nb_pages_to_flush;
  77 uint64_t *ram_pages_to_update;
  78 unsigned int nb_ram_pages_to_update;
  79 uint64_t *modified_ram_pages;
  80 unsigned int nb_modified_ram_pages;
  81 uint8_t *modified_ram_pages_table;
  82 int qpi_io_memory;
  83 uint32_t kqemu_comm_base; /* physical address of the QPI communication page */
  84 ram_addr_t kqemu_phys_ram_size;
  85 uint8_t *kqemu_phys_ram_base;
  86
  87 #define cpuid(index, eax, ebx, ecx, edx) \
  88   asm volatile ("cpuid" \
  89                 : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) \
  90                 : "0" (index))
  91
  92 #ifdef __x86_64__
  93 static int is_cpuid_supported(void)
  94 {
  95     return 1;
  96 }
  97 #else
  98 static int is_cpuid_supported(void)
  99 {
 100     int v0, v1;
 101     asm volatile ("pushf\n"
 102                   "popl %0\n"
 103                   "movl %0, %1\n"
 104                   "xorl $0x00200000, %0\n"
 105                   "pushl %0\n"
 106                   "popf\n"
 107                   "pushf\n"
 108                   "popl %0\n"
 109                   : "=a" (v0), "=d" (v1)
 110                   :
 111                   : "cc");
 112     return (v0 != v1);
 113 }
 114 #endif
 115
 116 static void kqemu_update_cpuid(CPUState *env)
 117 {
 118     int critical_features_mask, features, ext_features, ext_features_mask;
 119     uint32_t eax, ebx, ecx, edx;
 120
 121     /* the following features are kept identical on the host and
 122        target cpus because they are important for user code. Strictly
 123        speaking, only SSE really matters because the OS must support
 124        it if the user code uses it. */
 125     critical_features_mask =
 126         CPUID_CMOV | CPUID_CX8 |
 127         CPUID_FXSR | CPUID_MMX | CPUID_SSE |
 128         CPUID_SSE2 | CPUID_SEP;
 129     ext_features_mask = CPUID_EXT_SSE3 | CPUID_EXT_MONITOR;
 130     if (!is_cpuid_supported()) {
 131         features = 0;
 132         ext_features = 0;
 133     } else {
 134         cpuid(1, eax, ebx, ecx, edx);
 135         features = edx;
 136         ext_features = ecx;
 137     }
 138 #ifdef __x86_64__
 139     /* NOTE: on x86_64 CPUs, SYSENTER is not supported in
 140        compatibility mode, so in order to have the best performances
 141        it is better not to use it */
 142     features &= ~CPUID_SEP;
 143 #endif
 144     env->cpuid_features = (env->cpuid_features & ~critical_features_mask) |
 145         (features & critical_features_mask);
 146     env->cpuid_ext_features = (env->cpuid_ext_features & ~ext_features_mask) |
 147         (ext_features & ext_features_mask);
 148     /* XXX: we could update more of the target CPUID state so that the
 149        non accelerated code sees exactly the same CPU features as the
 150        accelerated code */
 151 }
 152
 153 int kqemu_init(CPUState *env)
 154 {
 155     struct kqemu_init kinit;
 156     int ret, version;
 157 #ifdef _WIN32
 158     DWORD temp;
 159 #endif
 160
 161     if (!kqemu_allowed)
 162         return -1;
 163
 164 #ifdef _WIN32
 165     kqemu_fd = CreateFile(KQEMU_DEVICE, GENERIC_WRITE | GENERIC_READ,
 166                           FILE_SHARE_READ | FILE_SHARE_WRITE,
 167                           NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
 168                           NULL);
 169     if (kqemu_fd == KQEMU_INVALID_FD) {
 170         fprintf(stderr, "Could not open '%s' - QEMU acceleration layer not activated: %lu\n",
 171                 KQEMU_DEVICE, GetLastError());
 172         return -1;
 173     }
 174 #else
 175     kqemu_fd = open(KQEMU_DEVICE, O_RDWR);
 176     if (kqemu_fd == KQEMU_INVALID_FD) {
 177         fprintf(stderr, "Could not open '%s' - QEMU acceleration layer not activated: %s\n",
 178                 KQEMU_DEVICE, strerror(errno));
 179         return -1;
 180     }
 181 #endif
 182     version = 0;
 183 #ifdef _WIN32
 184     DeviceIoControl(kqemu_fd, KQEMU_GET_VERSION, NULL, 0,
 185                     &version, sizeof(version), &temp, NULL);
 186 #else
 187     ioctl(kqemu_fd, KQEMU_GET_VERSION, &version);
 188 #endif
 189     if (version != KQEMU_VERSION) {
 190         fprintf(stderr, "Version mismatch between kqemu module and qemu (%08x %08x) - disabling kqemu use\n",
 191                 version, KQEMU_VERSION);
 192         goto fail;
 193     }
 194
 195     pages_to_flush = qemu_vmalloc(KQEMU_MAX_PAGES_TO_FLUSH *
 196                                   sizeof(uint64_t));
 197     if (!pages_to_flush)
 198         goto fail;
 199
 200     ram_pages_to_update = qemu_vmalloc(KQEMU_MAX_RAM_PAGES_TO_UPDATE *
 201                                        sizeof(uint64_t));
 202     if (!ram_pages_to_update)
 203         goto fail;
 204
 205     modified_ram_pages = qemu_vmalloc(KQEMU_MAX_MODIFIED_RAM_PAGES *
 206                                       sizeof(uint64_t));
 207     if (!modified_ram_pages)
 208         goto fail;
 209     modified_ram_pages_table =
 210         g_malloc0(kqemu_phys_ram_size >> TARGET_PAGE_BITS);
 211     if (!modified_ram_pages_table)
 212         goto fail;
 213
 214     memset(&kinit, 0, sizeof(kinit)); /* set the paddings to zero */
 215     kinit.ram_base = kqemu_phys_ram_base;
 216     kinit.ram_size = kqemu_phys_ram_size;
 217     kinit.ram_dirty = phys_ram_dirty;
 218     kinit.pages_to_flush = pages_to_flush;
 219     kinit.ram_pages_to_update = ram_pages_to_update;
 220     kinit.modified_ram_pages = modified_ram_pages;
 221 #ifdef _WIN32
 222     ret = DeviceIoControl(kqemu_fd, KQEMU_INIT, &kinit, sizeof(kinit),
 223                           NULL, 0, &temp, NULL) == TRUE ? 0 : -1;
 224 #else
 225     ret = ioctl(kqemu_fd, KQEMU_INIT, &kinit);
 226 #endif
 227     if (ret < 0) {
 228         fprintf(stderr, "Error %d while initializing QEMU acceleration layer - disabling it for now\n", ret);
 229     fail:
 230         kqemu_closefd(kqemu_fd);
 231         kqemu_fd = KQEMU_INVALID_FD;
 232         return -1;
 233     }
 234     kqemu_update_cpuid(env);
 235     env->kqemu_enabled = kqemu_allowed;
 236     nb_pages_to_flush = 0;
 237     nb_ram_pages_to_update = 0;
 238
 239     qpi_init();
 240     return 0;
 241 }
 242
 243 void kqemu_flush_page(CPUState *env, target_ulong addr)
 244 {
 245     LOG_INT("kqemu_flush_page: addr=" TARGET_FMT_lx "\n", addr);
 246     if (nb_pages_to_flush >= KQEMU_MAX_PAGES_TO_FLUSH)
 247         nb_pages_to_flush = KQEMU_FLUSH_ALL;
 248     else
 249         pages_to_flush[nb_pages_to_flush++] = addr;
 250 }
 251
 252 void kqemu_flush(CPUState *env, int global)
 253 {
 254     LOG_INT("kqemu_flush:\n");
 255     nb_pages_to_flush = KQEMU_FLUSH_ALL;
 256 }
 257
 258 void kqemu_set_notdirty(CPUState *env, ram_addr_t ram_addr)
 259 {
 260     LOG_INT("kqemu_set_notdirty: addr=%08lx\n",
 261                 (unsigned long)ram_addr);
 262     /* we only track transitions to dirty state */
 263     if (phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] != 0xff)
 264         return;
 265     if (nb_ram_pages_to_update >= KQEMU_MAX_RAM_PAGES_TO_UPDATE)
 266         nb_ram_pages_to_update = KQEMU_RAM_PAGES_UPDATE_ALL;
 267     else
 268         ram_pages_to_update[nb_ram_pages_to_update++] = ram_addr;
 269 }
 270
 271 static void kqemu_reset_modified_ram_pages(void)
 272 {
 273     int i;
 274     unsigned long page_index;
 275
 276     for(i = 0; i < nb_modified_ram_pages; i++) {
 277         page_index = modified_ram_pages[i] >> TARGET_PAGE_BITS;
 278         modified_ram_pages_table[page_index] = 0;
 279     }
 280     nb_modified_ram_pages = 0;
 281 }
 282
 283 void kqemu_modify_page(CPUState *env, ram_addr_t ram_addr)
 284 {
 285     unsigned long page_index;
 286     int ret;
 287 #ifdef _WIN32
 288     DWORD temp;
 289 #endif
 290
 291     page_index = ram_addr >> TARGET_PAGE_BITS;
 292     if (!modified_ram_pages_table[page_index]) {
 293 #if 0
 294         printf("%d: modify_page=%08lx\n", nb_modified_ram_pages, ram_addr);
 295 #endif
 296         modified_ram_pages_table[page_index] = 1;
 297         modified_ram_pages[nb_modified_ram_pages++] = ram_addr;
 298         if (nb_modified_ram_pages >= KQEMU_MAX_MODIFIED_RAM_PAGES) {
 299             /* flush */
 300 #ifdef _WIN32
 301             ret = DeviceIoControl(kqemu_fd, KQEMU_MODIFY_RAM_PAGES,
 302                                   &nb_modified_ram_pages,
 303                                   sizeof(nb_modified_ram_pages),
 304                                   NULL, 0, &temp, NULL);
 305 #else
 306             ret = ioctl(kqemu_fd, KQEMU_MODIFY_RAM_PAGES,
 307                         &nb_modified_ram_pages);
 308 #endif
 309             kqemu_reset_modified_ram_pages();
 310         }
 311     }
 312 }
 313
 314 void kqemu_set_phys_mem(uint64_t start_addr, ram_addr_t size,
 315                         ram_addr_t phys_offset)
 316 {
 317     struct kqemu_phys_mem kphys_mem1, *kphys_mem = &kphys_mem1;
 318     uint64_t end;
 319     int ret, io_index;
 320
 321     end = (start_addr + size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
 322     start_addr &= TARGET_PAGE_MASK;
 323     kphys_mem->phys_addr = start_addr;
 324     kphys_mem->size = end - start_addr;
 325     kphys_mem->ram_addr = phys_offset & TARGET_PAGE_MASK;
 326     io_index = phys_offset & ~TARGET_PAGE_MASK;
 327     switch(io_index) {
 328     case IO_MEM_RAM:
 329         kphys_mem->io_index = KQEMU_IO_MEM_RAM;
 330         break;
 331     case IO_MEM_ROM:
 332         kphys_mem->io_index = KQEMU_IO_MEM_ROM;
 333         break;
 334     default:
 335         if (qpi_io_memory == io_index) {
 336             kphys_mem->io_index = KQEMU_IO_MEM_COMM;
 337         } else {
 338             kphys_mem->io_index = KQEMU_IO_MEM_UNASSIGNED;
 339         }
 340         break;
 341     }
 342 #ifdef _WIN32
 343     {
 344         DWORD temp;
 345         ret = DeviceIoControl(kqemu_fd, KQEMU_SET_PHYS_MEM,
 346                               kphys_mem, sizeof(*kphys_mem),
 347                               NULL, 0, &temp, NULL) == TRUE ? 0 : -1;
 348     }
 349 #else
 350     ret = ioctl(kqemu_fd, KQEMU_SET_PHYS_MEM, kphys_mem);
 351 #endif
 352     if (ret < 0) {
 353         fprintf(stderr, "kqemu: KQEMU_SET_PHYS_PAGE error=%d: start_addr=0x%016" PRIx64 " size=0x%08lx phys_offset=0x%08lx\n",
 354                 ret, start_addr,
 355                 (unsigned long)size, (unsigned long)phys_offset);
 356     }
 357 }
 358
 359 struct fpstate {
 360     uint16_t fpuc;
 361     uint16_t dummy1;
 362     uint16_t fpus;
 363     uint16_t dummy2;
 364     uint16_t fptag;
 365     uint16_t dummy3;
 366
 367     uint32_t fpip;
 368     uint32_t fpcs;
 369     uint32_t fpoo;
 370     uint32_t fpos;
 371     uint8_t fpregs1[8 * 10];
 372 };
 373
 374 struct fpxstate {
 375     uint16_t fpuc;
 376     uint16_t fpus;
 377     uint16_t fptag;
 378     uint16_t fop;
 379     uint32_t fpuip;
 380     uint16_t cs_sel;
 381     uint16_t dummy0;
 382     uint32_t fpudp;
 383     uint16_t ds_sel;
 384     uint16_t dummy1;
 385     uint32_t mxcsr;
 386     uint32_t mxcsr_mask;
 387     uint8_t fpregs1[8 * 16];
 388     uint8_t xmm_regs[16 * 16];
 389     uint8_t dummy2[96];
 390 };
 391
 392 static struct fpxstate fpx1 __attribute__((aligned(16)));
 393
 394 static void restore_native_fp_frstor(CPUState *env)
 395 {
 396     int fptag, i, j;
 397     struct fpstate fp1, *fp = &fp1;
 398
 399     fp->fpuc = env->fpuc;
 400     fp->fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
 401     fptag = 0;
 402     for (i=7; i>=0; i--) {
 403         fptag <<= 2;
 404         if (env->fptags[i]) {
 405             fptag |= 3;
 406         } else {
 407             /* the FPU automatically computes it */
 408         }
 409     }
 410     fp->fptag = fptag;
 411     j = env->fpstt;
 412     for(i = 0;i < 8; i++) {
 413         memcpy(&fp->fpregs1[i * 10], &env->fpregs[j].d, 10);
 414         j = (j + 1) & 7;
 415     }
 416     asm volatile ("frstor %0" : "=m" (*fp));
 417 }
 418
 419 static void save_native_fp_fsave(CPUState *env)
 420 {
 421     int fptag, i, j;
 422     uint16_t fpuc;
 423     struct fpstate fp1, *fp = &fp1;
 424
 425     asm volatile ("fsave %0" : : "m" (*fp));
 426     env->fpuc = fp->fpuc;
 427     env->fpstt = (fp->fpus >> 11) & 7;
 428     env->fpus = fp->fpus & ~0x3800;
 429     fptag = fp->fptag;
 430     for(i = 0;i < 8; i++) {
 431         env->fptags[i] = ((fptag & 3) == 3);
 432         fptag >>= 2;
 433     }
 434     j = env->fpstt;
 435     for(i = 0;i < 8; i++) {
 436         memcpy(&env->fpregs[j].d, &fp->fpregs1[i * 10], 10);
 437         j = (j + 1) & 7;
 438     }
 439     /* we must restore the default rounding state */
 440     fpuc = 0x037f | (env->fpuc & (3 << 10));
 441     asm volatile("fldcw %0" : : "m" (fpuc));
 442 }
 443
 444 static void restore_native_fp_fxrstor(CPUState *env)
 445 {
 446     struct fpxstate *fp = &fpx1;
 447     int i, j, fptag;
 448
 449     fp->fpuc = env->fpuc;
 450     fp->fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
 451     fptag = 0;
 452     for(i = 0; i < 8; i++)
 453         fptag |= (env->fptags[i] << i);
 454     fp->fptag = fptag ^ 0xff;
 455
 456     j = env->fpstt;
 457     for(i = 0;i < 8; i++) {
 458         memcpy(&fp->fpregs1[i * 16], &env->fpregs[j].d, 10);
 459         j = (j + 1) & 7;
 460     }
 461     if (env->cpuid_features & CPUID_SSE) {
 462         fp->mxcsr = env->mxcsr;
 463         /* XXX: check if DAZ is not available */
 464         fp->mxcsr_mask = 0xffff;
 465         memcpy(fp->xmm_regs, env->xmm_regs, CPU_NB_REGS * 16);
 466     }
 467     asm volatile ("fxrstor %0" : "=m" (*fp));
 468 }
 469
 470 static void save_native_fp_fxsave(CPUState *env)
 471 {
 472     struct fpxstate *fp = &fpx1;
 473     int fptag, i, j;
 474     uint16_t fpuc;
 475
 476     asm volatile ("fxsave %0" : : "m" (*fp));
 477     env->fpuc = fp->fpuc;
 478     env->fpstt = (fp->fpus >> 11) & 7;
 479     env->fpus = fp->fpus & ~0x3800;
 480     fptag = fp->fptag ^ 0xff;
 481     for(i = 0;i < 8; i++) {
 482         env->fptags[i] = (fptag >> i) & 1;
 483     }
 484     j = env->fpstt;
 485     for(i = 0;i < 8; i++) {
 486         memcpy(&env->fpregs[j].d, &fp->fpregs1[i * 16], 10);
 487         j = (j + 1) & 7;
 488     }
 489     if (env->cpuid_features & CPUID_SSE) {
 490         env->mxcsr = fp->mxcsr;
 491         memcpy(env->xmm_regs, fp->xmm_regs, CPU_NB_REGS * 16);
 492     }
 493
 494     /* we must restore the default rounding state */
 495     asm volatile ("fninit");
 496     fpuc = 0x037f | (env->fpuc & (3 << 10));
 497     asm volatile("fldcw %0" : : "m" (fpuc));
 498 }
 499
 500 static int do_syscall(CPUState *env,
 501                       struct kqemu_cpu_state *kenv)
 502 {
 503     int selector;
 504
 505     selector = (env->star >> 32) & 0xffff;
 506 #ifdef TARGET_X86_64
 507     if (env->hflags & HF_LMA_MASK) {
 508         int code64;
 509
 510         env->regs[R_ECX] = kenv->next_eip;
 511         env->regs[11] = env->eflags;
 512
 513         code64 = env->hflags & HF_CS64_MASK;
 514
 515         cpu_x86_set_cpl(env, 0);
 516         cpu_x86_load_seg_cache(env, R_CS, selector & 0xfffc,
 517                                0, 0xffffffff,
 518                                DESC_G_MASK | DESC_P_MASK |
 519                                DESC_S_MASK |
 520                                DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK | DESC_L_MASK);
 521         cpu_x86_load_seg_cache(env, R_SS, (selector + 8) & 0xfffc,
 522                                0, 0xffffffff,
 523                                DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
 524                                DESC_S_MASK |
 525                                DESC_W_MASK | DESC_A_MASK);
 526         env->eflags &= ~env->fmask;
 527         if (code64)
 528             env->eip = env->lstar;
 529         else
 530             env->eip = env->cstar;
 531     } else
 532 #endif
 533     {
 534         env->regs[R_ECX] = (uint32_t)kenv->next_eip;
 535
 536         cpu_x86_set_cpl(env, 0);
 537         cpu_x86_load_seg_cache(env, R_CS, selector & 0xfffc,
 538                            0, 0xffffffff,
 539                                DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
 540                                DESC_S_MASK |
 541                                DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
 542         cpu_x86_load_seg_cache(env, R_SS, (selector + 8) & 0xfffc,
 543                                0, 0xffffffff,
 544                                DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
 545                                DESC_S_MASK |
 546                                DESC_W_MASK | DESC_A_MASK);
 547         env->eflags &= ~(IF_MASK | RF_MASK | VM_MASK);
 548         env->eip = (uint32_t)env->star;
 549     }
 550     return 2;
 551 }
 552
 553 #ifdef CONFIG_PROFILER
 554
 555 #define PC_REC_SIZE 1
 556 #define PC_REC_HASH_BITS 16
 557 #define PC_REC_HASH_SIZE (1 << PC_REC_HASH_BITS)
 558
 559 typedef struct PCRecord {
 560     unsigned long pc;
 561     int64_t count;
 562     struct PCRecord *next;
 563 } PCRecord;
 564
 565 static PCRecord *pc_rec_hash[PC_REC_HASH_SIZE];
 566 static int nb_pc_records;
 567
 568 static void kqemu_record_pc(unsigned long pc)
 569 {
 570     unsigned long h;
 571     PCRecord **pr, *r;
 572
 573     h = pc / PC_REC_SIZE;
 574     h = h ^ (h >> PC_REC_HASH_BITS);
 575     h &= (PC_REC_HASH_SIZE - 1);
 576     pr = &pc_rec_hash[h];
 577     for(;;) {
 578         r = *pr;
 579         if (r == NULL)
 580             break;
 581         if (r->pc == pc) {
 582             r->count++;
 583             return;
 584         }
 585         pr = &r->next;
 586     }
 587     r = malloc(sizeof(PCRecord));
 588     r->count = 1;
 589     r->pc = pc;
 590     r->next = NULL;
 591     *pr = r;
 592     nb_pc_records++;
 593 }
 594
 595 static int pc_rec_cmp(const void *p1, const void *p2)
 596 {
 597     PCRecord *r1 = *(PCRecord **)p1;
 598     PCRecord *r2 = *(PCRecord **)p2;
 599     if (r1->count < r2->count)
 600         return 1;
 601     else if (r1->count == r2->count)
 602         return 0;
 603     else
 604         return -1;
 605 }
 606
 607 static void kqemu_record_flush(void)
 608 {
 609     PCRecord *r, *r_next;
 610     int h;
 611
 612     for(h = 0; h < PC_REC_HASH_SIZE; h++) {
 613         for(r = pc_rec_hash[h]; r != NULL; r = r_next) {
 614             r_next = r->next;
 615             free(r);
 616         }
 617         pc_rec_hash[h] = NULL;
 618     }
 619     nb_pc_records = 0;
 620 }
 621
 622 void kqemu_record_dump(void)
 623 {
 624     PCRecord **pr, *r;
 625     int i, h;
 626     FILE *f;
 627     int64_t total, sum;
 628
 629     pr = malloc(sizeof(PCRecord *) * nb_pc_records);
 630     i = 0;
 631     total = 0;
 632     for(h = 0; h < PC_REC_HASH_SIZE; h++) {
 633         for(r = pc_rec_hash[h]; r != NULL; r = r->next) {
 634             pr[i++] = r;
 635             total += r->count;
 636         }
 637     }
 638     qsort(pr, nb_pc_records, sizeof(PCRecord *), pc_rec_cmp);
 639
 640     f = fopen("/tmp/kqemu.stats", "w");
 641     if (!f) {
 642         perror("/tmp/kqemu.stats");
 643         exit(1);
 644     }
 645     fprintf(f, "total: %" PRId64 "\n", total);
 646     sum = 0;
 647     for(i = 0; i < nb_pc_records; i++) {
 648         r = pr[i];
 649         sum += r->count;
 650         fprintf(f, "%08lx: %" PRId64 " %0.2f%% %0.2f%%\n",
 651                 r->pc,
 652                 r->count,
 653                 (double)r->count / (double)total * 100.0,
 654                 (double)sum / (double)total * 100.0);
 655     }
 656     fclose(f);
 657     free(pr);
 658
 659     kqemu_record_flush();
 660 }
 661 #endif
 662
 663 static inline void kqemu_load_seg(struct kqemu_segment_cache *ksc,
 664                                   const SegmentCache *sc)
 665 {
 666     ksc->selector = sc->selector;
 667     ksc->flags = sc->flags;
 668     ksc->limit = sc->limit;
 669     ksc->base = sc->base;
 670 }
 671
 672 static inline void kqemu_save_seg(SegmentCache *sc,
 673                                   const struct kqemu_segment_cache *ksc)
 674 {
 675     sc->selector = ksc->selector;
 676     sc->flags = ksc->flags;
 677     sc->limit = ksc->limit;
 678     sc->base = ksc->base;
 679 }
 680
 681 int kqemu_cpu_exec(CPUState *env)
 682 {
 683     struct kqemu_cpu_state kcpu_state, *kenv = &kcpu_state;
 684     int ret, cpl, i;
 685 #ifdef CONFIG_PROFILER
 686     int64_t ti;
 687 #endif
 688 #ifdef _WIN32
 689     DWORD temp;
 690 #endif
 691
 692 #ifdef CONFIG_PROFILER
 693     ti = profile_getclock();
 694 #endif
 695     LOG_INT("kqemu: cpu_exec: enter\n");
 696     LOG_INT_STATE(env);
 697     for(i = 0; i < CPU_NB_REGS; i++)
 698         kenv->regs[i] = env->regs[i];
 699     kenv->eip = env->eip;
 700     kenv->eflags = env->eflags;
 701     for(i = 0; i < 6; i++)
 702         kqemu_load_seg(&kenv->segs[i], &env->segs[i]);
 703     kqemu_load_seg(&kenv->ldt, &env->ldt);
 704     kqemu_load_seg(&kenv->tr, &env->tr);
 705     kqemu_load_seg(&kenv->gdt, &env->gdt);
 706     kqemu_load_seg(&kenv->idt, &env->idt);
 707     kenv->cr0 = env->cr[0];
 708     kenv->cr2 = env->cr[2];
 709     kenv->cr3 = env->cr[3];
 710     kenv->cr4 = env->cr[4];
 711     kenv->a20_mask = env->a20_mask;
 712     kenv->efer = env->efer;
 713     kenv->tsc_offset = 0;
 714     kenv->star = env->star;
 715     kenv->sysenter_cs = env->sysenter_cs;
 716     kenv->sysenter_esp = env->sysenter_esp;
 717     kenv->sysenter_eip = env->sysenter_eip;
 718 #ifdef TARGET_X86_64
 719     kenv->lstar = env->lstar;
 720     kenv->cstar = env->cstar;
 721     kenv->fmask = env->fmask;
 722     kenv->kernelgsbase = env->kernelgsbase;
 723 #endif
 724     if (env->dr[7] & 0xff) {
 725         kenv->dr7 = env->dr[7];
 726         kenv->dr0 = env->dr[0];
 727         kenv->dr1 = env->dr[1];
 728         kenv->dr2 = env->dr[2];
 729         kenv->dr3 = env->dr[3];
 730     } else {
 731         kenv->dr7 = 0;
 732     }
 733     kenv->dr6 = env->dr[6];
 734     cpl = (env->hflags & HF_CPL_MASK);
 735     kenv->cpl = cpl;
 736     kenv->nb_pages_to_flush = nb_pages_to_flush;
 737     kenv->user_only = (env->kqemu_enabled == 1);
 738     kenv->nb_ram_pages_to_update = nb_ram_pages_to_update;
 739     nb_ram_pages_to_update = 0;
 740     kenv->nb_modified_ram_pages = nb_modified_ram_pages;
 741
 742     kqemu_reset_modified_ram_pages();
 743
 744     if (env->cpuid_features & CPUID_FXSR)
 745         restore_native_fp_fxrstor(env);
 746     else
 747         restore_native_fp_frstor(env);
 748
 749 #ifdef _WIN32
 750     if (DeviceIoControl(kqemu_fd, KQEMU_EXEC,
 751                         kenv, sizeof(struct kqemu_cpu_state),
 752                         kenv, sizeof(struct kqemu_cpu_state),
 753                         &temp, NULL)) {
 754         ret = kenv->retval;
 755     } else {
 756         ret = -1;
 757     }
 758 #else
 759     ioctl(kqemu_fd, KQEMU_EXEC, kenv);
 760     ret = kenv->retval;
 761 #endif
 762     if (env->cpuid_features & CPUID_FXSR)
 763         save_native_fp_fxsave(env);
 764     else
 765         save_native_fp_fsave(env);
 766
 767     for(i = 0; i < CPU_NB_REGS; i++)
 768         env->regs[i] = kenv->regs[i];
 769     env->eip = kenv->eip;
 770     env->eflags = kenv->eflags;
 771     for(i = 0; i < 6; i++)
 772         kqemu_save_seg(&env->segs[i], &kenv->segs[i]);
 773     cpu_x86_set_cpl(env, kenv->cpl);
 774     kqemu_save_seg(&env->ldt, &kenv->ldt);
 775     env->cr[0] = kenv->cr0;
 776     env->cr[4] = kenv->cr4;
 777     env->cr[3] = kenv->cr3;
 778     env->cr[2] = kenv->cr2;
 779     env->dr[6] = kenv->dr6;
 780 #ifdef TARGET_X86_64
 781     env->kernelgsbase = kenv->kernelgsbase;
 782 #endif
 783
 784     /* flush pages as indicated by kqemu */
 785     if (kenv->nb_pages_to_flush >= KQEMU_FLUSH_ALL) {
 786         tlb_flush(env, 1);
 787     } else {
 788         for(i = 0; i < kenv->nb_pages_to_flush; i++) {
 789             tlb_flush_page(env, pages_to_flush[i]);
 790         }
 791     }
 792     nb_pages_to_flush = 0;
 793
 794 #ifdef CONFIG_PROFILER
 795     kqemu_time += profile_getclock() - ti;
 796     kqemu_exec_count++;
 797 #endif
 798
 799     if (kenv->nb_ram_pages_to_update > 0) {
 800         cpu_tlb_update_dirty(env);
 801     }
 802
 803     if (kenv->nb_modified_ram_pages > 0) {
 804         for(i = 0; i < kenv->nb_modified_ram_pages; i++) {
 805             unsigned long addr;
 806             addr = modified_ram_pages[i];
 807             tb_invalidate_phys_page_range(addr, addr + TARGET_PAGE_SIZE, 0);
 808         }
 809     }
 810
 811     /* restore the hidden flags */
 812     {
 813         unsigned int new_hflags;
 814 #ifdef TARGET_X86_64
 815         if ((env->hflags & HF_LMA_MASK) &&
 816             (env->segs[R_CS].flags & DESC_L_MASK)) {
 817             /* long mode */
 818             new_hflags = HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
 819         } else
 820 #endif
 821         {
 822             /* legacy / compatibility case */
 823             new_hflags = (env->segs[R_CS].flags & DESC_B_MASK)
 824                 >> (DESC_B_SHIFT - HF_CS32_SHIFT);
 825             new_hflags |= (env->segs[R_SS].flags & DESC_B_MASK)
 826                 >> (DESC_B_SHIFT - HF_SS32_SHIFT);
 827             if (!(env->cr[0] & CR0_PE_MASK) ||
 828                    (env->eflags & VM_MASK) ||
 829                    !(env->hflags & HF_CS32_MASK)) {
 830                 /* XXX: try to avoid this test. The problem comes from the
 831                    fact that is real mode or vm86 mode we only modify the
 832                    'base' and 'selector' fields of the segment cache to go
 833                    faster. A solution may be to force addseg to one in
 834                    translate-i386.c. */
 835                 new_hflags |= HF_ADDSEG_MASK;
 836             } else {
 837                 new_hflags |= ((env->segs[R_DS].base |
 838                                 env->segs[R_ES].base |
 839                                 env->segs[R_SS].base) != 0) <<
 840                     HF_ADDSEG_SHIFT;
 841             }
 842         }
 843         env->hflags = (env->hflags &
 844            ~(HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)) |
 845             new_hflags;
 846     }
 847     /* update FPU flags */
 848     env->hflags = (env->hflags & ~(HF_MP_MASK | HF_EM_MASK | HF_TS_MASK)) |
 849         ((env->cr[0] << (HF_MP_SHIFT - 1)) & (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK));
 850     if (env->cr[4] & CR4_OSFXSR_MASK)
 851         env->hflags |= HF_OSFXSR_MASK;
 852     else
 853         env->hflags &= ~HF_OSFXSR_MASK;
 854
 855     LOG_INT("kqemu: kqemu_cpu_exec: ret=0x%x\n", ret);
 856     if (ret == KQEMU_RET_SYSCALL) {
 857         /* syscall instruction */
 858         return do_syscall(env, kenv);
 859     } else
 860     if ((ret & 0xff00) == KQEMU_RET_INT) {
 861         env->exception_index = ret & 0xff;
 862         env->error_code = 0;
 863         env->exception_is_int = 1;
 864         env->exception_next_eip = kenv->next_eip;
 865 #ifdef CONFIG_PROFILER
 866         kqemu_ret_int_count++;
 867 #endif
 868         LOG_INT("kqemu: interrupt v=%02x:\n", env->exception_index);
 869         LOG_INT_STATE(env);
 870         return 1;
 871     } else if ((ret & 0xff00) == KQEMU_RET_EXCEPTION) {
 872         env->exception_index = ret & 0xff;
 873         env->error_code = kenv->error_code;
 874         env->exception_is_int = 0;
 875         env->exception_next_eip = 0;
 876 #ifdef CONFIG_PROFILER
 877         kqemu_ret_excp_count++;
 878 #endif
 879         LOG_INT("kqemu: exception v=%02x e=%04x:\n",
 880                     env->exception_index, env->error_code);
 881         LOG_INT_STATE(env);
 882         return 1;
 883     } else if (ret == KQEMU_RET_INTR) {
 884 #ifdef CONFIG_PROFILER
 885         kqemu_ret_intr_count++;
 886 #endif
 887         LOG_INT_STATE(env);
 888         return 0;
 889     } else if (ret == KQEMU_RET_SOFTMMU) {
 890 #ifdef CONFIG_PROFILER
 891         {
 892             unsigned long pc = env->eip + env->segs[R_CS].base;
 893             kqemu_record_pc(pc);
 894         }
 895 #endif
 896         LOG_INT_STATE(env);
 897         return 2;
 898     } else {
 899         cpu_dump_state(env, stderr, fprintf, 0);
 900         fprintf(stderr, "Unsupported return value: 0x%x\n", ret);
 901         exit(1);
 902     }
 903     return 0;
 904 }
 905
 906 void kqemu_cpu_interrupt(CPUState *env)
 907 {
 908 #if defined(_WIN32)
 909     /* cancelling the I/O request causes KQEMU to finish executing the
 910        current block and successfully returning. */
 911     CancelIo(kqemu_fd);
 912 #endif
 913 }
 914
 915 /*
 916    QEMU paravirtualization interface. The current interface only
 917    allows to modify the IF and IOPL flags when running in
 918    kqemu.
 919
 920    At this point it is not very satisfactory. I leave it for reference
 921    as it adds little complexity.
 922 */
 923
 924 #define QPI_COMM_PAGE_PHYS_ADDR 0xff000000
 925
 926 static uint32_t qpi_mem_readb(void *opaque, hwaddr addr)
 927 {
 928     return 0;
 929 }
 930
 931 static uint32_t qpi_mem_readw(void *opaque, hwaddr addr)
 932 {
 933     return 0;
 934 }
 935
 936 static void qpi_mem_writeb(void *opaque, hwaddr addr, uint32_t val)
 937 {
 938 }
 939
 940 static void qpi_mem_writew(void *opaque, hwaddr addr, uint32_t val)
 941 {
 942 }
 943
 944 static uint32_t qpi_mem_readl(void *opaque, hwaddr addr)
 945 {
 946     CPUState *env;
 947
 948     env = cpu_single_env;
 949     if (!env)
 950         return 0;
 951     return env->eflags & (IF_MASK | IOPL_MASK);
 952 }
 953
 954 /* Note: after writing to this address, the guest code must make sure
 955    it is exiting the current TB. pushf/popf can be used for that
 956    purpose. */
 957 static void qpi_mem_writel(void *opaque, hwaddr addr, uint32_t val)
 958 {
 959     CPUState *env;
 960
 961     env = cpu_single_env;
 962     if (!env)
 963         return;
 964     env->eflags = (env->eflags & ~(IF_MASK | IOPL_MASK)) |
 965         (val & (IF_MASK | IOPL_MASK));
 966 }
 967
 968 static CPUReadMemoryFunc * const qpi_mem_read[3] = {
 969     qpi_mem_readb,
 970     qpi_mem_readw,
 971     qpi_mem_readl,
 972 };
 973
 974 static CPUWriteMemoryFunc * const qpi_mem_write[3] = {
 975     qpi_mem_writeb,
 976     qpi_mem_writew,
 977     qpi_mem_writel,
 978 };
 979
 980 static void qpi_init(void)
 981 {
 982     kqemu_comm_base = 0xff000000 | 1;
 983     qpi_io_memory = cpu_register_io_memory(
 984                                            qpi_mem_read,
 985                                            qpi_mem_write, NULL);
 986     cpu_register_physical_memory(kqemu_comm_base & ~0xfff,
 987                                  0x1000, qpi_io_memory);
 988 }
 989 #endif