cpus.c

   1 /*
   2  * QEMU System Emulator
   3  *
   4  * Copyright (c) 2003-2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 /* Needed early for CONFIG_BSD etc. */
  26 #include "config-host.h"
  27
  28 #include "monitor.h"
  29 #include "sysemu.h"
  30 #include "gdbstub.h"
  31 #include "dma.h"
  32 #include "kvm.h"
  33
  34 #include "qemu-thread.h"
  35 #include "cpus.h"
  36
  37 #ifndef _WIN32
  38 #include "compatfd.h"
  39 #endif
  40
  41 #ifdef SIGRTMIN
  42 #define SIG_IPI (SIGRTMIN+4)
  43 #else
  44 #define SIG_IPI SIGUSR1
  45 #endif
  46
  47 #ifdef CONFIG_LINUX
  48
  49 #include <sys/prctl.h>
  50
  51 #ifndef PR_MCE_KILL
  52 #define PR_MCE_KILL 33
  53 #endif
  54
  55 #ifndef PR_MCE_KILL_SET
  56 #define PR_MCE_KILL_SET 1
  57 #endif
  58
  59 #ifndef PR_MCE_KILL_EARLY
  60 #define PR_MCE_KILL_EARLY 1
  61 #endif
  62
  63 #endif /* CONFIG_LINUX */
  64
  65 static CPUState *next_cpu;
  66
  67 /***********************************************************/
  68 void hw_error(const char *fmt, ...)
  69 {
  70     va_list ap;
  71     CPUState *env;
  72
  73     va_start(ap, fmt);
  74     fprintf(stderr, "qemu: hardware error: ");
  75     vfprintf(stderr, fmt, ap);
  76     fprintf(stderr, "\n");
  77     for(env = first_cpu; env != NULL; env = env->next_cpu) {
  78         fprintf(stderr, "CPU #%d:\n", env->cpu_index);
  79 #ifdef TARGET_I386
  80         cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU);
  81 #else
  82         cpu_dump_state(env, stderr, fprintf, 0);
  83 #endif
  84     }
  85     va_end(ap);
  86     abort();
  87 }
  88
  89 void cpu_synchronize_all_states(void)
  90 {
  91     CPUState *cpu;
  92
  93     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
  94         cpu_synchronize_state(cpu);
  95     }
  96 }
  97
  98 void cpu_synchronize_all_post_reset(void)
  99 {
 100     CPUState *cpu;
 101
 102     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 103         cpu_synchronize_post_reset(cpu);
 104     }
 105 }
 106
 107 void cpu_synchronize_all_post_init(void)
 108 {
 109     CPUState *cpu;
 110
 111     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 112         cpu_synchronize_post_init(cpu);
 113     }
 114 }
 115
 116 int cpu_is_stopped(CPUState *env)
 117 {
 118     return !vm_running || env->stopped;
 119 }
 120
 121 static void do_vm_stop(int reason)
 122 {
 123     if (vm_running) {
 124         cpu_disable_ticks();
 125         vm_running = 0;
 126         pause_all_vcpus();
 127         vm_state_notify(0, reason);
 128         qemu_aio_flush();
 129         bdrv_flush_all();
 130         monitor_protocol_event(QEVENT_STOP, NULL);
 131     }
 132 }
 133
 134 static int cpu_can_run(CPUState *env)
 135 {
 136     if (env->stop) {
 137         return 0;
 138     }
 139     if (env->stopped || !vm_running) {
 140         return 0;
 141     }
 142     return 1;
 143 }
 144
 145 static bool cpu_thread_is_idle(CPUState *env)
 146 {
 147     if (env->stop || env->queued_work_first) {
 148         return false;
 149     }
 150     if (env->stopped || !vm_running) {
 151         return true;
 152     }
 153     if (!env->halted || qemu_cpu_has_work(env) ||
 154         (kvm_enabled() && kvm_irqchip_in_kernel())) {
 155         return false;
 156     }
 157     return true;
 158 }
 159
 160 bool all_cpu_threads_idle(void)
 161 {
 162     CPUState *env;
 163
 164     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 165         if (!cpu_thread_is_idle(env)) {
 166             return false;
 167         }
 168     }
 169     return true;
 170 }
 171
 172 static void cpu_handle_guest_debug(CPUState *env)
 173 {
 174     gdb_set_stop_cpu(env);
 175     qemu_system_debug_request();
 176 #ifdef CONFIG_IOTHREAD
 177     env->stopped = 1;
 178 #endif
 179 }
 180
 181 #ifdef CONFIG_IOTHREAD
 182 static void cpu_signal(int sig)
 183 {
 184     if (cpu_single_env) {
 185         cpu_exit(cpu_single_env);
 186     }
 187     exit_request = 1;
 188 }
 189 #endif
 190
 191 #ifdef CONFIG_LINUX
 192 static void sigbus_reraise(void)
 193 {
 194     sigset_t set;
 195     struct sigaction action;
 196
 197     memset(&action, 0, sizeof(action));
 198     action.sa_handler = SIG_DFL;
 199     if (!sigaction(SIGBUS, &action, NULL)) {
 200         raise(SIGBUS);
 201         sigemptyset(&set);
 202         sigaddset(&set, SIGBUS);
 203         sigprocmask(SIG_UNBLOCK, &set, NULL);
 204     }
 205     perror("Failed to re-raise SIGBUS!\n");
 206     abort();
 207 }
 208
 209 static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo,
 210                            void *ctx)
 211 {
 212     if (kvm_on_sigbus(siginfo->ssi_code,
 213                       (void *)(intptr_t)siginfo->ssi_addr)) {
 214         sigbus_reraise();
 215     }
 216 }
 217
 218 static void qemu_init_sigbus(void)
 219 {
 220     struct sigaction action;
 221
 222     memset(&action, 0, sizeof(action));
 223     action.sa_flags = SA_SIGINFO;
 224     action.sa_sigaction = (void (*)(int, siginfo_t*, void*))sigbus_handler;
 225     sigaction(SIGBUS, &action, NULL);
 226
 227     prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
 228 }
 229
 230 static void qemu_kvm_eat_signals(CPUState *env)
 231 {
 232     struct timespec ts = { 0, 0 };
 233     siginfo_t siginfo;
 234     sigset_t waitset;
 235     sigset_t chkset;
 236     int r;
 237
 238     sigemptyset(&waitset);
 239     sigaddset(&waitset, SIG_IPI);
 240     sigaddset(&waitset, SIGBUS);
 241
 242     do {
 243         r = sigtimedwait(&waitset, &siginfo, &ts);
 244         if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
 245             perror("sigtimedwait");
 246             exit(1);
 247         }
 248
 249         switch (r) {
 250         case SIGBUS:
 251             if (kvm_on_sigbus_vcpu(env, siginfo.si_code, siginfo.si_addr)) {
 252                 sigbus_reraise();
 253             }
 254             break;
 255         default:
 256             break;
 257         }
 258
 259         r = sigpending(&chkset);
 260         if (r == -1) {
 261             perror("sigpending");
 262             exit(1);
 263         }
 264     } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS));
 265
 266 #ifndef CONFIG_IOTHREAD
 267     if (sigismember(&chkset, SIGIO) || sigismember(&chkset, SIGALRM)) {
 268         qemu_notify_event();
 269     }
 270 #endif
 271 }
 272
 273 #else /* !CONFIG_LINUX */
 274
 275 static void qemu_init_sigbus(void)
 276 {
 277 }
 278
 279 static void qemu_kvm_eat_signals(CPUState *env)
 280 {
 281 }
 282 #endif /* !CONFIG_LINUX */
 283
 284 #ifndef _WIN32
 285 static int io_thread_fd = -1;
 286
 287 static void qemu_event_increment(void)
 288 {
 289     /* Write 8 bytes to be compatible with eventfd.  */
 290     static const uint64_t val = 1;
 291     ssize_t ret;
 292
 293     if (io_thread_fd == -1) {
 294         return;
 295     }
 296     do {
 297         ret = write(io_thread_fd, &val, sizeof(val));
 298     } while (ret < 0 && errno == EINTR);
 299
 300     /* EAGAIN is fine, a read must be pending.  */
 301     if (ret < 0 && errno != EAGAIN) {
 302         fprintf(stderr, "qemu_event_increment: write() failed: %s\n",
 303                 strerror(errno));
 304         exit (1);
 305     }
 306 }
 307
 308 static void qemu_event_read(void *opaque)
 309 {
 310     int fd = (intptr_t)opaque;
 311     ssize_t len;
 312     char buffer[512];
 313
 314     /* Drain the notify pipe.  For eventfd, only 8 bytes will be read.  */
 315     do {
 316         len = read(fd, buffer, sizeof(buffer));
 317     } while ((len == -1 && errno == EINTR) || len == sizeof(buffer));
 318 }
 319
 320 static int qemu_event_init(void)
 321 {
 322     int err;
 323     int fds[2];
 324
 325     err = qemu_eventfd(fds);
 326     if (err == -1) {
 327         return -errno;
 328     }
 329     err = fcntl_setfl(fds[0], O_NONBLOCK);
 330     if (err < 0) {
 331         goto fail;
 332     }
 333     err = fcntl_setfl(fds[1], O_NONBLOCK);
 334     if (err < 0) {
 335         goto fail;
 336     }
 337     qemu_set_fd_handler2(fds[0], NULL, qemu_event_read, NULL,
 338                          (void *)(intptr_t)fds[0]);
 339
 340     io_thread_fd = fds[1];
 341     return 0;
 342
 343 fail:
 344     close(fds[0]);
 345     close(fds[1]);
 346     return err;
 347 }
 348
 349 static void dummy_signal(int sig)
 350 {
 351 }
 352
 353 /* If we have signalfd, we mask out the signals we want to handle and then
 354  * use signalfd to listen for them.  We rely on whatever the current signal
 355  * handler is to dispatch the signals when we receive them.
 356  */
 357 static void sigfd_handler(void *opaque)
 358 {
 359     int fd = (intptr_t)opaque;
 360     struct qemu_signalfd_siginfo info;
 361     struct sigaction action;
 362     ssize_t len;
 363
 364     while (1) {
 365         do {
 366             len = read(fd, &info, sizeof(info));
 367         } while (len == -1 && errno == EINTR);
 368
 369         if (len == -1 && errno == EAGAIN) {
 370             break;
 371         }
 372
 373         if (len != sizeof(info)) {
 374             printf("read from sigfd returned %zd: %m\n", len);
 375             return;
 376         }
 377
 378         sigaction(info.ssi_signo, NULL, &action);
 379         if ((action.sa_flags & SA_SIGINFO) && action.sa_sigaction) {
 380             action.sa_sigaction(info.ssi_signo,
 381                                 (siginfo_t *)&info, NULL);
 382         } else if (action.sa_handler) {
 383             action.sa_handler(info.ssi_signo);
 384         }
 385     }
 386 }
 387
 388 static int qemu_signal_init(void)
 389 {
 390     int sigfd;
 391     sigset_t set;
 392
 393 #ifdef CONFIG_IOTHREAD
 394     /* SIGUSR2 used by posix-aio-compat.c */
 395     sigemptyset(&set);
 396     sigaddset(&set, SIGUSR2);
 397     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 398
 399     /*
 400      * SIG_IPI must be blocked in the main thread and must not be caught
 401      * by sigwait() in the signal thread. Otherwise, the cpu thread will
 402      * not catch it reliably.
 403      */
 404     sigemptyset(&set);
 405     sigaddset(&set, SIG_IPI);
 406     pthread_sigmask(SIG_BLOCK, &set, NULL);
 407
 408     sigemptyset(&set);
 409     sigaddset(&set, SIGIO);
 410     sigaddset(&set, SIGALRM);
 411     sigaddset(&set, SIGBUS);
 412 #else
 413     sigemptyset(&set);
 414     sigaddset(&set, SIGBUS);
 415     if (kvm_enabled()) {
 416         /*
 417          * We need to process timer signals synchronously to avoid a race
 418          * between exit_request check and KVM vcpu entry.
 419          */
 420         sigaddset(&set, SIGIO);
 421         sigaddset(&set, SIGALRM);
 422     }
 423 #endif
 424     pthread_sigmask(SIG_BLOCK, &set, NULL);
 425
 426     sigfd = qemu_signalfd(&set);
 427     if (sigfd == -1) {
 428         fprintf(stderr, "failed to create signalfd\n");
 429         return -errno;
 430     }
 431
 432     fcntl_setfl(sigfd, O_NONBLOCK);
 433
 434     qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
 435                          (void *)(intptr_t)sigfd);
 436
 437     return 0;
 438 }
 439
 440 static void qemu_kvm_init_cpu_signals(CPUState *env)
 441 {
 442     int r;
 443     sigset_t set;
 444     struct sigaction sigact;
 445
 446     memset(&sigact, 0, sizeof(sigact));
 447     sigact.sa_handler = dummy_signal;
 448     sigaction(SIG_IPI, &sigact, NULL);
 449
 450 #ifdef CONFIG_IOTHREAD
 451     pthread_sigmask(SIG_BLOCK, NULL, &set);
 452     sigdelset(&set, SIG_IPI);
 453     sigdelset(&set, SIGBUS);
 454     r = kvm_set_signal_mask(env, &set);
 455     if (r) {
 456         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 457         exit(1);
 458     }
 459 #else
 460     sigemptyset(&set);
 461     sigaddset(&set, SIG_IPI);
 462     sigaddset(&set, SIGIO);
 463     sigaddset(&set, SIGALRM);
 464     pthread_sigmask(SIG_BLOCK, &set, NULL);
 465
 466     pthread_sigmask(SIG_BLOCK, NULL, &set);
 467     sigdelset(&set, SIGIO);
 468     sigdelset(&set, SIGALRM);
 469 #endif
 470     sigdelset(&set, SIG_IPI);
 471     sigdelset(&set, SIGBUS);
 472     r = kvm_set_signal_mask(env, &set);
 473     if (r) {
 474         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 475         exit(1);
 476     }
 477 }
 478
 479 static void qemu_tcg_init_cpu_signals(void)
 480 {
 481 #ifdef CONFIG_IOTHREAD
 482     sigset_t set;
 483     struct sigaction sigact;
 484
 485     memset(&sigact, 0, sizeof(sigact));
 486     sigact.sa_handler = cpu_signal;
 487     sigaction(SIG_IPI, &sigact, NULL);
 488
 489     sigemptyset(&set);
 490     sigaddset(&set, SIG_IPI);
 491     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 492 #endif
 493 }
 494
 495 #else /* _WIN32 */
 496
 497 HANDLE qemu_event_handle;
 498
 499 static void dummy_event_handler(void *opaque)
 500 {
 501 }
 502
 503 static int qemu_event_init(void)
 504 {
 505     qemu_event_handle = CreateEvent(NULL, FALSE, FALSE, NULL);
 506     if (!qemu_event_handle) {
 507         fprintf(stderr, "Failed CreateEvent: %ld\n", GetLastError());
 508         return -1;
 509     }
 510     qemu_add_wait_object(qemu_event_handle, dummy_event_handler, NULL);
 511     return 0;
 512 }
 513
 514 static void qemu_event_increment(void)
 515 {
 516     if (!SetEvent(qemu_event_handle)) {
 517         fprintf(stderr, "qemu_event_increment: SetEvent failed: %ld\n",
 518                 GetLastError());
 519         exit (1);
 520     }
 521 }
 522
 523 static int qemu_signal_init(void)
 524 {
 525     return 0;
 526 }
 527
 528 static void qemu_kvm_init_cpu_signals(CPUState *env)
 529 {
 530     abort();
 531 }
 532
 533 static void qemu_tcg_init_cpu_signals(void)
 534 {
 535 }
 536 #endif /* _WIN32 */
 537
 538 #ifndef CONFIG_IOTHREAD
 539 int qemu_init_main_loop(void)
 540 {
 541     int ret;
 542
 543     ret = qemu_signal_init();
 544     if (ret) {
 545         return ret;
 546     }
 547
 548     qemu_init_sigbus();
 549
 550     return qemu_event_init();
 551 }
 552
 553 void qemu_main_loop_start(void)
 554 {
 555 }
 556
 557 void qemu_init_vcpu(void *_env)
 558 {
 559     CPUState *env = _env;
 560     int r;
 561
 562     env->nr_cores = smp_cores;
 563     env->nr_threads = smp_threads;
 564
 565     if (kvm_enabled()) {
 566         r = kvm_init_vcpu(env);
 567         if (r < 0) {
 568             fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 569             exit(1);
 570         }
 571         qemu_kvm_init_cpu_signals(env);
 572     } else {
 573         qemu_tcg_init_cpu_signals();
 574     }
 575 }
 576
 577 int qemu_cpu_is_self(void *env)
 578 {
 579     return 1;
 580 }
 581
 582 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 583 {
 584     func(data);
 585 }
 586
 587 void resume_all_vcpus(void)
 588 {
 589 }
 590
 591 void pause_all_vcpus(void)
 592 {
 593 }
 594
 595 void qemu_cpu_kick(void *env)
 596 {
 597 }
 598
 599 void qemu_cpu_kick_self(void)
 600 {
 601 #ifndef _WIN32
 602     assert(cpu_single_env);
 603
 604     raise(SIG_IPI);
 605 #else
 606     abort();
 607 #endif
 608 }
 609
 610 void qemu_notify_event(void)
 611 {
 612     CPUState *env = cpu_single_env;
 613
 614     qemu_event_increment ();
 615     if (env) {
 616         cpu_exit(env);
 617     }
 618     if (next_cpu && env != next_cpu) {
 619         cpu_exit(next_cpu);
 620     }
 621     exit_request = 1;
 622 }
 623
 624 void qemu_mutex_lock_iothread(void) {}
 625 void qemu_mutex_unlock_iothread(void) {}
 626
 627 void cpu_stop_current(void)
 628 {
 629 }
 630
 631 void vm_stop(int reason)
 632 {
 633     do_vm_stop(reason);
 634 }
 635
 636 #else /* CONFIG_IOTHREAD */
 637
 638 QemuMutex qemu_global_mutex;
 639 static QemuCond qemu_io_proceeded_cond;
 640 static bool iothread_requesting_mutex;
 641
 642 static QemuThread io_thread;
 643
 644 static QemuThread *tcg_cpu_thread;
 645 static QemuCond *tcg_halt_cond;
 646
 647 /* cpu creation */
 648 static QemuCond qemu_cpu_cond;
 649 /* system init */
 650 static QemuCond qemu_pause_cond;
 651 static QemuCond qemu_work_cond;
 652
 653 int qemu_init_main_loop(void)
 654 {
 655     int ret;
 656
 657     qemu_init_sigbus();
 658
 659     ret = qemu_signal_init();
 660     if (ret) {
 661         return ret;
 662     }
 663
 664     /* Note eventfd must be drained before signalfd handlers run */
 665     ret = qemu_event_init();
 666     if (ret) {
 667         return ret;
 668     }
 669
 670     qemu_cond_init(&qemu_cpu_cond);
 671     qemu_cond_init(&qemu_pause_cond);
 672     qemu_cond_init(&qemu_work_cond);
 673     qemu_cond_init(&qemu_io_proceeded_cond);
 674     qemu_mutex_init(&qemu_global_mutex);
 675     qemu_mutex_lock(&qemu_global_mutex);
 676
 677     qemu_thread_get_self(&io_thread);
 678
 679     return 0;
 680 }
 681
 682 void qemu_main_loop_start(void)
 683 {
 684     resume_all_vcpus();
 685 }
 686
 687 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 688 {
 689     struct qemu_work_item wi;
 690
 691     if (qemu_cpu_is_self(env)) {
 692         func(data);
 693         return;
 694     }
 695
 696     wi.func = func;
 697     wi.data = data;
 698     if (!env->queued_work_first) {
 699         env->queued_work_first = &wi;
 700     } else {
 701         env->queued_work_last->next = &wi;
 702     }
 703     env->queued_work_last = &wi;
 704     wi.next = NULL;
 705     wi.done = false;
 706
 707     qemu_cpu_kick(env);
 708     while (!wi.done) {
 709         CPUState *self_env = cpu_single_env;
 710
 711         qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
 712         cpu_single_env = self_env;
 713     }
 714 }
 715
 716 static void flush_queued_work(CPUState *env)
 717 {
 718     struct qemu_work_item *wi;
 719
 720     if (!env->queued_work_first) {
 721         return;
 722     }
 723
 724     while ((wi = env->queued_work_first)) {
 725         env->queued_work_first = wi->next;
 726         wi->func(wi->data);
 727         wi->done = true;
 728     }
 729     env->queued_work_last = NULL;
 730     qemu_cond_broadcast(&qemu_work_cond);
 731 }
 732
 733 static void qemu_wait_io_event_common(CPUState *env)
 734 {
 735     if (env->stop) {
 736         env->stop = 0;
 737         env->stopped = 1;
 738         qemu_cond_signal(&qemu_pause_cond);
 739     }
 740     flush_queued_work(env);
 741     env->thread_kicked = false;
 742 }
 743
 744 static void qemu_tcg_wait_io_event(void)
 745 {
 746     CPUState *env;
 747
 748     while (all_cpu_threads_idle()) {
 749        /* Start accounting real time to the virtual clock if the CPUs
 750           are idle.  */
 751         qemu_clock_warp(vm_clock);
 752         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
 753     }
 754
 755     while (iothread_requesting_mutex) {
 756         qemu_cond_wait(&qemu_io_proceeded_cond, &qemu_global_mutex);
 757     }
 758
 759     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 760         qemu_wait_io_event_common(env);
 761     }
 762 }
 763
 764 static void qemu_kvm_wait_io_event(CPUState *env)
 765 {
 766     while (cpu_thread_is_idle(env)) {
 767         qemu_cond_wait(env->halt_cond, &qemu_global_mutex);
 768     }
 769
 770     qemu_kvm_eat_signals(env);
 771     qemu_wait_io_event_common(env);
 772 }
 773
 774 static void *qemu_kvm_cpu_thread_fn(void *arg)
 775 {
 776     CPUState *env = arg;
 777     int r;
 778
 779     qemu_mutex_lock(&qemu_global_mutex);
 780     qemu_thread_get_self(env->thread);
 781     env->thread_id = qemu_get_thread_id();
 782
 783     r = kvm_init_vcpu(env);
 784     if (r < 0) {
 785         fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 786         exit(1);
 787     }
 788
 789     qemu_kvm_init_cpu_signals(env);
 790
 791     /* signal CPU creation */
 792     env->created = 1;
 793     qemu_cond_signal(&qemu_cpu_cond);
 794
 795     while (1) {
 796         if (cpu_can_run(env)) {
 797             r = kvm_cpu_exec(env);
 798             if (r == EXCP_DEBUG) {
 799                 cpu_handle_guest_debug(env);
 800             }
 801         }
 802         qemu_kvm_wait_io_event(env);
 803     }
 804
 805     return NULL;
 806 }
 807
 808 static void *qemu_tcg_cpu_thread_fn(void *arg)
 809 {
 810     CPUState *env = arg;
 811
 812     qemu_tcg_init_cpu_signals();
 813     qemu_thread_get_self(env->thread);
 814
 815     /* signal CPU creation */
 816     qemu_mutex_lock(&qemu_global_mutex);
 817     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 818         env->thread_id = qemu_get_thread_id();
 819         env->created = 1;
 820     }
 821     qemu_cond_signal(&qemu_cpu_cond);
 822
 823     /* wait for initial kick-off after machine start */
 824     while (first_cpu->stopped) {
 825         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
 826     }
 827
 828     while (1) {
 829         cpu_exec_all();
 830         if (use_icount && qemu_next_icount_deadline() <= 0) {
 831             qemu_notify_event();
 832         }
 833         qemu_tcg_wait_io_event();
 834     }
 835
 836     return NULL;
 837 }
 838
 839 static void qemu_cpu_kick_thread(CPUState *env)
 840 {
 841 #ifndef _WIN32
 842     int err;
 843
 844     err = pthread_kill(env->thread->thread, SIG_IPI);
 845     if (err) {
 846         fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
 847         exit(1);
 848     }
 849 #else /* _WIN32 */
 850     if (!qemu_cpu_is_self(env)) {
 851         SuspendThread(env->thread->thread);
 852         cpu_signal(0);
 853         ResumeThread(env->thread->thread);
 854     }
 855 #endif
 856 }
 857
 858 void qemu_cpu_kick(void *_env)
 859 {
 860     CPUState *env = _env;
 861
 862     qemu_cond_broadcast(env->halt_cond);
 863     if (kvm_enabled() && !env->thread_kicked) {
 864         qemu_cpu_kick_thread(env);
 865         env->thread_kicked = true;
 866     }
 867 }
 868
 869 void qemu_cpu_kick_self(void)
 870 {
 871 #ifndef _WIN32
 872     assert(cpu_single_env);
 873
 874     if (!cpu_single_env->thread_kicked) {
 875         qemu_cpu_kick_thread(cpu_single_env);
 876         cpu_single_env->thread_kicked = true;
 877     }
 878 #else
 879     abort();
 880 #endif
 881 }
 882
 883 int qemu_cpu_is_self(void *_env)
 884 {
 885     CPUState *env = _env;
 886
 887     return qemu_thread_is_self(env->thread);
 888 }
 889
 890 void qemu_mutex_lock_iothread(void)
 891 {
 892     if (kvm_enabled()) {
 893         qemu_mutex_lock(&qemu_global_mutex);
 894     } else {
 895         iothread_requesting_mutex = true;
 896         if (qemu_mutex_trylock(&qemu_global_mutex)) {
 897             qemu_cpu_kick_thread(first_cpu);
 898             qemu_mutex_lock(&qemu_global_mutex);
 899         }
 900         iothread_requesting_mutex = false;
 901         qemu_cond_broadcast(&qemu_io_proceeded_cond);
 902     }
 903 }
 904
 905 void qemu_mutex_unlock_iothread(void)
 906 {
 907     qemu_mutex_unlock(&qemu_global_mutex);
 908 }
 909
 910 static int all_vcpus_paused(void)
 911 {
 912     CPUState *penv = first_cpu;
 913
 914     while (penv) {
 915         if (!penv->stopped) {
 916             return 0;
 917         }
 918         penv = (CPUState *)penv->next_cpu;
 919     }
 920
 921     return 1;
 922 }
 923
 924 void pause_all_vcpus(void)
 925 {
 926     CPUState *penv = first_cpu;
 927
 928     while (penv) {
 929         penv->stop = 1;
 930         qemu_cpu_kick(penv);
 931         penv = (CPUState *)penv->next_cpu;
 932     }
 933
 934     while (!all_vcpus_paused()) {
 935         qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex);
 936         penv = first_cpu;
 937         while (penv) {
 938             qemu_cpu_kick(penv);
 939             penv = (CPUState *)penv->next_cpu;
 940         }
 941     }
 942 }
 943
 944 void resume_all_vcpus(void)
 945 {
 946     CPUState *penv = first_cpu;
 947
 948     while (penv) {
 949         penv->stop = 0;
 950         penv->stopped = 0;
 951         qemu_cpu_kick(penv);
 952         penv = (CPUState *)penv->next_cpu;
 953     }
 954 }
 955
 956 static void qemu_tcg_init_vcpu(void *_env)
 957 {
 958     CPUState *env = _env;
 959
 960     /* share a single thread for all cpus with TCG */
 961     if (!tcg_cpu_thread) {
 962         env->thread = g_malloc0(sizeof(QemuThread));
 963         env->halt_cond = g_malloc0(sizeof(QemuCond));
 964         qemu_cond_init(env->halt_cond);
 965         tcg_halt_cond = env->halt_cond;
 966         qemu_thread_create(env->thread, qemu_tcg_cpu_thread_fn, env);
 967         while (env->created == 0) {
 968             qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
 969         }
 970         tcg_cpu_thread = env->thread;
 971     } else {
 972         env->thread = tcg_cpu_thread;
 973         env->halt_cond = tcg_halt_cond;
 974     }
 975 }
 976
 977 static void qemu_kvm_start_vcpu(CPUState *env)
 978 {
 979     env->thread = g_malloc0(sizeof(QemuThread));
 980     env->halt_cond = g_malloc0(sizeof(QemuCond));
 981     qemu_cond_init(env->halt_cond);
 982     qemu_thread_create(env->thread, qemu_kvm_cpu_thread_fn, env);
 983     while (env->created == 0) {
 984         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
 985     }
 986 }
 987
 988 void qemu_init_vcpu(void *_env)
 989 {
 990     CPUState *env = _env;
 991
 992     env->nr_cores = smp_cores;
 993     env->nr_threads = smp_threads;
 994     env->stopped = 1;
 995     if (kvm_enabled()) {
 996         qemu_kvm_start_vcpu(env);
 997     } else {
 998         qemu_tcg_init_vcpu(env);
 999     }
1000 }
1001
1002 void qemu_notify_event(void)
1003 {
1004     qemu_event_increment();
1005 }
1006
1007 void cpu_stop_current(void)
1008 {
1009     if (cpu_single_env) {
1010         cpu_single_env->stop = 0;
1011         cpu_single_env->stopped = 1;
1012         cpu_exit(cpu_single_env);
1013         qemu_cond_signal(&qemu_pause_cond);
1014     }
1015 }
1016
1017 void vm_stop(int reason)
1018 {
1019     if (!qemu_thread_is_self(&io_thread)) {
1020         qemu_system_vmstop_request(reason);
1021         /*
1022          * FIXME: should not return to device code in case
1023          * vm_stop() has been requested.
1024          */
1025         cpu_stop_current();
1026         return;
1027     }
1028     do_vm_stop(reason);
1029 }
1030
1031 #endif
1032
1033 static int tcg_cpu_exec(CPUState *env)
1034 {
1035     int ret;
1036 #ifdef CONFIG_PROFILER
1037     int64_t ti;
1038 #endif
1039
1040 #ifdef CONFIG_PROFILER
1041     ti = profile_getclock();
1042 #endif
1043     if (use_icount) {
1044         int64_t count;
1045         int decr;
1046         qemu_icount -= (env->icount_decr.u16.low + env->icount_extra);
1047         env->icount_decr.u16.low = 0;
1048         env->icount_extra = 0;
1049         count = qemu_icount_round(qemu_next_icount_deadline());
1050         qemu_icount += count;
1051         decr = (count > 0xffff) ? 0xffff : count;
1052         count -= decr;
1053         env->icount_decr.u16.low = decr;
1054         env->icount_extra = count;
1055     }
1056     ret = cpu_exec(env);
1057 #ifdef CONFIG_PROFILER
1058     qemu_time += profile_getclock() - ti;
1059 #endif
1060     if (use_icount) {
1061         /* Fold pending instructions back into the
1062            instruction counter, and clear the interrupt flag.  */
1063         qemu_icount -= (env->icount_decr.u16.low
1064                         + env->icount_extra);
1065         env->icount_decr.u32 = 0;
1066         env->icount_extra = 0;
1067     }
1068     return ret;
1069 }
1070
1071 bool cpu_exec_all(void)
1072 {
1073     int r;
1074
1075     /* Account partial waits to the vm_clock.  */
1076     qemu_clock_warp(vm_clock);
1077
1078     if (next_cpu == NULL) {
1079         next_cpu = first_cpu;
1080     }
1081     for (; next_cpu != NULL && !exit_request; next_cpu = next_cpu->next_cpu) {
1082         CPUState *env = next_cpu;
1083
1084         qemu_clock_enable(vm_clock,
1085                           (env->singlestep_enabled & SSTEP_NOTIMER) == 0);
1086
1087 #ifndef CONFIG_IOTHREAD
1088         if (qemu_alarm_pending()) {
1089             break;
1090         }
1091 #endif
1092         if (cpu_can_run(env)) {
1093             if (kvm_enabled()) {
1094                 r = kvm_cpu_exec(env);
1095                 qemu_kvm_eat_signals(env);
1096             } else {
1097                 r = tcg_cpu_exec(env);
1098             }
1099             if (r == EXCP_DEBUG) {
1100                 cpu_handle_guest_debug(env);
1101                 break;
1102             }
1103         } else if (env->stop || env->stopped) {
1104             break;
1105         }
1106     }
1107     exit_request = 0;
1108     return !all_cpu_threads_idle();
1109 }
1110
1111 void set_numa_modes(void)
1112 {
1113     CPUState *env;
1114     int i;
1115
1116     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1117         for (i = 0; i < nb_numa_nodes; i++) {
1118             if (node_cpumask[i] & (1 << env->cpu_index)) {
1119                 env->numa_node = i;
1120             }
1121         }
1122     }
1123 }
1124
1125 void set_cpu_log(const char *optarg)
1126 {
1127     int mask;
1128     const CPULogItem *item;
1129
1130     mask = cpu_str_to_log_mask(optarg);
1131     if (!mask) {
1132         printf("Log items (comma separated):\n");
1133         for (item = cpu_log_items; item->mask != 0; item++) {
1134             printf("%-10s %s\n", item->name, item->help);
1135         }
1136         exit(1);
1137     }
1138     cpu_set_log(mask);
1139 }
1140
1141 void set_cpu_log_filename(const char *optarg)
1142 {
1143     cpu_set_log_filename(optarg);
1144 }
1145
1146 /* Return the virtual CPU time, based on the instruction counter.  */
1147 int64_t cpu_get_icount(void)
1148 {
1149     int64_t icount;
1150     CPUState *env = cpu_single_env;;
1151
1152     icount = qemu_icount;
1153     if (env) {
1154         if (!can_do_io(env)) {
1155             fprintf(stderr, "Bad clock read\n");
1156         }
1157         icount -= (env->icount_decr.u16.low + env->icount_extra);
1158     }
1159     return qemu_icount_bias + (icount << icount_time_shift);
1160 }
1161
1162 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
1163 {
1164     /* XXX: implement xxx_cpu_list for targets that still miss it */
1165 #if defined(cpu_list_id)
1166     cpu_list_id(f, cpu_fprintf, optarg);
1167 #elif defined(cpu_list)
1168     cpu_list(f, cpu_fprintf); /* deprecated */
1169 #endif
1170 }