cpus.c

   1 /*
   2  * QEMU System Emulator
   3  *
   4  * Copyright (c) 2003-2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 /* Needed early for CONFIG_BSD etc. */
  26 #include "config-host.h"
  27
  28 #include "monitor.h"
  29 #include "sysemu.h"
  30 #include "gdbstub.h"
  31 #include "dma.h"
  32 #include "kvm.h"
  33
  34 #include "qemu-thread.h"
  35 #include "cpus.h"
  36
  37 #ifndef _WIN32
  38 #include "compatfd.h"
  39 #endif
  40
  41 #ifdef SIGRTMIN
  42 #define SIG_IPI (SIGRTMIN+4)
  43 #else
  44 #define SIG_IPI SIGUSR1
  45 #endif
  46
  47 #ifdef CONFIG_LINUX
  48
  49 #include <sys/prctl.h>
  50
  51 #ifndef PR_MCE_KILL
  52 #define PR_MCE_KILL 33
  53 #endif
  54
  55 #ifndef PR_MCE_KILL_SET
  56 #define PR_MCE_KILL_SET 1
  57 #endif
  58
  59 #ifndef PR_MCE_KILL_EARLY
  60 #define PR_MCE_KILL_EARLY 1
  61 #endif
  62
  63 #endif /* CONFIG_LINUX */
  64
  65 static CPUState *next_cpu;
  66
  67 /***********************************************************/
  68 void hw_error(const char *fmt, ...)
  69 {
  70     va_list ap;
  71     CPUState *env;
  72
  73     va_start(ap, fmt);
  74     fprintf(stderr, "qemu: hardware error: ");
  75     vfprintf(stderr, fmt, ap);
  76     fprintf(stderr, "\n");
  77     for(env = first_cpu; env != NULL; env = env->next_cpu) {
  78         fprintf(stderr, "CPU #%d:\n", env->cpu_index);
  79 #ifdef TARGET_I386
  80         cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU);
  81 #else
  82         cpu_dump_state(env, stderr, fprintf, 0);
  83 #endif
  84     }
  85     va_end(ap);
  86     abort();
  87 }
  88
  89 void cpu_synchronize_all_states(void)
  90 {
  91     CPUState *cpu;
  92
  93     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
  94         cpu_synchronize_state(cpu);
  95     }
  96 }
  97
  98 void cpu_synchronize_all_post_reset(void)
  99 {
 100     CPUState *cpu;
 101
 102     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 103         cpu_synchronize_post_reset(cpu);
 104     }
 105 }
 106
 107 void cpu_synchronize_all_post_init(void)
 108 {
 109     CPUState *cpu;
 110
 111     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 112         cpu_synchronize_post_init(cpu);
 113     }
 114 }
 115
 116 int cpu_is_stopped(CPUState *env)
 117 {
 118     return !vm_running || env->stopped;
 119 }
 120
 121 static void do_vm_stop(int reason)
 122 {
 123     if (vm_running) {
 124         cpu_disable_ticks();
 125         vm_running = 0;
 126         pause_all_vcpus();
 127         vm_state_notify(0, reason);
 128         qemu_aio_flush();
 129         bdrv_flush_all();
 130         monitor_protocol_event(QEVENT_STOP, NULL);
 131     }
 132 }
 133
 134 static int cpu_can_run(CPUState *env)
 135 {
 136     if (env->stop) {
 137         return 0;
 138     }
 139     if (env->stopped || !vm_running) {
 140         return 0;
 141     }
 142     return 1;
 143 }
 144
 145 static bool cpu_thread_is_idle(CPUState *env)
 146 {
 147     if (env->stop || env->queued_work_first) {
 148         return false;
 149     }
 150     if (env->stopped || !vm_running) {
 151         return true;
 152     }
 153     if (!env->halted || qemu_cpu_has_work(env) ||
 154         (kvm_enabled() && kvm_irqchip_in_kernel())) {
 155         return false;
 156     }
 157     return true;
 158 }
 159
 160 bool all_cpu_threads_idle(void)
 161 {
 162     CPUState *env;
 163
 164     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 165         if (!cpu_thread_is_idle(env)) {
 166             return false;
 167         }
 168     }
 169     return true;
 170 }
 171
 172 static void cpu_handle_guest_debug(CPUState *env)
 173 {
 174     gdb_set_stop_cpu(env);
 175     qemu_system_debug_request();
 176 #ifdef CONFIG_IOTHREAD
 177     env->stopped = 1;
 178 #endif
 179 }
 180
 181 #ifdef CONFIG_IOTHREAD
 182 static void cpu_signal(int sig)
 183 {
 184     if (cpu_single_env) {
 185         cpu_exit(cpu_single_env);
 186     }
 187     exit_request = 1;
 188 }
 189 #endif
 190
 191 #ifdef CONFIG_LINUX
 192 static void sigbus_reraise(void)
 193 {
 194     sigset_t set;
 195     struct sigaction action;
 196
 197     memset(&action, 0, sizeof(action));
 198     action.sa_handler = SIG_DFL;
 199     if (!sigaction(SIGBUS, &action, NULL)) {
 200         raise(SIGBUS);
 201         sigemptyset(&set);
 202         sigaddset(&set, SIGBUS);
 203         sigprocmask(SIG_UNBLOCK, &set, NULL);
 204     }
 205     perror("Failed to re-raise SIGBUS!\n");
 206     abort();
 207 }
 208
 209 static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo,
 210                            void *ctx)
 211 {
 212     if (kvm_on_sigbus(siginfo->ssi_code,
 213                       (void *)(intptr_t)siginfo->ssi_addr)) {
 214         sigbus_reraise();
 215     }
 216 }
 217
 218 static void qemu_init_sigbus(void)
 219 {
 220     struct sigaction action;
 221
 222     memset(&action, 0, sizeof(action));
 223     action.sa_flags = SA_SIGINFO;
 224     action.sa_sigaction = (void (*)(int, siginfo_t*, void*))sigbus_handler;
 225     sigaction(SIGBUS, &action, NULL);
 226
 227     prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
 228 }
 229
 230 static void qemu_kvm_eat_signals(CPUState *env)
 231 {
 232     struct timespec ts = { 0, 0 };
 233     siginfo_t siginfo;
 234     sigset_t waitset;
 235     sigset_t chkset;
 236     int r;
 237
 238     sigemptyset(&waitset);
 239     sigaddset(&waitset, SIG_IPI);
 240     sigaddset(&waitset, SIGBUS);
 241
 242     do {
 243         r = sigtimedwait(&waitset, &siginfo, &ts);
 244         if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
 245             perror("sigtimedwait");
 246             exit(1);
 247         }
 248
 249         switch (r) {
 250         case SIGBUS:
 251             if (kvm_on_sigbus_vcpu(env, siginfo.si_code, siginfo.si_addr)) {
 252                 sigbus_reraise();
 253             }
 254             break;
 255         default:
 256             break;
 257         }
 258
 259         r = sigpending(&chkset);
 260         if (r == -1) {
 261             perror("sigpending");
 262             exit(1);
 263         }
 264     } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS));
 265
 266 #ifndef CONFIG_IOTHREAD
 267     if (sigismember(&chkset, SIGIO) || sigismember(&chkset, SIGALRM)) {
 268         qemu_notify_event();
 269     }
 270 #endif
 271 }
 272
 273 #else /* !CONFIG_LINUX */
 274
 275 static void qemu_init_sigbus(void)
 276 {
 277 }
 278
 279 static void qemu_kvm_eat_signals(CPUState *env)
 280 {
 281 }
 282 #endif /* !CONFIG_LINUX */
 283
 284 #ifndef _WIN32
 285 static int io_thread_fd = -1;
 286
 287 static void qemu_event_increment(void)
 288 {
 289     /* Write 8 bytes to be compatible with eventfd.  */
 290     static const uint64_t val = 1;
 291     ssize_t ret;
 292
 293     if (io_thread_fd == -1) {
 294         return;
 295     }
 296     do {
 297         ret = write(io_thread_fd, &val, sizeof(val));
 298     } while (ret < 0 && errno == EINTR);
 299
 300     /* EAGAIN is fine, a read must be pending.  */
 301     if (ret < 0 && errno != EAGAIN) {
 302         fprintf(stderr, "qemu_event_increment: write() failed: %s\n",
 303                 strerror(errno));
 304         exit (1);
 305     }
 306 }
 307
 308 static void qemu_event_read(void *opaque)
 309 {
 310     int fd = (intptr_t)opaque;
 311     ssize_t len;
 312     char buffer[512];
 313
 314     /* Drain the notify pipe.  For eventfd, only 8 bytes will be read.  */
 315     do {
 316         len = read(fd, buffer, sizeof(buffer));
 317     } while ((len == -1 && errno == EINTR) || len == sizeof(buffer));
 318 }
 319
 320 static int qemu_event_init(void)
 321 {
 322     int err;
 323     int fds[2];
 324
 325     err = qemu_eventfd(fds);
 326     if (err == -1) {
 327         return -errno;
 328     }
 329     err = fcntl_setfl(fds[0], O_NONBLOCK);
 330     if (err < 0) {
 331         goto fail;
 332     }
 333     err = fcntl_setfl(fds[1], O_NONBLOCK);
 334     if (err < 0) {
 335         goto fail;
 336     }
 337     qemu_set_fd_handler2(fds[0], NULL, qemu_event_read, NULL,
 338                          (void *)(intptr_t)fds[0]);
 339
 340     io_thread_fd = fds[1];
 341     return 0;
 342
 343 fail:
 344     close(fds[0]);
 345     close(fds[1]);
 346     return err;
 347 }
 348
 349 static void dummy_signal(int sig)
 350 {
 351 }
 352
 353 /* If we have signalfd, we mask out the signals we want to handle and then
 354  * use signalfd to listen for them.  We rely on whatever the current signal
 355  * handler is to dispatch the signals when we receive them.
 356  */
 357 static void sigfd_handler(void *opaque)
 358 {
 359     int fd = (intptr_t)opaque;
 360     struct qemu_signalfd_siginfo info;
 361     struct sigaction action;
 362     ssize_t len;
 363
 364     while (1) {
 365         do {
 366             len = read(fd, &info, sizeof(info));
 367         } while (len == -1 && errno == EINTR);
 368
 369         if (len == -1 && errno == EAGAIN) {
 370             break;
 371         }
 372
 373         if (len != sizeof(info)) {
 374             printf("read from sigfd returned %zd: %m\n", len);
 375             return;
 376         }
 377
 378         sigaction(info.ssi_signo, NULL, &action);
 379         if ((action.sa_flags & SA_SIGINFO) && action.sa_sigaction) {
 380             action.sa_sigaction(info.ssi_signo,
 381                                 (siginfo_t *)&info, NULL);
 382         } else if (action.sa_handler) {
 383             action.sa_handler(info.ssi_signo);
 384         }
 385     }
 386 }
 387
 388 static int qemu_signal_init(void)
 389 {
 390     int sigfd;
 391     sigset_t set;
 392
 393 #ifdef CONFIG_IOTHREAD
 394     /* SIGUSR2 used by posix-aio-compat.c */
 395     sigemptyset(&set);
 396     sigaddset(&set, SIGUSR2);
 397     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 398
 399     /*
 400      * SIG_IPI must be blocked in the main thread and must not be caught
 401      * by sigwait() in the signal thread. Otherwise, the cpu thread will
 402      * not catch it reliably.
 403      */
 404     sigemptyset(&set);
 405     sigaddset(&set, SIG_IPI);
 406     pthread_sigmask(SIG_BLOCK, &set, NULL);
 407
 408     sigemptyset(&set);
 409     sigaddset(&set, SIGIO);
 410     sigaddset(&set, SIGALRM);
 411     sigaddset(&set, SIGBUS);
 412 #else
 413     sigemptyset(&set);
 414     sigaddset(&set, SIGBUS);
 415     if (kvm_enabled()) {
 416         /*
 417          * We need to process timer signals synchronously to avoid a race
 418          * between exit_request check and KVM vcpu entry.
 419          */
 420         sigaddset(&set, SIGIO);
 421         sigaddset(&set, SIGALRM);
 422     }
 423 #endif
 424     pthread_sigmask(SIG_BLOCK, &set, NULL);
 425
 426     sigfd = qemu_signalfd(&set);
 427     if (sigfd == -1) {
 428         fprintf(stderr, "failed to create signalfd\n");
 429         return -errno;
 430     }
 431
 432     fcntl_setfl(sigfd, O_NONBLOCK);
 433
 434     qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
 435                          (void *)(intptr_t)sigfd);
 436
 437     return 0;
 438 }
 439
 440 static void qemu_kvm_init_cpu_signals(CPUState *env)
 441 {
 442     int r;
 443     sigset_t set;
 444     struct sigaction sigact;
 445
 446     memset(&sigact, 0, sizeof(sigact));
 447     sigact.sa_handler = dummy_signal;
 448     sigaction(SIG_IPI, &sigact, NULL);
 449
 450 #ifdef CONFIG_IOTHREAD
 451     pthread_sigmask(SIG_BLOCK, NULL, &set);
 452     sigdelset(&set, SIG_IPI);
 453     sigdelset(&set, SIGBUS);
 454     r = kvm_set_signal_mask(env, &set);
 455     if (r) {
 456         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 457         exit(1);
 458     }
 459 #else
 460     sigemptyset(&set);
 461     sigaddset(&set, SIG_IPI);
 462     sigaddset(&set, SIGIO);
 463     sigaddset(&set, SIGALRM);
 464     pthread_sigmask(SIG_BLOCK, &set, NULL);
 465
 466     pthread_sigmask(SIG_BLOCK, NULL, &set);
 467     sigdelset(&set, SIGIO);
 468     sigdelset(&set, SIGALRM);
 469 #endif
 470     sigdelset(&set, SIG_IPI);
 471     sigdelset(&set, SIGBUS);
 472     r = kvm_set_signal_mask(env, &set);
 473     if (r) {
 474         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 475         exit(1);
 476     }
 477 }
 478
 479 static void qemu_tcg_init_cpu_signals(void)
 480 {
 481 #ifdef CONFIG_IOTHREAD
 482     sigset_t set;
 483     struct sigaction sigact;
 484
 485     memset(&sigact, 0, sizeof(sigact));
 486     sigact.sa_handler = cpu_signal;
 487     sigaction(SIG_IPI, &sigact, NULL);
 488
 489     sigemptyset(&set);
 490     sigaddset(&set, SIG_IPI);
 491     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 492 #endif
 493 }
 494
 495 #else /* _WIN32 */
 496
 497 HANDLE qemu_event_handle;
 498
 499 static void dummy_event_handler(void *opaque)
 500 {
 501 }
 502
 503 static int qemu_event_init(void)
 504 {
 505     qemu_event_handle = CreateEvent(NULL, FALSE, FALSE, NULL);
 506     if (!qemu_event_handle) {
 507         fprintf(stderr, "Failed CreateEvent: %ld\n", GetLastError());
 508         return -1;
 509     }
 510     qemu_add_wait_object(qemu_event_handle, dummy_event_handler, NULL);
 511     return 0;
 512 }
 513
 514 static void qemu_event_increment(void)
 515 {
 516     if (!SetEvent(qemu_event_handle)) {
 517         fprintf(stderr, "qemu_event_increment: SetEvent failed: %ld\n",
 518                 GetLastError());
 519         exit (1);
 520     }
 521 }
 522
 523 static int qemu_signal_init(void)
 524 {
 525     return 0;
 526 }
 527
 528 static void qemu_kvm_init_cpu_signals(CPUState *env)
 529 {
 530     abort();
 531 }
 532
 533 static void qemu_tcg_init_cpu_signals(void)
 534 {
 535 }
 536 #endif /* _WIN32 */
 537
 538 #ifndef CONFIG_IOTHREAD
 539 int qemu_init_main_loop(void)
 540 {
 541     int ret;
 542
 543     ret = qemu_signal_init();
 544     if (ret) {
 545         return ret;
 546     }
 547
 548     qemu_init_sigbus();
 549
 550     return qemu_event_init();
 551 }
 552
 553 void qemu_main_loop_start(void)
 554 {
 555 }
 556
 557 void qemu_init_vcpu(void *_env)
 558 {
 559     CPUState *env = _env;
 560     int r;
 561
 562     env->nr_cores = smp_cores;
 563     env->nr_threads = smp_threads;
 564
 565     if (kvm_enabled()) {
 566         r = kvm_init_vcpu(env);
 567         if (r < 0) {
 568             fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 569             exit(1);
 570         }
 571         qemu_kvm_init_cpu_signals(env);
 572     } else {
 573         qemu_tcg_init_cpu_signals();
 574     }
 575 }
 576
 577 int qemu_cpu_is_self(void *env)
 578 {
 579     return 1;
 580 }
 581
 582 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 583 {
 584     func(data);
 585 }
 586
 587 void resume_all_vcpus(void)
 588 {
 589 }
 590
 591 void pause_all_vcpus(void)
 592 {
 593 }
 594
 595 void qemu_cpu_kick(void *env)
 596 {
 597 }
 598
 599 void qemu_cpu_kick_self(void)
 600 {
 601 #ifndef _WIN32
 602     assert(cpu_single_env);
 603
 604     raise(SIG_IPI);
 605 #else
 606     abort();
 607 #endif
 608 }
 609
 610 void qemu_notify_event(void)
 611 {
 612     CPUState *env = cpu_single_env;
 613
 614     qemu_event_increment ();
 615     if (env) {
 616         cpu_exit(env);
 617     }
 618     if (next_cpu && env != next_cpu) {
 619         cpu_exit(next_cpu);
 620     }
 621     exit_request = 1;
 622 }
 623
 624 void qemu_mutex_lock_iothread(void) {}
 625 void qemu_mutex_unlock_iothread(void) {}
 626
 627 void cpu_stop_current(void)
 628 {
 629 }
 630
 631 void vm_stop(int reason)
 632 {
 633     do_vm_stop(reason);
 634 }
 635
 636 #else /* CONFIG_IOTHREAD */
 637
 638 QemuMutex qemu_global_mutex;
 639 static QemuMutex qemu_fair_mutex;
 640
 641 static QemuThread io_thread;
 642
 643 static QemuThread *tcg_cpu_thread;
 644 static QemuCond *tcg_halt_cond;
 645
 646 static int qemu_system_ready;
 647 /* cpu creation */
 648 static QemuCond qemu_cpu_cond;
 649 /* system init */
 650 static QemuCond qemu_system_cond;
 651 static QemuCond qemu_pause_cond;
 652 static QemuCond qemu_work_cond;
 653
 654 int qemu_init_main_loop(void)
 655 {
 656     int ret;
 657
 658     qemu_init_sigbus();
 659
 660     ret = qemu_signal_init();
 661     if (ret) {
 662         return ret;
 663     }
 664
 665     /* Note eventfd must be drained before signalfd handlers run */
 666     ret = qemu_event_init();
 667     if (ret) {
 668         return ret;
 669     }
 670
 671     qemu_cond_init(&qemu_cpu_cond);
 672     qemu_cond_init(&qemu_system_cond);
 673     qemu_cond_init(&qemu_pause_cond);
 674     qemu_cond_init(&qemu_work_cond);
 675     qemu_mutex_init(&qemu_fair_mutex);
 676     qemu_mutex_init(&qemu_global_mutex);
 677     qemu_mutex_lock(&qemu_global_mutex);
 678
 679     qemu_thread_get_self(&io_thread);
 680
 681     return 0;
 682 }
 683
 684 void qemu_main_loop_start(void)
 685 {
 686     qemu_system_ready = 1;
 687     qemu_cond_broadcast(&qemu_system_cond);
 688 }
 689
 690 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 691 {
 692     struct qemu_work_item wi;
 693
 694     if (qemu_cpu_is_self(env)) {
 695         func(data);
 696         return;
 697     }
 698
 699     wi.func = func;
 700     wi.data = data;
 701     if (!env->queued_work_first) {
 702         env->queued_work_first = &wi;
 703     } else {
 704         env->queued_work_last->next = &wi;
 705     }
 706     env->queued_work_last = &wi;
 707     wi.next = NULL;
 708     wi.done = false;
 709
 710     qemu_cpu_kick(env);
 711     while (!wi.done) {
 712         CPUState *self_env = cpu_single_env;
 713
 714         qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
 715         cpu_single_env = self_env;
 716     }
 717 }
 718
 719 static void flush_queued_work(CPUState *env)
 720 {
 721     struct qemu_work_item *wi;
 722
 723     if (!env->queued_work_first) {
 724         return;
 725     }
 726
 727     while ((wi = env->queued_work_first)) {
 728         env->queued_work_first = wi->next;
 729         wi->func(wi->data);
 730         wi->done = true;
 731     }
 732     env->queued_work_last = NULL;
 733     qemu_cond_broadcast(&qemu_work_cond);
 734 }
 735
 736 static void qemu_wait_io_event_common(CPUState *env)
 737 {
 738     if (env->stop) {
 739         env->stop = 0;
 740         env->stopped = 1;
 741         qemu_cond_signal(&qemu_pause_cond);
 742     }
 743     flush_queued_work(env);
 744     env->thread_kicked = false;
 745 }
 746
 747 static void qemu_tcg_wait_io_event(void)
 748 {
 749     CPUState *env;
 750
 751     while (all_cpu_threads_idle()) {
 752        /* Start accounting real time to the virtual clock if the CPUs
 753           are idle.  */
 754         qemu_clock_warp(vm_clock);
 755         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
 756     }
 757
 758     qemu_mutex_unlock(&qemu_global_mutex);
 759
 760     /*
 761      * Users of qemu_global_mutex can be starved, having no chance
 762      * to acquire it since this path will get to it first.
 763      * So use another lock to provide fairness.
 764      */
 765     qemu_mutex_lock(&qemu_fair_mutex);
 766     qemu_mutex_unlock(&qemu_fair_mutex);
 767
 768     qemu_mutex_lock(&qemu_global_mutex);
 769
 770     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 771         qemu_wait_io_event_common(env);
 772     }
 773 }
 774
 775 static void qemu_kvm_wait_io_event(CPUState *env)
 776 {
 777     while (cpu_thread_is_idle(env)) {
 778         qemu_cond_wait(env->halt_cond, &qemu_global_mutex);
 779     }
 780
 781     qemu_kvm_eat_signals(env);
 782     qemu_wait_io_event_common(env);
 783 }
 784
 785 static void *qemu_kvm_cpu_thread_fn(void *arg)
 786 {
 787     CPUState *env = arg;
 788     int r;
 789
 790     qemu_mutex_lock(&qemu_global_mutex);
 791     qemu_thread_get_self(env->thread);
 792     env->thread_id = qemu_get_thread_id();
 793
 794     r = kvm_init_vcpu(env);
 795     if (r < 0) {
 796         fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 797         exit(1);
 798     }
 799
 800     qemu_kvm_init_cpu_signals(env);
 801
 802     /* signal CPU creation */
 803     env->created = 1;
 804     qemu_cond_signal(&qemu_cpu_cond);
 805
 806     /* and wait for machine initialization */
 807     while (!qemu_system_ready) {
 808         qemu_cond_wait(&qemu_system_cond, &qemu_global_mutex);
 809     }
 810
 811     while (1) {
 812         if (cpu_can_run(env)) {
 813             r = kvm_cpu_exec(env);
 814             if (r == EXCP_DEBUG) {
 815                 cpu_handle_guest_debug(env);
 816             }
 817         }
 818         qemu_kvm_wait_io_event(env);
 819     }
 820
 821     return NULL;
 822 }
 823
 824 static void *qemu_tcg_cpu_thread_fn(void *arg)
 825 {
 826     CPUState *env = arg;
 827
 828     qemu_tcg_init_cpu_signals();
 829     qemu_thread_get_self(env->thread);
 830
 831     /* signal CPU creation */
 832     qemu_mutex_lock(&qemu_global_mutex);
 833     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 834         env->thread_id = qemu_get_thread_id();
 835         env->created = 1;
 836     }
 837     qemu_cond_signal(&qemu_cpu_cond);
 838
 839     /* and wait for machine initialization */
 840     while (!qemu_system_ready) {
 841         qemu_cond_wait(&qemu_system_cond, &qemu_global_mutex);
 842     }
 843
 844     while (1) {
 845         cpu_exec_all();
 846         if (use_icount && qemu_next_icount_deadline() <= 0) {
 847             qemu_notify_event();
 848         }
 849         qemu_tcg_wait_io_event();
 850     }
 851
 852     return NULL;
 853 }
 854
 855 static void qemu_cpu_kick_thread(CPUState *env)
 856 {
 857 #ifndef _WIN32
 858     int err;
 859
 860     err = pthread_kill(env->thread->thread, SIG_IPI);
 861     if (err) {
 862         fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
 863         exit(1);
 864     }
 865 #else /* _WIN32 */
 866     if (!qemu_cpu_is_self(env)) {
 867         SuspendThread(env->thread->thread);
 868         cpu_signal(0);
 869         ResumeThread(env->thread->thread);
 870     }
 871 #endif
 872 }
 873
 874 void qemu_cpu_kick(void *_env)
 875 {
 876     CPUState *env = _env;
 877
 878     qemu_cond_broadcast(env->halt_cond);
 879     if (!env->thread_kicked) {
 880         qemu_cpu_kick_thread(env);
 881         env->thread_kicked = true;
 882     }
 883 }
 884
 885 void qemu_cpu_kick_self(void)
 886 {
 887 #ifndef _WIN32
 888     assert(cpu_single_env);
 889
 890     if (!cpu_single_env->thread_kicked) {
 891         qemu_cpu_kick_thread(cpu_single_env);
 892         cpu_single_env->thread_kicked = true;
 893     }
 894 #else
 895     abort();
 896 #endif
 897 }
 898
 899 int qemu_cpu_is_self(void *_env)
 900 {
 901     CPUState *env = _env;
 902
 903     return qemu_thread_is_self(env->thread);
 904 }
 905
 906 void qemu_mutex_lock_iothread(void)
 907 {
 908     if (kvm_enabled()) {
 909         qemu_mutex_lock(&qemu_global_mutex);
 910     } else {
 911         qemu_mutex_lock(&qemu_fair_mutex);
 912         if (qemu_mutex_trylock(&qemu_global_mutex)) {
 913             qemu_cpu_kick_thread(first_cpu);
 914             qemu_mutex_lock(&qemu_global_mutex);
 915         }
 916         qemu_mutex_unlock(&qemu_fair_mutex);
 917     }
 918 }
 919
 920 void qemu_mutex_unlock_iothread(void)
 921 {
 922     qemu_mutex_unlock(&qemu_global_mutex);
 923 }
 924
 925 static int all_vcpus_paused(void)
 926 {
 927     CPUState *penv = first_cpu;
 928
 929     while (penv) {
 930         if (!penv->stopped) {
 931             return 0;
 932         }
 933         penv = (CPUState *)penv->next_cpu;
 934     }
 935
 936     return 1;
 937 }
 938
 939 void pause_all_vcpus(void)
 940 {
 941     CPUState *penv = first_cpu;
 942
 943     while (penv) {
 944         penv->stop = 1;
 945         qemu_cpu_kick(penv);
 946         penv = (CPUState *)penv->next_cpu;
 947     }
 948
 949     while (!all_vcpus_paused()) {
 950         qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex);
 951         penv = first_cpu;
 952         while (penv) {
 953             qemu_cpu_kick(penv);
 954             penv = (CPUState *)penv->next_cpu;
 955         }
 956     }
 957 }
 958
 959 void resume_all_vcpus(void)
 960 {
 961     CPUState *penv = first_cpu;
 962
 963     while (penv) {
 964         penv->stop = 0;
 965         penv->stopped = 0;
 966         qemu_cpu_kick(penv);
 967         penv = (CPUState *)penv->next_cpu;
 968     }
 969 }
 970
 971 static void qemu_tcg_init_vcpu(void *_env)
 972 {
 973     CPUState *env = _env;
 974
 975     /* share a single thread for all cpus with TCG */
 976     if (!tcg_cpu_thread) {
 977         env->thread = qemu_mallocz(sizeof(QemuThread));
 978         env->halt_cond = qemu_mallocz(sizeof(QemuCond));
 979         qemu_cond_init(env->halt_cond);
 980         qemu_thread_create(env->thread, qemu_tcg_cpu_thread_fn, env);
 981         while (env->created == 0) {
 982             qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
 983         }
 984         tcg_cpu_thread = env->thread;
 985         tcg_halt_cond = env->halt_cond;
 986     } else {
 987         env->thread = tcg_cpu_thread;
 988         env->halt_cond = tcg_halt_cond;
 989     }
 990 }
 991
 992 static void qemu_kvm_start_vcpu(CPUState *env)
 993 {
 994     env->thread = qemu_mallocz(sizeof(QemuThread));
 995     env->halt_cond = qemu_mallocz(sizeof(QemuCond));
 996     qemu_cond_init(env->halt_cond);
 997     qemu_thread_create(env->thread, qemu_kvm_cpu_thread_fn, env);
 998     while (env->created == 0) {
 999         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
1000     }
1001 }
1002
1003 void qemu_init_vcpu(void *_env)
1004 {
1005     CPUState *env = _env;
1006
1007     env->nr_cores = smp_cores;
1008     env->nr_threads = smp_threads;
1009     if (kvm_enabled()) {
1010         qemu_kvm_start_vcpu(env);
1011     } else {
1012         qemu_tcg_init_vcpu(env);
1013     }
1014 }
1015
1016 void qemu_notify_event(void)
1017 {
1018     qemu_event_increment();
1019 }
1020
1021 void cpu_stop_current(void)
1022 {
1023     if (cpu_single_env) {
1024         cpu_single_env->stop = 0;
1025         cpu_single_env->stopped = 1;
1026         cpu_exit(cpu_single_env);
1027         qemu_cond_signal(&qemu_pause_cond);
1028     }
1029 }
1030
1031 void vm_stop(int reason)
1032 {
1033     if (!qemu_thread_is_self(&io_thread)) {
1034         qemu_system_vmstop_request(reason);
1035         /*
1036          * FIXME: should not return to device code in case
1037          * vm_stop() has been requested.
1038          */
1039         cpu_stop_current();
1040         return;
1041     }
1042     do_vm_stop(reason);
1043 }
1044
1045 #endif
1046
1047 static int tcg_cpu_exec(CPUState *env)
1048 {
1049     int ret;
1050 #ifdef CONFIG_PROFILER
1051     int64_t ti;
1052 #endif
1053
1054 #ifdef CONFIG_PROFILER
1055     ti = profile_getclock();
1056 #endif
1057     if (use_icount) {
1058         int64_t count;
1059         int decr;
1060         qemu_icount -= (env->icount_decr.u16.low + env->icount_extra);
1061         env->icount_decr.u16.low = 0;
1062         env->icount_extra = 0;
1063         count = qemu_icount_round(qemu_next_icount_deadline());
1064         qemu_icount += count;
1065         decr = (count > 0xffff) ? 0xffff : count;
1066         count -= decr;
1067         env->icount_decr.u16.low = decr;
1068         env->icount_extra = count;
1069     }
1070     ret = cpu_exec(env);
1071 #ifdef CONFIG_PROFILER
1072     qemu_time += profile_getclock() - ti;
1073 #endif
1074     if (use_icount) {
1075         /* Fold pending instructions back into the
1076            instruction counter, and clear the interrupt flag.  */
1077         qemu_icount -= (env->icount_decr.u16.low
1078                         + env->icount_extra);
1079         env->icount_decr.u32 = 0;
1080         env->icount_extra = 0;
1081     }
1082     return ret;
1083 }
1084
1085 bool cpu_exec_all(void)
1086 {
1087     int r;
1088
1089     /* Account partial waits to the vm_clock.  */
1090     qemu_clock_warp(vm_clock);
1091
1092     if (next_cpu == NULL) {
1093         next_cpu = first_cpu;
1094     }
1095     for (; next_cpu != NULL && !exit_request; next_cpu = next_cpu->next_cpu) {
1096         CPUState *env = next_cpu;
1097
1098         qemu_clock_enable(vm_clock,
1099                           (env->singlestep_enabled & SSTEP_NOTIMER) == 0);
1100
1101 #ifndef CONFIG_IOTHREAD
1102         if (qemu_alarm_pending()) {
1103             break;
1104         }
1105 #endif
1106         if (cpu_can_run(env)) {
1107             if (kvm_enabled()) {
1108                 r = kvm_cpu_exec(env);
1109                 qemu_kvm_eat_signals(env);
1110             } else {
1111                 r = tcg_cpu_exec(env);
1112             }
1113             if (r == EXCP_DEBUG) {
1114                 cpu_handle_guest_debug(env);
1115                 break;
1116             }
1117         } else if (env->stop || env->stopped) {
1118             break;
1119         }
1120     }
1121     exit_request = 0;
1122     return !all_cpu_threads_idle();
1123 }
1124
1125 void set_numa_modes(void)
1126 {
1127     CPUState *env;
1128     int i;
1129
1130     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1131         for (i = 0; i < nb_numa_nodes; i++) {
1132             if (node_cpumask[i] & (1 << env->cpu_index)) {
1133                 env->numa_node = i;
1134             }
1135         }
1136     }
1137 }
1138
1139 void set_cpu_log(const char *optarg)
1140 {
1141     int mask;
1142     const CPULogItem *item;
1143
1144     mask = cpu_str_to_log_mask(optarg);
1145     if (!mask) {
1146         printf("Log items (comma separated):\n");
1147         for (item = cpu_log_items; item->mask != 0; item++) {
1148             printf("%-10s %s\n", item->name, item->help);
1149         }
1150         exit(1);
1151     }
1152     cpu_set_log(mask);
1153 }
1154
1155 void set_cpu_log_filename(const char *optarg)
1156 {
1157     cpu_set_log_filename(optarg);
1158 }
1159
1160 /* Return the virtual CPU time, based on the instruction counter.  */
1161 int64_t cpu_get_icount(void)
1162 {
1163     int64_t icount;
1164     CPUState *env = cpu_single_env;;
1165
1166     icount = qemu_icount;
1167     if (env) {
1168         if (!can_do_io(env)) {
1169             fprintf(stderr, "Bad clock read\n");
1170         }
1171         icount -= (env->icount_decr.u16.low + env->icount_extra);
1172     }
1173     return qemu_icount_bias + (icount << icount_time_shift);
1174 }
1175
1176 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
1177 {
1178     /* XXX: implement xxx_cpu_list for targets that still miss it */
1179 #if defined(cpu_list_id)
1180     cpu_list_id(f, cpu_fprintf, optarg);
1181 #elif defined(cpu_list)
1182     cpu_list(f, cpu_fprintf); /* deprecated */
1183 #endif
1184 }