cpus.c

   1 /*
   2  * QEMU System Emulator
   3  *
   4  * Copyright (c) 2003-2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 /* Needed early for CONFIG_BSD etc. */
  26 #include "config-host.h"
  27
  28 #include "monitor.h"
  29 #include "sysemu.h"
  30 #include "gdbstub.h"
  31 #include "dma.h"
  32 #include "kvm.h"
  33 #include "exec-all.h"
  34
  35 #include "cpus.h"
  36 #include "compatfd.h"
  37
  38 #ifdef SIGRTMIN
  39 #define SIG_IPI (SIGRTMIN+4)
  40 #else
  41 #define SIG_IPI SIGUSR1
  42 #endif
  43
  44 #ifdef CONFIG_LINUX
  45
  46 #include <sys/prctl.h>
  47
  48 #ifndef PR_MCE_KILL
  49 #define PR_MCE_KILL 33
  50 #endif
  51
  52 #ifndef PR_MCE_KILL_SET
  53 #define PR_MCE_KILL_SET 1
  54 #endif
  55
  56 #ifndef PR_MCE_KILL_EARLY
  57 #define PR_MCE_KILL_EARLY 1
  58 #endif
  59
  60 #endif /* CONFIG_LINUX */
  61
  62 static CPUState *next_cpu;
  63
  64 /***********************************************************/
  65 void hw_error(const char *fmt, ...)
  66 {
  67     va_list ap;
  68     CPUState *env;
  69
  70     va_start(ap, fmt);
  71     fprintf(stderr, "qemu: hardware error: ");
  72     vfprintf(stderr, fmt, ap);
  73     fprintf(stderr, "\n");
  74     for(env = first_cpu; env != NULL; env = env->next_cpu) {
  75         fprintf(stderr, "CPU #%d:\n", env->cpu_index);
  76 #ifdef TARGET_I386
  77         cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU);
  78 #else
  79         cpu_dump_state(env, stderr, fprintf, 0);
  80 #endif
  81     }
  82     va_end(ap);
  83     abort();
  84 }
  85
  86 void cpu_synchronize_all_states(void)
  87 {
  88     CPUState *cpu;
  89
  90     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
  91         cpu_synchronize_state(cpu);
  92     }
  93 }
  94
  95 void cpu_synchronize_all_post_reset(void)
  96 {
  97     CPUState *cpu;
  98
  99     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 100         cpu_synchronize_post_reset(cpu);
 101     }
 102 }
 103
 104 void cpu_synchronize_all_post_init(void)
 105 {
 106     CPUState *cpu;
 107
 108     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 109         cpu_synchronize_post_init(cpu);
 110     }
 111 }
 112
 113 int cpu_is_stopped(CPUState *env)
 114 {
 115     return !vm_running || env->stopped;
 116 }
 117
 118 static void do_vm_stop(int reason)
 119 {
 120     if (vm_running) {
 121         cpu_disable_ticks();
 122         vm_running = 0;
 123         pause_all_vcpus();
 124         vm_state_notify(0, reason);
 125         qemu_aio_flush();
 126         bdrv_flush_all();
 127         monitor_protocol_event(QEVENT_STOP, NULL);
 128     }
 129 }
 130
 131 static int cpu_can_run(CPUState *env)
 132 {
 133     if (env->stop) {
 134         return 0;
 135     }
 136     if (env->stopped || !vm_running) {
 137         return 0;
 138     }
 139     return 1;
 140 }
 141
 142 static bool cpu_thread_is_idle(CPUState *env)
 143 {
 144     if (env->stop || env->queued_work_first) {
 145         return false;
 146     }
 147     if (env->stopped || !vm_running) {
 148         return true;
 149     }
 150     if (!env->halted || qemu_cpu_has_work(env)) {
 151         return false;
 152     }
 153     return true;
 154 }
 155
 156 static bool all_cpu_threads_idle(void)
 157 {
 158     CPUState *env;
 159
 160     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 161         if (!cpu_thread_is_idle(env)) {
 162             return false;
 163         }
 164     }
 165     return true;
 166 }
 167
 168 static CPUDebugExcpHandler *debug_excp_handler;
 169
 170 CPUDebugExcpHandler *cpu_set_debug_excp_handler(CPUDebugExcpHandler *handler)
 171 {
 172     CPUDebugExcpHandler *old_handler = debug_excp_handler;
 173
 174     debug_excp_handler = handler;
 175     return old_handler;
 176 }
 177
 178 static void cpu_handle_debug_exception(CPUState *env)
 179 {
 180     CPUWatchpoint *wp;
 181
 182     if (!env->watchpoint_hit) {
 183         QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 184             wp->flags &= ~BP_WATCHPOINT_HIT;
 185         }
 186     }
 187     if (debug_excp_handler) {
 188         debug_excp_handler(env);
 189     }
 190
 191     gdb_set_stop_cpu(env);
 192     qemu_system_debug_request();
 193 #ifdef CONFIG_IOTHREAD
 194     env->stopped = 1;
 195 #endif
 196 }
 197
 198 #ifdef CONFIG_LINUX
 199 static void sigbus_reraise(void)
 200 {
 201     sigset_t set;
 202     struct sigaction action;
 203
 204     memset(&action, 0, sizeof(action));
 205     action.sa_handler = SIG_DFL;
 206     if (!sigaction(SIGBUS, &action, NULL)) {
 207         raise(SIGBUS);
 208         sigemptyset(&set);
 209         sigaddset(&set, SIGBUS);
 210         sigprocmask(SIG_UNBLOCK, &set, NULL);
 211     }
 212     perror("Failed to re-raise SIGBUS!\n");
 213     abort();
 214 }
 215
 216 static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo,
 217                            void *ctx)
 218 {
 219     if (kvm_on_sigbus(siginfo->ssi_code,
 220                       (void *)(intptr_t)siginfo->ssi_addr)) {
 221         sigbus_reraise();
 222     }
 223 }
 224
 225 static void qemu_init_sigbus(void)
 226 {
 227     struct sigaction action;
 228
 229     memset(&action, 0, sizeof(action));
 230     action.sa_flags = SA_SIGINFO;
 231     action.sa_sigaction = (void (*)(int, siginfo_t*, void*))sigbus_handler;
 232     sigaction(SIGBUS, &action, NULL);
 233
 234     prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
 235 }
 236
 237 #else /* !CONFIG_LINUX */
 238
 239 static void qemu_init_sigbus(void)
 240 {
 241 }
 242 #endif /* !CONFIG_LINUX */
 243
 244 #ifndef _WIN32
 245 static int io_thread_fd = -1;
 246
 247 static void qemu_event_increment(void)
 248 {
 249     /* Write 8 bytes to be compatible with eventfd.  */
 250     static const uint64_t val = 1;
 251     ssize_t ret;
 252
 253     if (io_thread_fd == -1) {
 254         return;
 255     }
 256     do {
 257         ret = write(io_thread_fd, &val, sizeof(val));
 258     } while (ret < 0 && errno == EINTR);
 259
 260     /* EAGAIN is fine, a read must be pending.  */
 261     if (ret < 0 && errno != EAGAIN) {
 262         fprintf(stderr, "qemu_event_increment: write() filed: %s\n",
 263                 strerror(errno));
 264         exit (1);
 265     }
 266 }
 267
 268 static void qemu_event_read(void *opaque)
 269 {
 270     int fd = (unsigned long)opaque;
 271     ssize_t len;
 272     char buffer[512];
 273
 274     /* Drain the notify pipe.  For eventfd, only 8 bytes will be read.  */
 275     do {
 276         len = read(fd, buffer, sizeof(buffer));
 277     } while ((len == -1 && errno == EINTR) || len == sizeof(buffer));
 278 }
 279
 280 static int qemu_event_init(void)
 281 {
 282     int err;
 283     int fds[2];
 284
 285     err = qemu_eventfd(fds);
 286     if (err == -1) {
 287         return -errno;
 288     }
 289     err = fcntl_setfl(fds[0], O_NONBLOCK);
 290     if (err < 0) {
 291         goto fail;
 292     }
 293     err = fcntl_setfl(fds[1], O_NONBLOCK);
 294     if (err < 0) {
 295         goto fail;
 296     }
 297     qemu_set_fd_handler2(fds[0], NULL, qemu_event_read, NULL,
 298                          (void *)(unsigned long)fds[0]);
 299
 300     io_thread_fd = fds[1];
 301     return 0;
 302
 303 fail:
 304     close(fds[0]);
 305     close(fds[1]);
 306     return err;
 307 }
 308
 309 #ifdef OBSOLETE_KVM_IMPL
 310 static void dummy_signal(int sig)
 311 {
 312 }
 313 #endif
 314
 315 /* If we have signalfd, we mask out the signals we want to handle and then
 316  * use signalfd to listen for them.  We rely on whatever the current signal
 317  * handler is to dispatch the signals when we receive them.
 318  */
 319 static void sigfd_handler(void *opaque)
 320 {
 321     int fd = (unsigned long) opaque;
 322     struct qemu_signalfd_siginfo info;
 323     struct sigaction action;
 324     ssize_t len;
 325
 326     while (1) {
 327         do {
 328             len = read(fd, &info, sizeof(info));
 329         } while (len == -1 && errno == EINTR);
 330
 331         if (len == -1 && errno == EAGAIN) {
 332             break;
 333         }
 334
 335         if (len != sizeof(info)) {
 336             printf("read from sigfd returned %zd: %m\n", len);
 337             return;
 338         }
 339
 340         sigaction(info.ssi_signo, NULL, &action);
 341         if ((action.sa_flags & SA_SIGINFO) && action.sa_sigaction) {
 342             action.sa_sigaction(info.ssi_signo,
 343                                 (siginfo_t *)&info, NULL);
 344         } else if (action.sa_handler) {
 345             action.sa_handler(info.ssi_signo);
 346         }
 347     }
 348 }
 349
 350 static int qemu_signalfd_init(sigset_t mask)
 351 {
 352     int sigfd;
 353
 354     sigfd = qemu_signalfd(&mask);
 355     if (sigfd == -1) {
 356         fprintf(stderr, "failed to create signalfd\n");
 357         return -errno;
 358     }
 359
 360     fcntl_setfl(sigfd, O_NONBLOCK);
 361
 362     qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
 363                          (void *)(unsigned long) sigfd);
 364
 365     return 0;
 366 }
 367
 368 static void qemu_kvm_eat_signals(CPUState *env)
 369 {
 370     struct timespec ts = { 0, 0 };
 371     siginfo_t siginfo;
 372     sigset_t waitset;
 373     sigset_t chkset;
 374     int r;
 375
 376     sigemptyset(&waitset);
 377     sigaddset(&waitset, SIG_IPI);
 378     sigaddset(&waitset, SIGBUS);
 379
 380     do {
 381         r = sigtimedwait(&waitset, &siginfo, &ts);
 382         if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
 383             perror("sigtimedwait");
 384             exit(1);
 385         }
 386
 387         switch (r) {
 388         case SIGBUS:
 389             if (kvm_on_sigbus_vcpu(env, siginfo.si_code, siginfo.si_addr)) {
 390                 sigbus_reraise();
 391             }
 392             break;
 393         default:
 394             break;
 395         }
 396
 397         r = sigpending(&chkset);
 398         if (r == -1) {
 399             perror("sigpending");
 400             exit(1);
 401         }
 402     } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS));
 403
 404 #ifndef CONFIG_IOTHREAD
 405     if (sigismember(&chkset, SIGIO) || sigismember(&chkset, SIGALRM)) {
 406         qemu_notify_event();
 407     }
 408 #endif
 409 }
 410
 411 #else /* _WIN32 */
 412
 413 HANDLE qemu_event_handle;
 414
 415 static void dummy_event_handler(void *opaque)
 416 {
 417 }
 418
 419 static int qemu_event_init(void)
 420 {
 421     qemu_event_handle = CreateEvent(NULL, FALSE, FALSE, NULL);
 422     if (!qemu_event_handle) {
 423         fprintf(stderr, "Failed CreateEvent: %ld\n", GetLastError());
 424         return -1;
 425     }
 426     qemu_add_wait_object(qemu_event_handle, dummy_event_handler, NULL);
 427     return 0;
 428 }
 429
 430 static void qemu_event_increment(void)
 431 {
 432     if (!SetEvent(qemu_event_handle)) {
 433         fprintf(stderr, "qemu_event_increment: SetEvent failed: %ld\n",
 434                 GetLastError());
 435         exit (1);
 436     }
 437 }
 438
 439 static void qemu_kvm_eat_signals(CPUState *env)
 440 {
 441 }
 442 #endif /* _WIN32 */
 443
 444 #ifndef CONFIG_IOTHREAD
 445 static void qemu_kvm_init_cpu_signals(CPUState *env)
 446 {
 447 #if 0 /* Causes regressions: autotest WinXP.64 migrate.tcp */
 448 #ifndef _WIN32
 449     int r;
 450     sigset_t set;
 451     struct sigaction sigact;
 452
 453     memset(&sigact, 0, sizeof(sigact));
 454     sigact.sa_handler = dummy_signal;
 455     sigaction(SIG_IPI, &sigact, NULL);
 456
 457     sigemptyset(&set);
 458     sigaddset(&set, SIG_IPI);
 459     sigaddset(&set, SIGIO);
 460     sigaddset(&set, SIGALRM);
 461     pthread_sigmask(SIG_BLOCK, &set, NULL);
 462
 463     pthread_sigmask(SIG_BLOCK, NULL, &set);
 464     sigdelset(&set, SIG_IPI);
 465     sigdelset(&set, SIGBUS);
 466     sigdelset(&set, SIGIO);
 467     sigdelset(&set, SIGALRM);
 468     r = kvm_set_signal_mask(env, &set);
 469     if (r) {
 470         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 471         exit(1);
 472     }
 473 #endif
 474 #endif
 475 }
 476
 477 #ifndef _WIN32
 478 static sigset_t block_synchronous_signals(void)
 479 {
 480     sigset_t set;
 481
 482     sigemptyset(&set);
 483     sigaddset(&set, SIGBUS);
 484     if (kvm_enabled()) {
 485         /*
 486          * We need to process timer signals synchronously to avoid a race
 487          * between exit_request check and KVM vcpu entry.
 488          */
 489         sigaddset(&set, SIGIO);
 490         sigaddset(&set, SIGALRM);
 491     }
 492
 493     return set;
 494 }
 495 #endif
 496
 497 int qemu_init_main_loop(void)
 498 {
 499 #ifndef _WIN32
 500     sigset_t blocked_signals;
 501     int ret;
 502
 503     blocked_signals = block_synchronous_signals();
 504
 505     ret = qemu_signalfd_init(blocked_signals);
 506     if (ret) {
 507         return ret;
 508     }
 509 #endif
 510
 511     qemu_init_sigbus();
 512
 513     return qemu_event_init();
 514 }
 515
 516 void qemu_main_loop_start(void)
 517 {
 518 }
 519
 520 void qemu_init_vcpu(void *_env)
 521 {
 522     CPUState *env = _env;
 523     int r;
 524
 525     env->nr_cores = smp_cores;
 526     env->nr_threads = smp_threads;
 527
 528     if (kvm_enabled()) {
 529         r = kvm_init_vcpu(env);
 530         if (r < 0) {
 531             fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 532             exit(1);
 533         }
 534         qemu_kvm_init_cpu_signals(env);
 535     }
 536 }
 537
 538 int qemu_cpu_self(void *env)
 539 {
 540     return 1;
 541 }
 542
 543 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 544 {
 545     func(data);
 546 }
 547
 548 void resume_all_vcpus(void)
 549 {
 550 }
 551
 552 void pause_all_vcpus(void)
 553 {
 554 }
 555
 556 void qemu_cpu_kick(void *env)
 557 {
 558 }
 559
 560 void qemu_cpu_kick_self(void)
 561 {
 562 #ifndef _WIN32
 563     assert(cpu_single_env);
 564
 565     raise(SIG_IPI);
 566 #else
 567     abort();
 568 #endif
 569 }
 570
 571 void qemu_notify_event(void)
 572 {
 573     CPUState *env = cpu_single_env;
 574
 575     qemu_event_increment ();
 576     if (env) {
 577         cpu_exit(env);
 578     }
 579     if (next_cpu && env != next_cpu) {
 580         cpu_exit(next_cpu);
 581     }
 582     exit_request = 1;
 583 }
 584
 585 #if defined(OBSOLETE_KVM_IMPL) || !defined(CONFIG_KVM)
 586 void qemu_mutex_lock_iothread(void) {}
 587 void qemu_mutex_unlock_iothread(void) {}
 588 #endif
 589
 590 void cpu_stop_current(void)
 591 {
 592 }
 593
 594 void vm_stop(int reason)
 595 {
 596     do_vm_stop(reason);
 597 }
 598
 599 #else /* CONFIG_IOTHREAD */
 600
 601 #include "qemu-thread.h"
 602
 603 QemuMutex qemu_global_mutex;
 604 static QemuMutex qemu_fair_mutex;
 605
 606 static QemuThread io_thread;
 607
 608 static QemuThread *tcg_cpu_thread;
 609 static QemuCond *tcg_halt_cond;
 610
 611 static int qemu_system_ready;
 612 /* cpu creation */
 613 static QemuCond qemu_cpu_cond;
 614 /* system init */
 615 static QemuCond qemu_system_cond;
 616 static QemuCond qemu_pause_cond;
 617 static QemuCond qemu_work_cond;
 618
 619 static void cpu_signal(int sig)
 620 {
 621     if (cpu_single_env) {
 622         cpu_exit(cpu_single_env);
 623     }
 624     exit_request = 1;
 625 }
 626
 627 static void qemu_kvm_init_cpu_signals(CPUState *env)
 628 {
 629     int r;
 630     sigset_t set;
 631     struct sigaction sigact;
 632
 633     memset(&sigact, 0, sizeof(sigact));
 634     sigact.sa_handler = dummy_signal;
 635     sigaction(SIG_IPI, &sigact, NULL);
 636
 637     pthread_sigmask(SIG_BLOCK, NULL, &set);
 638     sigdelset(&set, SIG_IPI);
 639     sigdelset(&set, SIGBUS);
 640     r = kvm_set_signal_mask(env, &set);
 641     if (r) {
 642         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 643         exit(1);
 644     }
 645 }
 646
 647 static void qemu_tcg_init_cpu_signals(void)
 648 {
 649     sigset_t set;
 650     struct sigaction sigact;
 651
 652     memset(&sigact, 0, sizeof(sigact));
 653     sigact.sa_handler = cpu_signal;
 654     sigaction(SIG_IPI, &sigact, NULL);
 655
 656     sigemptyset(&set);
 657     sigaddset(&set, SIG_IPI);
 658     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 659 }
 660
 661 static sigset_t block_io_signals(void)
 662 {
 663     sigset_t set;
 664
 665     /* SIGUSR2 used by posix-aio-compat.c */
 666     sigemptyset(&set);
 667     sigaddset(&set, SIGUSR2);
 668     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 669
 670     sigemptyset(&set);
 671     sigaddset(&set, SIGIO);
 672     sigaddset(&set, SIGALRM);
 673     sigaddset(&set, SIG_IPI);
 674     sigaddset(&set, SIGBUS);
 675     pthread_sigmask(SIG_BLOCK, &set, NULL);
 676
 677     return set;
 678 }
 679
 680 int qemu_init_main_loop(void)
 681 {
 682     int ret;
 683     sigset_t blocked_signals;
 684
 685     qemu_init_sigbus();
 686
 687     blocked_signals = block_io_signals();
 688
 689     ret = qemu_signalfd_init(blocked_signals);
 690     if (ret) {
 691         return ret;
 692     }
 693
 694     /* Note eventfd must be drained before signalfd handlers run */
 695     ret = qemu_event_init();
 696     if (ret) {
 697         return ret;
 698     }
 699
 700     qemu_cond_init(&qemu_cpu_cond);
 701     qemu_cond_init(&qemu_system_cond);
 702     qemu_cond_init(&qemu_pause_cond);
 703     qemu_cond_init(&qemu_work_cond);
 704     qemu_mutex_init(&qemu_fair_mutex);
 705     qemu_mutex_init(&qemu_global_mutex);
 706     qemu_mutex_lock(&qemu_global_mutex);
 707
 708     qemu_thread_self(&io_thread);
 709
 710     return 0;
 711 }
 712
 713 void qemu_main_loop_start(void)
 714 {
 715     qemu_system_ready = 1;
 716     qemu_cond_broadcast(&qemu_system_cond);
 717 }
 718
 719 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 720 {
 721     struct qemu_work_item wi;
 722
 723     if (qemu_cpu_self(env)) {
 724         func(data);
 725         return;
 726     }
 727
 728     wi.func = func;
 729     wi.data = data;
 730     if (!env->queued_work_first) {
 731         env->queued_work_first = &wi;
 732     } else {
 733         env->queued_work_last->next = &wi;
 734     }
 735     env->queued_work_last = &wi;
 736     wi.next = NULL;
 737     wi.done = false;
 738
 739     qemu_cpu_kick(env);
 740     while (!wi.done) {
 741         CPUState *self_env = cpu_single_env;
 742
 743         qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
 744         cpu_single_env = self_env;
 745     }
 746 }
 747
 748 static void flush_queued_work(CPUState *env)
 749 {
 750     struct qemu_work_item *wi;
 751
 752     if (!env->queued_work_first) {
 753         return;
 754     }
 755
 756     while ((wi = env->queued_work_first)) {
 757         env->queued_work_first = wi->next;
 758         wi->func(wi->data);
 759         wi->done = true;
 760     }
 761     env->queued_work_last = NULL;
 762     qemu_cond_broadcast(&qemu_work_cond);
 763 }
 764
 765 static void qemu_wait_io_event_common(CPUState *env)
 766 {
 767     if (env->stop) {
 768         env->stop = 0;
 769         env->stopped = 1;
 770         qemu_cond_signal(&qemu_pause_cond);
 771     }
 772     flush_queued_work(env);
 773     env->thread_kicked = false;
 774 }
 775
 776 static void qemu_tcg_wait_io_event(void)
 777 {
 778     CPUState *env;
 779
 780     while (all_cpu_threads_idle()) {
 781         qemu_cond_timedwait(tcg_halt_cond, &qemu_global_mutex, 1000);
 782     }
 783
 784     qemu_mutex_unlock(&qemu_global_mutex);
 785
 786     /*
 787      * Users of qemu_global_mutex can be starved, having no chance
 788      * to acquire it since this path will get to it first.
 789      * So use another lock to provide fairness.
 790      */
 791     qemu_mutex_lock(&qemu_fair_mutex);
 792     qemu_mutex_unlock(&qemu_fair_mutex);
 793
 794     qemu_mutex_lock(&qemu_global_mutex);
 795
 796     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 797         qemu_wait_io_event_common(env);
 798     }
 799 }
 800
 801 static void qemu_kvm_wait_io_event(CPUState *env)
 802 {
 803     while (cpu_thread_is_idle(env)) {
 804         qemu_cond_timedwait(env->halt_cond, &qemu_global_mutex, 1000);
 805     }
 806
 807     qemu_kvm_eat_signals(env);
 808     qemu_wait_io_event_common(env);
 809 }
 810
 811 static void *qemu_kvm_cpu_thread_fn(void *arg)
 812 {
 813     CPUState *env = arg;
 814     int r;
 815
 816     qemu_mutex_lock(&qemu_global_mutex);
 817     qemu_thread_self(env->thread);
 818
 819     r = kvm_init_vcpu(env);
 820     if (r < 0) {
 821         fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 822         exit(1);
 823     }
 824
 825     qemu_kvm_init_cpu_signals(env);
 826
 827     /* signal CPU creation */
 828     env->created = 1;
 829     qemu_cond_signal(&qemu_cpu_cond);
 830
 831     /* and wait for machine initialization */
 832     while (!qemu_system_ready) {
 833         qemu_cond_timedwait(&qemu_system_cond, &qemu_global_mutex, 100);
 834     }
 835
 836     while (1) {
 837         if (cpu_can_run(env)) {
 838             r = kvm_cpu_exec(env);
 839             if (r == EXCP_DEBUG) {
 840                 cpu_handle_debug_exception(env);
 841             }
 842         }
 843         qemu_kvm_wait_io_event(env);
 844     }
 845
 846     return NULL;
 847 }
 848
 849 static void *qemu_tcg_cpu_thread_fn(void *arg)
 850 {
 851     CPUState *env = arg;
 852
 853     qemu_tcg_init_cpu_signals();
 854     qemu_thread_self(env->thread);
 855
 856     /* signal CPU creation */
 857     qemu_mutex_lock(&qemu_global_mutex);
 858     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 859         env->created = 1;
 860     }
 861     qemu_cond_signal(&qemu_cpu_cond);
 862
 863     /* and wait for machine initialization */
 864     while (!qemu_system_ready) {
 865         qemu_cond_timedwait(&qemu_system_cond, &qemu_global_mutex, 100);
 866     }
 867
 868     while (1) {
 869         cpu_exec_all();
 870         qemu_tcg_wait_io_event();
 871     }
 872
 873     return NULL;
 874 }
 875
 876 void qemu_cpu_kick(void *_env)
 877 {
 878     CPUState *env = _env;
 879
 880     qemu_cond_broadcast(env->halt_cond);
 881     if (!env->thread_kicked) {
 882         qemu_thread_signal(env->thread, SIG_IPI);
 883         env->thread_kicked = true;
 884     }
 885 }
 886
 887 void qemu_cpu_kick_self(void)
 888 {
 889     assert(cpu_single_env);
 890
 891     if (!cpu_single_env->thread_kicked) {
 892         qemu_thread_signal(cpu_single_env->thread, SIG_IPI);
 893         cpu_single_env->thread_kicked = true;
 894     }
 895 }
 896
 897 int qemu_cpu_self(void *_env)
 898 {
 899     CPUState *env = _env;
 900     QemuThread this;
 901
 902     qemu_thread_self(&this);
 903
 904     return qemu_thread_equal(&this, env->thread);
 905 }
 906
 907 void qemu_mutex_lock_iothread(void)
 908 {
 909     if (kvm_enabled()) {
 910         qemu_mutex_lock(&qemu_global_mutex);
 911     } else {
 912         qemu_mutex_lock(&qemu_fair_mutex);
 913         if (qemu_mutex_trylock(&qemu_global_mutex)) {
 914             qemu_thread_signal(tcg_cpu_thread, SIG_IPI);
 915             qemu_mutex_lock(&qemu_global_mutex);
 916         }
 917         qemu_mutex_unlock(&qemu_fair_mutex);
 918     }
 919 }
 920
 921 void qemu_mutex_unlock_iothread(void)
 922 {
 923     qemu_mutex_unlock(&qemu_global_mutex);
 924 }
 925
 926 static int all_vcpus_paused(void)
 927 {
 928     CPUState *penv = first_cpu;
 929
 930     while (penv) {
 931         if (!penv->stopped) {
 932             return 0;
 933         }
 934         penv = (CPUState *)penv->next_cpu;
 935     }
 936
 937     return 1;
 938 }
 939
 940 void pause_all_vcpus(void)
 941 {
 942     CPUState *penv = first_cpu;
 943
 944     while (penv) {
 945         penv->stop = 1;
 946         qemu_cpu_kick(penv);
 947         penv = (CPUState *)penv->next_cpu;
 948     }
 949
 950     while (!all_vcpus_paused()) {
 951         qemu_cond_timedwait(&qemu_pause_cond, &qemu_global_mutex, 100);
 952         penv = first_cpu;
 953         while (penv) {
 954             qemu_cpu_kick(penv);
 955             penv = (CPUState *)penv->next_cpu;
 956         }
 957     }
 958 }
 959
 960 void resume_all_vcpus(void)
 961 {
 962     CPUState *penv = first_cpu;
 963
 964     while (penv) {
 965         penv->stop = 0;
 966         penv->stopped = 0;
 967         qemu_cpu_kick(penv);
 968         penv = (CPUState *)penv->next_cpu;
 969     }
 970 }
 971
 972 static void qemu_tcg_init_vcpu(void *_env)
 973 {
 974     CPUState *env = _env;
 975
 976     /* share a single thread for all cpus with TCG */
 977     if (!tcg_cpu_thread) {
 978         env->thread = qemu_mallocz(sizeof(QemuThread));
 979         env->halt_cond = qemu_mallocz(sizeof(QemuCond));
 980         qemu_cond_init(env->halt_cond);
 981         qemu_thread_create(env->thread, qemu_tcg_cpu_thread_fn, env);
 982         while (env->created == 0) {
 983             qemu_cond_timedwait(&qemu_cpu_cond, &qemu_global_mutex, 100);
 984         }
 985         tcg_cpu_thread = env->thread;
 986         tcg_halt_cond = env->halt_cond;
 987     } else {
 988         env->thread = tcg_cpu_thread;
 989         env->halt_cond = tcg_halt_cond;
 990     }
 991 }
 992
 993 static void qemu_kvm_start_vcpu(CPUState *env)
 994 {
 995     env->thread = qemu_mallocz(sizeof(QemuThread));
 996     env->halt_cond = qemu_mallocz(sizeof(QemuCond));
 997     qemu_cond_init(env->halt_cond);
 998     qemu_thread_create(env->thread, qemu_kvm_cpu_thread_fn, env);
 999     while (env->created == 0) {
1000         qemu_cond_timedwait(&qemu_cpu_cond, &qemu_global_mutex, 100);
1001     }
1002 }
1003
1004 void qemu_init_vcpu(void *_env)
1005 {
1006     CPUState *env = _env;
1007
1008     env->nr_cores = smp_cores;
1009     env->nr_threads = smp_threads;
1010     if (kvm_enabled()) {
1011         qemu_kvm_start_vcpu(env);
1012     } else {
1013         qemu_tcg_init_vcpu(env);
1014     }
1015 }
1016
1017 void qemu_notify_event(void)
1018 {
1019     qemu_event_increment();
1020 }
1021
1022 void cpu_stop_current(void)
1023 {
1024     if (cpu_single_env) {
1025         cpu_single_env->stopped = 1;
1026         cpu_exit(cpu_single_env);
1027     }
1028 }
1029
1030 void vm_stop(int reason)
1031 {
1032     QemuThread me;
1033     qemu_thread_self(&me);
1034
1035     if (!qemu_thread_equal(&me, &io_thread)) {
1036         qemu_system_vmstop_request(reason);
1037         /*
1038          * FIXME: should not return to device code in case
1039          * vm_stop() has been requested.
1040          */
1041         cpu_stop_current();
1042         return;
1043     }
1044     do_vm_stop(reason);
1045 }
1046
1047 #endif
1048
1049 static int tcg_cpu_exec(CPUState *env)
1050 {
1051     int ret;
1052 #ifdef CONFIG_PROFILER
1053     int64_t ti;
1054 #endif
1055
1056 #ifdef CONFIG_PROFILER
1057     ti = profile_getclock();
1058 #endif
1059     if (use_icount) {
1060         int64_t count;
1061         int decr;
1062         qemu_icount -= (env->icount_decr.u16.low + env->icount_extra);
1063         env->icount_decr.u16.low = 0;
1064         env->icount_extra = 0;
1065         count = qemu_icount_round (qemu_next_deadline());
1066         qemu_icount += count;
1067         decr = (count > 0xffff) ? 0xffff : count;
1068         count -= decr;
1069         env->icount_decr.u16.low = decr;
1070         env->icount_extra = count;
1071     }
1072     ret = cpu_exec(env);
1073 #ifdef CONFIG_PROFILER
1074     qemu_time += profile_getclock() - ti;
1075 #endif
1076     if (use_icount) {
1077         /* Fold pending instructions back into the
1078            instruction counter, and clear the interrupt flag.  */
1079         qemu_icount -= (env->icount_decr.u16.low
1080                         + env->icount_extra);
1081         env->icount_decr.u32 = 0;
1082         env->icount_extra = 0;
1083     }
1084     return ret;
1085 }
1086
1087 bool cpu_exec_all(void)
1088 {
1089     int r;
1090
1091     if (next_cpu == NULL) {
1092         next_cpu = first_cpu;
1093     }
1094     for (; next_cpu != NULL && !exit_request; next_cpu = next_cpu->next_cpu) {
1095         CPUState *env = next_cpu;
1096
1097         qemu_clock_enable(vm_clock,
1098                           (env->singlestep_enabled & SSTEP_NOTIMER) == 0);
1099
1100         if (qemu_alarm_pending()) {
1101             break;
1102         }
1103         if (cpu_can_run(env)) {
1104             if (kvm_enabled()) {
1105                 r = kvm_cpu_exec(env);
1106                 qemu_kvm_eat_signals(env);
1107             } else {
1108                 r = tcg_cpu_exec(env);
1109             }
1110             if (r == EXCP_DEBUG) {
1111                 cpu_handle_debug_exception(env);
1112                 break;
1113             }
1114         } else if (env->stop) {
1115             break;
1116         }
1117     }
1118     exit_request = 0;
1119     return !all_cpu_threads_idle();
1120 }
1121
1122 void set_numa_modes(void)
1123 {
1124     CPUState *env;
1125     int i;
1126
1127     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1128         for (i = 0; i < nb_numa_nodes; i++) {
1129             if (node_cpumask[i] & (1 << env->cpu_index)) {
1130                 env->numa_node = i;
1131             }
1132         }
1133     }
1134 }
1135
1136 void set_cpu_log(const char *optarg)
1137 {
1138     int mask;
1139     const CPULogItem *item;
1140
1141     mask = cpu_str_to_log_mask(optarg);
1142     if (!mask) {
1143         printf("Log items (comma separated):\n");
1144         for (item = cpu_log_items; item->mask != 0; item++) {
1145             printf("%-10s %s\n", item->name, item->help);
1146         }
1147         exit(1);
1148     }
1149     cpu_set_log(mask);
1150 }
1151
1152 /* Return the virtual CPU time, based on the instruction counter.  */
1153 int64_t cpu_get_icount(void)
1154 {
1155     int64_t icount;
1156     CPUState *env = cpu_single_env;;
1157
1158     icount = qemu_icount;
1159     if (env) {
1160         if (!can_do_io(env)) {
1161             fprintf(stderr, "Bad clock read\n");
1162         }
1163         icount -= (env->icount_decr.u16.low + env->icount_extra);
1164     }
1165     return qemu_icount_bias + (icount << icount_time_shift);
1166 }
1167
1168 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
1169 {
1170     /* XXX: implement xxx_cpu_list for targets that still miss it */
1171 #if defined(cpu_list_id)
1172     cpu_list_id(f, cpu_fprintf, optarg);
1173 #elif defined(cpu_list)
1174     cpu_list(f, cpu_fprintf); /* deprecated */
1175 #endif
1176 }