cpus.c

   1 /*
   2  * QEMU System Emulator
   3  *
   4  * Copyright (c) 2003-2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 /* Needed early for CONFIG_BSD etc. */
  26 #include "config-host.h"
  27
  28 #include "monitor.h"
  29 #include "sysemu.h"
  30 #include "gdbstub.h"
  31 #include "dma.h"
  32 #include "kvm.h"
  33
  34 #include "qemu-thread.h"
  35 #include "cpus.h"
  36 #include "compatfd.h"
  37
  38 #ifdef SIGRTMIN
  39 #define SIG_IPI (SIGRTMIN+4)
  40 #else
  41 #define SIG_IPI SIGUSR1
  42 #endif
  43
  44 #ifdef CONFIG_LINUX
  45
  46 #include <sys/prctl.h>
  47
  48 #ifndef PR_MCE_KILL
  49 #define PR_MCE_KILL 33
  50 #endif
  51
  52 #ifndef PR_MCE_KILL_SET
  53 #define PR_MCE_KILL_SET 1
  54 #endif
  55
  56 #ifndef PR_MCE_KILL_EARLY
  57 #define PR_MCE_KILL_EARLY 1
  58 #endif
  59
  60 #endif /* CONFIG_LINUX */
  61
  62 static CPUState *next_cpu;
  63
  64 /***********************************************************/
  65 void hw_error(const char *fmt, ...)
  66 {
  67     va_list ap;
  68     CPUState *env;
  69
  70     va_start(ap, fmt);
  71     fprintf(stderr, "qemu: hardware error: ");
  72     vfprintf(stderr, fmt, ap);
  73     fprintf(stderr, "\n");
  74     for(env = first_cpu; env != NULL; env = env->next_cpu) {
  75         fprintf(stderr, "CPU #%d:\n", env->cpu_index);
  76 #ifdef TARGET_I386
  77         cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU);
  78 #else
  79         cpu_dump_state(env, stderr, fprintf, 0);
  80 #endif
  81     }
  82     va_end(ap);
  83     abort();
  84 }
  85
  86 void cpu_synchronize_all_states(void)
  87 {
  88     CPUState *cpu;
  89
  90     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
  91         cpu_synchronize_state(cpu);
  92     }
  93 }
  94
  95 void cpu_synchronize_all_post_reset(void)
  96 {
  97     CPUState *cpu;
  98
  99     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 100         cpu_synchronize_post_reset(cpu);
 101     }
 102 }
 103
 104 void cpu_synchronize_all_post_init(void)
 105 {
 106     CPUState *cpu;
 107
 108     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 109         cpu_synchronize_post_init(cpu);
 110     }
 111 }
 112
 113 int cpu_is_stopped(CPUState *env)
 114 {
 115     return !vm_running || env->stopped;
 116 }
 117
 118 static void do_vm_stop(int reason)
 119 {
 120     if (vm_running) {
 121         cpu_disable_ticks();
 122         vm_running = 0;
 123         pause_all_vcpus();
 124         vm_state_notify(0, reason);
 125         qemu_aio_flush();
 126         bdrv_flush_all();
 127         monitor_protocol_event(QEVENT_STOP, NULL);
 128     }
 129 }
 130
 131 static int cpu_can_run(CPUState *env)
 132 {
 133     if (env->stop) {
 134         return 0;
 135     }
 136     if (env->stopped || !vm_running) {
 137         return 0;
 138     }
 139     return 1;
 140 }
 141
 142 static bool cpu_thread_is_idle(CPUState *env)
 143 {
 144     if (env->stop || env->queued_work_first) {
 145         return false;
 146     }
 147     if (env->stopped || !vm_running) {
 148         return true;
 149     }
 150     if (!env->halted || qemu_cpu_has_work(env) ||
 151         (kvm_enabled() && kvm_irqchip_in_kernel())) {
 152         return false;
 153     }
 154     return true;
 155 }
 156
 157 bool all_cpu_threads_idle(void)
 158 {
 159     CPUState *env;
 160
 161     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 162         if (!cpu_thread_is_idle(env)) {
 163             return false;
 164         }
 165     }
 166     return true;
 167 }
 168
 169 static void cpu_handle_guest_debug(CPUState *env)
 170 {
 171     gdb_set_stop_cpu(env);
 172     qemu_system_debug_request();
 173 #ifdef CONFIG_IOTHREAD
 174     env->stopped = 1;
 175 #endif
 176 }
 177
 178 #ifdef CONFIG_IOTHREAD
 179 static void cpu_signal(int sig)
 180 {
 181     if (cpu_single_env) {
 182         cpu_exit(cpu_single_env);
 183     }
 184     exit_request = 1;
 185 }
 186 #endif
 187
 188 #ifdef CONFIG_LINUX
 189 static void sigbus_reraise(void)
 190 {
 191     sigset_t set;
 192     struct sigaction action;
 193
 194     memset(&action, 0, sizeof(action));
 195     action.sa_handler = SIG_DFL;
 196     if (!sigaction(SIGBUS, &action, NULL)) {
 197         raise(SIGBUS);
 198         sigemptyset(&set);
 199         sigaddset(&set, SIGBUS);
 200         sigprocmask(SIG_UNBLOCK, &set, NULL);
 201     }
 202     perror("Failed to re-raise SIGBUS!\n");
 203     abort();
 204 }
 205
 206 static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo,
 207                            void *ctx)
 208 {
 209     if (kvm_on_sigbus(siginfo->ssi_code,
 210                       (void *)(intptr_t)siginfo->ssi_addr)) {
 211         sigbus_reraise();
 212     }
 213 }
 214
 215 static void qemu_init_sigbus(void)
 216 {
 217     struct sigaction action;
 218
 219     memset(&action, 0, sizeof(action));
 220     action.sa_flags = SA_SIGINFO;
 221     action.sa_sigaction = (void (*)(int, siginfo_t*, void*))sigbus_handler;
 222     sigaction(SIGBUS, &action, NULL);
 223
 224     prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
 225 }
 226
 227 static void qemu_kvm_eat_signals(CPUState *env)
 228 {
 229     struct timespec ts = { 0, 0 };
 230     siginfo_t siginfo;
 231     sigset_t waitset;
 232     sigset_t chkset;
 233     int r;
 234
 235     sigemptyset(&waitset);
 236     sigaddset(&waitset, SIG_IPI);
 237     sigaddset(&waitset, SIGBUS);
 238
 239     do {
 240         r = sigtimedwait(&waitset, &siginfo, &ts);
 241         if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
 242             perror("sigtimedwait");
 243             exit(1);
 244         }
 245
 246         switch (r) {
 247         case SIGBUS:
 248             if (kvm_on_sigbus_vcpu(env, siginfo.si_code, siginfo.si_addr)) {
 249                 sigbus_reraise();
 250             }
 251             break;
 252         default:
 253             break;
 254         }
 255
 256         r = sigpending(&chkset);
 257         if (r == -1) {
 258             perror("sigpending");
 259             exit(1);
 260         }
 261     } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS));
 262
 263 #ifndef CONFIG_IOTHREAD
 264     if (sigismember(&chkset, SIGIO) || sigismember(&chkset, SIGALRM)) {
 265         qemu_notify_event();
 266     }
 267 #endif
 268 }
 269
 270 #else /* !CONFIG_LINUX */
 271
 272 static void qemu_init_sigbus(void)
 273 {
 274 }
 275
 276 static void qemu_kvm_eat_signals(CPUState *env)
 277 {
 278 }
 279 #endif /* !CONFIG_LINUX */
 280
 281 #ifndef _WIN32
 282 static int io_thread_fd = -1;
 283
 284 static void qemu_event_increment(void)
 285 {
 286     /* Write 8 bytes to be compatible with eventfd.  */
 287     static const uint64_t val = 1;
 288     ssize_t ret;
 289
 290     if (io_thread_fd == -1) {
 291         return;
 292     }
 293     do {
 294         ret = write(io_thread_fd, &val, sizeof(val));
 295     } while (ret < 0 && errno == EINTR);
 296
 297     /* EAGAIN is fine, a read must be pending.  */
 298     if (ret < 0 && errno != EAGAIN) {
 299         fprintf(stderr, "qemu_event_increment: write() failed: %s\n",
 300                 strerror(errno));
 301         exit (1);
 302     }
 303 }
 304
 305 static void qemu_event_read(void *opaque)
 306 {
 307     int fd = (intptr_t)opaque;
 308     ssize_t len;
 309     char buffer[512];
 310
 311     /* Drain the notify pipe.  For eventfd, only 8 bytes will be read.  */
 312     do {
 313         len = read(fd, buffer, sizeof(buffer));
 314     } while ((len == -1 && errno == EINTR) || len == sizeof(buffer));
 315 }
 316
 317 static int qemu_event_init(void)
 318 {
 319     int err;
 320     int fds[2];
 321
 322     err = qemu_eventfd(fds);
 323     if (err == -1) {
 324         return -errno;
 325     }
 326     err = fcntl_setfl(fds[0], O_NONBLOCK);
 327     if (err < 0) {
 328         goto fail;
 329     }
 330     err = fcntl_setfl(fds[1], O_NONBLOCK);
 331     if (err < 0) {
 332         goto fail;
 333     }
 334     qemu_set_fd_handler2(fds[0], NULL, qemu_event_read, NULL,
 335                          (void *)(intptr_t)fds[0]);
 336
 337     io_thread_fd = fds[1];
 338     return 0;
 339
 340 fail:
 341     close(fds[0]);
 342     close(fds[1]);
 343     return err;
 344 }
 345
 346 static void dummy_signal(int sig)
 347 {
 348 }
 349
 350 /* If we have signalfd, we mask out the signals we want to handle and then
 351  * use signalfd to listen for them.  We rely on whatever the current signal
 352  * handler is to dispatch the signals when we receive them.
 353  */
 354 static void sigfd_handler(void *opaque)
 355 {
 356     int fd = (intptr_t)opaque;
 357     struct qemu_signalfd_siginfo info;
 358     struct sigaction action;
 359     ssize_t len;
 360
 361     while (1) {
 362         do {
 363             len = read(fd, &info, sizeof(info));
 364         } while (len == -1 && errno == EINTR);
 365
 366         if (len == -1 && errno == EAGAIN) {
 367             break;
 368         }
 369
 370         if (len != sizeof(info)) {
 371             printf("read from sigfd returned %zd: %m\n", len);
 372             return;
 373         }
 374
 375         sigaction(info.ssi_signo, NULL, &action);
 376         if ((action.sa_flags & SA_SIGINFO) && action.sa_sigaction) {
 377             action.sa_sigaction(info.ssi_signo,
 378                                 (siginfo_t *)&info, NULL);
 379         } else if (action.sa_handler) {
 380             action.sa_handler(info.ssi_signo);
 381         }
 382     }
 383 }
 384
 385 static int qemu_signal_init(void)
 386 {
 387     int sigfd;
 388     sigset_t set;
 389
 390 #ifdef CONFIG_IOTHREAD
 391     /* SIGUSR2 used by posix-aio-compat.c */
 392     sigemptyset(&set);
 393     sigaddset(&set, SIGUSR2);
 394     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 395
 396     sigemptyset(&set);
 397     sigaddset(&set, SIGIO);
 398     sigaddset(&set, SIGALRM);
 399     sigaddset(&set, SIG_IPI);
 400     sigaddset(&set, SIGBUS);
 401     pthread_sigmask(SIG_BLOCK, &set, NULL);
 402 #else
 403     sigemptyset(&set);
 404     sigaddset(&set, SIGBUS);
 405     if (kvm_enabled()) {
 406         /*
 407          * We need to process timer signals synchronously to avoid a race
 408          * between exit_request check and KVM vcpu entry.
 409          */
 410         sigaddset(&set, SIGIO);
 411         sigaddset(&set, SIGALRM);
 412     }
 413 #endif
 414
 415     sigfd = qemu_signalfd(&set);
 416     if (sigfd == -1) {
 417         fprintf(stderr, "failed to create signalfd\n");
 418         return -errno;
 419     }
 420
 421     fcntl_setfl(sigfd, O_NONBLOCK);
 422
 423     qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
 424                          (void *)(intptr_t)sigfd);
 425
 426     return 0;
 427 }
 428
 429 static void qemu_kvm_init_cpu_signals(CPUState *env)
 430 {
 431     int r;
 432     sigset_t set;
 433     struct sigaction sigact;
 434
 435     memset(&sigact, 0, sizeof(sigact));
 436     sigact.sa_handler = dummy_signal;
 437     sigaction(SIG_IPI, &sigact, NULL);
 438
 439 #ifdef CONFIG_IOTHREAD
 440     pthread_sigmask(SIG_BLOCK, NULL, &set);
 441     sigdelset(&set, SIG_IPI);
 442     sigdelset(&set, SIGBUS);
 443     r = kvm_set_signal_mask(env, &set);
 444     if (r) {
 445         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 446         exit(1);
 447     }
 448 #else
 449     sigemptyset(&set);
 450     sigaddset(&set, SIG_IPI);
 451     sigaddset(&set, SIGIO);
 452     sigaddset(&set, SIGALRM);
 453     pthread_sigmask(SIG_BLOCK, &set, NULL);
 454
 455     pthread_sigmask(SIG_BLOCK, NULL, &set);
 456     sigdelset(&set, SIGIO);
 457     sigdelset(&set, SIGALRM);
 458 #endif
 459     sigdelset(&set, SIG_IPI);
 460     sigdelset(&set, SIGBUS);
 461     r = kvm_set_signal_mask(env, &set);
 462     if (r) {
 463         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 464         exit(1);
 465     }
 466 }
 467
 468 static void qemu_tcg_init_cpu_signals(void)
 469 {
 470 #ifdef CONFIG_IOTHREAD
 471     sigset_t set;
 472     struct sigaction sigact;
 473
 474     memset(&sigact, 0, sizeof(sigact));
 475     sigact.sa_handler = cpu_signal;
 476     sigaction(SIG_IPI, &sigact, NULL);
 477
 478     sigemptyset(&set);
 479     sigaddset(&set, SIG_IPI);
 480     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 481 #endif
 482 }
 483
 484 #else /* _WIN32 */
 485
 486 HANDLE qemu_event_handle;
 487
 488 static void dummy_event_handler(void *opaque)
 489 {
 490 }
 491
 492 static int qemu_event_init(void)
 493 {
 494     qemu_event_handle = CreateEvent(NULL, FALSE, FALSE, NULL);
 495     if (!qemu_event_handle) {
 496         fprintf(stderr, "Failed CreateEvent: %ld\n", GetLastError());
 497         return -1;
 498     }
 499     qemu_add_wait_object(qemu_event_handle, dummy_event_handler, NULL);
 500     return 0;
 501 }
 502
 503 static void qemu_event_increment(void)
 504 {
 505     if (!SetEvent(qemu_event_handle)) {
 506         fprintf(stderr, "qemu_event_increment: SetEvent failed: %ld\n",
 507                 GetLastError());
 508         exit (1);
 509     }
 510 }
 511
 512 static int qemu_signal_init(void)
 513 {
 514     return 0;
 515 }
 516
 517 static void qemu_kvm_init_cpu_signals(CPUState *env)
 518 {
 519     abort();
 520 }
 521
 522 static void qemu_tcg_init_cpu_signals(void)
 523 {
 524 }
 525 #endif /* _WIN32 */
 526
 527 #ifndef CONFIG_IOTHREAD
 528 int qemu_init_main_loop(void)
 529 {
 530     int ret;
 531
 532     ret = qemu_signal_init();
 533     if (ret) {
 534         return ret;
 535     }
 536
 537     qemu_init_sigbus();
 538
 539     return qemu_event_init();
 540 }
 541
 542 void qemu_main_loop_start(void)
 543 {
 544 }
 545
 546 void qemu_init_vcpu(void *_env)
 547 {
 548     CPUState *env = _env;
 549     int r;
 550
 551     env->nr_cores = smp_cores;
 552     env->nr_threads = smp_threads;
 553
 554     if (kvm_enabled()) {
 555         r = kvm_init_vcpu(env);
 556         if (r < 0) {
 557             fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 558             exit(1);
 559         }
 560         qemu_kvm_init_cpu_signals(env);
 561     } else {
 562         qemu_tcg_init_cpu_signals();
 563     }
 564 }
 565
 566 int qemu_cpu_is_self(void *env)
 567 {
 568     return 1;
 569 }
 570
 571 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 572 {
 573     func(data);
 574 }
 575
 576 void resume_all_vcpus(void)
 577 {
 578 }
 579
 580 void pause_all_vcpus(void)
 581 {
 582 }
 583
 584 void qemu_cpu_kick(void *env)
 585 {
 586 }
 587
 588 void qemu_cpu_kick_self(void)
 589 {
 590 #ifndef _WIN32
 591     assert(cpu_single_env);
 592
 593     raise(SIG_IPI);
 594 #else
 595     abort();
 596 #endif
 597 }
 598
 599 void qemu_notify_event(void)
 600 {
 601     CPUState *env = cpu_single_env;
 602
 603     qemu_event_increment ();
 604     if (env) {
 605         cpu_exit(env);
 606     }
 607     if (next_cpu && env != next_cpu) {
 608         cpu_exit(next_cpu);
 609     }
 610     exit_request = 1;
 611 }
 612
 613 void qemu_mutex_lock_iothread(void) {}
 614 void qemu_mutex_unlock_iothread(void) {}
 615
 616 void cpu_stop_current(void)
 617 {
 618 }
 619
 620 void vm_stop(int reason)
 621 {
 622     do_vm_stop(reason);
 623 }
 624
 625 #else /* CONFIG_IOTHREAD */
 626
 627 QemuMutex qemu_global_mutex;
 628 static QemuMutex qemu_fair_mutex;
 629
 630 static QemuThread io_thread;
 631
 632 static QemuThread *tcg_cpu_thread;
 633 static QemuCond *tcg_halt_cond;
 634
 635 static int qemu_system_ready;
 636 /* cpu creation */
 637 static QemuCond qemu_cpu_cond;
 638 /* system init */
 639 static QemuCond qemu_system_cond;
 640 static QemuCond qemu_pause_cond;
 641 static QemuCond qemu_work_cond;
 642
 643 int qemu_init_main_loop(void)
 644 {
 645     int ret;
 646
 647     qemu_init_sigbus();
 648
 649     ret = qemu_signal_init();
 650     if (ret) {
 651         return ret;
 652     }
 653
 654     /* Note eventfd must be drained before signalfd handlers run */
 655     ret = qemu_event_init();
 656     if (ret) {
 657         return ret;
 658     }
 659
 660     qemu_cond_init(&qemu_cpu_cond);
 661     qemu_cond_init(&qemu_system_cond);
 662     qemu_cond_init(&qemu_pause_cond);
 663     qemu_cond_init(&qemu_work_cond);
 664     qemu_mutex_init(&qemu_fair_mutex);
 665     qemu_mutex_init(&qemu_global_mutex);
 666     qemu_mutex_lock(&qemu_global_mutex);
 667
 668     qemu_thread_get_self(&io_thread);
 669
 670     return 0;
 671 }
 672
 673 void qemu_main_loop_start(void)
 674 {
 675     qemu_system_ready = 1;
 676     qemu_cond_broadcast(&qemu_system_cond);
 677 }
 678
 679 void run_on_cpu(CPUState *env, void (*func)(void *data), void *data)
 680 {
 681     struct qemu_work_item wi;
 682
 683     if (qemu_cpu_is_self(env)) {
 684         func(data);
 685         return;
 686     }
 687
 688     wi.func = func;
 689     wi.data = data;
 690     if (!env->queued_work_first) {
 691         env->queued_work_first = &wi;
 692     } else {
 693         env->queued_work_last->next = &wi;
 694     }
 695     env->queued_work_last = &wi;
 696     wi.next = NULL;
 697     wi.done = false;
 698
 699     qemu_cpu_kick(env);
 700     while (!wi.done) {
 701         CPUState *self_env = cpu_single_env;
 702
 703         qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
 704         cpu_single_env = self_env;
 705     }
 706 }
 707
 708 static void flush_queued_work(CPUState *env)
 709 {
 710     struct qemu_work_item *wi;
 711
 712     if (!env->queued_work_first) {
 713         return;
 714     }
 715
 716     while ((wi = env->queued_work_first)) {
 717         env->queued_work_first = wi->next;
 718         wi->func(wi->data);
 719         wi->done = true;
 720     }
 721     env->queued_work_last = NULL;
 722     qemu_cond_broadcast(&qemu_work_cond);
 723 }
 724
 725 static void qemu_wait_io_event_common(CPUState *env)
 726 {
 727     if (env->stop) {
 728         env->stop = 0;
 729         env->stopped = 1;
 730         qemu_cond_signal(&qemu_pause_cond);
 731     }
 732     flush_queued_work(env);
 733     env->thread_kicked = false;
 734 }
 735
 736 static void qemu_tcg_wait_io_event(void)
 737 {
 738     CPUState *env;
 739
 740     while (all_cpu_threads_idle()) {
 741        /* Start accounting real time to the virtual clock if the CPUs
 742           are idle.  */
 743         qemu_clock_warp(vm_clock);
 744         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
 745     }
 746
 747     qemu_mutex_unlock(&qemu_global_mutex);
 748
 749     /*
 750      * Users of qemu_global_mutex can be starved, having no chance
 751      * to acquire it since this path will get to it first.
 752      * So use another lock to provide fairness.
 753      */
 754     qemu_mutex_lock(&qemu_fair_mutex);
 755     qemu_mutex_unlock(&qemu_fair_mutex);
 756
 757     qemu_mutex_lock(&qemu_global_mutex);
 758
 759     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 760         qemu_wait_io_event_common(env);
 761     }
 762 }
 763
 764 static void qemu_kvm_wait_io_event(CPUState *env)
 765 {
 766     while (cpu_thread_is_idle(env)) {
 767         qemu_cond_wait(env->halt_cond, &qemu_global_mutex);
 768     }
 769
 770     qemu_kvm_eat_signals(env);
 771     qemu_wait_io_event_common(env);
 772 }
 773
 774 static void *qemu_kvm_cpu_thread_fn(void *arg)
 775 {
 776     CPUState *env = arg;
 777     int r;
 778
 779     qemu_mutex_lock(&qemu_global_mutex);
 780     qemu_thread_get_self(env->thread);
 781     env->thread_id = qemu_get_thread_id();
 782
 783     r = kvm_init_vcpu(env);
 784     if (r < 0) {
 785         fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 786         exit(1);
 787     }
 788
 789     qemu_kvm_init_cpu_signals(env);
 790
 791     /* signal CPU creation */
 792     env->created = 1;
 793     qemu_cond_signal(&qemu_cpu_cond);
 794
 795     /* and wait for machine initialization */
 796     while (!qemu_system_ready) {
 797         qemu_cond_wait(&qemu_system_cond, &qemu_global_mutex);
 798     }
 799
 800     while (1) {
 801         if (cpu_can_run(env)) {
 802             r = kvm_cpu_exec(env);
 803             if (r == EXCP_DEBUG) {
 804                 cpu_handle_guest_debug(env);
 805             }
 806         }
 807         qemu_kvm_wait_io_event(env);
 808     }
 809
 810     return NULL;
 811 }
 812
 813 static void *qemu_tcg_cpu_thread_fn(void *arg)
 814 {
 815     CPUState *env = arg;
 816
 817     qemu_tcg_init_cpu_signals();
 818     qemu_thread_get_self(env->thread);
 819
 820     /* signal CPU creation */
 821     qemu_mutex_lock(&qemu_global_mutex);
 822     for (env = first_cpu; env != NULL; env = env->next_cpu) {
 823         env->thread_id = qemu_get_thread_id();
 824         env->created = 1;
 825     }
 826     qemu_cond_signal(&qemu_cpu_cond);
 827
 828     /* and wait for machine initialization */
 829     while (!qemu_system_ready) {
 830         qemu_cond_wait(&qemu_system_cond, &qemu_global_mutex);
 831     }
 832
 833     while (1) {
 834         cpu_exec_all();
 835         if (use_icount && qemu_next_icount_deadline() <= 0) {
 836             qemu_notify_event();
 837         }
 838         qemu_tcg_wait_io_event();
 839     }
 840
 841     return NULL;
 842 }
 843
 844 static void qemu_cpu_kick_thread(CPUState *env)
 845 {
 846 #ifndef _WIN32
 847     int err;
 848
 849     err = pthread_kill(env->thread->thread, SIG_IPI);
 850     if (err) {
 851         fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
 852         exit(1);
 853     }
 854 #else /* _WIN32 */
 855     if (!qemu_cpu_is_self(env)) {
 856         SuspendThread(env->thread->thread);
 857         cpu_signal(0);
 858         ResumeThread(env->thread->thread);
 859     }
 860 #endif
 861 }
 862
 863 void qemu_cpu_kick(void *_env)
 864 {
 865     CPUState *env = _env;
 866
 867     qemu_cond_broadcast(env->halt_cond);
 868     if (!env->thread_kicked) {
 869         qemu_cpu_kick_thread(env);
 870         env->thread_kicked = true;
 871     }
 872 }
 873
 874 void qemu_cpu_kick_self(void)
 875 {
 876 #ifndef _WIN32
 877     assert(cpu_single_env);
 878
 879     if (!cpu_single_env->thread_kicked) {
 880         qemu_cpu_kick_thread(cpu_single_env);
 881         cpu_single_env->thread_kicked = true;
 882     }
 883 #else
 884     abort();
 885 #endif
 886 }
 887
 888 int qemu_cpu_is_self(void *_env)
 889 {
 890     CPUState *env = _env;
 891
 892     return qemu_thread_is_self(env->thread);
 893 }
 894
 895 void qemu_mutex_lock_iothread(void)
 896 {
 897     if (kvm_enabled()) {
 898         qemu_mutex_lock(&qemu_global_mutex);
 899     } else {
 900         qemu_mutex_lock(&qemu_fair_mutex);
 901         if (qemu_mutex_trylock(&qemu_global_mutex)) {
 902             qemu_cpu_kick_thread(first_cpu);
 903             qemu_mutex_lock(&qemu_global_mutex);
 904         }
 905         qemu_mutex_unlock(&qemu_fair_mutex);
 906     }
 907 }
 908
 909 void qemu_mutex_unlock_iothread(void)
 910 {
 911     qemu_mutex_unlock(&qemu_global_mutex);
 912 }
 913
 914 static int all_vcpus_paused(void)
 915 {
 916     CPUState *penv = first_cpu;
 917
 918     while (penv) {
 919         if (!penv->stopped) {
 920             return 0;
 921         }
 922         penv = (CPUState *)penv->next_cpu;
 923     }
 924
 925     return 1;
 926 }
 927
 928 void pause_all_vcpus(void)
 929 {
 930     CPUState *penv = first_cpu;
 931
 932     while (penv) {
 933         penv->stop = 1;
 934         qemu_cpu_kick(penv);
 935         penv = (CPUState *)penv->next_cpu;
 936     }
 937
 938     while (!all_vcpus_paused()) {
 939         qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex);
 940         penv = first_cpu;
 941         while (penv) {
 942             qemu_cpu_kick(penv);
 943             penv = (CPUState *)penv->next_cpu;
 944         }
 945     }
 946 }
 947
 948 void resume_all_vcpus(void)
 949 {
 950     CPUState *penv = first_cpu;
 951
 952     while (penv) {
 953         penv->stop = 0;
 954         penv->stopped = 0;
 955         qemu_cpu_kick(penv);
 956         penv = (CPUState *)penv->next_cpu;
 957     }
 958 }
 959
 960 static void qemu_tcg_init_vcpu(void *_env)
 961 {
 962     CPUState *env = _env;
 963
 964     /* share a single thread for all cpus with TCG */
 965     if (!tcg_cpu_thread) {
 966         env->thread = qemu_mallocz(sizeof(QemuThread));
 967         env->halt_cond = qemu_mallocz(sizeof(QemuCond));
 968         qemu_cond_init(env->halt_cond);
 969         qemu_thread_create(env->thread, qemu_tcg_cpu_thread_fn, env);
 970         while (env->created == 0) {
 971             qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
 972         }
 973         tcg_cpu_thread = env->thread;
 974         tcg_halt_cond = env->halt_cond;
 975     } else {
 976         env->thread = tcg_cpu_thread;
 977         env->halt_cond = tcg_halt_cond;
 978     }
 979 }
 980
 981 static void qemu_kvm_start_vcpu(CPUState *env)
 982 {
 983     env->thread = qemu_mallocz(sizeof(QemuThread));
 984     env->halt_cond = qemu_mallocz(sizeof(QemuCond));
 985     qemu_cond_init(env->halt_cond);
 986     qemu_thread_create(env->thread, qemu_kvm_cpu_thread_fn, env);
 987     while (env->created == 0) {
 988         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
 989     }
 990 }
 991
 992 void qemu_init_vcpu(void *_env)
 993 {
 994     CPUState *env = _env;
 995
 996     env->nr_cores = smp_cores;
 997     env->nr_threads = smp_threads;
 998     if (kvm_enabled()) {
 999         qemu_kvm_start_vcpu(env);
1000     } else {
1001         qemu_tcg_init_vcpu(env);
1002     }
1003 }
1004
1005 void qemu_notify_event(void)
1006 {
1007     qemu_event_increment();
1008 }
1009
1010 void cpu_stop_current(void)
1011 {
1012     if (cpu_single_env) {
1013         cpu_single_env->stop = 0;
1014         cpu_single_env->stopped = 1;
1015         cpu_exit(cpu_single_env);
1016         qemu_cond_signal(&qemu_pause_cond);
1017     }
1018 }
1019
1020 void vm_stop(int reason)
1021 {
1022     if (!qemu_thread_is_self(&io_thread)) {
1023         qemu_system_vmstop_request(reason);
1024         /*
1025          * FIXME: should not return to device code in case
1026          * vm_stop() has been requested.
1027          */
1028         cpu_stop_current();
1029         return;
1030     }
1031     do_vm_stop(reason);
1032 }
1033
1034 #endif
1035
1036 static int tcg_cpu_exec(CPUState *env)
1037 {
1038     int ret;
1039 #ifdef CONFIG_PROFILER
1040     int64_t ti;
1041 #endif
1042
1043 #ifdef CONFIG_PROFILER
1044     ti = profile_getclock();
1045 #endif
1046     if (use_icount) {
1047         int64_t count;
1048         int decr;
1049         qemu_icount -= (env->icount_decr.u16.low + env->icount_extra);
1050         env->icount_decr.u16.low = 0;
1051         env->icount_extra = 0;
1052         count = qemu_icount_round(qemu_next_icount_deadline());
1053         qemu_icount += count;
1054         decr = (count > 0xffff) ? 0xffff : count;
1055         count -= decr;
1056         env->icount_decr.u16.low = decr;
1057         env->icount_extra = count;
1058     }
1059     ret = cpu_exec(env);
1060 #ifdef CONFIG_PROFILER
1061     qemu_time += profile_getclock() - ti;
1062 #endif
1063     if (use_icount) {
1064         /* Fold pending instructions back into the
1065            instruction counter, and clear the interrupt flag.  */
1066         qemu_icount -= (env->icount_decr.u16.low
1067                         + env->icount_extra);
1068         env->icount_decr.u32 = 0;
1069         env->icount_extra = 0;
1070     }
1071     return ret;
1072 }
1073
1074 bool cpu_exec_all(void)
1075 {
1076     int r;
1077
1078     /* Account partial waits to the vm_clock.  */
1079     qemu_clock_warp(vm_clock);
1080
1081     if (next_cpu == NULL) {
1082         next_cpu = first_cpu;
1083     }
1084     for (; next_cpu != NULL && !exit_request; next_cpu = next_cpu->next_cpu) {
1085         CPUState *env = next_cpu;
1086
1087         qemu_clock_enable(vm_clock,
1088                           (env->singlestep_enabled & SSTEP_NOTIMER) == 0);
1089
1090 #ifndef CONFIG_IOTHREAD
1091         if (qemu_alarm_pending()) {
1092             break;
1093         }
1094 #endif
1095         if (cpu_can_run(env)) {
1096             if (kvm_enabled()) {
1097                 r = kvm_cpu_exec(env);
1098                 qemu_kvm_eat_signals(env);
1099             } else {
1100                 r = tcg_cpu_exec(env);
1101             }
1102             if (r == EXCP_DEBUG) {
1103                 cpu_handle_guest_debug(env);
1104                 break;
1105             }
1106         } else if (env->stop || env->stopped) {
1107             break;
1108         }
1109     }
1110     exit_request = 0;
1111     return !all_cpu_threads_idle();
1112 }
1113
1114 void set_numa_modes(void)
1115 {
1116     CPUState *env;
1117     int i;
1118
1119     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1120         for (i = 0; i < nb_numa_nodes; i++) {
1121             if (node_cpumask[i] & (1 << env->cpu_index)) {
1122                 env->numa_node = i;
1123             }
1124         }
1125     }
1126 }
1127
1128 void set_cpu_log(const char *optarg)
1129 {
1130     int mask;
1131     const CPULogItem *item;
1132
1133     mask = cpu_str_to_log_mask(optarg);
1134     if (!mask) {
1135         printf("Log items (comma separated):\n");
1136         for (item = cpu_log_items; item->mask != 0; item++) {
1137             printf("%-10s %s\n", item->name, item->help);
1138         }
1139         exit(1);
1140     }
1141     cpu_set_log(mask);
1142 }
1143
1144 void set_cpu_log_filename(const char *optarg)
1145 {
1146     cpu_set_log_filename(optarg);
1147 }
1148
1149 /* Return the virtual CPU time, based on the instruction counter.  */
1150 int64_t cpu_get_icount(void)
1151 {
1152     int64_t icount;
1153     CPUState *env = cpu_single_env;;
1154
1155     icount = qemu_icount;
1156     if (env) {
1157         if (!can_do_io(env)) {
1158             fprintf(stderr, "Bad clock read\n");
1159         }
1160         icount -= (env->icount_decr.u16.low + env->icount_extra);
1161     }
1162     return qemu_icount_bias + (icount << icount_time_shift);
1163 }
1164
1165 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
1166 {
1167     /* XXX: implement xxx_cpu_list for targets that still miss it */
1168 #if defined(cpu_list_id)
1169     cpu_list_id(f, cpu_fprintf, optarg);
1170 #elif defined(cpu_list)
1171     cpu_list(f, cpu_fprintf); /* deprecated */
1172 #endif
1173 }