migration.c

   1 /*
   2  * QEMU migration support
   3  *
   4  * Copyright (C) 2007 Anthony Liguori <anthony@codemonkey.ws>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 #include "vl.h"
  26 #include "qemu_socket.h"
  27
  28 #include <sys/wait.h>
  29
  30 #define MIN_FINALIZE_SIZE       (200 << 10)
  31
  32 typedef struct MigrationState
  33 {
  34     int fd;
  35     int throttle_count;
  36     int bps;
  37     int updated_pages;
  38     int last_updated_pages;
  39     int iteration;
  40     int n_buffer;
  41     int throttled;
  42     int *has_error;
  43     char buffer[TARGET_PAGE_SIZE + 4];
  44     target_ulong addr;
  45     QEMUTimer *timer;
  46     void *opaque;
  47     int detach;
  48     int (*release)(void *opaque);
  49 } MigrationState;
  50
  51 static uint32_t max_throttle = (32 << 20);
  52 static MigrationState *current_migration;
  53
  54 /* QEMUFile migration implementation */
  55
  56 static void migrate_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, int size)
  57 {
  58     MigrationState *s = opaque;
  59     int offset = 0;
  60
  61     if (*s->has_error)
  62         return;
  63
  64     while (offset < size) {
  65         ssize_t len;
  66
  67         len = write(s->fd, buf + offset, size - offset);
  68         if (len == -1) {
  69             if (errno == EAGAIN || errno == EINTR)
  70                 continue;
  71             term_printf("migration: write failed (%s)\n", strerror(errno));
  72             *s->has_error = 10;
  73             break;
  74         } else if (len == 0) {
  75             term_printf("migration: other side closed connection\n");
  76             *s->has_error = 11;
  77             break;
  78         }
  79
  80         offset += len;
  81     }
  82 }
  83
  84 static void migrate_close(void *opaque)
  85 {
  86     MigrationState *s = opaque;
  87
  88     if (s->release && s->release(s->opaque))
  89         *s->has_error = 12;
  90
  91     qemu_free(s);
  92     current_migration = NULL;
  93 }
  94
  95 /* Outgoing migration routines */
  96
  97 static void migrate_finish(MigrationState *s)
  98 {
  99     QEMUFile *f;
 100     int ret;
 101     int *has_error = s->has_error;
 102
 103     fcntl(s->fd, F_SETFL, 0);
 104
 105     if (! *has_error) {
 106         f = qemu_fopen(s, migrate_put_buffer, NULL, migrate_close);
 107         qemu_aio_flush();
 108         vm_stop(0);
 109         qemu_put_be32(f, 1);
 110         ret = qemu_live_savevm_state(f);
 111         qemu_fclose(f);
 112     }
 113     if (ret != 0 || *has_error) {
 114         term_printf("Migration failed! ret=%d error=%d\n", ret, *has_error);
 115         vm_start();
 116     }
 117     if (!s->detach)
 118         monitor_resume();
 119     qemu_free(has_error);
 120     cpu_physical_memory_set_dirty_tracking(0);
 121 }
 122
 123 static int migrate_write_buffer(MigrationState *s)
 124 {
 125     if (*s->has_error)
 126         return 0;
 127
 128     if (s->n_buffer != sizeof(s->buffer)) {
 129         ssize_t len;
 130     again:
 131         len = write(s->fd, s->buffer + s->n_buffer, sizeof(s->buffer) - s->n_buffer);
 132         if (len == -1) {
 133             if (errno == EINTR)
 134                 goto again;
 135             if (errno == EAGAIN)
 136                 return 1;
 137             *s->has_error = 13;
 138             return 0;
 139         }
 140         if (len == 0) {
 141             *s->has_error = 14;
 142             return 0;
 143         }
 144
 145         s->throttle_count += len;
 146         s->n_buffer += len;
 147         if (s->n_buffer != sizeof(s->buffer))
 148             goto again;
 149     }
 150
 151     if (s->throttle_count > max_throttle) {
 152         s->throttled = 1;
 153         qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
 154         return 1;
 155     }
 156
 157     return 0;
 158 }
 159
 160 static int migrate_check_convergence(MigrationState *s)
 161 {
 162     target_ulong addr;
 163     int dirty_count = 0;
 164
 165     for (addr = 0; addr < phys_ram_size; addr += TARGET_PAGE_SIZE) {
 166 #ifdef USE_KVM
 167         if (kvm_allowed && (addr>=0xa0000) && (addr<0xc0000)) /* do not access video-addresses */
 168             continue;
 169 #endif
 170         if (cpu_physical_memory_get_dirty(addr, MIGRATION_DIRTY_FLAG))
 171             dirty_count++;
 172     }
 173
 174     return ((dirty_count * TARGET_PAGE_SIZE) < MIN_FINALIZE_SIZE);
 175 }
 176
 177 static void migrate_write(void *opaque)
 178 {
 179     MigrationState *s = opaque;
 180
 181     if (migrate_write_buffer(s))
 182         return;
 183
 184     if (migrate_check_convergence(s) || *s->has_error) {
 185         qemu_del_timer(s->timer);
 186         qemu_free_timer(s->timer);
 187         qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
 188         migrate_finish(s);
 189         return;
 190     }
 191
 192     while (s->addr < phys_ram_size) {
 193 #ifdef USE_KVM
 194         if (kvm_allowed && (s->addr>=0xa0000) && (s->addr<0xc0000)) /* do not access video-addresses */
 195             s->addr = 0xc0000;
 196 #endif
 197
 198         if (cpu_physical_memory_get_dirty(s->addr, MIGRATION_DIRTY_FLAG)) {
 199             uint32_t value = cpu_to_be32(s->addr);
 200
 201             memcpy(s->buffer, &value, 4);
 202             memcpy(s->buffer + 4, phys_ram_base + s->addr, TARGET_PAGE_SIZE);
 203             s->n_buffer = 0;
 204
 205             cpu_physical_memory_reset_dirty(s->addr, s->addr + TARGET_PAGE_SIZE, MIGRATION_DIRTY_FLAG);
 206
 207             s->addr += TARGET_PAGE_SIZE;
 208
 209             s->updated_pages++;
 210
 211             if (migrate_write_buffer(s))
 212                 return;
 213         } else
 214             s->addr += TARGET_PAGE_SIZE;
 215     }
 216
 217     s->last_updated_pages = s->updated_pages;
 218     s->updated_pages = 0;
 219     s->addr = 0;
 220     s->iteration++;
 221 }
 222
 223 static void migrate_reset_throttle(void *opaque)
 224 {
 225     MigrationState *s = opaque;
 226
 227     s->bps = s->throttle_count;
 228
 229     if (s->throttled) {
 230         s->throttled = 0;
 231         qemu_set_fd_handler2(s->fd, NULL, NULL, migrate_write, s);
 232     }
 233     s->throttle_count = 0;
 234     qemu_mod_timer(s->timer, qemu_get_clock(rt_clock) + 1000);
 235 }
 236
 237 static int start_migration(MigrationState *s)
 238 {
 239     uint32_t value = cpu_to_be32(phys_ram_size);
 240     target_phys_addr_t addr;
 241     size_t offset = 0;
 242
 243     while (offset != 4) {
 244         ssize_t len = write(s->fd, ((char *)&value) + offset, 4 - offset);
 245         if (len == -1 && errno == EINTR)
 246             continue;
 247
 248         if (len < 1)
 249             return -EIO;
 250
 251         offset += len;
 252     }
 253
 254     fcntl(s->fd, F_SETFL, O_NONBLOCK);
 255
 256     for (addr = 0; addr < phys_ram_size; addr += TARGET_PAGE_SIZE) {
 257 #ifdef USE_KVM
 258         if (kvm_allowed && (addr>=0xa0000) && (addr<0xc0000)) /* do not access video-addresses */
 259             continue;
 260 #endif
 261         if (!cpu_physical_memory_get_dirty(addr, MIGRATION_DIRTY_FLAG))
 262             cpu_physical_memory_set_dirty(addr);
 263     }
 264
 265     cpu_physical_memory_set_dirty_tracking(1);
 266
 267     s->addr = 0;
 268     s->iteration = 0;
 269     s->updated_pages = 0;
 270     s->last_updated_pages = 0;
 271     s->n_buffer = sizeof(s->buffer);
 272     s->timer = qemu_new_timer(rt_clock, migrate_reset_throttle, s);
 273
 274     qemu_mod_timer(s->timer, qemu_get_clock(rt_clock));
 275     qemu_set_fd_handler2(s->fd, NULL, NULL, migrate_write, s);
 276 }
 277
 278 static MigrationState *migration_init_fd(int detach, int fd)
 279 {
 280     MigrationState *s;
 281
 282     s = qemu_mallocz(sizeof(MigrationState));
 283     if (s == NULL) {
 284         term_printf("Allocation error\n");
 285         return NULL;
 286     }
 287
 288     s->fd = fd;
 289     s->has_error = qemu_mallocz(sizeof(int));
 290     if (s->has_error == NULL) {
 291         term_printf("malloc failed (for has_error)\n");
 292         return NULL;
 293     }
 294     s->detach = detach;
 295
 296     current_migration = s;
 297
 298     if (start_migration(s) == -1) {
 299         term_printf("Could not start migration\n");
 300         return NULL;
 301     }
 302
 303     if (!detach)
 304         monitor_suspend();
 305
 306     return s;
 307 }
 308
 309 typedef struct MigrationCmdState
 310 {
 311     int fd;
 312     pid_t pid;
 313 } MigrationCmdState;
 314
 315 static int cmd_release(void *opaque)
 316 {
 317     MigrationCmdState *c = opaque;
 318     int status, ret;
 319
 320     close(c->fd);
 321
 322 again:
 323     ret = waitpid(c->pid, &status, 0);
 324     if (ret == -1 && errno == EINTR)
 325         goto again;
 326
 327     if (ret == -1) {
 328         term_printf("migration: waitpid failed (%s)\n", strerror(errno));
 329         return -1;
 330     }
 331     /* FIXME: check and uncomment
 332      * if (WIFEXITED(status))
 333      *     status = WEXITSTATUS(status);
 334      */
 335     return status;
 336 }
 337
 338 static MigrationState *migration_init_cmd(int detach, const char *command, char **argv)
 339 {
 340     int fds[2];
 341     pid_t pid;
 342     int i;
 343     MigrationState *s;
 344
 345     if (pipe(fds) == -1) {
 346         term_printf("pipe() (%s)\n", strerror(errno));
 347         return NULL;
 348     }
 349
 350     pid = fork();
 351     if (pid == -1) {
 352         close(fds[0]);
 353         close(fds[1]);
 354         term_printf("fork error (%s)\n", strerror(errno));
 355         return NULL;
 356     }
 357     if (pid == 0) {
 358         close(fds[1]);
 359         dup2(fds[0], STDIN_FILENO);
 360         execvp(command, argv);
 361         exit(1);
 362     } else
 363         close(fds[0]);
 364
 365     for (i = 0; argv[i]; i++)
 366         qemu_free(argv[i]);
 367     qemu_free(argv);
 368
 369     s = migration_init_fd(detach, fds[1]);
 370     if (s) {
 371         MigrationCmdState *c = qemu_mallocz(sizeof(*c));
 372         c->pid = pid;
 373         c->fd = fds[1];
 374         s->release = cmd_release;
 375         s->opaque = c;
 376     }
 377
 378     return s;
 379 }
 380
 381 static MigrationState *migration_init_exec(int detach, const char *command)
 382 {
 383     char **argv = NULL;
 384
 385     argv = qemu_mallocz(sizeof(char *) * 4);
 386     argv[0] = strdup("sh");
 387     argv[1] = strdup("-c");
 388     argv[2] = strdup(command);
 389     argv[3] = NULL;
 390
 391     return migration_init_cmd(detach, "/bin/sh", argv);
 392 }
 393
 394 static MigrationState *migration_init_ssh(int detach, const char *host)
 395 {
 396     int qemu_argc, daemonize = 0, argc, i;
 397     char **qemu_argv, **argv;
 398     const char *incoming = NULL;
 399
 400     qemu_get_launch_info(&qemu_argc, &qemu_argv, &daemonize, &incoming);
 401
 402     argc = 3 + qemu_argc;
 403     if (!daemonize)
 404         argc++;
 405     if (!incoming)
 406         argc+=2;
 407
 408     argv = qemu_mallocz(sizeof(char *) * (argc + 1));
 409     argv[0] = strdup("ssh");
 410     argv[1] = strdup("-XC");
 411     argv[2] = strdup(host);
 412
 413     for (i = 0; i < qemu_argc; i++)
 414         argv[3 + i] = strdup(qemu_argv[i]);
 415
 416     if (!daemonize)
 417         argv[3 + i++] = strdup("-daemonize");
 418     if (!incoming) {
 419         argv[3 + i++] = strdup("-incoming");
 420         argv[3 + i++] = strdup("stdio");
 421     }
 422
 423     argv[3 + i] = NULL;
 424
 425     return migration_init_cmd(detach, "ssh", argv);
 426 }
 427
 428 static int tcp_release(void *opaque)
 429 {
 430     MigrationState *s = opaque;
 431     uint8_t status = 0;
 432     ssize_t len;
 433
 434 again:
 435     len = read(s->fd, &status, 1);
 436     if (len == -1 && errno == EINTR)
 437         goto again;
 438
 439     close(s->fd);
 440
 441     return (len != 1 || status != 0);
 442 }
 443
 444 static MigrationState *migration_init_tcp(int detach, const char *host)
 445 {
 446     int fd;
 447     struct sockaddr_in addr;
 448     MigrationState *s;
 449
 450     fd = socket(PF_INET, SOCK_STREAM, 0);
 451     if (fd == -1) {
 452         term_printf("socket() failed %s\n", strerror(errno));
 453         return NULL;
 454     }
 455
 456     addr.sin_family = AF_INET;
 457     if (parse_host_port(&addr, host) == -1) {
 458         term_printf("parse_host_port() FAILED for %s\n", host);
 459         close(fd);
 460         return NULL;
 461     }
 462
 463     if (connect(fd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
 464         term_printf("connect() failed %s\n", strerror(errno));
 465         close(fd);
 466         return NULL;
 467     }
 468
 469     s = migration_init_fd(detach, fd);
 470     if (s) {
 471         s->opaque = s;
 472         s->release = tcp_release;
 473     }
 474     return s;
 475 }
 476
 477 /* Incoming migration */
 478
 479 static int migrate_incoming_fd(int fd)
 480 {
 481     int ret;
 482     QEMUFile *f = qemu_fopen_fd(fd);
 483     uint32_t addr;
 484     extern void qemu_announce_self(void);
 485
 486     if (qemu_get_be32(f) != phys_ram_size)
 487         return 101;
 488
 489     do {
 490         int l;
 491         addr = qemu_get_be32(f);
 492         if (addr == 1)
 493             break;
 494         l = qemu_get_buffer(f, phys_ram_base + addr, TARGET_PAGE_SIZE);
 495         if (l != TARGET_PAGE_SIZE)
 496             return 102;
 497     } while (1);
 498
 499
 500     qemu_aio_flush();
 501     vm_stop(0);
 502     ret = qemu_live_loadvm_state(f);
 503     qemu_fclose(f);
 504
 505     return ret;
 506 }
 507
 508 static int migrate_incoming_tcp(const char *host)
 509 {
 510     struct sockaddr_in addr;
 511     socklen_t addrlen = sizeof(addr);
 512     int fd, sfd;
 513     ssize_t len;
 514     uint8_t status = 0;
 515     int reuse = 1;
 516     int rc;
 517
 518     addr.sin_family = AF_INET;
 519     if (parse_host_port(&addr, host) == -1) {
 520         fprintf(stderr, "parse_host_port() failed for %s\n", host);
 521         rc = 201;
 522         goto error;
 523     }
 524
 525     fd = socket(PF_INET, SOCK_STREAM, 0);
 526     if (fd == -1) {
 527         perror("socket failed");
 528         rc = 202;
 529         goto error;
 530     }
 531
 532     if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)) == -1) {
 533         perror("setsockopt() failed");
 534         rc = 203;
 535         goto error_socket;
 536     }
 537
 538     if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
 539         perror("bind() failed");
 540         rc = 204;
 541         goto error_socket;
 542     }
 543
 544     if (listen(fd, 1) == -1) {
 545         perror("listen() failed");
 546         rc = 205;
 547         goto error_socket;
 548     }
 549
 550 again:
 551     sfd = accept(fd, (struct sockaddr *)&addr, &addrlen);
 552     if (sfd == -1) {
 553         if (errno == EINTR)
 554             goto again;
 555         perror("accept() failed");
 556         rc = 206;
 557         goto error_socket;
 558     }
 559
 560     rc = migrate_incoming_fd(sfd);
 561     if (rc != 0) {
 562         rc = 207;
 563         fprintf(stderr, "migrate_incoming_fd failed (rc=%d)\n", rc);
 564         goto error_accept;
 565     }
 566
 567 again1:
 568     len = write(sfd, &status, 1);
 569     if (len == -1 && errno == EAGAIN)
 570         goto again1;
 571     if (len != 1) {
 572         rc = 208;
 573         goto error_accept;
 574
 575     }
 576
 577 error_accept:
 578     close(sfd);
 579 error_socket:
 580     close(fd);
 581 error:
 582     return rc;
 583 }
 584
 585 int migrate_incoming(const char *device)
 586 {
 587     const char *ptr;
 588     int ret = 0;
 589
 590     if (strcmp(device, "stdio") == 0)
 591         ret = migrate_incoming_fd(STDIN_FILENO);
 592     else if (strstart(device, "tcp://", &ptr)) {
 593         char *host, *end;
 594         host = strdup(ptr);
 595         end = strchr(host, '/');
 596         if (end) *end = 0;
 597         ret = migrate_incoming_tcp(host);
 598         qemu_free(host);
 599     } else {
 600         errno = EINVAL;
 601         ret = -1;
 602     }
 603
 604     return ret;
 605 }
 606
 607 /* Migration monitor command */
 608
 609 /* TODO:
 610    1) audit all error paths
 611 */
 612
 613 void do_migrate(int detach, const char *uri)
 614 {
 615     const char *ptr;
 616
 617     if (strstart(uri, "exec:", &ptr)) {
 618         char *command = urldecode(ptr);
 619         migration_init_exec(detach, command);
 620         free(command);
 621     } else if (strstart(uri, "ssh://", &ptr)) {
 622         char *host, *end;
 623
 624         host = strdup(ptr);
 625         end = strchr(host, '/');
 626         if (end) *end = 0;
 627         migration_init_ssh(detach, host);
 628         qemu_free(host);
 629     } else if (strstart(uri, "tcp://", &ptr)) {
 630         char *host, *end;
 631
 632         host = strdup(ptr);
 633         end = strchr(host, '/');
 634         if (end) *end = 0;
 635
 636         if (migration_init_tcp(detach, host) == NULL)
 637             term_printf("migration failed (migration_init_tcp for %s failed)\n", host);
 638         free(host);
 639     } else {
 640         term_printf("Unknown migration protocol '%s'\n", uri);
 641         return;
 642     }
 643 }
 644
 645 void do_migrate_set_speed(const char *value)
 646 {
 647     double d;
 648     char *ptr;
 649
 650     d = strtod(value, &ptr);
 651     switch (*ptr) {
 652     case 'G': case 'g':
 653         d *= 1024;
 654     case 'M': case 'm':
 655         d *= 1024;
 656     case 'K': case 'k':
 657         d *= 1024;
 658     default:
 659         break;
 660     }
 661
 662     max_throttle = (uint32_t)d;
 663 }
 664
 665 void do_info_migration(void)
 666 {
 667     MigrationState *s = current_migration;
 668
 669     if (s) {
 670         term_printf("Migration active\n");
 671         if (s->bps < (1 << 20))
 672             term_printf("Transfer rate %3.1f kb/s\n",
 673                         (double)s->bps / 1024);
 674         else
 675             term_printf("Transfer rate %3.1f mb/s\n",
 676                         (double)s->bps / (1024 * 1024));
 677         term_printf("Iteration %d\n", s->iteration);
 678         term_printf("Transferred %d/%d pages\n", s->updated_pages, phys_ram_size >> TARGET_PAGE_BITS);
 679         if (s->iteration)
 680             term_printf("Last iteration found %d dirty pages\n", s->last_updated_pages);
 681     } else
 682         term_printf("Migration inactive\n");
 683
 684     term_printf("Maximum migration speed is ");
 685     if (max_throttle < (1 << 20))
 686         term_printf("%3.1f kb/s\n", (double)max_throttle / 1024);
 687     else
 688         term_printf("%3.1f mb/s\n", (double)max_throttle / (1024 * 1024));
 689 }
 690
 691 void do_migrate_cancel(void)
 692 {
 693     MigrationState *s = current_migration;
 694
 695     if (s)
 696         *s->has_error = 20;
 697 }