tests/migration-test.c

   1 /*
   2  * QTest testcase for migration
   3  *
   4  * Copyright (c) 2016 Red Hat, Inc. and/or its affiliates
   5  *   based on the vhost-user-test.c that is:
   6  *      Copyright (c) 2014 Virtual Open Systems Sarl.
   7  *
   8  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   9  * See the COPYING file in the top-level directory.
  10  *
  11  */
  12
  13 #include "qemu/osdep.h"
  14
  15 #include "libqtest.h"
  16 #include "qemu/option.h"
  17 #include "qemu/range.h"
  18 #include "qemu/sockets.h"
  19 #include "chardev/char.h"
  20 #include "sysemu/sysemu.h"
  21 #include "hw/nvram/chrp_nvram.h"
  22
  23 #define MIN_NVRAM_SIZE 8192 /* from spapr_nvram.c */
  24
  25 const unsigned start_address = 1024 * 1024;
  26 const unsigned end_address = 100 * 1024 * 1024;
  27 bool got_stop;
  28
  29 #if defined(__linux__)
  30 #include <sys/syscall.h>
  31 #include <sys/vfs.h>
  32 #endif
  33
  34 #if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD)
  35 #include <sys/eventfd.h>
  36 #include <sys/ioctl.h>
  37 #include <linux/userfaultfd.h>
  38
  39 static bool ufd_version_check(void)
  40 {
  41     struct uffdio_api api_struct;
  42     uint64_t ioctl_mask;
  43
  44     int ufd = syscall(__NR_userfaultfd, O_CLOEXEC);
  45
  46     if (ufd == -1) {
  47         g_test_message("Skipping test: userfaultfd not available");
  48         return false;
  49     }
  50
  51     api_struct.api = UFFD_API;
  52     api_struct.features = 0;
  53     if (ioctl(ufd, UFFDIO_API, &api_struct)) {
  54         g_test_message("Skipping test: UFFDIO_API failed");
  55         return false;
  56     }
  57
  58     ioctl_mask = (__u64)1 << _UFFDIO_REGISTER |
  59                  (__u64)1 << _UFFDIO_UNREGISTER;
  60     if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) {
  61         g_test_message("Skipping test: Missing userfault feature");
  62         return false;
  63     }
  64
  65     return true;
  66 }
  67
  68 #else
  69 static bool ufd_version_check(void)
  70 {
  71     g_test_message("Skipping test: Userfault not available (builtdtime)");
  72     return false;
  73 }
  74
  75 #endif
  76
  77 static const char *tmpfs;
  78
  79 /* A simple PC boot sector that modifies memory (1-100MB) quickly
  80  * outputing a 'B' every so often if it's still running.
  81  */
  82 unsigned char bootsect[] = {
  83   0xfa, 0x0f, 0x01, 0x16, 0x74, 0x7c, 0x66, 0xb8, 0x01, 0x00, 0x00, 0x00,
  84   0x0f, 0x22, 0xc0, 0x66, 0xea, 0x20, 0x7c, 0x00, 0x00, 0x08, 0x00, 0x00,
  85   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x92, 0x0c, 0x02,
  86   0xe6, 0x92, 0xb8, 0x10, 0x00, 0x00, 0x00, 0x8e, 0xd8, 0x66, 0xb8, 0x41,
  87   0x00, 0x66, 0xba, 0xf8, 0x03, 0xee, 0xb3, 0x00, 0xb8, 0x00, 0x00, 0x10,
  88   0x00, 0xfe, 0x00, 0x05, 0x00, 0x10, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x40,
  89   0x06, 0x7c, 0xf2, 0xfe, 0xc3, 0x75, 0xe9, 0x66, 0xb8, 0x42, 0x00, 0x66,
  90   0xba, 0xf8, 0x03, 0xee, 0xeb, 0xde, 0x66, 0x90, 0x00, 0x00, 0x00, 0x00,
  91   0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x9a, 0xcf, 0x00,
  92   0xff, 0xff, 0x00, 0x00, 0x00, 0x92, 0xcf, 0x00, 0x27, 0x00, 0x5c, 0x7c,
  93   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  94   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  95   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  96   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  97   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  98   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  99   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 100   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 101   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 102   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 103   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 104   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 105   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 106   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 107   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 108   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 109   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 110   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 111   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 112   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 113   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 114   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 115   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 116   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 117   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 118   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 119   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 120   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 121   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 122   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 123   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 124   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 125   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0xaa
 126 };
 127
 128 static void init_bootfile_x86(const char *bootpath)
 129 {
 130     FILE *bootfile = fopen(bootpath, "wb");
 131
 132     g_assert_cmpint(fwrite(bootsect, 512, 1, bootfile), ==, 1);
 133     fclose(bootfile);
 134 }
 135
 136 static void init_bootfile_ppc(const char *bootpath)
 137 {
 138     FILE *bootfile;
 139     char buf[MIN_NVRAM_SIZE];
 140     ChrpNvramPartHdr *header = (ChrpNvramPartHdr *)buf;
 141
 142     memset(buf, 0, MIN_NVRAM_SIZE);
 143
 144     /* Create a "common" partition in nvram to store boot-command property */
 145
 146     header->signature = CHRP_NVPART_SYSTEM;
 147     memcpy(header->name, "common", 6);
 148     chrp_nvram_finish_partition(header, MIN_NVRAM_SIZE);
 149
 150     /* FW_MAX_SIZE is 4MB, but slof.bin is only 900KB,
 151      * so let's modify memory between 1MB and 100MB
 152      * to do like PC bootsector
 153      */
 154
 155     sprintf(buf + 16,
 156             "boot-command=hex .\" _\" begin %x %x do i c@ 1 + i c! 1000 +loop "
 157             ".\" B\" 0 until", end_address, start_address);
 158
 159     /* Write partition to the NVRAM file */
 160
 161     bootfile = fopen(bootpath, "wb");
 162     g_assert_cmpint(fwrite(buf, MIN_NVRAM_SIZE, 1, bootfile), ==, 1);
 163     fclose(bootfile);
 164 }
 165
 166 /*
 167  * Wait for some output in the serial output file,
 168  * we get an 'A' followed by an endless string of 'B's
 169  * but on the destination we won't have the A.
 170  */
 171 static void wait_for_serial(const char *side)
 172 {
 173     char *serialpath = g_strdup_printf("%s/%s", tmpfs, side);
 174     FILE *serialfile = fopen(serialpath, "r");
 175     const char *arch = qtest_get_arch();
 176     int started = (strcmp(side, "src_serial") == 0 &&
 177                    strcmp(arch, "ppc64") == 0) ? 0 : 1;
 178
 179     g_free(serialpath);
 180     do {
 181         int readvalue = fgetc(serialfile);
 182
 183         if (!started) {
 184             /* SLOF prints its banner before starting test,
 185              * to ignore it, mark the start of the test with '_',
 186              * ignore all characters until this marker
 187              */
 188             switch (readvalue) {
 189             case '_':
 190                 started = 1;
 191                 break;
 192             case EOF:
 193                 fseek(serialfile, 0, SEEK_SET);
 194                 usleep(1000);
 195                 break;
 196             }
 197             continue;
 198         }
 199         switch (readvalue) {
 200         case 'A':
 201             /* Fine */
 202             break;
 203
 204         case 'B':
 205             /* It's alive! */
 206             fclose(serialfile);
 207             return;
 208
 209         case EOF:
 210             started = (strcmp(side, "src_serial") == 0 &&
 211                        strcmp(arch, "ppc64") == 0) ? 0 : 1;
 212             fseek(serialfile, 0, SEEK_SET);
 213             usleep(1000);
 214             break;
 215
 216         default:
 217             fprintf(stderr, "Unexpected %d on %s serial\n", readvalue, side);
 218             g_assert_not_reached();
 219         }
 220     } while (true);
 221 }
 222
 223 /*
 224  * Events can get in the way of responses we are actually waiting for.
 225  */
 226 static QDict *return_or_event(QDict *response)
 227 {
 228     const char *event_string;
 229     if (!qdict_haskey(response, "event")) {
 230         return response;
 231     }
 232
 233     /* OK, it was an event */
 234     event_string = qdict_get_str(response, "event");
 235     if (!strcmp(event_string, "STOP")) {
 236         got_stop = true;
 237     }
 238     QDECREF(response);
 239     return return_or_event(qtest_qmp_receive(global_qtest));
 240 }
 241
 242
 243 /*
 244  * It's tricky to use qemu's migration event capability with qtest,
 245  * events suddenly appearing confuse the qmp()/hmp() responses.
 246  */
 247
 248 static uint64_t get_migration_pass(void)
 249 {
 250     QDict *rsp, *rsp_return, *rsp_ram;
 251     uint64_t result;
 252
 253     rsp = return_or_event(qmp("{ 'execute': 'query-migrate' }"));
 254     rsp_return = qdict_get_qdict(rsp, "return");
 255     if (!qdict_haskey(rsp_return, "ram")) {
 256         /* Still in setup */
 257         result = 0;
 258     } else {
 259         rsp_ram = qdict_get_qdict(rsp_return, "ram");
 260         result = qdict_get_try_int(rsp_ram, "dirty-sync-count", 0);
 261     }
 262     QDECREF(rsp);
 263     return result;
 264 }
 265
 266 static void wait_for_migration_complete(void)
 267 {
 268     QDict *rsp, *rsp_return;
 269     bool completed;
 270
 271     do {
 272         const char *status;
 273
 274         rsp = return_or_event(qmp("{ 'execute': 'query-migrate' }"));
 275         rsp_return = qdict_get_qdict(rsp, "return");
 276         status = qdict_get_str(rsp_return, "status");
 277         completed = strcmp(status, "completed") == 0;
 278         g_assert_cmpstr(status, !=,  "failed");
 279         QDECREF(rsp);
 280         usleep(1000 * 100);
 281     } while (!completed);
 282 }
 283
 284 static void wait_for_migration_pass(void)
 285 {
 286     uint64_t initial_pass = get_migration_pass();
 287     uint64_t pass;
 288
 289     /* Wait for the 1st sync */
 290     do {
 291         initial_pass = get_migration_pass();
 292         if (got_stop || initial_pass) {
 293             break;
 294         }
 295         usleep(1000 * 100);
 296     } while (true);
 297
 298     do {
 299         usleep(1000 * 100);
 300         pass = get_migration_pass();
 301     } while (pass == initial_pass && !got_stop);
 302 }
 303
 304 static void check_guests_ram(void)
 305 {
 306     /* Our ASM test will have been incrementing one byte from each page from
 307      * 1MB to <100MB in order.
 308      * This gives us a constraint that any page's byte should be equal or less
 309      * than the previous pages byte (mod 256); and they should all be equal
 310      * except for one transition at the point where we meet the incrementer.
 311      * (We're running this with the guest stopped).
 312      */
 313     unsigned address;
 314     uint8_t first_byte;
 315     uint8_t last_byte;
 316     bool hit_edge = false;
 317     bool bad = false;
 318
 319     qtest_memread(global_qtest, start_address, &first_byte, 1);
 320     last_byte = first_byte;
 321
 322     for (address = start_address + 4096; address < end_address; address += 4096)
 323     {
 324         uint8_t b;
 325         qtest_memread(global_qtest, address, &b, 1);
 326         if (b != last_byte) {
 327             if (((b + 1) % 256) == last_byte && !hit_edge) {
 328                 /* This is OK, the guest stopped at the point of
 329                  * incrementing the previous page but didn't get
 330                  * to us yet.
 331                  */
 332                 hit_edge = true;
 333             } else {
 334                 fprintf(stderr, "Memory content inconsistency at %x"
 335                                 " first_byte = %x last_byte = %x current = %x"
 336                                 " hit_edge = %x\n",
 337                                 address, first_byte, last_byte, b, hit_edge);
 338                 bad = true;
 339             }
 340         }
 341         last_byte = b;
 342     }
 343     g_assert_false(bad);
 344 }
 345
 346 static void cleanup(const char *filename)
 347 {
 348     char *path = g_strdup_printf("%s/%s", tmpfs, filename);
 349
 350     unlink(path);
 351     g_free(path);
 352 }
 353
 354 static void migrate_set_downtime(QTestState *who, const char *value)
 355 {
 356     QDict *rsp;
 357     gchar *cmd;
 358
 359     cmd = g_strdup_printf("{ 'execute': 'migrate_set_downtime',"
 360                           "'arguments': { 'value': %s } }", value);
 361     rsp = qtest_qmp(who, cmd);
 362     g_free(cmd);
 363     g_assert(qdict_haskey(rsp, "return"));
 364     QDECREF(rsp);
 365 }
 366
 367 static void migrate_set_speed(QTestState *who, const char *value)
 368 {
 369     QDict *rsp;
 370     gchar *cmd;
 371
 372     cmd = g_strdup_printf("{ 'execute': 'migrate_set_speed',"
 373                           "'arguments': { 'value': %s } }", value);
 374     rsp = qtest_qmp(who, cmd);
 375     g_free(cmd);
 376     g_assert(qdict_haskey(rsp, "return"));
 377     QDECREF(rsp);
 378 }
 379
 380 static void migrate_set_capability(QTestState *who, const char *capability,
 381                                    const char *value)
 382 {
 383     QDict *rsp;
 384     gchar *cmd;
 385
 386     cmd = g_strdup_printf("{ 'execute': 'migrate-set-capabilities',"
 387                           "'arguments': { "
 388                           "'capabilities': [ { "
 389                           "'capability': '%s', 'state': %s } ] } }",
 390                           capability, value);
 391     rsp = qtest_qmp(who, cmd);
 392     g_free(cmd);
 393     g_assert(qdict_haskey(rsp, "return"));
 394     QDECREF(rsp);
 395 }
 396
 397 static void migrate(QTestState *who, const char *uri)
 398 {
 399     QDict *rsp;
 400     gchar *cmd;
 401
 402     cmd = g_strdup_printf("{ 'execute': 'migrate',"
 403                           "'arguments': { 'uri': '%s' } }",
 404                           uri);
 405     rsp = qtest_qmp(who, cmd);
 406     g_free(cmd);
 407     g_assert(qdict_haskey(rsp, "return"));
 408     QDECREF(rsp);
 409 }
 410
 411 static void test_migrate(void)
 412 {
 413     char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs);
 414     QTestState *global = global_qtest, *from, *to;
 415     unsigned char dest_byte_a, dest_byte_b, dest_byte_c, dest_byte_d;
 416     gchar *cmd_src, *cmd_dst;
 417     QDict *rsp;
 418
 419     char *bootpath = g_strdup_printf("%s/bootsect", tmpfs);
 420     const char *arch = qtest_get_arch();
 421
 422     got_stop = false;
 423
 424     if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
 425         init_bootfile_x86(bootpath);
 426         cmd_src = g_strdup_printf("-machine accel=kvm:tcg -m 150M"
 427                                   " -name pcsource,debug-threads=on"
 428                                   " -serial file:%s/src_serial"
 429                                   " -drive file=%s,format=raw",
 430                                   tmpfs, bootpath);
 431         cmd_dst = g_strdup_printf("-machine accel=kvm:tcg -m 150M"
 432                                   " -name pcdest,debug-threads=on"
 433                                   " -serial file:%s/dest_serial"
 434                                   " -drive file=%s,format=raw"
 435                                   " -incoming %s",
 436                                   tmpfs, bootpath, uri);
 437     } else if (strcmp(arch, "ppc64") == 0) {
 438         const char *accel;
 439
 440         /* On ppc64, the test only works with kvm-hv, but not with kvm-pr */
 441         accel = access("/sys/module/kvm_hv", F_OK) ? "tcg" : "kvm:tcg";
 442         init_bootfile_ppc(bootpath);
 443         cmd_src = g_strdup_printf("-machine accel=%s -m 256M"
 444                                   " -name pcsource,debug-threads=on"
 445                                   " -serial file:%s/src_serial"
 446                                   " -drive file=%s,if=pflash,format=raw",
 447                                   accel, tmpfs, bootpath);
 448         cmd_dst = g_strdup_printf("-machine accel=%s -m 256M"
 449                                   " -name pcdest,debug-threads=on"
 450                                   " -serial file:%s/dest_serial"
 451                                   " -incoming %s",
 452                                   accel, tmpfs, uri);
 453     } else {
 454         g_assert_not_reached();
 455     }
 456
 457     g_free(bootpath);
 458
 459     from = qtest_start(cmd_src);
 460     g_free(cmd_src);
 461
 462     to = qtest_init(cmd_dst);
 463     g_free(cmd_dst);
 464
 465     migrate_set_capability(from, "postcopy-ram", "true");
 466     migrate_set_capability(to, "postcopy-ram", "true");
 467
 468     /* We want to pick a speed slow enough that the test completes
 469      * quickly, but that it doesn't complete precopy even on a slow
 470      * machine, so also set the downtime.
 471      */
 472     migrate_set_speed(from, "100000000");
 473     migrate_set_downtime(from, "0.001");
 474
 475     /* Wait for the first serial output from the source */
 476     wait_for_serial("src_serial");
 477
 478     migrate(from, uri);
 479
 480     global_qtest = from;
 481     wait_for_migration_pass();
 482
 483     rsp = return_or_event(qmp("{ 'execute': 'migrate-start-postcopy' }"));
 484     g_assert(qdict_haskey(rsp, "return"));
 485     QDECREF(rsp);
 486
 487     if (!got_stop) {
 488         qmp_eventwait("STOP");
 489     }
 490
 491     global_qtest = to;
 492     qmp_eventwait("RESUME");
 493
 494     wait_for_serial("dest_serial");
 495     global_qtest = from;
 496     wait_for_migration_complete();
 497
 498     qtest_quit(from);
 499
 500     global_qtest = to;
 501
 502     qtest_memread(to, start_address, &dest_byte_a, 1);
 503
 504     /* Destination still running, wait for a byte to change */
 505     do {
 506         qtest_memread(to, start_address, &dest_byte_b, 1);
 507         usleep(10 * 1000);
 508     } while (dest_byte_a == dest_byte_b);
 509
 510     qmp_discard_response("{ 'execute' : 'stop'}");
 511     /* With it stopped, check nothing changes */
 512     qtest_memread(to, start_address, &dest_byte_c, 1);
 513     sleep(1);
 514     qtest_memread(to, start_address, &dest_byte_d, 1);
 515     g_assert_cmpint(dest_byte_c, ==, dest_byte_d);
 516
 517     check_guests_ram();
 518
 519     qtest_quit(to);
 520     g_free(uri);
 521
 522     global_qtest = global;
 523
 524     cleanup("bootsect");
 525     cleanup("migsocket");
 526     cleanup("src_serial");
 527     cleanup("dest_serial");
 528 }
 529
 530 int main(int argc, char **argv)
 531 {
 532     char template[] = "/tmp/migration-test-XXXXXX";
 533     int ret;
 534
 535     g_test_init(&argc, &argv, NULL);
 536
 537     if (!ufd_version_check()) {
 538         return 0;
 539     }
 540
 541     tmpfs = mkdtemp(template);
 542     if (!tmpfs) {
 543         g_test_message("mkdtemp on path (%s): %s\n", template, strerror(errno));
 544     }
 545     g_assert(tmpfs);
 546
 547     module_call_init(MODULE_INIT_QOM);
 548
 549     qtest_add_func("/migration/postcopy/unix", test_migrate);
 550
 551     ret = g_test_run();
 552
 553     g_assert_cmpint(ret, ==, 0);
 554
 555     ret = rmdir(tmpfs);
 556     if (ret != 0) {
 557         g_test_message("unable to rmdir: path (%s): %s\n",
 558                        tmpfs, strerror(errno));
 559     }
 560
 561     return ret;
 562 }