usr/src/boot/sys/boot/common/multiboot2.c

   1 /*
   2  * This file and its contents are supplied under the terms of the
   3  * Common Development and Distribution License ("CDDL"), version 1.0.
   4  * You may only use this file in accordance with the terms of version
   5  * 1.0 of the CDDL.
   6  *
   7  * A full copy of the text of the CDDL should have accompanied this
   8  * source.  A copy of the CDDL is also available via the Internet at
   9  * http://www.illumos.org/license/CDDL.
  10  */
  11
  12 /*
  13  * Copyright 2017 Toomas Soome <tsoome@me.com>
  14  */
  15
  16 /*
  17  * This module adds support for loading and booting illumos multiboot2
  18  * kernel. This code is only built to support the illumos kernel, it does
  19  * not support xen.
  20  */
  21
  22 #include <sys/cdefs.h>
  23 #include <sys/stddef.h>
  24
  25 #include <sys/param.h>
  26 #include <sys/exec.h>
  27 #include <sys/linker.h>
  28 #include <sys/module.h>
  29 #include <sys/stdint.h>
  30 #include <sys/multiboot2.h>
  31 #include <stand.h>
  32 #include <stdbool.h>
  33 #include <machine/elf.h>
  34 #include "libzfs.h"
  35
  36 #include "bootstrap.h"
  37 #include <sys/consplat.h>
  38
  39 #include <machine/metadata.h>
  40 #include <machine/pc/bios.h>
  41
  42 #define SUPPORT_DHCP
  43 #include <bootp.h>
  44
  45 #if !defined(EFI)
  46 #include "../i386/btx/lib/btxv86.h"
  47 #include "libi386.h"
  48 #include "vbe.h"
  49
  50 #else
  51 #include <efi.h>
  52 #include <efilib.h>
  53 #include "loader_efi.h"
  54
  55 static void (*trampoline)(uint32_t, struct relocator *, uint64_t);
  56 #endif
  57
  58 #include "platform/acfreebsd.h"
  59 #include "acconfig.h"
  60 #define ACPI_SYSTEM_XFACE
  61 #include "actypes.h"
  62 #include "actbl.h"
  63
  64 extern ACPI_TABLE_RSDP *rsdp;
  65
  66 /* MB data heap pointer. */
  67 static vm_offset_t last_addr;
  68
  69 static int multiboot2_loadfile(char *, u_int64_t, struct preloaded_file **);
  70 static int multiboot2_exec(struct preloaded_file *);
  71
  72 struct file_format multiboot2 = { multiboot2_loadfile, multiboot2_exec };
  73 static bool keep_bs = false;
  74 static bool have_framebuffer = false;
  75 static vm_offset_t load_addr;
  76 static vm_offset_t entry_addr;
  77
  78 /*
  79  * Validate tags in info request. This function is provided just to
  80  * recognize the current tag list and only serves as a limited
  81  * safe guard against possibly corrupt information.
  82  */
  83 static bool
  84 is_info_request_valid(multiboot_header_tag_information_request_t *rtag)
  85 {
  86         int i;
  87
  88         /*
  89          * If the tag is optional and we do not support it, we do not
  90          * have to do anything special, so we skip optional tags.
  91          */
  92         if (rtag->mbh_flags & MULTIBOOT_HEADER_TAG_OPTIONAL)
  93                 return (true);
  94
  95         for (i = 0; i < (rtag->mbh_size - sizeof (*rtag)) /
  96             sizeof (rtag->mbh_requests[0]); i++)
  97                 switch (rtag->mbh_requests[i]) {
  98                 case MULTIBOOT_TAG_TYPE_END:
  99                 case MULTIBOOT_TAG_TYPE_CMDLINE:
 100                 case MULTIBOOT_TAG_TYPE_BOOT_LOADER_NAME:
 101                 case MULTIBOOT_TAG_TYPE_MODULE:
 102                 case MULTIBOOT_TAG_TYPE_BASIC_MEMINFO:
 103                 case MULTIBOOT_TAG_TYPE_BOOTDEV:
 104                 case MULTIBOOT_TAG_TYPE_MMAP:
 105                 case MULTIBOOT_TAG_TYPE_FRAMEBUFFER:
 106                 case MULTIBOOT_TAG_TYPE_VBE:
 107                 case MULTIBOOT_TAG_TYPE_ELF_SECTIONS:
 108                 case MULTIBOOT_TAG_TYPE_APM:
 109                 case MULTIBOOT_TAG_TYPE_EFI32:
 110                 case MULTIBOOT_TAG_TYPE_EFI64:
 111                 case MULTIBOOT_TAG_TYPE_ACPI_OLD:
 112                 case MULTIBOOT_TAG_TYPE_ACPI_NEW:
 113                 case MULTIBOOT_TAG_TYPE_NETWORK:
 114                 case MULTIBOOT_TAG_TYPE_EFI_MMAP:
 115                 case MULTIBOOT_TAG_TYPE_EFI_BS:
 116                 case MULTIBOOT_TAG_TYPE_EFI32_IH:
 117                 case MULTIBOOT_TAG_TYPE_EFI64_IH:
 118                 case MULTIBOOT_TAG_TYPE_LOAD_BASE_ADDR:
 119                         break;
 120                 default:
 121                         printf("unsupported information tag: 0x%x\n",
 122                             rtag->mbh_requests[i]);
 123                         return (false);
 124                 }
 125         return (true);
 126 }
 127
 128 static int
 129 multiboot2_loadfile(char *filename, u_int64_t dest,
 130     struct preloaded_file **result)
 131 {
 132         int fd, error;
 133         uint32_t i;
 134         struct stat st;
 135         caddr_t header_search;
 136         multiboot2_header_t *header;
 137         multiboot_header_tag_t *tag;
 138         multiboot_header_tag_address_t *addr_tag = NULL;
 139         multiboot_header_tag_entry_address_t *entry_tag = NULL;
 140         struct preloaded_file *fp;
 141
 142         /* This allows to check other file formats from file_formats array. */
 143         error = EFTYPE;
 144         if (filename == NULL)
 145                 return (error);
 146
 147         /* is kernel already loaded? */
 148         fp = file_findfile(NULL, NULL);
 149         if (fp != NULL)
 150                 return (error);
 151
 152         if ((fd = open(filename, O_RDONLY)) == -1)
 153                 return (errno);
 154
 155         /*
 156          * Read MULTIBOOT_SEARCH size in order to search for the
 157          * multiboot magic header.
 158          */
 159         header_search = malloc(MULTIBOOT_SEARCH);
 160         if (header_search == NULL) {
 161                 close(fd);
 162                 return (ENOMEM);
 163         }
 164
 165         if (read(fd, header_search, MULTIBOOT_SEARCH) != MULTIBOOT_SEARCH)
 166                 goto out;
 167
 168         header = NULL;
 169         for (i = 0; i <= (MULTIBOOT_SEARCH - sizeof (multiboot2_header_t));
 170             i += MULTIBOOT_HEADER_ALIGN) {
 171                 header = (multiboot2_header_t *)(header_search + i);
 172
 173                 /* Do we have match on magic? */
 174                 if (header->mb2_magic != MULTIBOOT2_HEADER_MAGIC) {
 175                         header = NULL;
 176                         continue;
 177                 }
 178                 /*
 179                  * Validate checksum, the sum of magic + architecture +
 180                  * header_length + checksum must equal 0.
 181                  */
 182                 if (header->mb2_magic + header->mb2_architecture +
 183                     header->mb2_header_length + header->mb2_checksum != 0) {
 184                         header = NULL;
 185                         continue;
 186                 }
 187                 /*
 188                  * Finally, the entire header must fit within MULTIBOOT_SEARCH.
 189                  */
 190                 if (i + header->mb2_header_length > MULTIBOOT_SEARCH) {
 191                         header = NULL;
 192                         continue;
 193                 }
 194                 break;
 195         }
 196
 197         if (header == NULL)
 198                 goto out;
 199
 200         have_framebuffer = false;
 201         for (tag = header->mb2_tags; tag->mbh_type != MULTIBOOT_TAG_TYPE_END;
 202             tag = (multiboot_header_tag_t *)((uintptr_t)tag +
 203             roundup2(tag->mbh_size, MULTIBOOT_TAG_ALIGN))) {
 204                 switch (tag->mbh_type) {
 205                 case MULTIBOOT_HEADER_TAG_INFORMATION_REQUEST:
 206                         if (is_info_request_valid((void*)tag) == false)
 207                                 goto out;
 208                         break;
 209                 case MULTIBOOT_HEADER_TAG_ADDRESS:
 210                         addr_tag = (multiboot_header_tag_address_t *)tag;
 211                         break;
 212                 case MULTIBOOT_HEADER_TAG_ENTRY_ADDRESS:
 213                         entry_tag =
 214                             (multiboot_header_tag_entry_address_t *)tag;
 215                         break;
 216                 case MULTIBOOT_HEADER_TAG_CONSOLE_FLAGS:
 217                         break;
 218                 case MULTIBOOT_HEADER_TAG_FRAMEBUFFER:
 219                         have_framebuffer = true;
 220                         break;
 221                 case MULTIBOOT_HEADER_TAG_MODULE_ALIGN:
 222                         /* we always align modules */
 223                         break;
 224                 case MULTIBOOT_HEADER_TAG_EFI_BS:
 225                         keep_bs = true;
 226                         break;
 227                 default:
 228                         if (!(tag->mbh_flags & MULTIBOOT_HEADER_TAG_OPTIONAL)) {
 229                                 printf("unsupported tag: 0x%x\n",
 230                                     tag->mbh_type);
 231                                 goto out;
 232                         }
 233                 }
 234         }
 235
 236         /*
 237          * We must have addr_tag and entry_tag to load a 64-bit kernel.
 238          * If these tags are missing, we either have a 32-bit kernel, or
 239          * this is not our kernel at all.
 240          */
 241         if (addr_tag != NULL && entry_tag != NULL) {
 242                 fp = file_alloc();
 243                 if (fp == NULL) {
 244                         error = ENOMEM;
 245                         goto out;
 246                 }
 247                 if (lseek(fd, 0, SEEK_SET) == -1) {
 248                         printf("lseek failed\n");
 249                         error = EIO;
 250                         file_discard(fp);
 251                         goto out;
 252                 }
 253                 if (fstat(fd, &st) < 0) {
 254                         printf("fstat failed\n");
 255                         error = EIO;
 256                         file_discard(fp);
 257                         goto out;
 258                 }
 259
 260                 load_addr = addr_tag->mbh_load_addr;
 261                 entry_addr = entry_tag->mbh_entry_addr;
 262                 fp->f_addr = archsw.arch_loadaddr(LOAD_KERN, filename,
 263                     addr_tag->mbh_load_addr);
 264                 if (fp->f_addr == 0) {
 265                         error = ENOMEM;
 266                         file_discard(fp);
 267                         goto out;
 268                 }
 269                 fp->f_size = archsw.arch_readin(fd, fp->f_addr, st.st_size);
 270
 271                 if (fp->f_size != st.st_size) {
 272                         printf("error reading: %s", strerror(errno));
 273                         file_discard(fp);
 274                         error = EIO;
 275                         goto out;
 276                 }
 277
 278                 fp->f_name = strdup(filename);
 279                 fp->f_type = strdup("aout multiboot2 kernel");
 280                 if (fp->f_name == NULL || fp->f_type == NULL) {
 281                         error = ENOMEM;
 282                         file_discard(fp);
 283                         goto out;
 284                 }
 285
 286                 fp->f_metadata = NULL;
 287                 error = 0;
 288         } else {
 289 #if defined(EFI)
 290                 /* 32-bit kernel is not yet supported for EFI */
 291                 printf("32-bit kernel is not supported by UEFI loader\n");
 292                 error = ENOTSUP;
 293                 goto out;
 294 #endif
 295                 /* elf32_loadfile_raw will fill the attributes in fp. */
 296                 error = elf32_loadfile_raw(filename, dest, &fp, 2);
 297                 if (error != 0) {
 298                         printf("elf32_loadfile_raw failed: %d unable to "
 299                             "load multiboot2 kernel\n", error);
 300                         goto out;
 301                 }
 302                 entry_addr = fp->f_addr;
 303                 /*
 304                  * We want the load_addr to have some legal value,
 305                  * so we set it same as the entry_addr.
 306                  * The distinction is important with UEFI, but not
 307                  * with BIOS version, because BIOS version does not use
 308                  * staging area.
 309                  */
 310                 load_addr = fp->f_addr;
 311         }
 312
 313         setenv("kernelname", fp->f_name, 1);
 314 #if defined(EFI)
 315         efi_addsmapdata(fp);
 316 #else
 317         bios_addsmapdata(fp);
 318 #endif
 319         *result = fp;
 320 out:
 321         free(header_search);
 322         close(fd);
 323         return (error);
 324 }
 325
 326 /*
 327  * Search the command line for named property.
 328  *
 329  * Return codes:
 330  *      0       The name is found, we return the data in value and len.
 331  *      ENOENT  The name is not found.
 332  *      EINVAL  The provided command line is badly formed.
 333  */
 334 static int
 335 find_property_value(const char *cmd, const char *name, const char **value,
 336     size_t *len)
 337 {
 338         const char *namep, *valuep;
 339         size_t name_len, value_len;
 340         int quoted;
 341
 342         *value = NULL;
 343         *len = 0;
 344
 345         if (cmd == NULL)
 346                 return (ENOENT);
 347
 348         while (*cmd != '\0') {
 349                 if (cmd[0] != '-' || cmd[1] != 'B') {
 350                         cmd++;
 351                         continue;
 352                 }
 353                 cmd += 2;       /* Skip -B */
 354                 while (cmd[0] == ' ' || cmd[0] == '\t')
 355                         cmd++;  /* Skip whitespaces. */
 356                 while (*cmd != '\0' && cmd[0] != ' ' && cmd[0] != '\t') {
 357                         namep = cmd;
 358                         valuep = strchr(cmd, '=');
 359                         if (valuep == NULL)
 360                                 break;
 361                         name_len = valuep - namep;
 362                         valuep++;
 363                         value_len = 0;
 364                         quoted = 0;
 365                         for (; ; ++value_len) {
 366                                 if (valuep[value_len] == '\0')
 367                                         break;
 368
 369                                 /* Is this value quoted? */
 370                                 if (value_len == 0 &&
 371                                     (valuep[0] == '\'' || valuep[0] == '"')) {
 372                                         quoted = valuep[0];
 373                                         ++value_len;
 374                                 }
 375
 376                                 /*
 377                                  * In the quote accept any character,
 378                                  * but look for ending quote.
 379                                  */
 380                                 if (quoted != 0) {
 381                                         if (valuep[value_len] == quoted)
 382                                                 quoted = 0;
 383                                         continue;
 384                                 }
 385
 386                                 /* A comma or white space ends the value. */
 387                                 if (valuep[value_len] == ',' ||
 388                                     valuep[value_len] == ' ' ||
 389                                     valuep[value_len] == '\t')
 390                                         break;
 391                         }
 392                         if (quoted != 0) {
 393                                 printf("Missing closing '%c' in \"%s\"\n",
 394                                     quoted, valuep);
 395                                 return (EINVAL);
 396                         }
 397                         if (value_len != 0) {
 398                                 if (strncmp(namep, name, name_len) == 0) {
 399                                         *value = valuep;
 400                                         *len = value_len;
 401                                         return (0);
 402                                 }
 403                         }
 404                         cmd = valuep + value_len;
 405                         while (*cmd == ',')
 406                                 cmd++;
 407                 }
 408         }
 409         return (ENOENT);
 410 }
 411
 412 /*
 413  * If command line has " -B ", insert property after "-B ", otherwise
 414  * append to command line.
 415  */
 416 static char *
 417 insert_cmdline(const char *head, const char *prop)
 418 {
 419         const char *prop_opt = " -B ";
 420         char *cmdline, *tail;
 421         int len = 0;
 422
 423         tail = strstr(head, prop_opt);
 424         if (tail != NULL) {
 425                 ptrdiff_t diff;
 426                 tail += strlen(prop_opt);
 427                 diff = tail - head;
 428                 if (diff >= INT_MAX)
 429                         return (NULL);
 430                 len = (int)diff;
 431         }
 432
 433         if (tail == NULL)
 434                 asprintf(&cmdline, "%s%s%s", head, prop_opt, prop);
 435         else
 436                 asprintf(&cmdline, "%.*s%s,%s", len, head, prop, tail);
 437
 438         return (cmdline);
 439 }
 440
 441 /*
 442  * Since we have no way to pass the environment to the mb1 kernel other than
 443  * through arguments, we need to take care of console setup.
 444  *
 445  * If the console is in mirror mode, set the kernel console from $os_console.
 446  * If it's unset, use first item from $console.
 447  * If $console is "ttyX", also pass $ttyX-mode, since it may have been set by
 448  * the user.
 449  *
 450  * In case of memory allocation errors, just return the original command line
 451  * so we have a chance of booting.
 452  *
 453  * On success, cl will be freed and a new, allocated command line string is
 454  * returned.
 455  *
 456  * For the mb2 kernel, we only set command line console if os_console is set.
 457  * We can not overwrite console in the environment, as it can disrupt the
 458  * loader console messages, and we do not want to deal with the os_console
 459  * in the kernel.
 460  */
 461 static char *
 462 update_cmdline(char *cl, bool mb2)
 463 {
 464         char *os_console = getenv("os_console");
 465         char *ttymode = NULL;
 466         char mode[10];
 467         char *tmp;
 468         const char *prop;
 469         size_t plen;
 470         int rv;
 471
 472         if (mb2 == true && os_console == NULL)
 473                 return (cl);
 474
 475         if (os_console == NULL) {
 476                 tmp = strdup(getenv("console"));
 477                 os_console = strsep(&tmp, ", ");
 478         } else {
 479                 os_console = strdup(os_console);
 480         }
 481
 482         if (os_console == NULL)
 483                 return (cl);
 484
 485         if (mb2 == false && strncmp(os_console, "tty", 3) == 0) {
 486                 snprintf(mode, sizeof (mode), "%s-mode", os_console);
 487                 /*
 488                  * The ttyX-mode variable is set by our serial console
 489                  * driver for ttya-ttyd. However, since the os_console
 490                  * values are not verified, it is possible we get bogus
 491                  * name and no mode variable. If so, we do not set console
 492                  * property and let the kernel use defaults.
 493                  */
 494                 if ((ttymode = getenv(mode)) == NULL)
 495                         return (cl);
 496         }
 497
 498         rv = find_property_value(cl, "console", &prop, &plen);
 499         if (rv != 0 && rv != ENOENT) {
 500                 free(os_console);
 501                 return (cl);
 502         }
 503
 504         /* If console is set and this is MB2 boot, we are done. */
 505         if (rv == 0 && mb2 == true) {
 506                 free(os_console);
 507                 return (cl);
 508         }
 509
 510         /* If console is set, do we need to set tty mode? */
 511         if (rv == 0) {
 512                 const char *ttyp = NULL;
 513                 size_t ttylen;
 514
 515                 free(os_console);
 516                 os_console = NULL;
 517                 *mode = '\0';
 518                 if (strncmp(prop, "tty", 3) == 0 && plen == 4) {
 519                         strncpy(mode, prop, plen);
 520                         mode[plen] = '\0';
 521                         strncat(mode, "-mode", 5);
 522                         find_property_value(cl, mode, &ttyp, &ttylen);
 523                 }
 524
 525                 if (*mode != '\0' && ttyp == NULL)
 526                         ttymode = getenv(mode);
 527                 else
 528                         return (cl);
 529         }
 530
 531         /* Build updated command line. */
 532         if (os_console != NULL) {
 533                 char *propstr;
 534
 535                 asprintf(&propstr, "console=%s", os_console);
 536                 free(os_console);
 537                 if (propstr == NULL) {
 538                         return (cl);
 539                 }
 540
 541                 tmp = insert_cmdline(cl, propstr);
 542                 free(propstr);
 543                 if (tmp == NULL)
 544                         return (cl);
 545
 546                 free(cl);
 547                 cl = tmp;
 548         }
 549         if (ttymode != NULL) {
 550                 char *propstr;
 551
 552                 asprintf(&propstr, "%s=\"%s\"", mode, ttymode);
 553                 if (propstr == NULL)
 554                         return (cl);
 555
 556                 tmp = insert_cmdline(cl, propstr);
 557                 free(propstr);
 558                 if (tmp == NULL)
 559                         return (cl);
 560                 free(cl);
 561                 cl = tmp;
 562         }
 563
 564         return (cl);
 565 }
 566
 567 /*
 568  * Build the kernel command line. Shared function between MB1 and MB2.
 569  *
 570  * In both cases, if fstype is set and is not zfs, we do not set up
 571  * zfs-bootfs property. But we set kernel file name and options.
 572  *
 573  * For the MB1, we only can pass properties on command line, so
 574  * we will set console, ttyX-mode (for serial console) and zfs-bootfs.
 575  *
 576  * For the MB2, we can pass properties in environment, but if os_console
 577  * is set in environment, we need to add console property on the kernel
 578  * command line.
 579  *
 580  * The console properties are managed in update_cmdline().
 581  */
 582 int
 583 mb_kernel_cmdline(struct preloaded_file *fp, struct devdesc *rootdev,
 584     char **line)
 585 {
 586         const char *fs = getenv("fstype");
 587         char *cmdline;
 588         size_t len;
 589         bool zfs_root = false;
 590         bool mb2;
 591         int rv;
 592
 593         /*
 594          * 64-bit kernel has aout header, 32-bit kernel is elf, and the
 595          * type strings are different. Lets just search for "multiboot2".
 596          */
 597         if (strstr(fp->f_type, "multiboot2") == NULL)
 598                 mb2 = false;
 599         else
 600                 mb2 = true;
 601
 602         if (rootdev->d_dev->dv_type == DEVT_ZFS)
 603                 zfs_root = true;
 604
 605         /* If we have fstype set in env, reset zfs_root if needed. */
 606         if (fs != NULL && strcmp(fs, "zfs") != 0)
 607                 zfs_root = false;
 608
 609         /*
 610          * If we have fstype set on the command line,
 611          * reset zfs_root if needed.
 612          */
 613         rv = find_property_value(fp->f_args, "fstype", &fs, &len);
 614         if (rv != 0 && rv != ENOENT)
 615                 return (rv);
 616
 617         if (fs != NULL && strncmp(fs, "zfs", len) != 0)
 618                 zfs_root = false;
 619
 620         /* zfs_bootfs() will set the environment, it must be called. */
 621         if (zfs_root == true)
 622                 fs = zfs_bootfs(rootdev);
 623
 624         if (fp->f_args == NULL)
 625                 cmdline = strdup(fp->f_name);
 626         else
 627                 asprintf(&cmdline, "%s %s", fp->f_name, fp->f_args);
 628
 629         if (cmdline == NULL)
 630                 return (ENOMEM);
 631
 632         /* Append zfs-bootfs for MB1 command line. */
 633         if (mb2 == false && zfs_root == true) {
 634                 char *tmp;
 635
 636                 tmp = insert_cmdline(cmdline, fs);
 637                 free(cmdline);
 638                 if (tmp == NULL)
 639                         return (ENOMEM);
 640                 cmdline = tmp;
 641         }
 642
 643         *line = update_cmdline(cmdline, mb2);
 644         return (0);
 645 }
 646
 647 /*
 648  * Returns allocated virtual address from MB info area.
 649  */
 650 static vm_offset_t
 651 mb_malloc(size_t n)
 652 {
 653         vm_offset_t ptr = last_addr;
 654         last_addr = roundup(last_addr + n, MULTIBOOT_TAG_ALIGN);
 655         return (ptr);
 656 }
 657
 658 /*
 659  * Calculate size for module tag list.
 660  */
 661 static size_t
 662 module_size(struct preloaded_file *fp)
 663 {
 664         size_t len, size;
 665         struct preloaded_file *mfp;
 666
 667         size = 0;
 668         for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
 669                 len = strlen(mfp->f_name) + 1;
 670                 len += strlen(mfp->f_type) + 5 + 1; /* 5 is for "type=" */
 671                 if (mfp->f_args != NULL)
 672                         len += strlen(mfp->f_args) + 1;
 673                 size += sizeof (multiboot_tag_module_t) + len;
 674                 size = roundup(size, MULTIBOOT_TAG_ALIGN);
 675         }
 676         return (size);
 677 }
 678
 679 #if defined (EFI)
 680 /*
 681  * Calculate size for UEFI memory map tag.
 682  */
 683 static int
 684 efimemmap_size(void)
 685 {
 686         UINTN size, cur_size, desc_size;
 687         EFI_MEMORY_DESCRIPTOR *mmap;
 688         EFI_STATUS ret;
 689
 690         size = EFI_PAGE_SIZE;           /* Start with 4k. */
 691         while (1) {
 692                 cur_size = size;
 693                 mmap = malloc(cur_size);
 694                 if (mmap == NULL)
 695                         return (0);
 696                 ret = BS->GetMemoryMap(&cur_size, mmap, NULL, &desc_size, NULL);
 697                 free(mmap);
 698                 if (ret == EFI_SUCCESS)
 699                         break;
 700                 if (ret == EFI_BUFFER_TOO_SMALL) {
 701                         if (size < cur_size)
 702                                 size = cur_size;
 703                         size += (EFI_PAGE_SIZE);
 704                 } else
 705                         return (0);
 706         }
 707
 708         /* EFI MMAP will grow when we allocate MBI, set some buffer. */
 709         size += (3 << EFI_PAGE_SHIFT);
 710         size = roundup(size, desc_size);
 711         return (sizeof (multiboot_tag_efi_mmap_t) + size);
 712 }
 713 #endif
 714
 715 /*
 716  * Calculate size for bios smap tag.
 717  */
 718 static size_t
 719 biossmap_size(struct preloaded_file *fp)
 720 {
 721         int num;
 722         struct file_metadata *md;
 723
 724         md = file_findmetadata(fp, MODINFOMD_SMAP);
 725         if (md == NULL)
 726                 return (0);
 727
 728         num = md->md_size / sizeof(struct bios_smap); /* number of entries */
 729         return (sizeof (multiboot_tag_mmap_t) +
 730             num * sizeof (multiboot_mmap_entry_t));
 731 }
 732
 733 static size_t
 734 mbi_size(struct preloaded_file *fp, char *cmdline)
 735 {
 736         size_t size;
 737 #if !defined (EFI)
 738         extern multiboot_tag_framebuffer_t gfx_fb;
 739 #endif
 740
 741         size = sizeof (uint32_t) * 2; /* first 2 fields from MBI header */
 742         size += sizeof (multiboot_tag_string_t) + strlen(cmdline) + 1;
 743         size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 744         size += sizeof (multiboot_tag_string_t) + strlen(bootprog_info) + 1;
 745         size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 746 #if !defined (EFI)
 747         size += sizeof (multiboot_tag_basic_meminfo_t);
 748         size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 749 #endif
 750         size += module_size(fp);
 751         size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 752 #if defined (EFI)
 753         size += sizeof (multiboot_tag_efi64_t);
 754         size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 755         size += efimemmap_size();
 756         size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 757
 758         if (have_framebuffer == true) {
 759                 size += sizeof (multiboot_tag_framebuffer_t);
 760                 size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 761         }
 762 #endif
 763
 764         size += biossmap_size(fp);
 765         size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 766
 767 #if !defined (EFI)
 768         if (gfx_fb.framebuffer_common.framebuffer_type ==
 769             MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED) {
 770                 size += sizeof (struct multiboot_tag_framebuffer_common);
 771                 size += gfx_fb.u.fb1.framebuffer_palette_num_colors *
 772                     sizeof (multiboot_color_t);
 773         } else {
 774                 size += sizeof (multiboot_tag_framebuffer_t);
 775         }
 776         size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 777
 778         size += sizeof (multiboot_tag_vbe_t);
 779         size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 780 #endif
 781
 782         if (bootp_response != NULL) {
 783                 size += sizeof(multiboot_tag_network_t) + bootp_response_size;
 784                 size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 785         }
 786
 787         if (rsdp != NULL) {
 788                 if (rsdp->Revision == 0) {
 789                         size += sizeof (multiboot_tag_old_acpi_t) +
 790                             sizeof(ACPI_RSDP_COMMON);
 791                 } else {
 792                         size += sizeof (multiboot_tag_new_acpi_t) +
 793                             rsdp->Length;
 794                 }
 795                 size = roundup2(size, MULTIBOOT_TAG_ALIGN);
 796         }
 797         size += sizeof(multiboot_tag_t);
 798
 799         return (size);
 800 }
 801
 802 static int
 803 multiboot2_exec(struct preloaded_file *fp)
 804 {
 805         struct preloaded_file *mfp;
 806         multiboot2_info_header_t *mbi;
 807         char *cmdline = NULL;
 808         struct devdesc *rootdev;
 809         struct file_metadata *md;
 810         int i, error, num;
 811         int rootfs = 0;
 812         size_t size;
 813         struct bios_smap *smap;
 814 #if defined (EFI)
 815         multiboot_tag_module_t *module, *mp;
 816         EFI_MEMORY_DESCRIPTOR *map;
 817         UINTN map_size, desc_size;
 818         struct relocator *relocator;
 819         struct chunk_head *head;
 820         struct chunk *chunk;
 821         vm_offset_t tmp;
 822
 823         efi_getdev((void **)(&rootdev), NULL, NULL);
 824 #else
 825         i386_getdev((void **)(&rootdev), NULL, NULL);
 826
 827         if (have_framebuffer == false) {
 828                 /* make sure we have text mode */
 829                 bios_set_text_mode(VGA_TEXT_MODE);
 830         }
 831 #endif
 832
 833         mbi = NULL;
 834         error = EINVAL;
 835         if (rootdev == NULL) {
 836                 printf("can't determine root device\n");
 837                 goto error;
 838         }
 839
 840         /*
 841          * Set the image command line.
 842          */
 843         if (fp->f_args == NULL) {
 844                 cmdline = getenv("boot-args");
 845                 if (cmdline != NULL) {
 846                         fp->f_args = strdup(cmdline);
 847                         if (fp->f_args == NULL) {
 848                                 error = ENOMEM;
 849                                 goto error;
 850                         }
 851                 }
 852         }
 853
 854         error = mb_kernel_cmdline(fp, rootdev, &cmdline);
 855         if (error != 0)
 856                 goto error;
 857
 858         /* mb_kernel_cmdline() updates the environment. */
 859         build_environment_module();
 860
 861         if (have_framebuffer == true) {
 862                 /* Pass the loaded console font for kernel. */
 863                 build_font_module();
 864         }
 865
 866         size = mbi_size(fp, cmdline);   /* Get the size for MBI. */
 867
 868         /* Set up the base for mb_malloc. */
 869         i = 0;
 870         for (mfp = fp; mfp->f_next != NULL; mfp = mfp->f_next)
 871                 i++;
 872
 873 #if defined (EFI)
 874         /* We need space for kernel + MBI + # modules */
 875         num = (EFI_PAGE_SIZE - offsetof(struct relocator, rel_chunklist)) /
 876             sizeof (struct chunk);
 877         if (i + 2 >= num) {
 878                 printf("Too many modules, do not have space for relocator.\n");
 879                 error = ENOMEM;
 880                 goto error;
 881         }
 882
 883         last_addr = efi_loadaddr(LOAD_MEM, &size, mfp->f_addr + mfp->f_size);
 884         mbi = (multiboot2_info_header_t *)last_addr;
 885         if (mbi == NULL) {
 886                 error = ENOMEM;
 887                 goto error;
 888         }
 889         last_addr = (vm_offset_t)mbi->mbi_tags;
 890 #else
 891         /* Start info block from the new page. */
 892         last_addr = i386_loadaddr(LOAD_MEM, &size, mfp->f_addr + mfp->f_size);
 893
 894         /* Do we have space for multiboot info? */
 895         if (last_addr + size >= memtop_copyin) {
 896                 error = ENOMEM;
 897                 goto error;
 898         }
 899
 900         mbi = (multiboot2_info_header_t *)PTOV(last_addr);
 901         last_addr = (vm_offset_t)mbi->mbi_tags;
 902 #endif  /* EFI */
 903
 904         {
 905                 multiboot_tag_string_t *tag;
 906                 i = sizeof (multiboot_tag_string_t) + strlen(cmdline) + 1;
 907                 tag = (multiboot_tag_string_t *) mb_malloc(i);
 908
 909                 tag->mb_type = MULTIBOOT_TAG_TYPE_CMDLINE;
 910                 tag->mb_size = i;
 911                 memcpy(tag->mb_string, cmdline, strlen(cmdline) + 1);
 912                 free(cmdline);
 913                 cmdline = NULL;
 914         }
 915
 916         {
 917                 multiboot_tag_string_t *tag;
 918                 i = sizeof (multiboot_tag_string_t) + strlen(bootprog_info) + 1;
 919                 tag = (multiboot_tag_string_t *) mb_malloc(i);
 920
 921                 tag->mb_type = MULTIBOOT_TAG_TYPE_BOOT_LOADER_NAME;
 922                 tag->mb_size = i;
 923                 memcpy(tag->mb_string, bootprog_info,
 924                     strlen(bootprog_info) + 1);
 925         }
 926
 927 #if !defined (EFI)
 928         /* Only set in case of BIOS. */
 929         {
 930                 multiboot_tag_basic_meminfo_t *tag;
 931                 tag = (multiboot_tag_basic_meminfo_t *)
 932                     mb_malloc(sizeof (*tag));
 933
 934                 tag->mb_type = MULTIBOOT_TAG_TYPE_BASIC_MEMINFO;
 935                 tag->mb_size = sizeof (*tag);
 936                 tag->mb_mem_lower = bios_basemem / 1024;
 937                 tag->mb_mem_upper = bios_extmem / 1024;
 938         }
 939 #endif
 940
 941         num = 0;
 942         for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
 943                 num++;
 944                 if (mfp->f_type != NULL && strcmp(mfp->f_type, "rootfs") == 0)
 945                         rootfs++;
 946         }
 947
 948         if (num == 0 || rootfs == 0) {
 949                 /* We need at least one module - rootfs. */
 950                 printf("No rootfs module provided, aborting\n");
 951                 error = EINVAL;
 952                 goto error;
 953         }
 954
 955         /*
 956          * Set the stage for physical memory layout:
 957          * - We have kernel at load_addr.
 958          * - Modules are aligned to page boundary.
 959          * - MBI is aligned to page boundary.
 960          * - Set the tmp to point to physical address of the first module.
 961          * - tmp != mfp->f_addr only in case of EFI.
 962          */
 963 #if defined (EFI)
 964         tmp = roundup2(load_addr + fp->f_size + 1, MULTIBOOT_MOD_ALIGN);
 965         module = (multiboot_tag_module_t *)last_addr;
 966 #endif
 967
 968         for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
 969                 multiboot_tag_module_t *tag;
 970
 971                 num = strlen(mfp->f_name) + 1;
 972                 num += strlen(mfp->f_type) + 5 + 1;
 973                 if (mfp->f_args != NULL) {
 974                         num += strlen(mfp->f_args) + 1;
 975                 }
 976                 cmdline = malloc(num);
 977                 if (cmdline == NULL) {
 978                         error = ENOMEM;
 979                         goto error;
 980                 }
 981
 982                 if (mfp->f_args != NULL)
 983                         snprintf(cmdline, num, "%s type=%s %s",
 984                             mfp->f_name, mfp->f_type, mfp->f_args);
 985                 else
 986                         snprintf(cmdline, num, "%s type=%s",
 987                             mfp->f_name, mfp->f_type);
 988
 989                 tag = (multiboot_tag_module_t *)mb_malloc(sizeof (*tag) + num);
 990
 991                 tag->mb_type = MULTIBOOT_TAG_TYPE_MODULE;
 992                 tag->mb_size = sizeof (*tag) + num;
 993 #if defined (EFI)
 994                 /*
 995                  * We can assign module addresses only after BS have been
 996                  * switched off.
 997                  */
 998                 tag->mb_mod_start = 0;
 999                 tag->mb_mod_end = mfp->f_size;
1000 #else
1001                 tag->mb_mod_start = mfp->f_addr;
1002                 tag->mb_mod_end = mfp->f_addr + mfp->f_size;
1003 #endif
1004                 memcpy(tag->mb_cmdline, cmdline, num);
1005                 free(cmdline);
1006                 cmdline = NULL;
1007         }
1008
1009         md = file_findmetadata(fp, MODINFOMD_SMAP);
1010         if (md == NULL) {
1011                 printf("no memory smap\n");
1012                 error = EINVAL;
1013                 goto error;
1014         }
1015
1016         smap = (struct bios_smap *)md->md_data;
1017         num = md->md_size / sizeof(struct bios_smap); /* number of entries */
1018
1019         {
1020                 multiboot_tag_mmap_t *tag;
1021                 multiboot_mmap_entry_t *mmap_entry;
1022
1023                 tag = (multiboot_tag_mmap_t *)
1024                     mb_malloc(sizeof (*tag) +
1025                     num * sizeof (multiboot_mmap_entry_t));
1026
1027                 tag->mb_type = MULTIBOOT_TAG_TYPE_MMAP;
1028                 tag->mb_size = sizeof (*tag) +
1029                     num * sizeof (multiboot_mmap_entry_t);
1030                 tag->mb_entry_size = sizeof (multiboot_mmap_entry_t);
1031                 tag->mb_entry_version = 0;
1032                 mmap_entry = (multiboot_mmap_entry_t *)tag->mb_entries;
1033
1034                 for (i = 0; i < num; i++) {
1035                         mmap_entry[i].mmap_addr = smap[i].base;
1036                         mmap_entry[i].mmap_len = smap[i].length;
1037                         mmap_entry[i].mmap_type = smap[i].type;
1038                         mmap_entry[i].mmap_reserved = 0;
1039                 }
1040         }
1041
1042         if (bootp_response != NULL) {
1043                 multiboot_tag_network_t *tag;
1044                 tag = (multiboot_tag_network_t *)
1045                     mb_malloc(sizeof(*tag) + bootp_response_size);
1046
1047                 tag->mb_type = MULTIBOOT_TAG_TYPE_NETWORK;
1048                 tag->mb_size = sizeof(*tag) + bootp_response_size;
1049                 memcpy(tag->mb_dhcpack, bootp_response, bootp_response_size);
1050         }
1051
1052 #if !defined (EFI)
1053         {
1054                 multiboot_tag_vbe_t *tag;
1055                 extern multiboot_tag_vbe_t vbestate;
1056
1057                 if (VBE_VALID_MODE(vbestate.vbe_mode)) {
1058                         tag = (multiboot_tag_vbe_t *)mb_malloc(sizeof(*tag));
1059                         memcpy(tag, &vbestate, sizeof(*tag));
1060                         tag->mb_type = MULTIBOOT_TAG_TYPE_VBE;
1061                         tag->mb_size = sizeof(*tag);
1062                 }
1063         }
1064 #endif
1065
1066         if (rsdp != NULL) {
1067                 multiboot_tag_new_acpi_t *ntag;
1068                 multiboot_tag_old_acpi_t *otag;
1069                 uint32_t tsize;
1070
1071                 if (rsdp->Revision == 0) {
1072                         tsize = sizeof (*otag) + sizeof (ACPI_RSDP_COMMON);
1073                         otag = (multiboot_tag_old_acpi_t *)mb_malloc(tsize);
1074                         otag->mb_type = MULTIBOOT_TAG_TYPE_ACPI_OLD;
1075                         otag->mb_size = tsize;
1076                         memcpy(otag->mb_rsdp, rsdp, sizeof (ACPI_RSDP_COMMON));
1077                 } else {
1078                         tsize = sizeof (*ntag) + rsdp->Length;
1079                         ntag = (multiboot_tag_new_acpi_t *)mb_malloc(tsize);
1080                         ntag->mb_type = MULTIBOOT_TAG_TYPE_ACPI_NEW;
1081                         ntag->mb_size = tsize;
1082                         memcpy(ntag->mb_rsdp, rsdp, rsdp->Length);
1083                 }
1084         }
1085
1086 #if defined (EFI)
1087 #ifdef  __LP64__
1088         {
1089                 multiboot_tag_efi64_t *tag;
1090                 tag = (multiboot_tag_efi64_t *)
1091                     mb_malloc(sizeof (*tag));
1092
1093                 tag->mb_type = MULTIBOOT_TAG_TYPE_EFI64;
1094                 tag->mb_size = sizeof (*tag);
1095                 tag->mb_pointer = (uint64_t)(uintptr_t)ST;
1096         }
1097 #else
1098         {
1099                 multiboot_tag_efi32_t *tag;
1100                 tag = (multiboot_tag_efi32_t *)
1101                     mb_malloc(sizeof (*tag));
1102
1103                 tag->mb_type = MULTIBOOT_TAG_TYPE_EFI32;
1104                 tag->mb_size = sizeof (*tag);
1105                 tag->mb_pointer = (uint32_t)ST;
1106         }
1107 #endif /* __LP64__ */
1108 #endif /* EFI */
1109
1110         if (have_framebuffer == true) {
1111                 multiboot_tag_framebuffer_t *tag;
1112                 extern multiboot_tag_framebuffer_t gfx_fb;
1113 #if defined (EFI)
1114
1115                 tag = (multiboot_tag_framebuffer_t *) mb_malloc(sizeof (*tag));
1116                 memcpy(tag, &gfx_fb, sizeof (*tag));
1117                 tag->framebuffer_common.mb_type =
1118                     MULTIBOOT_TAG_TYPE_FRAMEBUFFER;
1119                 tag->framebuffer_common.mb_size = sizeof (*tag);
1120 #else
1121                 extern multiboot_color_t *cmap;
1122                 uint32_t size;
1123
1124                 if (gfx_fb.framebuffer_common.framebuffer_type ==
1125                     MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED) {
1126                         uint16_t nc;
1127                         nc = gfx_fb.u.fb1.framebuffer_palette_num_colors;
1128                         size = sizeof (struct multiboot_tag_framebuffer_common)
1129                             + sizeof (nc)
1130                             + nc * sizeof (multiboot_color_t);
1131                 } else {
1132                         size = sizeof (gfx_fb);
1133                 }
1134
1135                 tag = (multiboot_tag_framebuffer_t *) mb_malloc(size);
1136                 memcpy(tag, &gfx_fb, sizeof (*tag));
1137
1138                 tag->framebuffer_common.mb_type =
1139                     MULTIBOOT_TAG_TYPE_FRAMEBUFFER;
1140                 tag->framebuffer_common.mb_size = size;
1141
1142                 if (gfx_fb.framebuffer_common.framebuffer_type ==
1143                     MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED) {
1144                         memcpy(tag->u.fb1.framebuffer_palette, cmap,
1145                             sizeof (multiboot_color_t) *
1146                             gfx_fb.u.fb1.framebuffer_palette_num_colors);
1147                 }
1148 #endif /* EFI */
1149         }
1150
1151 #if defined (EFI)
1152         /* Leave EFI memmap last as we will also switch off the BS. */
1153         {
1154                 multiboot_tag_efi_mmap_t *tag;
1155                 UINTN key;
1156                 EFI_STATUS status;
1157
1158                 tag = (multiboot_tag_efi_mmap_t *)
1159                     mb_malloc(sizeof (*tag));
1160
1161                 map_size = 0;
1162                 status = BS->GetMemoryMap(&map_size,
1163                     (EFI_MEMORY_DESCRIPTOR *)tag->mb_efi_mmap, &key,
1164                     &desc_size, &tag->mb_descr_vers);
1165                 if (status != EFI_BUFFER_TOO_SMALL) {
1166                         error = EINVAL;
1167                         goto error;
1168                 }
1169                 status = BS->GetMemoryMap(&map_size,
1170                     (EFI_MEMORY_DESCRIPTOR *)tag->mb_efi_mmap, &key,
1171                     &desc_size, &tag->mb_descr_vers);
1172                 if (EFI_ERROR(status)) {
1173                         error = EINVAL;
1174                         goto error;
1175                 }
1176                 tag->mb_type = MULTIBOOT_TAG_TYPE_EFI_MMAP;
1177                 tag->mb_size = sizeof (*tag) + map_size;
1178                 tag->mb_descr_size = (uint32_t) desc_size;
1179
1180                 /*
1181                  * Find relocater pages. We assume we have free pages
1182                  * below kernel load address.
1183                  * In this version we are using 5 pages:
1184                  * relocator data, trampoline, copy, memmove, stack.
1185                  */
1186                 for (i = 0, map = (EFI_MEMORY_DESCRIPTOR *)tag->mb_efi_mmap;
1187                     i < map_size / desc_size;
1188                     i++, map = NextMemoryDescriptor(map, desc_size)) {
1189                         if (map->PhysicalStart == 0)
1190                                 continue;
1191                         if (map->Type != EfiConventionalMemory)
1192                                 continue;
1193                         if (map->PhysicalStart < load_addr &&
1194                             map->NumberOfPages > 5)
1195                                 break;
1196                 }
1197                 if (map->PhysicalStart == 0)
1198                         panic("Could not find memory for relocater\n");
1199
1200                 if (keep_bs == 0) {
1201                         status = BS->ExitBootServices(IH, key);
1202                         if (EFI_ERROR(status)) {
1203                                 printf("Call to ExitBootServices failed\n");
1204                                 error = EINVAL;
1205                                 goto error;
1206                         }
1207                 }
1208
1209                 last_addr += map_size;
1210                 last_addr = roundup2(last_addr, MULTIBOOT_TAG_ALIGN);
1211         }
1212 #endif /* EFI */
1213
1214         /*
1215          * MB tag list end marker.
1216          */
1217         {
1218                 multiboot_tag_t *tag = (multiboot_tag_t *)
1219                     mb_malloc(sizeof(*tag));
1220                 tag->mb_type = MULTIBOOT_TAG_TYPE_END;
1221                 tag->mb_size = sizeof(*tag);
1222         }
1223
1224         mbi->mbi_total_size = last_addr - (vm_offset_t)mbi;
1225         mbi->mbi_reserved = 0;
1226
1227 #if defined (EFI)
1228         /* At this point we have load_addr pointing to kernel load
1229          * address, module list in MBI having physical addresses,
1230          * module list in fp having logical addresses and tmp pointing to
1231          * physical address for MBI.
1232          * Now we must move all pieces to place and start the kernel.
1233          */
1234         relocator = (struct relocator *)(uintptr_t)map->PhysicalStart;
1235         head = &relocator->rel_chunk_head;
1236         STAILQ_INIT(head);
1237
1238         i = 0;
1239         chunk = &relocator->rel_chunklist[i++];
1240         chunk->chunk_vaddr = fp->f_addr;
1241         chunk->chunk_paddr = load_addr;
1242         chunk->chunk_size = fp->f_size;
1243
1244         STAILQ_INSERT_TAIL(head, chunk, chunk_next);
1245
1246         mp = module;
1247         for (mfp = fp->f_next; mfp != NULL; mfp = mfp->f_next) {
1248                 chunk = &relocator->rel_chunklist[i++];
1249                 chunk->chunk_vaddr = mfp->f_addr;
1250
1251                 /*
1252                  * fix the mb_mod_start and mb_mod_end.
1253                  */
1254                 mp->mb_mod_start = efi_physaddr(module, tmp, map,
1255                     map_size / desc_size, desc_size, mp->mb_mod_end);
1256                 if (mp->mb_mod_start == 0)
1257                         panic("Could not find memory for module\n");
1258
1259                 mp->mb_mod_end += mp->mb_mod_start;
1260                 chunk->chunk_paddr = mp->mb_mod_start;
1261                 chunk->chunk_size = mfp->f_size;
1262                 STAILQ_INSERT_TAIL(head, chunk, chunk_next);
1263
1264                 mp = (multiboot_tag_module_t *)
1265                     roundup2((uintptr_t)mp + mp->mb_size,
1266                     MULTIBOOT_TAG_ALIGN);
1267         }
1268         chunk = &relocator->rel_chunklist[i++];
1269         chunk->chunk_vaddr = (EFI_VIRTUAL_ADDRESS)(uintptr_t)mbi;
1270         chunk->chunk_paddr = efi_physaddr(module, tmp, map,
1271                     map_size / desc_size, desc_size, mbi->mbi_total_size);
1272         chunk->chunk_size = mbi->mbi_total_size;
1273         STAILQ_INSERT_TAIL(head, chunk, chunk_next);
1274
1275         trampoline = (void *)(uintptr_t)relocator + EFI_PAGE_SIZE;
1276         memmove(trampoline, multiboot_tramp, EFI_PAGE_SIZE);
1277
1278         relocator->rel_copy = (uintptr_t)trampoline + EFI_PAGE_SIZE;
1279         memmove((void *)relocator->rel_copy, efi_copy_finish, EFI_PAGE_SIZE);
1280
1281         relocator->rel_memmove = (uintptr_t)relocator->rel_copy + EFI_PAGE_SIZE;
1282         memmove((void *)relocator->rel_memmove, memmove, EFI_PAGE_SIZE);
1283         relocator->rel_stack = relocator->rel_memmove + EFI_PAGE_SIZE - 8;
1284
1285         trampoline(MULTIBOOT2_BOOTLOADER_MAGIC, relocator, entry_addr);
1286 #else
1287         dev_cleanup();
1288         __exec((void *)VTOP(multiboot_tramp), MULTIBOOT2_BOOTLOADER_MAGIC,
1289             (void *)entry_addr, (void *)VTOP(mbi));
1290 #endif /* EFI */
1291         panic("exec returned");
1292
1293 error:
1294         if (cmdline != NULL)
1295                 free(cmdline);
1296 #if defined (EFI)
1297         if (mbi != NULL)
1298                 efi_free_loadaddr((vm_offset_t)mbi, EFI_SIZE_TO_PAGES(size));
1299 #endif
1300         return (error);
1301 }