libbe: be_do_installboot() fails with vdev replacing

[unleashed.git] / usr / src / lib / libbe / common / be_activate.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 */
/*
 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
 */

/*
 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
 * Copyright 2016 Toomas Soome <tsoome@me.com>
 */

#include <assert.h>
#include <libintl.h>
#include <libnvpair.h>
#include <libzfs.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <errno.h>
#include <sys/mnttab.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/efi_partition.h>

#include <libbe.h>
#include <libbe_priv.h>

char    *mnttab = MNTTAB;

/*
 * Private function prototypes
 */
static int set_bootfs(char *boot_rpool, char *be_root_ds);
static int set_canmount(be_node_list_t *, char *);
static boolean_t be_do_install_mbr(char *, nvlist_t *);
static int be_do_installboot_helper(zpool_handle_t *, nvlist_t *, char *,
    char *, uint16_t);
static int be_do_installboot(be_transaction_data_t *, uint16_t);
static int be_promote_zone_ds(char *, char *);
static int be_promote_ds_callback(zfs_handle_t *, void *);

/* ******************************************************************** */
/*                      Public Functions                                */
/* ******************************************************************** */

/*
 * Function:    be_activate
 * Description: Calls _be_activate which activates the BE named in the
 *              attributes passed in through be_attrs. The process of
 *              activation sets the bootfs property of the root pool, resets
 *              the canmount property to noauto, and sets the default in the
 *              menu to the entry corresponding to the entry for the named BE.
 * Parameters:
 *              be_attrs - pointer to nvlist_t of attributes being passed in.
 *                      The follow attribute values are used by this function:
 *
 *                      BE_ATTR_ORIG_BE_NAME            *required
 * Return:
 *              BE_SUCCESS - Success
 *              be_errno_t - Failure
 * Scope:
 *              Public
 */
int
be_activate(nvlist_t *be_attrs)
{
        int     ret = BE_SUCCESS;
        char    *be_name = NULL;

        /* Initialize libzfs handle */
        if (!be_zfs_init())
                return (BE_ERR_INIT);

        /* Get the BE name to activate */
        if (nvlist_lookup_string(be_attrs, BE_ATTR_ORIG_BE_NAME, &be_name)
            != 0) {
                be_print_err(gettext("be_activate: failed to "
                    "lookup BE_ATTR_ORIG_BE_NAME attribute\n"));
                be_zfs_fini();
                return (BE_ERR_INVAL);
        }

        /* Validate BE name */
        if (!be_valid_be_name(be_name)) {
                be_print_err(gettext("be_activate: invalid BE name %s\n"),
                    be_name);
                be_zfs_fini();
                return (BE_ERR_INVAL);
        }

        ret = _be_activate(be_name);

        be_zfs_fini();

        return (ret);
}

/*
 * Function:    be_installboot
 * Description: Calls be_do_installboot to install/update bootloader on
 *              pool passed in through be_attrs. The primary consumer is
 *              bootadm command to avoid duplication of the code.
 * Parameters:
 *              be_attrs - pointer to nvlist_t of attributes being passed in.
 *                      The following attribute values are used:
 *
 *                      BE_ATTR_ORIG_BE_NAME            *required
 *                      BE_ATTR_ORIG_BE_POOL            *required
 *                      BE_ATTR_ORIG_BE_ROOT            *required
 *                      BE_ATTR_INSTALL_FLAGS           optional
 *
 * Return:
 *              BE_SUCCESS - Success
 *              be_errno_t - Failure
 * Scope:
 *              Public
 */
int
be_installboot(nvlist_t *be_attrs)
{
        int             ret = BE_SUCCESS;
        uint16_t        flags = 0;
        uint16_t        verbose;
        be_transaction_data_t bt = { 0 };

        /* Get flags */
        if (nvlist_lookup_pairs(be_attrs, NV_FLAG_NOENTOK,
            BE_ATTR_INSTALL_FLAGS, DATA_TYPE_UINT16, &flags, NULL) != 0) {
                be_print_err(gettext("be_installboot: failed to lookup "
                    "BE_ATTR_INSTALL_FLAGS attribute\n"));
                return (BE_ERR_INVAL);
        }

        /* Set verbose early, so we get all messages */
        verbose = flags & BE_INSTALLBOOT_FLAG_VERBOSE;
        if (verbose == BE_INSTALLBOOT_FLAG_VERBOSE)
                libbe_print_errors(B_TRUE);

        ret = nvlist_lookup_string(be_attrs, BE_ATTR_ORIG_BE_NAME,
            &bt.obe_name);
        if (ret != 0) {
                be_print_err(gettext("be_installboot: failed to "
                    "lookup BE_ATTR_ORIG_BE_NAME attribute\n"));
                return (BE_ERR_INVAL);
        }

        ret = nvlist_lookup_string(be_attrs, BE_ATTR_ORIG_BE_POOL,
            &bt.obe_zpool);
        if (ret != 0) {
                be_print_err(gettext("be_installboot: failed to "
                    "lookup BE_ATTR_ORIG_BE_POOL attribute\n"));
                return (BE_ERR_INVAL);
        }

        ret = nvlist_lookup_string(be_attrs, BE_ATTR_ORIG_BE_ROOT,
            &bt.obe_root_ds);
        if (ret != 0) {
                be_print_err(gettext("be_installboot: failed to "
                    "lookup BE_ATTR_ORIG_BE_ROOT attribute\n"));
                return (BE_ERR_INVAL);
        }

        /* Initialize libzfs handle */
        if (!be_zfs_init())
                return (BE_ERR_INIT);

        ret = be_do_installboot(&bt, flags);

        be_zfs_fini();

        return (ret);
}

/* ******************************************************************** */
/*                      Semi Private Functions                          */
/* ******************************************************************** */

/*
 * Function:    _be_activate
 * Description: This does the actual work described in be_activate.
 * Parameters:
 *              be_name - pointer to the name of BE to activate.
 *
 * Return:
 *              BE_SUCCESS - Success
 *              be_errnot_t - Failure
 * Scope:
 *              Public
 */
int
_be_activate(char *be_name)
{
        be_transaction_data_t cb = { 0 };
        zfs_handle_t    *zhp = NULL;
        char            root_ds[MAXPATHLEN];
        char            active_ds[MAXPATHLEN];
        be_node_list_t  *be_nodes = NULL;
        uuid_t          uu = {0};
        int             entry, ret = BE_SUCCESS;
        int             zret = 0;

        /*
         * TODO: The BE needs to be validated to make sure that it is actually
         * a bootable BE.
         */

        if (be_name == NULL)
                return (BE_ERR_INVAL);

        /* Set obe_name to be_name in the cb structure */
        cb.obe_name = be_name;

        /* find which zpool the be is in */
        if ((zret = zpool_iter(g_zfs, be_find_zpool_callback, &cb)) == 0) {
                be_print_err(gettext("be_activate: failed to "
                    "find zpool for BE (%s)\n"), cb.obe_name);
                return (BE_ERR_BE_NOENT);
        } else if (zret < 0) {
                be_print_err(gettext("be_activate: "
                    "zpool_iter failed: %s\n"),
                    libzfs_error_description(g_zfs));
                ret = zfs_err_to_be_err(g_zfs);
                return (ret);
        }

        be_make_root_ds(cb.obe_zpool, cb.obe_name, root_ds, sizeof (root_ds));
        cb.obe_root_ds = strdup(root_ds);

        if (getzoneid() == GLOBAL_ZONEID) {
                ret = be_do_installboot(&cb, BE_INSTALLBOOT_FLAG_NULL);
                if (ret != BE_SUCCESS)
                        return (ret);

                if (!be_has_menu_entry(root_ds, cb.obe_zpool, &entry)) {
                        if ((ret = be_append_menu(cb.obe_name, cb.obe_zpool,
                            NULL, NULL, NULL)) != BE_SUCCESS) {
                                be_print_err(gettext("be_activate: Failed to "
                                    "add BE (%s) to the menu\n"),
                                    cb.obe_name);
                                goto done;
                        }
                }
        }

        if ((ret = _be_list(cb.obe_name, &be_nodes)) != BE_SUCCESS) {
                return (ret);
        }

        if ((ret = set_canmount(be_nodes, "noauto")) != BE_SUCCESS) {
                be_print_err(gettext("be_activate: failed to set "
                    "canmount dataset property\n"));
                goto done;
        }

        if (getzoneid() == GLOBAL_ZONEID) {
                if ((ret = set_bootfs(be_nodes->be_rpool,
                    root_ds)) != BE_SUCCESS) {
                        be_print_err(gettext("be_activate: failed to set "
                            "bootfs pool property for %s\n"), root_ds);
                        goto done;
                }
        }

        if ((zhp = zfs_open(g_zfs, root_ds, ZFS_TYPE_FILESYSTEM)) != NULL) {
                /*
                 * We don't need to close the zfs handle at this
                 * point because The callback funtion
                 * be_promote_ds_callback() will close it for us.
                 */
                if (be_promote_ds_callback(zhp, NULL) != 0) {
                        be_print_err(gettext("be_activate: "
                            "failed to activate the "
                            "datasets for %s: %s\n"),
                            root_ds,
                            libzfs_error_description(g_zfs));
                        ret = BE_ERR_PROMOTE;
                        goto done;
                }
        } else {
                be_print_err(gettext("be_activate: failed to open "
                    "dataset (%s): %s\n"), root_ds,
                    libzfs_error_description(g_zfs));
                ret = zfs_err_to_be_err(g_zfs);
                goto done;
        }

        if (getzoneid() == GLOBAL_ZONEID &&
            be_get_uuid(cb.obe_root_ds, &uu) == BE_SUCCESS &&
            (ret = be_promote_zone_ds(cb.obe_name, cb.obe_root_ds))
            != BE_SUCCESS) {
                be_print_err(gettext("be_activate: failed to promote "
                    "the active zonepath datasets for zones in BE %s\n"),
                    cb.obe_name);
        }

        if (getzoneid() != GLOBAL_ZONEID) {
                if (!be_zone_compare_uuids(root_ds)) {
                        be_print_err(gettext("be_activate: activating zone "
                            "root dataset from non-active global BE is not "
                            "supported\n"));
                        ret = BE_ERR_NOTSUP;
                        goto done;
                }
                if ((zhp = zfs_open(g_zfs, root_ds,
                    ZFS_TYPE_FILESYSTEM)) == NULL) {
                        be_print_err(gettext("be_activate: failed to open "
                            "dataset (%s): %s\n"), root_ds,
                            libzfs_error_description(g_zfs));
                        ret = zfs_err_to_be_err(g_zfs);
                        goto done;
                }
                /* Find current active zone root dataset */
                if ((ret = be_find_active_zone_root(zhp, cb.obe_zpool,
                    active_ds, sizeof (active_ds))) != BE_SUCCESS) {
                        be_print_err(gettext("be_activate: failed to find "
                            "active zone root dataset\n"));
                        ZFS_CLOSE(zhp);
                        goto done;
                }
                /* Do nothing if requested BE is already active */
                if (strcmp(root_ds, active_ds) == 0) {
                        ret = BE_SUCCESS;
                        ZFS_CLOSE(zhp);
                        goto done;
                }

                /* Set active property for BE */
                if (zfs_prop_set(zhp, BE_ZONE_ACTIVE_PROPERTY, "on") != 0) {
                        be_print_err(gettext("be_activate: failed to set "
                            "active property (%s): %s\n"), root_ds,
                            libzfs_error_description(g_zfs));
                        ret = zfs_err_to_be_err(g_zfs);
                        ZFS_CLOSE(zhp);
                        goto done;
                }
                ZFS_CLOSE(zhp);

                /* Unset active property for old active root dataset */
                if ((zhp = zfs_open(g_zfs, active_ds,
                    ZFS_TYPE_FILESYSTEM)) == NULL) {
                        be_print_err(gettext("be_activate: failed to open "
                            "dataset (%s): %s\n"), active_ds,
                            libzfs_error_description(g_zfs));
                        ret = zfs_err_to_be_err(g_zfs);
                        goto done;
                }
                if (zfs_prop_set(zhp, BE_ZONE_ACTIVE_PROPERTY, "off") != 0) {
                        be_print_err(gettext("be_activate: failed to unset "
                            "active property (%s): %s\n"), active_ds,
                            libzfs_error_description(g_zfs));
                        ret = zfs_err_to_be_err(g_zfs);
                        ZFS_CLOSE(zhp);
                        goto done;
                }
                ZFS_CLOSE(zhp);
        }
done:
        be_free_list(be_nodes);
        return (ret);
}

/*
 * Function:    be_activate_current_be
 * Description: Set the currently "active" BE to be "active on boot"
 * Paramters:
 *              none
 * Returns:
 *              BE_SUCCESS - Success
 *              be_errnot_t - Failure
 * Scope:
 *              Semi-private (library wide use only)
 */
int
be_activate_current_be(void)
{
        int ret = BE_SUCCESS;
        be_transaction_data_t bt = { 0 };

        if ((ret = be_find_current_be(&bt)) != BE_SUCCESS) {
                return (ret);
        }

        if ((ret = _be_activate(bt.obe_name)) != BE_SUCCESS) {
                be_print_err(gettext("be_activate_current_be: failed to "
                    "activate %s\n"), bt.obe_name);
                return (ret);
        }

        return (BE_SUCCESS);
}

/*
 * Function:    be_is_active_on_boot
 * Description: Checks if the BE name passed in has the "active on boot"
 *              property set to B_TRUE.
 * Paramters:
 *              be_name - the name of the BE to check
 * Returns:
 *              B_TRUE - if active on boot.
 *              B_FALSE - if not active on boot.
 * Scope:
 *              Semi-private (library wide use only)
 */
boolean_t
be_is_active_on_boot(char *be_name)
{
        be_node_list_t *be_node = NULL;

        if (be_name == NULL) {
                be_print_err(gettext("be_is_active_on_boot: "
                    "be_name must not be NULL\n"));
                return (B_FALSE);
        }

        if (_be_list(be_name, &be_node) != BE_SUCCESS) {
                return (B_FALSE);
        }

        if (be_node == NULL) {
                return (B_FALSE);
        }

        if (be_node->be_active_on_boot) {
                be_free_list(be_node);
                return (B_TRUE);
        } else {
                be_free_list(be_node);
                return (B_FALSE);
        }
}

/* ******************************************************************** */
/*                      Private Functions                               */
/* ******************************************************************** */

/*
 * Function:    set_bootfs
 * Description: Sets the bootfs property on the boot pool to be the
 *              root dataset of the activated BE.
 * Parameters:
 *              boot_pool - The pool we're setting bootfs in.
 *              be_root_ds - The main dataset for the BE.
 * Return:
 *              BE_SUCCESS - Success
 *              be_errno_t - Failure
 * Scope:
 *              Private
 */
static int
set_bootfs(char *boot_rpool, char *be_root_ds)
{
        zpool_handle_t *zhp;
        int err = BE_SUCCESS;

        if ((zhp = zpool_open(g_zfs, boot_rpool)) == NULL) {
                be_print_err(gettext("set_bootfs: failed to open pool "
                    "(%s): %s\n"), boot_rpool, libzfs_error_description(g_zfs));
                err = zfs_err_to_be_err(g_zfs);
                return (err);
        }

        err = zpool_set_prop(zhp, "bootfs", be_root_ds);
        if (err) {
                be_print_err(gettext("set_bootfs: failed to set "
                    "bootfs property for pool %s: %s\n"), boot_rpool,
                    libzfs_error_description(g_zfs));
                err = zfs_err_to_be_err(g_zfs);
                zpool_close(zhp);
                return (err);
        }

        zpool_close(zhp);
        return (BE_SUCCESS);
}

/*
 * Function:    set_canmount
 * Description: Sets the canmount property on the datasets of the
 *              activated BE.
 * Parameters:
 *              be_nodes - The be_node_t returned from be_list
 *              value - The value of canmount we setting, on|off|noauto.
 * Return:
 *              BE_SUCCESS - Success
 *              be_errno_t - Failure
 * Scope:
 *              Private
 */
static int
set_canmount(be_node_list_t *be_nodes, char *value)
{
        char            ds_path[MAXPATHLEN];
        zfs_handle_t    *zhp = NULL;
        be_node_list_t  *list = be_nodes;
        int             err = BE_SUCCESS;

        while (list != NULL) {
                be_dataset_list_t *datasets = list->be_node_datasets;

                be_make_root_ds(list->be_rpool, list->be_node_name, ds_path,
                    sizeof (ds_path));

                if ((zhp = zfs_open(g_zfs, ds_path, ZFS_TYPE_DATASET)) ==
                    NULL) {
                        be_print_err(gettext("set_canmount: failed to open "
                            "dataset (%s): %s\n"), ds_path,
                            libzfs_error_description(g_zfs));
                        err = zfs_err_to_be_err(g_zfs);
                        return (err);
                }
                if (zfs_prop_get_int(zhp, ZFS_PROP_MOUNTED)) {
                        /*
                         * it's already mounted so we can't change the
                         * canmount property anyway.
                         */
                        err = BE_SUCCESS;
                } else {
                        err = zfs_prop_set(zhp,
                            zfs_prop_to_name(ZFS_PROP_CANMOUNT), value);
                        if (err) {
                                ZFS_CLOSE(zhp);
                                be_print_err(gettext("set_canmount: failed to "
                                    "set dataset property (%s): %s\n"),
                                    ds_path, libzfs_error_description(g_zfs));
                                err = zfs_err_to_be_err(g_zfs);
                                return (err);
                        }
                }
                ZFS_CLOSE(zhp);

                while (datasets != NULL) {
                        be_make_root_ds(list->be_rpool,
                            datasets->be_dataset_name, ds_path,
                            sizeof (ds_path));

                        if ((zhp = zfs_open(g_zfs, ds_path, ZFS_TYPE_DATASET))
                            == NULL) {
                                be_print_err(gettext("set_canmount: failed to "
                                    "open dataset %s: %s\n"), ds_path,
                                    libzfs_error_description(g_zfs));
                                err = zfs_err_to_be_err(g_zfs);
                                return (err);
                        }
                        if (zfs_prop_get_int(zhp, ZFS_PROP_MOUNTED)) {
                                /*
                                 * it's already mounted so we can't change the
                                 * canmount property anyway.
                                 */
                                err = BE_SUCCESS;
                                ZFS_CLOSE(zhp);
                                break;
                        }
                        err = zfs_prop_set(zhp,
                            zfs_prop_to_name(ZFS_PROP_CANMOUNT), value);
                        if (err) {
                                ZFS_CLOSE(zhp);
                                be_print_err(gettext("set_canmount: "
                                    "Failed to set property value %s "
                                    "for dataset %s: %s\n"), value, ds_path,
                                    libzfs_error_description(g_zfs));
                                err = zfs_err_to_be_err(g_zfs);
                                return (err);
                        }
                        ZFS_CLOSE(zhp);
                        datasets = datasets->be_next_dataset;
                }
                list = list->be_next_node;
        }
        return (err);
}

/*
 * To be able to boot EFI labeled disks, stage1 needs to be written
 * into the MBR. We do not do this if we're on disks with a traditional
 * fdisk partition table only, or if any foreign EFI partitions exist.
 * In the trivial case of a whole-disk vdev we always write stage1 into
 * the MBR.
 */
static boolean_t
be_do_install_mbr(char *diskname, nvlist_t *child)
{
        struct uuid allowed_uuids[] = {
                EFI_UNUSED,
                EFI_RESV1,
                EFI_BOOT,
                EFI_ROOT,
                EFI_SWAP,
                EFI_USR,
                EFI_BACKUP,
                EFI_RESV2,
                EFI_VAR,
                EFI_HOME,
                EFI_ALTSCTR,
                EFI_RESERVED,
                EFI_SYSTEM,
                EFI_BIOS_BOOT,
                EFI_SYMC_PUB,
                EFI_SYMC_CDS
        };

        uint64_t whole;
        struct dk_gpt *gpt;
        struct uuid *u;
        int fd, npart, i, j;

        (void) nvlist_lookup_uint64(child, ZPOOL_CONFIG_WHOLE_DISK,
            &whole);

        if (whole)
                return (B_TRUE);

        if ((fd = open(diskname, O_RDONLY|O_NDELAY)) < 0)
                return (B_FALSE);

        if ((npart = efi_alloc_and_read(fd, &gpt)) <= 0)
                return (B_FALSE);

        for (i = 0; i != npart; i++) {
                int match = 0;

                u = &gpt->efi_parts[i].p_guid;

                for (j = 0;
                    j != sizeof (allowed_uuids) / sizeof (struct uuid);
                    j++)
                        if (bcmp(u, &allowed_uuids[j],
                            sizeof (struct uuid)) == 0)
                                match++;

                if (match == 0)
                        return (B_FALSE);
        }

        return (B_TRUE);
}

static int
be_do_installboot_helper(zpool_handle_t *zphp, nvlist_t *child, char *stage1,
    char *stage2, uint16_t flags)
{
        char install_cmd[MAXPATHLEN];
        char be_run_cmd_errbuf[BUFSIZ];
        char be_run_cmd_outbuf[BUFSIZ];
        char diskname[MAXPATHLEN];
        char *vname;
        char *path, *type, *dsk_ptr;
        char *flag = "";
        int ret;
        vdev_stat_t *vs;
        uint_t vsc;

        if (nvlist_lookup_string(child, ZPOOL_CONFIG_TYPE, &type) != 0) {
                be_print_err(gettext("%s: failed to get device type\n"),
                    __func__);
                return (BE_ERR_NODEV);
        }
        /* Skip indirect devices. */
        if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
                return (BE_ERR_NOTSUP);

        if (nvlist_lookup_string(child, ZPOOL_CONFIG_PATH, &path) != 0) {
                be_print_err(gettext("%s: failed to get device path\n"),
                    __func__);
                return (BE_ERR_NODEV);
        }

        if ((nvlist_lookup_uint64_array(child, ZPOOL_CONFIG_VDEV_STATS,
            (uint64_t **)&vs, &vsc) != 0) ||
            vs->vs_state < VDEV_STATE_DEGRADED) {
                /*
                 * Don't try to run installboot on a vdev that is not ONLINE
                 * or DEGRADED. Try to print a warning for each such vdev.
                 */
                be_print_err(gettext("%s: vdev %s is %s, can't install "
                    "boot loader\n"), __func__, path,
                    zpool_state_to_name(vs->vs_state, vs->vs_aux));
                return (BE_SUCCESS);
        }

        /*
         * Modify the vdev path to point to the raw disk.
         */
        path = strdup(path);
        if (path == NULL)
                return (BE_ERR_NOMEM);

        dsk_ptr = strstr(path, "/dsk/");
        if (dsk_ptr != NULL) {
                *dsk_ptr = '\0';
                dsk_ptr++;
        } else {
                dsk_ptr = "";
        }

        (void) snprintf(diskname, sizeof (diskname), "%s/r%s", path, dsk_ptr);
        free(path);

        vname = zpool_vdev_name(g_zfs, zphp, child, B_FALSE);
        if (vname == NULL) {
                be_print_err(gettext("%s: failed to get device name: %s\n"),
                    __func__, libzfs_error_description(g_zfs));
                return (zfs_err_to_be_err(g_zfs));
        }

        if (be_is_isa("i386")) {
                uint16_t force = flags & BE_INSTALLBOOT_FLAG_FORCE;
                uint16_t mbr = flags & BE_INSTALLBOOT_FLAG_MBR;

                if (force == BE_INSTALLBOOT_FLAG_FORCE) {
                        if (mbr == BE_INSTALLBOOT_FLAG_MBR ||
                            be_do_install_mbr(diskname, child))
                                flag = "-F -m -f";
                        else
                                flag = "-F";
                } else {
                        if (mbr == BE_INSTALLBOOT_FLAG_MBR ||
                            be_do_install_mbr(diskname, child))
                                flag = "-m -f";
                }

                (void) snprintf(install_cmd, sizeof (install_cmd),
                    "%s %s %s %s %s", BE_INSTALL_BOOT, flag,
                    stage1, stage2, diskname);
        } else if (be_is_isa("sparc")) {
                if ((flags & BE_INSTALLBOOT_FLAG_FORCE) ==
                    BE_INSTALLBOOT_FLAG_FORCE)
                        flag = "-f -F zfs";
                else
                        flag = "-F zfs";

                (void) snprintf(install_cmd, sizeof (install_cmd),
                    "%s %s %s %s", BE_INSTALL_BOOT, flag, stage2, diskname);
        } else {
                be_print_err(gettext("%s: unsupported architecture.\n"),
                    __func__);
                return (BE_ERR_BOOTFILE_INST);
        }

        *be_run_cmd_outbuf = '\0';
        *be_run_cmd_errbuf = '\0';

        ret = be_run_cmd(install_cmd, be_run_cmd_errbuf, BUFSIZ,
            be_run_cmd_outbuf, BUFSIZ);

        if (ret != BE_SUCCESS) {
                be_print_err(gettext("%s: install failed for device %s.\n"),
                    __func__, vname);
                ret = BE_ERR_BOOTFILE_INST;
        }

        be_print_err(gettext("  Command: \"%s\"\n"), install_cmd);
        if (be_run_cmd_outbuf[0] != 0) {
                be_print_err(gettext("  Output:\n"));
                be_print_err("%s", be_run_cmd_outbuf);
        }

        if (be_run_cmd_errbuf[0] != 0) {
                be_print_err(gettext("  Errors:\n"));
                be_print_err("%s", be_run_cmd_errbuf);
        }
        free(vname);

        return (ret);
}

static int
be_do_installboot_walk(zpool_handle_t *zphp, nvlist_t *nv, char *stage1,
    char *stage2, uint16_t flags)
{
        boolean_t verbose = do_print;
        nvlist_t **child;
        uint_t children = 0;
        int ret = -1;

        /* It is OK to have no children. */
        (void) nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
            &children);

        for (int c = 0; c < children; c++) {
                char *vname;
                int rv;

                /* ensure update on child status */
                vname = zpool_vdev_name(g_zfs, zphp, child[c], verbose);
                if (vname == NULL) {
                        be_print_err(gettext("%s: "
                            "failed to get device name: %s\n"), __func__,
                            libzfs_error_description(g_zfs));
                        return (zfs_err_to_be_err(g_zfs));
                } else {
                        be_print_err(gettext("%s: child %d of %d device %s\n"),
                            __func__, c, children, vname);
                }

                rv = be_do_installboot_walk(zphp, child[c], stage1, stage2,
                    flags);
                switch (rv) {
                case BE_ERR_NOTSUP:
                        /* ignore unsupported devices */
                        be_print_err(
                            gettext("%s: device %s is not supported\n"),
                            __func__, vname);
                        break;
                case BE_SUCCESS:
                        /* catch at least one success */
                        ret = rv;
                        break;
                default:
                        if (ret == -1)
                                ret = rv;
                        break;
                }
                free(vname);
        }

        if (children > 0)
                return (ret == -1? BE_ERR_NOTSUP : ret);
        return (be_do_installboot_helper(zphp, nv, stage1, stage2, flags));
}

/*
 * Function:    be_do_installboot
 * Description: This function runs installboot using the boot
 *              loader files from the BE we're activating and installing
 *              them on the pool the BE lives in.
 *
 * Parameters:
 *              bt - The transaction data for the BE we're activating.
 *              flags - flags for bootloader install
 * Return:
 *              BE_SUCCESS - Success
 *              be_errno_t - Failure
 *
 * Scope:
 *              Private
 */
static int
be_do_installboot(be_transaction_data_t *bt, uint16_t flags)
{
        zpool_handle_t  *zphp = NULL;
        zfs_handle_t    *zhp = NULL;
        nvlist_t *nv, *config;
        char *tmp_mntpt = NULL;
        char stage1[MAXPATHLEN];
        char stage2[MAXPATHLEN];
        int ret = BE_SUCCESS;
        boolean_t be_mounted = B_FALSE;

        if ((zhp = zfs_open(g_zfs, bt->obe_root_ds, ZFS_TYPE_FILESYSTEM)) ==
            NULL) {
                be_print_err(gettext("%s: failed to "
                    "open BE root dataset (%s): %s\n"), __func__,
                    bt->obe_root_ds, libzfs_error_description(g_zfs));
                ret = zfs_err_to_be_err(g_zfs);
                return (ret);
        }
        if (!zfs_is_mounted(zhp, &tmp_mntpt)) {
                if ((ret = _be_mount(bt->obe_name, &tmp_mntpt,
                    BE_MOUNT_FLAG_NO_ZONES)) != BE_SUCCESS) {
                        be_print_err(gettext("%s: failed to "
                            "mount BE (%s)\n"), __func__, bt->obe_name);
                        ZFS_CLOSE(zhp);
                        return (ret);
                }
                be_mounted = B_TRUE;
        }
        ZFS_CLOSE(zhp);

        if (be_is_isa("i386")) {
                (void) snprintf(stage1, sizeof (stage1), "%s%s",
                    tmp_mntpt, BE_LOADER_STAGE_1);
                (void) snprintf(stage2, sizeof (stage2), "%s%s",
                    tmp_mntpt, BE_LOADER_STAGE_2);
        } else if (be_is_isa("sparc")) {
                char *platform = be_get_platform();

                if (platform == NULL) {
                        be_print_err(gettext("%s: failed to detect system "
                            "platform name\n"), __func__);
                        if (be_mounted)
                                (void) _be_unmount(bt->obe_name, 0);
                        free(tmp_mntpt);
                        return (BE_ERR_BOOTFILE_INST);
                }
                stage1[0] = '\0';       /* sparc has no stage1 */
                (void) snprintf(stage2, sizeof (stage2),
                    "%s/usr/platform/%s%s", tmp_mntpt,
                    platform, BE_SPARC_BOOTBLK);
        } else {
                be_print_err(gettext("%s: unsupported architecture.\n"),
                    __func__);
                return (BE_ERR_BOOTFILE_INST);
        }

        if ((zphp = zpool_open(g_zfs, bt->obe_zpool)) == NULL) {
                be_print_err(gettext("%s: failed to open "
                    "pool (%s): %s\n"), __func__, bt->obe_zpool,
                    libzfs_error_description(g_zfs));
                ret = zfs_err_to_be_err(g_zfs);
                if (be_mounted)
                        (void) _be_unmount(bt->obe_name, 0);
                free(tmp_mntpt);
                return (ret);
        }

        if ((config = zpool_get_config(zphp, NULL)) == NULL) {
                be_print_err(gettext("%s: failed to get zpool "
                    "configuration information. %s\n"), __func__,
                    libzfs_error_description(g_zfs));
                ret = zfs_err_to_be_err(g_zfs);
                goto done;
        }

        /*
         * Get the vdev tree
         */
        if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nv) != 0) {
                be_print_err(gettext("%s: failed to get vdev "
                    "tree: %s\n"), __func__, libzfs_error_description(g_zfs));
                ret = zfs_err_to_be_err(g_zfs);
                goto done;
        }

        ret = be_do_installboot_walk(zphp, nv, stage1, stage2, flags);

done:
        ZFS_CLOSE(zhp);
        if (be_mounted)
                (void) _be_unmount(bt->obe_name, 0);
        zpool_close(zphp);
        free(tmp_mntpt);
        return (ret);
}

/*
 * Function:    be_promote_zone_ds
 * Description: This function finds the zones for the BE being activated
 *              and the active zonepath dataset for each zone. Then each
 *              active zonepath dataset is promoted.
 *
 * Parameters:
 *              be_name - the name of the global zone BE that we need to
 *                       find the zones for.
 *              be_root_ds - the root dataset for be_name.
 * Return:
 *              BE_SUCCESS - Success
 *              be_errno_t - Failure
 *
 * Scope:
 *              Private
 */
static int
be_promote_zone_ds(char *be_name, char *be_root_ds)
{
        char            *zone_ds = NULL;
        char            *temp_mntpt = NULL;
        char            origin[MAXPATHLEN];
        char            zoneroot_ds[MAXPATHLEN];
        zfs_handle_t    *zhp = NULL;
        zfs_handle_t    *z_zhp = NULL;
        boolean_t       be_mounted = B_FALSE;
        int             err = BE_SUCCESS;
        FILE            *cookie;
        struct zoneent  *ze;

        if ((zhp = zfs_open(g_zfs, be_root_ds,
            ZFS_TYPE_FILESYSTEM)) == NULL) {
                be_print_err(gettext("be_promote_zone_ds: Failed to open "
                    "dataset (%s): %s\n"), be_root_ds,
                    libzfs_error_description(g_zfs));
                err = zfs_err_to_be_err(g_zfs);
                return (err);
        }

        if (!zfs_is_mounted(zhp, &temp_mntpt)) {
                if ((err = _be_mount(be_name, &temp_mntpt,
                    BE_MOUNT_FLAG_NO_ZONES)) != BE_SUCCESS) {
                        be_print_err(gettext("be_promote_zone_ds: failed to "
                            "mount the BE for zones procesing.\n"));
                        ZFS_CLOSE(zhp);
                        return (err);
                }
                be_mounted = B_TRUE;
        }

        /*
         * Set the zone root to the temp mount point for the BE we just mounted.
         */
        zonecfg_set_root((const char *)temp_mntpt);

        cookie = setzoneent();
        while((ze = getzoneent_private(cookie)) != NULL) {

                if (strcmp(ze->zone_name, "global") == 0)
                        continue;

                /* Skip zones that aren't at least installed */
                if (ze->zone_state < ZONE_STATE_INSTALLED)
                        continue;

                if (((zone_ds = be_get_ds_from_dir(ze->zone_path)) == NULL) ||
                    !be_zone_supported(zone_ds)) {
                        free(zone_ds);
                        free(ze);
                        continue;
                }

                if (be_find_active_zone_root(zhp, zone_ds,
                    zoneroot_ds, sizeof (zoneroot_ds)) != 0) {
                        be_print_err(gettext("be_promote_zone_ds: "
                            "Zone does not have an active root "
                            "dataset, skipping this zone.\n"));
                        continue;
                }

                if ((z_zhp = zfs_open(g_zfs, zoneroot_ds,
                    ZFS_TYPE_FILESYSTEM)) == NULL) {
                        be_print_err(gettext("be_promote_zone_ds: "
                            "Failed to open dataset "
                            "(%s): %s\n"), zoneroot_ds,
                            libzfs_error_description(g_zfs));
                        err = zfs_err_to_be_err(g_zfs);
                        goto done;
                }

                if (zfs_prop_get(z_zhp, ZFS_PROP_ORIGIN, origin,
                    sizeof (origin), NULL, NULL, 0, B_FALSE) != 0) {
                        ZFS_CLOSE(z_zhp);
                        continue;
                }

                /*
                 * We don't need to close the zfs handle at this
                 * point because the callback funtion
                 * be_promote_ds_callback() will close it for us.
                 */
                if (be_promote_ds_callback(z_zhp, NULL) != 0) {
                        be_print_err(gettext("be_promote_zone_ds: "
                            "failed to activate the "
                            "datasets for %s: %s\n"),
                            zoneroot_ds,
                            libzfs_error_description(g_zfs));
                        err = BE_ERR_PROMOTE;
                        goto done;
                }
                free(ze);
        }
        endzoneent(cookie);

done:
        if (be_mounted)
                (void) _be_unmount(be_name, 0);
        ZFS_CLOSE(zhp);
        free(temp_mntpt);
        return (err);
}

/*
 * Function:    be_promote_ds_callback
 * Description: This function is used to promote the datasets for the BE
 *              being activated as well as the datasets for the zones BE
 *              being activated.
 *
 * Parameters:
 *              zhp - the zfs handle for zone BE being activated.
 *              data - not used.
 * Return:
 *              0 - Success
 *              be_errno_t - Failure
 *
 * Scope:
 *              Private
 */
static int
/* LINTED */
be_promote_ds_callback(zfs_handle_t *zhp, void *data)
{
        char    origin[MAXPATHLEN];
        char    *sub_dataset = NULL;
        int     ret = 0;

        if (zhp != NULL) {
                sub_dataset = strdup(zfs_get_name(zhp));
                if (sub_dataset == NULL) {
                        ret = BE_ERR_NOMEM;
                        goto done;
                }
        } else {
                be_print_err(gettext("be_promote_ds_callback: "
                    "Invalid zfs handle passed into function\n"));
                ret = BE_ERR_INVAL;
                goto done;
        }

        /*
         * This loop makes sure that we promote the dataset to the
         * top of the tree so that it is no longer a decendent of any
         * dataset. The ZFS close and then open is used to make sure that
         * the promotion is updated before we move on.
         */
        while (zfs_prop_get(zhp, ZFS_PROP_ORIGIN, origin,
            sizeof (origin), NULL, NULL, 0, B_FALSE) == 0) {
                if (zfs_promote(zhp) != 0) {
                        if (libzfs_errno(g_zfs) != EZFS_EXISTS) {
                                be_print_err(gettext("be_promote_ds_callback: "
                                    "promote of %s failed: %s\n"),
                                    zfs_get_name(zhp),
                                    libzfs_error_description(g_zfs));
                                ret = zfs_err_to_be_err(g_zfs);
                                goto done;
                        } else {
                                /*
                                 * If the call to zfs_promote returns the
                                 * error EZFS_EXISTS we've hit a snapshot name
                                 * collision. This means we're probably
                                 * attemping to promote a zone dataset above a
                                 * parent dataset that belongs to another zone
                                 * which this zone was cloned from.
                                 *
                                 * TODO: If this is a zone dataset at some
                                 * point we should skip this if the zone
                                 * paths for the dataset and the snapshot
                                 * don't match.
                                 */
                                be_print_err(gettext("be_promote_ds_callback: "
                                    "promote of %s failed due to snapshot "
                                    "name collision: %s\n"), zfs_get_name(zhp),
                                    libzfs_error_description(g_zfs));
                                ret = zfs_err_to_be_err(g_zfs);
                                goto done;
                        }
                }
                ZFS_CLOSE(zhp);
                if ((zhp = zfs_open(g_zfs, sub_dataset,
                    ZFS_TYPE_FILESYSTEM)) == NULL) {
                        be_print_err(gettext("be_promote_ds_callback: "
                            "Failed to open dataset (%s): %s\n"), sub_dataset,
                            libzfs_error_description(g_zfs));
                        ret = zfs_err_to_be_err(g_zfs);
                        goto done;
                }
        }

        /* Iterate down this dataset's children and promote them */
        ret = zfs_iter_filesystems(zhp, be_promote_ds_callback, NULL);

done:
        free(sub_dataset);
        ZFS_CLOSE(zhp);
        return (ret);
}