5045 use atomic_{inc,dec}_* instead of atomic_add_*

[illumos-gate.git] / usr / src / uts / common / os / mmapobj.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/kmem.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/mman.h>
#include <sys/cmn_err.h>
#include <sys/cred.h>
#include <sys/vmsystm.h>
#include <sys/machsystm.h>
#include <sys/debug.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <sys/vmparam.h>
#include <sys/vfs.h>
#include <sys/elf.h>
#include <sys/machelf.h>
#include <sys/corectl.h>
#include <sys/exec.h>
#include <sys/exechdr.h>
#include <sys/autoconf.h>
#include <sys/mem.h>
#include <vm/seg_dev.h>
#include <sys/vmparam.h>
#include <sys/mmapobj.h>
#include <sys/atomic.h>

/*
 * Theory statement:
 *
 * The main driving force behind mmapobj is to interpret and map ELF files
 * inside of the kernel instead of having the linker be responsible for this.
 *
 * mmapobj also supports the AOUT 4.x binary format as well as flat files in
 * a read only manner.
 *
 * When interpreting and mapping an ELF file, mmapobj will map each PT_LOAD
 * or PT_SUNWBSS segment according to the ELF standard.  Refer to the "Linker
 * and Libraries Guide" for more information about the standard and mapping
 * rules.
 *
 * Having mmapobj interpret and map objects will allow the kernel to make the
 * best decision for where to place the mappings for said objects.  Thus, we
 * can make optimizations inside of the kernel for specific platforms or
 * cache mapping information to make mapping objects faster.
 *
 * The lib_va_hash will be one such optimization.  For each ELF object that
 * mmapobj is asked to interpret, we will attempt to cache the information
 * about the PT_LOAD and PT_SUNWBSS sections to speed up future mappings of
 * the same objects.  We will cache up to LIBVA_CACHED_SEGS (see below) program
 * headers which should cover a majority of the libraries out there without
 * wasting space.  In order to make sure that the cached information is valid,
 * we check the passed in vnode's mtime and ctime to make sure the vnode
 * has not been modified since the last time we used it.
 *
 * In addition, the lib_va_hash may contain a preferred starting VA for the
 * object which can be useful for platforms which support a shared context.
 * This will increase the likelyhood that library text can be shared among
 * many different processes.  We limit the reserved VA space for 32 bit objects
 * in order to minimize fragmenting the processes address space.
 *
 * In addition to the above, the mmapobj interface allows for padding to be
 * requested before the first mapping and after the last mapping created.
 * When padding is requested, no additional optimizations will be made for
 * that request.
 */

/*
 * Threshold to prevent allocating too much kernel memory to read in the
 * program headers for an object.  If it requires more than below,
 * we will use a KM_NOSLEEP allocation to allocate memory to hold all of the
 * program headers which could possibly fail.  If less memory than below is
 * needed, then we use a KM_SLEEP allocation and are willing to wait for the
 * memory if we need to.
 */
size_t mmapobj_alloc_threshold = 65536;

/* Debug stats for test coverage */
#ifdef DEBUG
struct mobj_stats {
        uint_t  mobjs_unmap_called;
        uint_t  mobjs_remap_devnull;
        uint_t  mobjs_lookup_start;
        uint_t  mobjs_alloc_start;
        uint_t  mobjs_alloc_vmem;
        uint_t  mobjs_add_collision;
        uint_t  mobjs_get_addr;
        uint_t  mobjs_map_flat_no_padding;
        uint_t  mobjs_map_flat_padding;
        uint_t  mobjs_map_ptload_text;
        uint_t  mobjs_map_ptload_initdata;
        uint_t  mobjs_map_ptload_preread;
        uint_t  mobjs_map_ptload_unaligned_text;
        uint_t  mobjs_map_ptload_unaligned_map_fail;
        uint_t  mobjs_map_ptload_unaligned_read_fail;
        uint_t  mobjs_zfoddiff;
        uint_t  mobjs_zfoddiff_nowrite;
        uint_t  mobjs_zfodextra;
        uint_t  mobjs_ptload_failed;
        uint_t  mobjs_map_elf_no_holes;
        uint_t  mobjs_unmap_hole;
        uint_t  mobjs_nomem_header;
        uint_t  mobjs_inval_header;
        uint_t  mobjs_overlap_header;
        uint_t  mobjs_np2_align;
        uint_t  mobjs_np2_align_overflow;
        uint_t  mobjs_exec_padding;
        uint_t  mobjs_exec_addr_mapped;
        uint_t  mobjs_exec_addr_devnull;
        uint_t  mobjs_exec_addr_in_use;
        uint_t  mobjs_lvp_found;
        uint_t  mobjs_no_loadable_yet;
        uint_t  mobjs_nothing_to_map;
        uint_t  mobjs_e2big;
        uint_t  mobjs_dyn_pad_align;
        uint_t  mobjs_dyn_pad_noalign;
        uint_t  mobjs_alloc_start_fail;
        uint_t  mobjs_lvp_nocache;
        uint_t  mobjs_extra_padding;
        uint_t  mobjs_lvp_not_needed;
        uint_t  mobjs_no_mem_map_sz;
        uint_t  mobjs_check_exec_failed;
        uint_t  mobjs_lvp_used;
        uint_t  mobjs_wrong_model;
        uint_t  mobjs_noexec_fs;
        uint_t  mobjs_e2big_et_rel;
        uint_t  mobjs_et_rel_mapped;
        uint_t  mobjs_unknown_elf_type;
        uint_t  mobjs_phent32_too_small;
        uint_t  mobjs_phent64_too_small;
        uint_t  mobjs_inval_elf_class;
        uint_t  mobjs_too_many_phdrs;
        uint_t  mobjs_no_phsize;
        uint_t  mobjs_phsize_large;
        uint_t  mobjs_phsize_xtralarge;
        uint_t  mobjs_fast_wrong_model;
        uint_t  mobjs_fast_e2big;
        uint_t  mobjs_fast;
        uint_t  mobjs_fast_success;
        uint_t  mobjs_fast_not_now;
        uint_t  mobjs_small_file;
        uint_t  mobjs_read_error;
        uint_t  mobjs_unsupported;
        uint_t  mobjs_flat_e2big;
        uint_t  mobjs_phent_align32;
        uint_t  mobjs_phent_align64;
        uint_t  mobjs_lib_va_find_hit;
        uint_t  mobjs_lib_va_find_delay_delete;
        uint_t  mobjs_lib_va_find_delete;
        uint_t  mobjs_lib_va_add_delay_delete;
        uint_t  mobjs_lib_va_add_delete;
        uint_t  mobjs_lib_va_create_failure;
        uint_t  mobjs_min_align;
#if defined(__sparc)
        uint_t  mobjs_aout_uzero_fault;
        uint_t  mobjs_aout_64bit_try;
        uint_t  mobjs_aout_noexec;
        uint_t  mobjs_aout_e2big;
        uint_t  mobjs_aout_lib;
        uint_t  mobjs_aout_fixed;
        uint_t  mobjs_aout_zfoddiff;
        uint_t  mobjs_aout_map_bss;
        uint_t  mobjs_aout_bss_fail;
        uint_t  mobjs_aout_nlist;
        uint_t  mobjs_aout_addr_in_use;
#endif
} mobj_stats;

#define MOBJ_STAT_ADD(stat)             ((mobj_stats.mobjs_##stat)++)
#else
#define MOBJ_STAT_ADD(stat)
#endif

/*
 * Check if addr is at or above the address space reserved for the stack.
 * The stack is at the top of the address space for all sparc processes
 * and 64 bit x86 processes.  For 32 bit x86, the stack is not at the top
 * of the address space and thus this check wil always return false for
 * 32 bit x86 processes.
 */
#if defined(__sparc)
#define OVERLAPS_STACK(addr, p)                                         \
        (addr >= (p->p_usrstack - ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK)))
#elif defined(__amd64)
#define OVERLAPS_STACK(addr, p)                                         \
        ((p->p_model == DATAMODEL_LP64) &&                              \
        (addr >= (p->p_usrstack - ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK))))
#elif defined(__i386)
#define OVERLAPS_STACK(addr, p) 0
#endif

/* lv_flags values - bitmap */
#define LV_ELF32        0x1             /* 32 bit ELF file */
#define LV_ELF64        0x2             /* 64 bit ELF file */
#define LV_DEL          0x4             /* delete when lv_refcnt hits zero */

/*
 * Note: lv_num_segs will denote how many segments this file has and will
 * only be set after the lv_mps array has been filled out.
 * lv_mps can only be valid if lv_num_segs is non-zero.
 */
struct lib_va {
        struct lib_va           *lv_next;
        caddr_t                 lv_base_va;     /* start va for library */
        ssize_t                 lv_len;         /* total va span of library */
        size_t                  lv_align;       /* minimum alignment */
        uint64_t                lv_nodeid;      /* filesystem node id */
        uint64_t                lv_fsid;        /* filesystem id */
        timestruc_t             lv_ctime;       /* last time file was changed */
        timestruc_t             lv_mtime;       /* or modified */
        mmapobj_result_t        lv_mps[LIBVA_CACHED_SEGS]; /* cached pheaders */
        int                     lv_num_segs;    /* # segs for this file */
        int                     lv_flags;
        uint_t                  lv_refcnt;      /* number of holds on struct */
};

#define LIB_VA_SIZE     1024
#define LIB_VA_MASK     (LIB_VA_SIZE - 1)
#define LIB_VA_MUTEX_SHIFT      3

#if (LIB_VA_SIZE & (LIB_VA_SIZE - 1))
#error  "LIB_VA_SIZE is not a power of 2"
#endif

static struct lib_va *lib_va_hash[LIB_VA_SIZE];
static kmutex_t lib_va_hash_mutex[LIB_VA_SIZE >> LIB_VA_MUTEX_SHIFT];

#define LIB_VA_HASH_MUTEX(index)                                        \
        (&lib_va_hash_mutex[index >> LIB_VA_MUTEX_SHIFT])

#define LIB_VA_HASH(nodeid)                                             \
        (((nodeid) ^ ((nodeid) << 7) ^ ((nodeid) << 13)) & LIB_VA_MASK)

#define LIB_VA_MATCH_ID(arg1, arg2)                                     \
        ((arg1)->lv_nodeid == (arg2)->va_nodeid &&                      \
        (arg1)->lv_fsid == (arg2)->va_fsid)

#define LIB_VA_MATCH_TIME(arg1, arg2)                                   \
        ((arg1)->lv_ctime.tv_sec == (arg2)->va_ctime.tv_sec &&          \
        (arg1)->lv_mtime.tv_sec == (arg2)->va_mtime.tv_sec &&           \
        (arg1)->lv_ctime.tv_nsec == (arg2)->va_ctime.tv_nsec &&         \
        (arg1)->lv_mtime.tv_nsec == (arg2)->va_mtime.tv_nsec)

#define LIB_VA_MATCH(arg1, arg2)                                        \
        (LIB_VA_MATCH_ID(arg1, arg2) && LIB_VA_MATCH_TIME(arg1, arg2))

/*
 * lib_va will be used for optimized allocation of address ranges for
 * libraries, such that subsequent mappings of the same library will attempt
 * to use the same VA as previous mappings of that library.
 * In order to map libraries at the same VA in many processes, we need to carve
 * out our own address space for them which is unique across many processes.
 * We use different arenas for 32 bit and 64 bit libraries.
 *
 * Since the 32 bit address space is relatively small, we limit the number of
 * libraries which try to use consistent virtual addresses to lib_threshold.
 * For 64 bit libraries there is no such limit since the address space is large.
 */
static vmem_t *lib_va_32_arena;
static vmem_t *lib_va_64_arena;
uint_t lib_threshold = 20;      /* modifiable via /etc/system */

static kmutex_t lib_va_init_mutex;      /* no need to initialize */

/*
 * Number of 32 bit and 64 bit libraries in lib_va hash.
 */
static uint_t libs_mapped_32 = 0;
static uint_t libs_mapped_64 = 0;

/*
 * Free up the resources associated with lvp as well as lvp itself.
 * We also decrement the number of libraries mapped via a lib_va
 * cached virtual address.
 */
void
lib_va_free(struct lib_va *lvp)
{
        int is_64bit = lvp->lv_flags & LV_ELF64;
        ASSERT(lvp->lv_refcnt == 0);

        if (lvp->lv_base_va != NULL) {
                vmem_xfree(is_64bit ? lib_va_64_arena : lib_va_32_arena,
                    lvp->lv_base_va, lvp->lv_len);
                if (is_64bit) {
                        atomic_dec_32(&libs_mapped_64);
                } else {
                        atomic_dec_32(&libs_mapped_32);
                }
        }
        kmem_free(lvp, sizeof (struct lib_va));
}

/*
 * See if the file associated with the vap passed in is in the lib_va hash.
 * If it is and the file has not been modified since last use, then
 * return a pointer to that data.  Otherwise, return NULL if the file has
 * changed or the file was not found in the hash.
 */
static struct lib_va *
lib_va_find(vattr_t *vap)
{
        struct lib_va *lvp;
        struct lib_va *del = NULL;
        struct lib_va **tmp;
        uint_t index;
        index = LIB_VA_HASH(vap->va_nodeid);

        mutex_enter(LIB_VA_HASH_MUTEX(index));
        tmp = &lib_va_hash[index];
        while (*tmp != NULL) {
                lvp = *tmp;
                if (LIB_VA_MATCH_ID(lvp, vap)) {
                        if (LIB_VA_MATCH_TIME(lvp, vap)) {
                                ASSERT((lvp->lv_flags & LV_DEL) == 0);
                                lvp->lv_refcnt++;
                                MOBJ_STAT_ADD(lib_va_find_hit);
                        } else {
                                /*
                                 * file was updated since last use.
                                 * need to remove it from list.
                                 */
                                del = lvp;
                                *tmp = del->lv_next;
                                del->lv_next = NULL;
                                /*
                                 * If we can't delete it now, mark it for later
                                 */
                                if (del->lv_refcnt) {
                                        MOBJ_STAT_ADD(lib_va_find_delay_delete);
                                        del->lv_flags |= LV_DEL;
                                        del = NULL;
                                }
                                lvp = NULL;
                        }
                        mutex_exit(LIB_VA_HASH_MUTEX(index));
                        if (del) {
                                ASSERT(del->lv_refcnt == 0);
                                MOBJ_STAT_ADD(lib_va_find_delete);
                                lib_va_free(del);
                        }
                        return (lvp);
                }
                tmp = &lvp->lv_next;
        }
        mutex_exit(LIB_VA_HASH_MUTEX(index));
        return (NULL);
}

/*
 * Add a new entry to the lib_va hash.
 * Search the hash while holding the appropriate mutex to make sure that the
 * data is not already in the cache.  If we find data that is in the cache
 * already and has not been modified since last use, we return NULL.  If it
 * has been modified since last use, we will remove that entry from
 * the hash and it will be deleted once it's reference count reaches zero.
 * If there is no current entry in the hash we will add the new entry and
 * return it to the caller who is responsible for calling lib_va_release to
 * drop their reference count on it.
 *
 * lv_num_segs will be set to zero since the caller needs to add that
 * information to the data structure.
 */
static struct lib_va *
lib_va_add_hash(caddr_t base_va, ssize_t len, size_t align, vattr_t *vap)
{
        struct lib_va *lvp;
        uint_t index;
        model_t model;
        struct lib_va **tmp;
        struct lib_va *del = NULL;

        model = get_udatamodel();
        index = LIB_VA_HASH(vap->va_nodeid);

        lvp = kmem_alloc(sizeof (struct lib_va), KM_SLEEP);

        mutex_enter(LIB_VA_HASH_MUTEX(index));

        /*
         * Make sure not adding same data a second time.
         * The hash chains should be relatively short and adding
         * is a relatively rare event, so it's worth the check.
         */
        tmp = &lib_va_hash[index];
        while (*tmp != NULL) {
                if (LIB_VA_MATCH_ID(*tmp, vap)) {
                        if (LIB_VA_MATCH_TIME(*tmp, vap)) {
                                mutex_exit(LIB_VA_HASH_MUTEX(index));
                                kmem_free(lvp, sizeof (struct lib_va));
                                return (NULL);
                        }

                        /*
                         * We have the same nodeid and fsid but the file has
                         * been modified since we last saw it.
                         * Need to remove the old node and add this new
                         * one.
                         * Could probably use a callback mechanism to make
                         * this cleaner.
                         */
                        ASSERT(del == NULL);
                        del = *tmp;
                        *tmp = del->lv_next;
                        del->lv_next = NULL;

                        /*
                         * Check to see if we can free it.  If lv_refcnt
                         * is greater than zero, than some other thread
                         * has a reference to the one we want to delete
                         * and we can not delete it.  All of this is done
                         * under the lib_va_hash_mutex lock so it is atomic.
                         */
                        if (del->lv_refcnt) {
                                MOBJ_STAT_ADD(lib_va_add_delay_delete);
                                del->lv_flags |= LV_DEL;
                                del = NULL;
                        }
                        /* tmp is already advanced */
                        continue;
                }
                tmp = &((*tmp)->lv_next);
        }

        lvp->lv_base_va = base_va;
        lvp->lv_len = len;
        lvp->lv_align = align;
        lvp->lv_nodeid = vap->va_nodeid;
        lvp->lv_fsid = vap->va_fsid;
        lvp->lv_ctime.tv_sec = vap->va_ctime.tv_sec;
        lvp->lv_ctime.tv_nsec = vap->va_ctime.tv_nsec;
        lvp->lv_mtime.tv_sec = vap->va_mtime.tv_sec;
        lvp->lv_mtime.tv_nsec = vap->va_mtime.tv_nsec;
        lvp->lv_next = NULL;
        lvp->lv_refcnt = 1;

        /* Caller responsible for filling this and lv_mps out */
        lvp->lv_num_segs = 0;

        if (model == DATAMODEL_LP64) {
                lvp->lv_flags = LV_ELF64;
        } else {
                ASSERT(model == DATAMODEL_ILP32);
                lvp->lv_flags = LV_ELF32;
        }

        if (base_va != NULL) {
                if (model == DATAMODEL_LP64) {
                        atomic_inc_32(&libs_mapped_64);
                } else {
                        ASSERT(model == DATAMODEL_ILP32);
                        atomic_inc_32(&libs_mapped_32);
                }
        }
        ASSERT(*tmp == NULL);
        *tmp = lvp;
        mutex_exit(LIB_VA_HASH_MUTEX(index));
        if (del) {
                ASSERT(del->lv_refcnt == 0);
                MOBJ_STAT_ADD(lib_va_add_delete);
                lib_va_free(del);
        }
        return (lvp);
}

/*
 * Release the hold on lvp which was acquired by lib_va_find or lib_va_add_hash.
 * In addition, if this is the last hold and lvp is marked for deletion,
 * free up it's reserved address space and free the structure.
 */
static void
lib_va_release(struct lib_va *lvp)
{
        uint_t index;
        int to_del = 0;

        ASSERT(lvp->lv_refcnt > 0);

        index = LIB_VA_HASH(lvp->lv_nodeid);
        mutex_enter(LIB_VA_HASH_MUTEX(index));
        if (--lvp->lv_refcnt == 0 && (lvp->lv_flags & LV_DEL)) {
                to_del = 1;
        }
        mutex_exit(LIB_VA_HASH_MUTEX(index));
        if (to_del) {
                ASSERT(lvp->lv_next == 0);
                lib_va_free(lvp);
        }
}

/*
 * Dummy function for mapping through /dev/null
 * Normally I would have used mmmmap in common/io/mem.c
 * but that is a static function, and for /dev/null, it
 * just returns -1.
 */
/* ARGSUSED */
static int
mmapobj_dummy(dev_t dev, off_t off, int prot)
{
        return (-1);
}

/*
 * Called when an error occurred which requires mmapobj to return failure.
 * All mapped objects will be unmapped and /dev/null mappings will be
 * reclaimed if necessary.
 * num_mapped is the number of elements of mrp which have been mapped, and
 * num_segs is the total number of elements in mrp.
 * For e_type ET_EXEC, we need to unmap all of the elements in mrp since
 * we had already made reservations for them.
 * If num_mapped equals num_segs, then we know that we had fully mapped
 * the file and only need to clean up the segments described.
 * If they are not equal, then for ET_DYN we will unmap the range from the
 * end of the last mapped segment to the end of the last segment in mrp
 * since we would have made a reservation for that memory earlier.
 * If e_type is passed in as zero, num_mapped must equal num_segs.
 */
void
mmapobj_unmap(mmapobj_result_t *mrp, int num_mapped, int num_segs,
    ushort_t e_type)
{
        int i;
        struct as *as = curproc->p_as;
        caddr_t addr;
        size_t size;

        if (e_type == ET_EXEC) {
                num_mapped = num_segs;
        }
#ifdef DEBUG
        if (e_type == 0) {
                ASSERT(num_mapped == num_segs);
        }
#endif

        MOBJ_STAT_ADD(unmap_called);
        for (i = 0; i < num_mapped; i++) {

                /*
                 * If we are going to have to create a mapping we need to
                 * make sure that no one else will use the address we
                 * need to remap between the time it is unmapped and
                 * mapped below.
                 */
                if (mrp[i].mr_flags & MR_RESV) {
                        as_rangelock(as);
                }
                /* Always need to unmap what we mapped */
                (void) as_unmap(as, mrp[i].mr_addr, mrp[i].mr_msize);

                /* Need to reclaim /dev/null reservation from earlier */
                if (mrp[i].mr_flags & MR_RESV) {
                        struct segdev_crargs dev_a;

                        ASSERT(e_type != ET_DYN);
                        /*
                         * Use seg_dev segment driver for /dev/null mapping.
                         */
                        dev_a.mapfunc = mmapobj_dummy;
                        dev_a.dev = makedevice(mm_major, M_NULL);
                        dev_a.offset = 0;
                        dev_a.type = 0;         /* neither PRIVATE nor SHARED */
                        dev_a.prot = dev_a.maxprot = (uchar_t)PROT_NONE;
                        dev_a.hat_attr = 0;
                        dev_a.hat_flags = 0;

                        (void) as_map(as, mrp[i].mr_addr, mrp[i].mr_msize,
                            segdev_create, &dev_a);
                        MOBJ_STAT_ADD(remap_devnull);
                        as_rangeunlock(as);
                }
        }

        if (num_mapped != num_segs) {
                ASSERT(e_type == ET_DYN);
                /* Need to unmap any reservation made after last mapped seg */
                if (num_mapped == 0) {
                        addr = mrp[0].mr_addr;
                } else {
                        addr = mrp[num_mapped - 1].mr_addr +
                            mrp[num_mapped - 1].mr_msize;
                }
                size = (size_t)mrp[num_segs - 1].mr_addr +
                    mrp[num_segs - 1].mr_msize - (size_t)addr;
                (void) as_unmap(as, addr, size);

                /*
                 * Now we need to unmap the holes between mapped segs.
                 * Note that we have not mapped all of the segments and thus
                 * the holes between segments would not have been unmapped
                 * yet.  If num_mapped == num_segs, then all of the holes
                 * between segments would have already been unmapped.
                 */

                for (i = 1; i < num_mapped; i++) {
                        addr = mrp[i - 1].mr_addr + mrp[i - 1].mr_msize;
                        size = mrp[i].mr_addr - addr;
                        (void) as_unmap(as, addr, size);
                }
        }
}

/*
 * We need to add the start address into mrp so that the unmap function
 * has absolute addresses to use.
 */
static void
mmapobj_unmap_exec(mmapobj_result_t *mrp, int num_mapped, caddr_t start_addr)
{
        int i;

        for (i = 0; i < num_mapped; i++) {
                mrp[i].mr_addr += (size_t)start_addr;
        }
        mmapobj_unmap(mrp, num_mapped, num_mapped, ET_EXEC);
}

static caddr_t
mmapobj_lookup_start_addr(struct lib_va *lvp)
{
        proc_t *p = curproc;
        struct as *as = p->p_as;
        struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
        int error;
        uint_t ma_flags = _MAP_LOW32;
        caddr_t base = NULL;
        size_t len;
        size_t align;

        ASSERT(lvp != NULL);
        MOBJ_STAT_ADD(lookup_start);

        as_rangelock(as);

        base = lvp->lv_base_va;
        len = lvp->lv_len;

        /*
         * If we don't have an expected base address, or the one that we want
         * to use is not available or acceptable, go get an acceptable
         * address range.
         */
        if (base == NULL || as_gap(as, len, &base, &len, 0, NULL) ||
            valid_usr_range(base, len, PROT_ALL, as, as->a_userlimit) !=
            RANGE_OKAY || OVERLAPS_STACK(base + len, p)) {
                if (lvp->lv_flags & LV_ELF64) {
                        ma_flags = 0;
                }

                align = lvp->lv_align;
                if (align > 1) {
                        ma_flags |= MAP_ALIGN;
                }

                base = (caddr_t)align;
                map_addr(&base, len, 0, 1, ma_flags);
        }

        /*
         * Need to reserve the address space we're going to use.
         * Don't reserve swap space since we'll be mapping over this.
         */
        if (base != NULL) {
                crargs.flags |= MAP_NORESERVE;
                error = as_map(as, base, len, segvn_create, &crargs);
                if (error) {
                        base = NULL;
                }
        }

        as_rangeunlock(as);
        return (base);
}

/*
 * Get the starting address for a given file to be mapped and return it
 * to the caller.  If we're using lib_va and we need to allocate an address,
 * we will attempt to allocate it from the global reserved pool such that the
 * same address can be used in the future for this file.  If we can't use the
 * reserved address then we just get one that will fit in our address space.
 *
 * Returns the starting virtual address for the range to be mapped or NULL
 * if an error is encountered. If we successfully insert the requested info
 * into the lib_va hash, then *lvpp will be set to point to this lib_va
 * structure.  The structure will have a hold on it and thus lib_va_release
 * needs to be called on it by the caller.  This function will not fill out
 * lv_mps or lv_num_segs since it does not have enough information to do so.
 * The caller is responsible for doing this making sure that any modifications
 * to lv_mps are visible before setting lv_num_segs.
 */
static caddr_t
mmapobj_alloc_start_addr(struct lib_va **lvpp, size_t len, int use_lib_va,
    size_t align, vattr_t *vap)
{
        proc_t *p = curproc;
        struct as *as = p->p_as;
        struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
        int error;
        model_t model;
        uint_t ma_flags = _MAP_LOW32;
        caddr_t base = NULL;
        vmem_t *model_vmem;
        size_t lib_va_start;
        size_t lib_va_end;
        size_t lib_va_len;

        ASSERT(lvpp != NULL);

        MOBJ_STAT_ADD(alloc_start);
        model = get_udatamodel();

        if (model == DATAMODEL_LP64) {
                ma_flags = 0;
                model_vmem = lib_va_64_arena;
        } else {
                ASSERT(model == DATAMODEL_ILP32);
                model_vmem = lib_va_32_arena;
        }

        if (align > 1) {
                ma_flags |= MAP_ALIGN;
        }
        if (use_lib_va) {
                /*
                 * The first time through, we need to setup the lib_va arenas.
                 * We call map_addr to find a suitable range of memory to map
                 * the given library, and we will set the highest address
                 * in our vmem arena to the end of this adddress range.
                 * We allow up to half of the address space to be used
                 * for lib_va addresses but we do not prevent any allocations
                 * in this range from other allocation paths.
                 */
                if (lib_va_64_arena == NULL && model == DATAMODEL_LP64) {
                        mutex_enter(&lib_va_init_mutex);
                        if (lib_va_64_arena == NULL) {
                                base = (caddr_t)align;
                                as_rangelock(as);
                                map_addr(&base, len, 0, 1, ma_flags);
                                as_rangeunlock(as);
                                if (base == NULL) {
                                        mutex_exit(&lib_va_init_mutex);
                                        MOBJ_STAT_ADD(lib_va_create_failure);
                                        goto nolibva;
                                }
                                lib_va_end = (size_t)base + len;
                                lib_va_len = lib_va_end >> 1;
                                lib_va_len = P2ROUNDUP(lib_va_len, PAGESIZE);
                                lib_va_start = lib_va_end - lib_va_len;

                                /*
                                 * Need to make sure we avoid the address hole.
                                 * We know lib_va_end is valid but we need to
                                 * make sure lib_va_start is as well.
                                 */
                                if ((lib_va_end > (size_t)hole_end) &&
                                    (lib_va_start < (size_t)hole_end)) {
                                        lib_va_start = P2ROUNDUP(
                                            (size_t)hole_end, PAGESIZE);
                                        lib_va_len = lib_va_end - lib_va_start;
                                }
                                lib_va_64_arena = vmem_create("lib_va_64",
                                    (void *)lib_va_start, lib_va_len, PAGESIZE,
                                    NULL, NULL, NULL, 0,
                                    VM_NOSLEEP | VMC_IDENTIFIER);
                                if (lib_va_64_arena == NULL) {
                                        mutex_exit(&lib_va_init_mutex);
                                        goto nolibva;
                                }
                        }
                        model_vmem = lib_va_64_arena;
                        mutex_exit(&lib_va_init_mutex);
                } else if (lib_va_32_arena == NULL &&
                    model == DATAMODEL_ILP32) {
                        mutex_enter(&lib_va_init_mutex);
                        if (lib_va_32_arena == NULL) {
                                base = (caddr_t)align;
                                as_rangelock(as);
                                map_addr(&base, len, 0, 1, ma_flags);
                                as_rangeunlock(as);
                                if (base == NULL) {
                                        mutex_exit(&lib_va_init_mutex);
                                        MOBJ_STAT_ADD(lib_va_create_failure);
                                        goto nolibva;
                                }
                                lib_va_end = (size_t)base + len;
                                lib_va_len = lib_va_end >> 1;
                                lib_va_len = P2ROUNDUP(lib_va_len, PAGESIZE);
                                lib_va_start = lib_va_end - lib_va_len;
                                lib_va_32_arena = vmem_create("lib_va_32",
                                    (void *)lib_va_start, lib_va_len, PAGESIZE,
                                    NULL, NULL, NULL, 0,
                                    VM_NOSLEEP | VMC_IDENTIFIER);
                                if (lib_va_32_arena == NULL) {
                                        mutex_exit(&lib_va_init_mutex);
                                        goto nolibva;
                                }
                        }
                        model_vmem = lib_va_32_arena;
                        mutex_exit(&lib_va_init_mutex);
                }

                if (model == DATAMODEL_LP64 || libs_mapped_32 < lib_threshold) {
                        base = vmem_xalloc(model_vmem, len, align, 0, 0, NULL,
                            NULL, VM_NOSLEEP | VM_ENDALLOC);
                        MOBJ_STAT_ADD(alloc_vmem);
                }

                /*
                 * Even if the address fails to fit in our address space,
                 * or we can't use a reserved address,
                 * we should still save it off in lib_va_hash.
                 */
                *lvpp = lib_va_add_hash(base, len, align, vap);

                /*
                 * Check for collision on insertion and free up our VA space.
                 * This is expected to be rare, so we'll just reset base to
                 * NULL instead of looking it up in the lib_va hash.
                 */
                if (*lvpp == NULL) {
                        if (base != NULL) {
                                vmem_xfree(model_vmem, base, len);
                                base = NULL;
                                MOBJ_STAT_ADD(add_collision);
                        }
                }
        }

nolibva:
        as_rangelock(as);

        /*
         * If we don't have an expected base address, or the one that we want
         * to use is not available or acceptable, go get an acceptable
         * address range.
         */
        if (base == NULL || as_gap(as, len, &base, &len, 0, NULL) ||
            valid_usr_range(base, len, PROT_ALL, as, as->a_userlimit) !=
            RANGE_OKAY || OVERLAPS_STACK(base + len, p)) {
                MOBJ_STAT_ADD(get_addr);
                base = (caddr_t)align;
                map_addr(&base, len, 0, 1, ma_flags);
        }

        /*
         * Need to reserve the address space we're going to use.
         * Don't reserve swap space since we'll be mapping over this.
         */
        if (base != NULL) {
                /* Don't reserve swap space since we'll be mapping over this */
                crargs.flags |= MAP_NORESERVE;
                error = as_map(as, base, len, segvn_create, &crargs);
                if (error) {
                        base = NULL;
                }
        }

        as_rangeunlock(as);
        return (base);
}

/*
 * Map the file associated with vp into the address space as a single
 * read only private mapping.
 * Returns 0 for success, and non-zero for failure to map the file.
 */
static int
mmapobj_map_flat(vnode_t *vp, mmapobj_result_t *mrp, size_t padding,
    cred_t *fcred)
{
        int error = 0;
        struct as *as = curproc->p_as;
        caddr_t addr = NULL;
        caddr_t start_addr;
        size_t len;
        size_t pad_len;
        int prot = PROT_USER | PROT_READ;
        uint_t ma_flags = _MAP_LOW32;
        vattr_t vattr;
        struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);

        if (get_udatamodel() == DATAMODEL_LP64) {
                ma_flags = 0;
        }

        vattr.va_mask = AT_SIZE;
        error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
        if (error) {
                return (error);
        }

        len = vattr.va_size;

        ma_flags |= MAP_PRIVATE;
        if (padding == 0) {
                MOBJ_STAT_ADD(map_flat_no_padding);
                error = VOP_MAP(vp, 0, as, &addr, len, prot, PROT_ALL,
                    ma_flags, fcred, NULL);
                if (error == 0) {
                        mrp[0].mr_addr = addr;
                        mrp[0].mr_msize = len;
                        mrp[0].mr_fsize = len;
                        mrp[0].mr_offset = 0;
                        mrp[0].mr_prot = prot;
                        mrp[0].mr_flags = 0;
                }
                return (error);
        }

        /* padding was requested so there's more work to be done */
        MOBJ_STAT_ADD(map_flat_padding);

        /* No need to reserve swap space now since it will be reserved later */
        crargs.flags |= MAP_NORESERVE;

        /* Need to setup padding which can only be in PAGESIZE increments. */
        ASSERT((padding & PAGEOFFSET) == 0);
        pad_len = len + (2 * padding);

        as_rangelock(as);
        map_addr(&addr, pad_len, 0, 1, ma_flags);
        error = as_map(as, addr, pad_len, segvn_create, &crargs);
        as_rangeunlock(as);
        if (error) {
                return (error);
        }
        start_addr = addr;
        addr += padding;
        ma_flags |= MAP_FIXED;
        error = VOP_MAP(vp, 0, as, &addr, len, prot, PROT_ALL, ma_flags,
            fcred, NULL);
        if (error == 0) {
                mrp[0].mr_addr = start_addr;
                mrp[0].mr_msize = padding;
                mrp[0].mr_fsize = 0;
                mrp[0].mr_offset = 0;
                mrp[0].mr_prot = 0;
                mrp[0].mr_flags = MR_PADDING;

                mrp[1].mr_addr = addr;
                mrp[1].mr_msize = len;
                mrp[1].mr_fsize = len;
                mrp[1].mr_offset = 0;
                mrp[1].mr_prot = prot;
                mrp[1].mr_flags = 0;

                mrp[2].mr_addr = addr + P2ROUNDUP(len, PAGESIZE);
                mrp[2].mr_msize = padding;
                mrp[2].mr_fsize = 0;
                mrp[2].mr_offset = 0;
                mrp[2].mr_prot = 0;
                mrp[2].mr_flags = MR_PADDING;
        } else {
                /* Need to cleanup the as_map from earlier */
                (void) as_unmap(as, start_addr, pad_len);
        }
        return (error);
}

/*
 * Map a PT_LOAD or PT_SUNWBSS section of an executable file into the user's
 * address space.
 * vp - vnode to be mapped in
 * addr - start address
 * len - length of vp to be mapped
 * zfodlen - length of zero filled memory after len above
 * offset - offset into file where mapping should start
 * prot - protections for this mapping
 * fcred - credentials for the file associated with vp at open time.
 */
static int
mmapobj_map_ptload(struct vnode *vp, caddr_t addr, size_t len, size_t zfodlen,
    off_t offset, int prot, cred_t *fcred)
{
        int error = 0;
        caddr_t zfodbase, oldaddr;
        size_t oldlen;
        size_t end;
        size_t zfoddiff;
        label_t ljb;
        struct as *as = curproc->p_as;
        model_t model;
        int full_page;

        /*
         * See if addr and offset are aligned such that we can map in
         * full pages instead of partial pages.
         */
        full_page = (((uintptr_t)addr & PAGEOFFSET) ==
            ((uintptr_t)offset & PAGEOFFSET));

        model = get_udatamodel();

        oldaddr = addr;
        addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
        if (len) {
                spgcnt_t availm, npages;
                int preread;
                uint_t mflag = MAP_PRIVATE | MAP_FIXED;

                if (model == DATAMODEL_ILP32) {
                        mflag |= _MAP_LOW32;
                }
                /* We may need to map in extra bytes */
                oldlen = len;
                len += ((size_t)oldaddr & PAGEOFFSET);

                if (full_page) {
                        offset = (off_t)((uintptr_t)offset & PAGEMASK);
                        if ((prot & (PROT_WRITE | PROT_EXEC)) == PROT_EXEC) {
                                mflag |= MAP_TEXT;
                                MOBJ_STAT_ADD(map_ptload_text);
                        } else {
                                mflag |= MAP_INITDATA;
                                MOBJ_STAT_ADD(map_ptload_initdata);
                        }

                        /*
                         * maxprot is passed as PROT_ALL so that mdb can
                         * write to this segment.
                         */
                        if (error = VOP_MAP(vp, (offset_t)offset, as, &addr,
                            len, prot, PROT_ALL, mflag, fcred, NULL)) {
                                return (error);
                        }

                        /*
                         * If the segment can fit and is relatively small, then
                         * we prefault the entire segment in.  This is based
                         * on the model that says the best working set of a
                         * small program is all of its pages.
                         * We only do this if freemem will not drop below
                         * lotsfree since we don't want to induce paging.
                         */
                        npages = (spgcnt_t)btopr(len);
                        availm = freemem - lotsfree;
                        preread = (npages < availm && len < PGTHRESH) ? 1 : 0;

                        /*
                         * If we aren't prefaulting the segment,
                         * increment "deficit", if necessary to ensure
                         * that pages will become available when this
                         * process starts executing.
                         */
                        if (preread == 0 && npages > availm &&
                            deficit < lotsfree) {
                                deficit += MIN((pgcnt_t)(npages - availm),
                                    lotsfree - deficit);
                        }

                        if (preread) {
                                (void) as_faulta(as, addr, len);
                                MOBJ_STAT_ADD(map_ptload_preread);
                        }
                } else {
                        /*
                         * addr and offset were not aligned such that we could
                         * use VOP_MAP, thus we need to as_map the memory we
                         * need and then read the data in from disk.
                         * This code path is a corner case which should never
                         * be taken, but hand crafted binaries could trigger
                         * this logic and it needs to work correctly.
                         */
                        MOBJ_STAT_ADD(map_ptload_unaligned_text);
                        as_rangelock(as);
                        (void) as_unmap(as, addr, len);

                        /*
                         * We use zfod_argsp because we need to be able to
                         * write to the mapping and then we'll change the
                         * protections later if they are incorrect.
                         */
                        error = as_map(as, addr, len, segvn_create, zfod_argsp);
                        as_rangeunlock(as);
                        if (error) {
                                MOBJ_STAT_ADD(map_ptload_unaligned_map_fail);
                                return (error);
                        }

                        /* Now read in the data from disk */
                        error = vn_rdwr(UIO_READ, vp, oldaddr, oldlen, offset,
                            UIO_USERSPACE, 0, (rlim64_t)0, fcred, NULL);
                        if (error) {
                                MOBJ_STAT_ADD(map_ptload_unaligned_read_fail);
                                return (error);
                        }

                        /*
                         * Now set protections.
                         */
                        if (prot != PROT_ZFOD) {
                                (void) as_setprot(as, addr, len, prot);
                        }
                }
        }

        if (zfodlen) {
                end = (size_t)addr + len;
                zfodbase = (caddr_t)P2ROUNDUP(end, PAGESIZE);
                zfoddiff = (uintptr_t)zfodbase - end;
                if (zfoddiff) {
                        MOBJ_STAT_ADD(zfoddiff);
                        if ((prot & PROT_WRITE) == 0) {
                                (void) as_setprot(as, (caddr_t)end,
                                    zfoddiff, prot | PROT_WRITE);
                                MOBJ_STAT_ADD(zfoddiff_nowrite);
                        }
                        if (on_fault(&ljb)) {
                                no_fault();
                                if ((prot & PROT_WRITE) == 0) {
                                        (void) as_setprot(as, (caddr_t)end,
                                            zfoddiff, prot);
                                }
                                return (EFAULT);
                        }
                        uzero((void *)end, zfoddiff);
                        no_fault();

                        /*
                         * Remove write protection to return to original state
                         */
                        if ((prot & PROT_WRITE) == 0) {
                                (void) as_setprot(as, (caddr_t)end,
                                    zfoddiff, prot);
                        }
                }
                if (zfodlen > zfoddiff) {
                        struct segvn_crargs crargs =
                            SEGVN_ZFOD_ARGS(prot, PROT_ALL);

                        MOBJ_STAT_ADD(zfodextra);
                        zfodlen -= zfoddiff;
                        crargs.szc = AS_MAP_NO_LPOOB;


                        as_rangelock(as);
                        (void) as_unmap(as, (caddr_t)zfodbase, zfodlen);
                        error = as_map(as, (caddr_t)zfodbase,
                            zfodlen, segvn_create, &crargs);
                        as_rangeunlock(as);
                        if (error) {
                                return (error);
                        }
                }
        }
        return (0);
}

/*
 * Map the ELF file represented by vp into the users address space.  The
 * first mapping will start at start_addr and there will be num_elements
 * mappings.  The mappings are described by the data in mrp which may be
 * modified upon returning from this function.
 * Returns 0 for success or errno for failure.
 */
static int
mmapobj_map_elf(struct vnode *vp, caddr_t start_addr, mmapobj_result_t *mrp,
    int num_elements, cred_t *fcred, ushort_t e_type)
{
        int i;
        int ret;
        caddr_t lo;
        caddr_t hi;
        struct as *as = curproc->p_as;

        for (i = 0; i < num_elements; i++) {
                caddr_t addr;
                size_t p_memsz;
                size_t p_filesz;
                size_t zfodlen;
                offset_t p_offset;
                size_t dif;
                int prot;

                /* Always need to adjust mr_addr */
                addr = start_addr + (size_t)(mrp[i].mr_addr);
                mrp[i].mr_addr =
                    (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);

                /* Padding has already been mapped */
                if (MR_GET_TYPE(mrp[i].mr_flags) == MR_PADDING) {
                        continue;
                }
                p_memsz = mrp[i].mr_msize;
                p_filesz = mrp[i].mr_fsize;
                zfodlen = p_memsz - p_filesz;
                p_offset = mrp[i].mr_offset;
                dif = (uintptr_t)(addr) & PAGEOFFSET;
                prot = mrp[i].mr_prot | PROT_USER;
                ret = mmapobj_map_ptload(vp, addr, p_filesz, zfodlen,
                    p_offset, prot, fcred);
                if (ret != 0) {
                        MOBJ_STAT_ADD(ptload_failed);
                        mmapobj_unmap(mrp, i, num_elements, e_type);
                        return (ret);
                }

                /* Need to cleanup mrp to reflect the actual values used */
                mrp[i].mr_msize += dif;
                mrp[i].mr_offset = (size_t)addr & PAGEOFFSET;
        }

        /* Also need to unmap any holes created above */
        if (num_elements == 1) {
                MOBJ_STAT_ADD(map_elf_no_holes);
                return (0);
        }
        if (e_type == ET_EXEC) {
                return (0);
        }

        as_rangelock(as);
        lo = start_addr;
        hi = mrp[0].mr_addr;

        /* Remove holes made by the rest of the segments */
        for (i = 0; i < num_elements - 1; i++) {
                lo = (caddr_t)P2ROUNDUP((size_t)(mrp[i].mr_addr) +
                    mrp[i].mr_msize, PAGESIZE);
                hi = mrp[i + 1].mr_addr;
                if (lo < hi) {
                        /*
                         * If as_unmap fails we just use up a bit of extra
                         * space
                         */
                        (void) as_unmap(as, (caddr_t)lo,
                            (size_t)hi - (size_t)lo);
                        MOBJ_STAT_ADD(unmap_hole);
                }
        }
        as_rangeunlock(as);

        return (0);
}

/* Ugly hack to get STRUCT_* macros to work below */
struct myphdr {
        Phdr            x;      /* native version */
};

struct myphdr32 {
        Elf32_Phdr      x;
};

/*
 * Calculate and return the number of loadable segments in the ELF Phdr
 * represented by phdrbase as well as the len of the total mapping and
 * the max alignment that is needed for a given segment.  On success,
 * 0 is returned, and *len, *loadable and *align have been filled out.
 * On failure, errno will be returned, which in this case is ENOTSUP
 * if we were passed an ELF file with overlapping segments.
 */
static int
calc_loadable(Ehdr *ehdrp, caddr_t phdrbase, int nphdrs, size_t *len,
    int *loadable, size_t *align)
{
        int i;
        int hsize;
        model_t model;
        ushort_t e_type = ehdrp->e_type;        /* same offset 32 and 64 bit */
        uint_t p_type;
        offset_t p_offset;
        size_t p_memsz;
        size_t p_align;
        caddr_t vaddr;
        int num_segs = 0;
        caddr_t start_addr = NULL;
        caddr_t p_end = NULL;
        size_t max_align = 0;
        size_t min_align = PAGESIZE;    /* needed for vmem_xalloc */
        STRUCT_HANDLE(myphdr, mph);
#if defined(__sparc)
        extern int vac_size;

        /*
         * Want to prevent aliasing by making the start address at least be
         * aligned to vac_size.
         */
        min_align = MAX(PAGESIZE, vac_size);
#endif

        model = get_udatamodel();
        STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);

        /* hsize alignment should have been checked before calling this func */
        if (model == DATAMODEL_LP64) {
                hsize = ehdrp->e_phentsize;
                if (hsize & 7) {
                        return (ENOTSUP);
                }
        } else {
                ASSERT(model == DATAMODEL_ILP32);
                hsize = ((Elf32_Ehdr *)ehdrp)->e_phentsize;
                if (hsize & 3) {
                        return (ENOTSUP);
                }
        }

        /*
         * Determine the span of all loadable segments and calculate the
         * number of loadable segments.
         */
        for (i = 0; i < nphdrs; i++) {
                p_type = STRUCT_FGET(mph, x.p_type);
                if (p_type == PT_LOAD || p_type == PT_SUNWBSS) {
                        vaddr = (caddr_t)(uintptr_t)STRUCT_FGET(mph, x.p_vaddr);
                        p_memsz = STRUCT_FGET(mph, x.p_memsz);

                        /*
                         * Skip this header if it requests no memory to be
                         * mapped.
                         */
                        if (p_memsz == 0) {
                                STRUCT_SET_HANDLE(mph, model,
                                    (struct myphdr *)((size_t)STRUCT_BUF(mph) +
                                    hsize));
                                MOBJ_STAT_ADD(nomem_header);
                                continue;
                        }
                        if (num_segs++ == 0) {
                                /*
                                 * The p_vaddr of the first PT_LOAD segment
                                 * must either be NULL or within the first
                                 * page in order to be interpreted.
                                 * Otherwise, its an invalid file.
                                 */
                                if (e_type == ET_DYN &&
                                    ((caddr_t)((uintptr_t)vaddr &
                                    (uintptr_t)PAGEMASK) != NULL)) {
                                        MOBJ_STAT_ADD(inval_header);
                                        return (ENOTSUP);
                                }
                                start_addr = vaddr;
                                /*
                                 * For the first segment, we need to map from
                                 * the beginning of the file, so we will
                                 * adjust the size of the mapping to include
                                 * this memory.
                                 */
                                p_offset = STRUCT_FGET(mph, x.p_offset);
                        } else {
                                p_offset = 0;
                        }
                        /*
                         * Check to make sure that this mapping wouldn't
                         * overlap a previous mapping.
                         */
                        if (vaddr < p_end) {
                                MOBJ_STAT_ADD(overlap_header);
                                return (ENOTSUP);
                        }

                        p_end = vaddr + p_memsz + p_offset;
                        p_end = (caddr_t)P2ROUNDUP((size_t)p_end, PAGESIZE);

                        p_align = STRUCT_FGET(mph, x.p_align);
                        if (p_align > 1 && p_align > max_align) {
                                max_align = p_align;
                                if (max_align < min_align) {
                                        max_align = min_align;
                                        MOBJ_STAT_ADD(min_align);
                                }
                        }
                }
                STRUCT_SET_HANDLE(mph, model,
                    (struct myphdr *)((size_t)STRUCT_BUF(mph) + hsize));
        }

        /*
         * The alignment should be a power of 2, if it isn't we forgive it
         * and round up.  On overflow, we'll set the alignment to max_align
         * rounded down to the nearest power of 2.
         */
        if (max_align > 0 && !ISP2(max_align)) {
                MOBJ_STAT_ADD(np2_align);
                *align = 2 * (1L << (highbit(max_align) - 1));
                if (*align < max_align ||
                    (*align > UINT_MAX && model == DATAMODEL_ILP32)) {
                        MOBJ_STAT_ADD(np2_align_overflow);
                        *align = 1L << (highbit(max_align) - 1);
                }
        } else {
                *align = max_align;
        }

        ASSERT(*align >= PAGESIZE || *align == 0);

        *loadable = num_segs;
        *len = p_end - start_addr;
        return (0);
}

/*
 * Check the address space to see if the virtual addresses to be used are
 * available.  If they are not, return errno for failure.  On success, 0
 * will be returned, and the virtual addresses for each mmapobj_result_t
 * will be reserved.  Note that a reservation could have earlier been made
 * for a given segment via a /dev/null mapping.  If that is the case, then
 * we can use that VA space for our mappings.
 * Note: this function will only be used for ET_EXEC binaries.
 */
int
check_exec_addrs(int loadable, mmapobj_result_t *mrp, caddr_t start_addr)
{
        int i;
        struct as *as = curproc->p_as;
        struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
        int ret;
        caddr_t myaddr;
        size_t mylen;
        struct seg *seg;

        /* No need to reserve swap space now since it will be reserved later */
        crargs.flags |= MAP_NORESERVE;
        as_rangelock(as);
        for (i = 0; i < loadable; i++) {

                myaddr = start_addr + (size_t)mrp[i].mr_addr;
                mylen = mrp[i].mr_msize;

                /* See if there is a hole in the as for this range */
                if (as_gap(as, mylen, &myaddr, &mylen, 0, NULL) == 0) {
                        ASSERT(myaddr == start_addr + (size_t)mrp[i].mr_addr);
                        ASSERT(mylen == mrp[i].mr_msize);

#ifdef DEBUG
                        if (MR_GET_TYPE(mrp[i].mr_flags) == MR_PADDING) {
                                MOBJ_STAT_ADD(exec_padding);
                        }
#endif
                        ret = as_map(as, myaddr, mylen, segvn_create, &crargs);
                        if (ret) {
                                as_rangeunlock(as);
                                mmapobj_unmap_exec(mrp, i, start_addr);
                                return (ret);
                        }
                } else {
                        /*
                         * There is a mapping that exists in the range
                         * so check to see if it was a "reservation"
                         * from /dev/null.  The mapping is from
                         * /dev/null if the mapping comes from
                         * segdev and the type is neither MAP_SHARED
                         * nor MAP_PRIVATE.
                         */
                        AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
                        seg = as_findseg(as, myaddr, 0);
                        MOBJ_STAT_ADD(exec_addr_mapped);
                        if (seg && seg->s_ops == &segdev_ops &&
                            ((SEGOP_GETTYPE(seg, myaddr) &
                            (MAP_SHARED | MAP_PRIVATE)) == 0) &&
                            myaddr >= seg->s_base &&
                            myaddr + mylen <=
                            seg->s_base + seg->s_size) {
                                MOBJ_STAT_ADD(exec_addr_devnull);
                                AS_LOCK_EXIT(as, &as->a_lock);
                                (void) as_unmap(as, myaddr, mylen);
                                ret = as_map(as, myaddr, mylen, segvn_create,
                                    &crargs);
                                mrp[i].mr_flags |= MR_RESV;
                                if (ret) {
                                        as_rangeunlock(as);
                                        /* Need to remap what we unmapped */
                                        mmapobj_unmap_exec(mrp, i + 1,
                                            start_addr);
                                        return (ret);
                                }
                        } else {
                                AS_LOCK_EXIT(as, &as->a_lock);
                                as_rangeunlock(as);
                                mmapobj_unmap_exec(mrp, i, start_addr);
                                MOBJ_STAT_ADD(exec_addr_in_use);
                                return (EADDRINUSE);
                        }
                }
        }
        as_rangeunlock(as);
        return (0);
}

/*
 * Walk through the ELF program headers and extract all useful information
 * for PT_LOAD and PT_SUNWBSS segments into mrp.
 * Return 0 on success or error on failure.
 */
static int
process_phdr(Ehdr *ehdrp, caddr_t phdrbase, int nphdrs, mmapobj_result_t *mrp,
    vnode_t *vp, uint_t *num_mapped, size_t padding, cred_t *fcred)
{
        int i;
        caddr_t start_addr = NULL;
        caddr_t vaddr;
        size_t len = 0;
        size_t lib_len = 0;
        int ret;
        int prot;
        struct lib_va *lvp = NULL;
        vattr_t vattr;
        struct as *as = curproc->p_as;
        int error;
        int loadable = 0;
        int current = 0;
        int use_lib_va = 1;
        size_t align = 0;
        size_t add_pad = 0;
        int hdr_seen = 0;
        ushort_t e_type = ehdrp->e_type;        /* same offset 32 and 64 bit */
        uint_t p_type;
        offset_t p_offset;
        size_t p_memsz;
        size_t p_filesz;
        uint_t p_flags;
        int hsize;
        model_t model;
        STRUCT_HANDLE(myphdr, mph);

        model = get_udatamodel();
        STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);

        /*
         * Need to make sure that hsize is aligned properly.
         * For 32bit processes, 4 byte alignment is required.
         * For 64bit processes, 8 byte alignment is required.
         * If the alignment isn't correct, we need to return failure
         * since it could cause an alignment error panic while walking
         * the phdr array.
         */
        if (model == DATAMODEL_LP64) {
                hsize = ehdrp->e_phentsize;
                if (hsize & 7) {
                        MOBJ_STAT_ADD(phent_align64);
                        return (ENOTSUP);
                }
        } else {
                ASSERT(model == DATAMODEL_ILP32);
                hsize = ((Elf32_Ehdr *)ehdrp)->e_phentsize;
                if (hsize & 3) {
                        MOBJ_STAT_ADD(phent_align32);
                        return (ENOTSUP);
                }
        }

        if (padding != 0) {
                use_lib_va = 0;
        }
        if (e_type == ET_DYN) {
                vattr.va_mask = AT_FSID | AT_NODEID | AT_CTIME | AT_MTIME;
                error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
                if (error) {
                        return (error);
                }
                /* Check to see if we already have a description for this lib */
                lvp = lib_va_find(&vattr);

                if (lvp != NULL) {
                        MOBJ_STAT_ADD(lvp_found);
                        if (use_lib_va) {
                                start_addr = mmapobj_lookup_start_addr(lvp);
                                if (start_addr == NULL) {
                                        lib_va_release(lvp);
                                        return (ENOMEM);
                                }
                        }

                        /*
                         * loadable may be zero if the original allocator
                         * of lvp hasn't finished setting it up but the rest
                         * of the fields will be accurate.
                         */
                        loadable = lvp->lv_num_segs;
                        len = lvp->lv_len;
                        align = lvp->lv_align;
                }
        }

        /*
         * Determine the span of all loadable segments and calculate the
         * number of loadable segments, the total len spanned by the mappings
         * and the max alignment, if we didn't get them above.
         */
        if (loadable == 0) {
                MOBJ_STAT_ADD(no_loadable_yet);
                ret = calc_loadable(ehdrp, phdrbase, nphdrs, &len,
                    &loadable, &align);
                if (ret != 0) {
                        /*
                         * Since it'd be an invalid file, we shouldn't have
                         * cached it previously.
                         */
                        ASSERT(lvp == NULL);
                        return (ret);
                }
#ifdef DEBUG
                if (lvp) {
                        ASSERT(len == lvp->lv_len);
                        ASSERT(align == lvp->lv_align);
                }
#endif
        }

        /* Make sure there's something to map. */
        if (len == 0 || loadable == 0) {
                /*
                 * Since it'd be an invalid file, we shouldn't have
                 * cached it previously.
                 */
                ASSERT(lvp == NULL);
                MOBJ_STAT_ADD(nothing_to_map);
                return (ENOTSUP);
        }

        lib_len = len;
        if (padding != 0) {
                loadable += 2;
        }
        if (loadable > *num_mapped) {
                *num_mapped = loadable;
                /* cleanup previous reservation */
                if (start_addr) {
                        (void) as_unmap(as, start_addr, lib_len);
                }
                MOBJ_STAT_ADD(e2big);
                if (lvp) {
                        lib_va_release(lvp);
                }
                return (E2BIG);
        }

        /*
         * We now know the size of the object to map and now we need to
         * get the start address to map it at.  It's possible we already
         * have it if we found all the info we need in the lib_va cache.
         */
        if (e_type == ET_DYN && start_addr == NULL) {
                /*
                 * Need to make sure padding does not throw off
                 * required alignment.  We can only specify an
                 * alignment for the starting address to be mapped,
                 * so we round padding up to the alignment and map
                 * from there and then throw out the extra later.
                 */
                if (padding != 0) {
                        if (align > 1) {
                                add_pad = P2ROUNDUP(padding, align);
                                len += add_pad;
                                MOBJ_STAT_ADD(dyn_pad_align);
                        } else {
                                MOBJ_STAT_ADD(dyn_pad_noalign);
                                len += padding; /* at beginning */
                        }
                        len += padding; /* at end of mapping */
                }
                /*
                 * At this point, if lvp is non-NULL, then above we
                 * already found it in the cache but did not get
                 * the start address since we were not going to use lib_va.
                 * Since we know that lib_va will not be used, it's safe
                 * to call mmapobj_alloc_start_addr and know that lvp
                 * will not be modified.
                 */
                ASSERT(lvp ? use_lib_va == 0 : 1);
                start_addr = mmapobj_alloc_start_addr(&lvp, len,
                    use_lib_va, align, &vattr);
                if (start_addr == NULL) {
                        if (lvp) {
                                lib_va_release(lvp);
                        }
                        MOBJ_STAT_ADD(alloc_start_fail);
                        return (ENOMEM);
                }
                /*
                 * If we can't cache it, no need to hang on to it.
                 * Setting lv_num_segs to non-zero will make that
                 * field active and since there are too many segments
                 * to cache, all future users will not try to use lv_mps.
                 */
                if (lvp != NULL && loadable > LIBVA_CACHED_SEGS && use_lib_va) {
                        lvp->lv_num_segs = loadable;
                        lib_va_release(lvp);
                        lvp = NULL;
                        MOBJ_STAT_ADD(lvp_nocache);
                }
                /*
                 * Free the beginning of the mapping if the padding
                 * was not aligned correctly.
                 */
                if (padding != 0 && add_pad != padding) {
                        (void) as_unmap(as, start_addr,
                            add_pad - padding);
                        start_addr += (add_pad - padding);
                        MOBJ_STAT_ADD(extra_padding);
                }
        }

        /*
         * At this point, we have reserved the virtual address space
         * for our mappings.  Now we need to start filling out the mrp
         * array to describe all of the individual mappings we are going
         * to return.
         * For ET_EXEC there has been no memory reservation since we are
         * using fixed addresses.  While filling in the mrp array below,
         * we will have the first segment biased to start at addr 0
         * and the rest will be biased by this same amount.  Thus if there
         * is padding, the first padding will start at addr 0, and the next
         * segment will start at the value of padding.
         */

        /* We'll fill out padding later, so start filling in mrp at index 1 */
        if (padding != 0) {
                current = 1;
        }

        /* If we have no more need for lvp let it go now */
        if (lvp != NULL && use_lib_va == 0) {
                lib_va_release(lvp);
                MOBJ_STAT_ADD(lvp_not_needed);
                lvp = NULL;
        }

        /* Now fill out the mrp structs from the program headers */
        STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
        for (i = 0; i < nphdrs; i++) {
                p_type = STRUCT_FGET(mph, x.p_type);
                if (p_type == PT_LOAD || p_type == PT_SUNWBSS) {
                        vaddr = (caddr_t)(uintptr_t)STRUCT_FGET(mph, x.p_vaddr);
                        p_memsz = STRUCT_FGET(mph, x.p_memsz);
                        p_filesz = STRUCT_FGET(mph, x.p_filesz);
                        p_offset = STRUCT_FGET(mph, x.p_offset);
                        p_flags = STRUCT_FGET(mph, x.p_flags);

                        /*
                         * Skip this header if it requests no memory to be
                         * mapped.
                         */
                        if (p_memsz == 0) {
                                STRUCT_SET_HANDLE(mph, model,
                                    (struct myphdr *)((size_t)STRUCT_BUF(mph) +
                                    hsize));
                                MOBJ_STAT_ADD(no_mem_map_sz);
                                continue;
                        }

                        prot = 0;
                        if (p_flags & PF_R)
                                prot |= PROT_READ;
                        if (p_flags & PF_W)
                                prot |= PROT_WRITE;
                        if (p_flags & PF_X)
                                prot |= PROT_EXEC;

                        ASSERT(current < loadable);
                        mrp[current].mr_msize = p_memsz;
                        mrp[current].mr_fsize = p_filesz;
                        mrp[current].mr_offset = p_offset;
                        mrp[current].mr_prot = prot;

                        if (hdr_seen == 0 && p_filesz != 0) {
                                mrp[current].mr_flags = MR_HDR_ELF;
                                /*
                                 * We modify mr_offset because we
                                 * need to map the ELF header as well, and if
                                 * we didn't then the header could be left out
                                 * of the mapping that we will create later.
                                 * Since we're removing the offset, we need to
                                 * account for that in the other fields as well
                                 * since we will be mapping the memory from 0
                                 * to p_offset.
                                 */
                                if (e_type == ET_DYN) {
                                        mrp[current].mr_offset = 0;
                                        mrp[current].mr_msize += p_offset;
                                        mrp[current].mr_fsize += p_offset;
                                } else {
                                        ASSERT(e_type == ET_EXEC);
                                        /*
                                         * Save off the start addr which will be
                                         * our bias for the rest of the
                                         * ET_EXEC mappings.
                                         */
                                        start_addr = vaddr - padding;
                                }
                                mrp[current].mr_addr = (caddr_t)padding;
                                hdr_seen = 1;
                        } else {
                                if (e_type == ET_EXEC) {
                                        /* bias mr_addr */
                                        mrp[current].mr_addr =
                                            vaddr - (size_t)start_addr;
                                } else {
                                        mrp[current].mr_addr = vaddr + padding;
                                }
                                mrp[current].mr_flags = 0;
                        }
                        current++;
                }

                /* Move to next phdr */
                STRUCT_SET_HANDLE(mph, model,
                    (struct myphdr *)((size_t)STRUCT_BUF(mph) +
                    hsize));
        }

        /* Now fill out the padding segments */
        if (padding != 0) {
                mrp[0].mr_addr = NULL;
                mrp[0].mr_msize = padding;
                mrp[0].mr_fsize = 0;
                mrp[0].mr_offset = 0;
                mrp[0].mr_prot = 0;
                mrp[0].mr_flags = MR_PADDING;

                /* Setup padding for the last segment */
                ASSERT(current == loadable - 1);
                mrp[current].mr_addr = (caddr_t)lib_len + padding;
                mrp[current].mr_msize = padding;
                mrp[current].mr_fsize = 0;
                mrp[current].mr_offset = 0;
                mrp[current].mr_prot = 0;
                mrp[current].mr_flags = MR_PADDING;
        }

        /*
         * Need to make sure address ranges desired are not in use or
         * are previously allocated reservations from /dev/null.  For
         * ET_DYN, we already made sure our address range was free.
         */
        if (e_type == ET_EXEC) {
                ret = check_exec_addrs(loadable, mrp, start_addr);
                if (ret != 0) {
                        ASSERT(lvp == NULL);
                        MOBJ_STAT_ADD(check_exec_failed);
                        return (ret);
                }
        }

        /* Finish up our business with lvp. */
        if (lvp) {
                ASSERT(e_type == ET_DYN);
                if (lvp->lv_num_segs == 0 && loadable <= LIBVA_CACHED_SEGS) {
                        bcopy(mrp, lvp->lv_mps,
                            loadable * sizeof (mmapobj_result_t));
                        membar_producer();
                }
                /*
                 * Setting lv_num_segs to a non-zero value indicates that
                 * lv_mps is now valid and can be used by other threads.
                 * So, the above stores need to finish before lv_num_segs
                 * is updated. lv_mps is only valid if lv_num_segs is
                 * greater than LIBVA_CACHED_SEGS.
                 */
                lvp->lv_num_segs = loadable;
                lib_va_release(lvp);
                MOBJ_STAT_ADD(lvp_used);
        }

        /* Now that we have mrp completely filled out go map it */
        ret = mmapobj_map_elf(vp, start_addr, mrp, loadable, fcred, e_type);
        if (ret == 0) {
                *num_mapped = loadable;
        }

        return (ret);
}

/*
 * Take the ELF file passed in, and do the work of mapping it.
 * num_mapped in - # elements in user buffer
 * num_mapped out - # sections mapped and length of mrp array if
 *                      no errors.
 */
static int
doelfwork(Ehdr *ehdrp, vnode_t *vp, mmapobj_result_t *mrp,
    uint_t *num_mapped, size_t padding, cred_t *fcred)
{
        int error;
        offset_t phoff;
        int nphdrs;
        unsigned char ei_class;
        unsigned short phentsize;
        ssize_t phsizep;
        caddr_t phbasep;
        int to_map;
        model_t model;

        ei_class = ehdrp->e_ident[EI_CLASS];
        model = get_udatamodel();
        if ((model == DATAMODEL_ILP32 && ei_class == ELFCLASS64) ||
            (model == DATAMODEL_LP64 && ei_class == ELFCLASS32)) {
                MOBJ_STAT_ADD(wrong_model);
                return (ENOTSUP);
        }

        /* Can't execute code from "noexec" mounted filesystem. */
        if (ehdrp->e_type == ET_EXEC &&
            (vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0) {
                MOBJ_STAT_ADD(noexec_fs);
                return (EACCES);
        }

        /*
         * Relocatable and core files are mapped as a single flat file
         * since no interpretation is done on them by mmapobj.
         */
        if (ehdrp->e_type == ET_REL || ehdrp->e_type == ET_CORE) {
                to_map = padding ? 3 : 1;
                if (*num_mapped < to_map) {
                        *num_mapped = to_map;
                        MOBJ_STAT_ADD(e2big_et_rel);
                        return (E2BIG);
                }
                error = mmapobj_map_flat(vp, mrp, padding, fcred);
                if (error == 0) {
                        *num_mapped = to_map;
                        mrp[padding ? 1 : 0].mr_flags = MR_HDR_ELF;
                        MOBJ_STAT_ADD(et_rel_mapped);
                }
                return (error);
        }

        /* Check for an unknown ELF type */
        if (ehdrp->e_type != ET_EXEC && ehdrp->e_type != ET_DYN) {
                MOBJ_STAT_ADD(unknown_elf_type);
                return (ENOTSUP);
        }

        if (ei_class == ELFCLASS32) {
                Elf32_Ehdr *e32hdr = (Elf32_Ehdr *)ehdrp;
                ASSERT(model == DATAMODEL_ILP32);
                nphdrs = e32hdr->e_phnum;
                phentsize = e32hdr->e_phentsize;
                if (phentsize < sizeof (Elf32_Phdr)) {
                        MOBJ_STAT_ADD(phent32_too_small);
                        return (ENOTSUP);
                }
                phoff = e32hdr->e_phoff;
        } else if (ei_class == ELFCLASS64) {
                Elf64_Ehdr *e64hdr = (Elf64_Ehdr *)ehdrp;
                ASSERT(model == DATAMODEL_LP64);
                nphdrs = e64hdr->e_phnum;
                phentsize = e64hdr->e_phentsize;
                if (phentsize < sizeof (Elf64_Phdr)) {
                        MOBJ_STAT_ADD(phent64_too_small);
                        return (ENOTSUP);
                }
                phoff = e64hdr->e_phoff;
        } else {
                /* fallthrough case for an invalid ELF class */
                MOBJ_STAT_ADD(inval_elf_class);
                return (ENOTSUP);
        }

        /*
         * nphdrs should only have this value for core files which are handled
         * above as a single mapping.  If other file types ever use this
         * sentinel, then we'll add the support needed to handle this here.
         */
        if (nphdrs == PN_XNUM) {
                MOBJ_STAT_ADD(too_many_phdrs);
                return (ENOTSUP);
        }

        phsizep = nphdrs * phentsize;

        if (phsizep == 0) {
                MOBJ_STAT_ADD(no_phsize);
                return (ENOTSUP);
        }

        /* Make sure we only wait for memory if it's a reasonable request */
        if (phsizep > mmapobj_alloc_threshold) {
                MOBJ_STAT_ADD(phsize_large);
                if ((phbasep = kmem_alloc(phsizep, KM_NOSLEEP)) == NULL) {
                        MOBJ_STAT_ADD(phsize_xtralarge);
                        return (ENOMEM);
                }
        } else {
                phbasep = kmem_alloc(phsizep, KM_SLEEP);
        }

        if ((error = vn_rdwr(UIO_READ, vp, phbasep, phsizep,
            (offset_t)phoff, UIO_SYSSPACE, 0, (rlim64_t)0,
            fcred, NULL)) != 0) {
                kmem_free(phbasep, phsizep);
                return (error);
        }

        /* Now process the phdr's */
        error = process_phdr(ehdrp, phbasep, nphdrs, mrp, vp, num_mapped,
            padding, fcred);
        kmem_free(phbasep, phsizep);
        return (error);
}

#if defined(__sparc)
/*
 * Hack to support 64 bit kernels running AOUT 4.x programs.
 * This is the sizeof (struct nlist) for a 32 bit kernel.
 * Since AOUT programs are 32 bit only, they will never use the 64 bit
 * sizeof (struct nlist) and thus creating a #define is the simplest
 * way around this since this is a format which is not being updated.
 * This will be used in the place of sizeof (struct nlist) below.
 */
#define NLIST_SIZE      (0xC)

static int
doaoutwork(vnode_t *vp, mmapobj_result_t *mrp,
    uint_t *num_mapped, struct exec *hdr, cred_t *fcred)
{
        int error;
        size_t size;
        size_t osize;
        size_t nsize;   /* nlist size */
        size_t msize;
        size_t zfoddiff;
        caddr_t addr;
        caddr_t start_addr;
        struct as *as = curproc->p_as;
        int prot = PROT_USER | PROT_READ | PROT_EXEC;
        uint_t mflag = MAP_PRIVATE | _MAP_LOW32;
        offset_t off = 0;
        int segnum = 0;
        uint_t to_map;
        int is_library = 0;
        struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);

        /* Only 32bit apps supported by this file format */
        if (get_udatamodel() != DATAMODEL_ILP32) {
                MOBJ_STAT_ADD(aout_64bit_try);
                return (ENOTSUP);
        }

        /* Check to see if this is a library */
        if (hdr->a_magic == ZMAGIC && hdr->a_entry < PAGESIZE) {
                is_library = 1;
        }

        /* Can't execute code from "noexec" mounted filesystem. */
        if (((vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0) && (is_library == 0)) {
                MOBJ_STAT_ADD(aout_noexec);
                return (EACCES);
        }

        /*
         * There are 2 ways to calculate the mapped size of executable:
         * 1) rounded text size + data size + bss size.
         * 2) starting offset for text + text size + data size + text relocation
         *    size + data relocation size + room for nlist data structure.
         *
         * The larger of the two sizes will be used to map this binary.
         */
        osize = P2ROUNDUP(hdr->a_text, PAGESIZE) + hdr->a_data + hdr->a_bss;

        off = hdr->a_magic == ZMAGIC ? 0 : sizeof (struct exec);

        nsize = off + hdr->a_text + hdr->a_data + hdr->a_trsize +
            hdr->a_drsize + NLIST_SIZE;

        size = MAX(osize, nsize);
        if (size != nsize) {
                nsize = 0;
        }

        /*
         * 1 seg for text and 1 seg for initialized data.
         * 1 seg for bss (if can't fit in leftover space of init data)
         * 1 seg for nlist if needed.
         */
        to_map = 2 + (nsize ? 1 : 0) +
            (hdr->a_bss > PAGESIZE - P2PHASE(hdr->a_data, PAGESIZE) ? 1 : 0);
        if (*num_mapped < to_map) {
                *num_mapped = to_map;
                MOBJ_STAT_ADD(aout_e2big);
                return (E2BIG);
        }

        /* Reserve address space for the whole mapping */
        if (is_library) {
                /* We'll let VOP_MAP below pick our address for us */
                addr = NULL;
                MOBJ_STAT_ADD(aout_lib);
        } else {
                /*
                 * default start address for fixed binaries from AOUT 4.x
                 * standard.
                 */
                MOBJ_STAT_ADD(aout_fixed);
                mflag |= MAP_FIXED;
                addr = (caddr_t)0x2000;
                as_rangelock(as);
                if (as_gap(as, size, &addr, &size, 0, NULL) != 0) {
                        as_rangeunlock(as);
                        MOBJ_STAT_ADD(aout_addr_in_use);
                        return (EADDRINUSE);
                }
                crargs.flags |= MAP_NORESERVE;
                error = as_map(as, addr, size, segvn_create, &crargs);
                ASSERT(addr == (caddr_t)0x2000);
                as_rangeunlock(as);
        }

        start_addr = addr;
        osize = size;

        /*
         * Map as large as we need, backed by file, this will be text, and
         * possibly the nlist segment.  We map over this mapping for bss and
         * initialized data segments.
         */
        error = VOP_MAP(vp, off, as, &addr, size, prot, PROT_ALL,
            mflag, fcred, NULL);
        if (error) {
                if (!is_library) {
                        (void) as_unmap(as, start_addr, osize);
                }
                return (error);
        }

        /* pickup the value of start_addr and osize for libraries */
        start_addr = addr;
        osize = size;

        /*
         * We have our initial reservation/allocation so we need to use fixed
         * addresses from now on.
         */
        mflag |= MAP_FIXED;

        mrp[0].mr_addr = addr;
        mrp[0].mr_msize = hdr->a_text;
        mrp[0].mr_fsize = hdr->a_text;
        mrp[0].mr_offset = 0;
        mrp[0].mr_prot = PROT_READ | PROT_EXEC;
        mrp[0].mr_flags = MR_HDR_AOUT;


        /*
         * Map initialized data. We are mapping over a portion of the
         * previous mapping which will be unmapped in VOP_MAP below.
         */
        off = P2ROUNDUP((offset_t)(hdr->a_text), PAGESIZE);
        msize = off;
        addr += off;
        size = hdr->a_data;
        error = VOP_MAP(vp, off, as, &addr, size, PROT_ALL, PROT_ALL,
            mflag, fcred, NULL);
        if (error) {
                (void) as_unmap(as, start_addr, osize);
                return (error);
        }
        msize += size;
        mrp[1].mr_addr = addr;
        mrp[1].mr_msize = size;
        mrp[1].mr_fsize = size;
        mrp[1].mr_offset = 0;
        mrp[1].mr_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
        mrp[1].mr_flags = 0;

        /* Need to zero out remainder of page */
        addr += hdr->a_data;
        zfoddiff = P2PHASE((size_t)addr, PAGESIZE);
        if (zfoddiff) {
                label_t ljb;

                MOBJ_STAT_ADD(aout_zfoddiff);
                zfoddiff = PAGESIZE - zfoddiff;
                if (on_fault(&ljb)) {
                        no_fault();
                        MOBJ_STAT_ADD(aout_uzero_fault);
                        (void) as_unmap(as, start_addr, osize);
                        return (EFAULT);
                }
                uzero(addr, zfoddiff);
                no_fault();
        }
        msize += zfoddiff;
        segnum = 2;

        /* Map bss */
        if (hdr->a_bss > zfoddiff) {
                struct segvn_crargs crargs =
                    SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
                MOBJ_STAT_ADD(aout_map_bss);
                addr += zfoddiff;
                size = hdr->a_bss - zfoddiff;
                as_rangelock(as);
                (void) as_unmap(as, addr, size);
                error = as_map(as, addr, size, segvn_create, &crargs);
                as_rangeunlock(as);
                msize += size;

                if (error) {
                        MOBJ_STAT_ADD(aout_bss_fail);
                        (void) as_unmap(as, start_addr, osize);
                        return (error);
                }
                mrp[2].mr_addr = addr;
                mrp[2].mr_msize = size;
                mrp[2].mr_fsize = 0;
                mrp[2].mr_offset = 0;
                mrp[2].mr_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
                mrp[2].mr_flags = 0;

                addr += size;
                segnum = 3;
        }

        /*
         * If we have extra bits left over, we need to include that in how
         * much we mapped to make sure the nlist logic is correct
         */
        msize = P2ROUNDUP(msize, PAGESIZE);

        if (nsize && msize < nsize) {
                MOBJ_STAT_ADD(aout_nlist);
                mrp[segnum].mr_addr = addr;
                mrp[segnum].mr_msize = nsize - msize;
                mrp[segnum].mr_fsize = 0;
                mrp[segnum].mr_offset = 0;
                mrp[segnum].mr_prot = PROT_READ | PROT_EXEC;
                mrp[segnum].mr_flags = 0;
        }

        *num_mapped = to_map;
        return (0);
}
#endif

/*
 * These are the two types of files that we can interpret and we want to read
 * in enough info to cover both types when looking at the initial header.
 */
#define MAX_HEADER_SIZE (MAX(sizeof (Ehdr), sizeof (struct exec)))

/*
 * Map vp passed in in an interpreted manner.  ELF and AOUT files will be
 * interpreted and mapped appropriately for execution.
 * num_mapped in - # elements in mrp
 * num_mapped out - # sections mapped and length of mrp array if
 *                  no errors or E2BIG returned.
 *
 * Returns 0 on success, errno value on failure.
 */
static int
mmapobj_map_interpret(vnode_t *vp, mmapobj_result_t *mrp,
    uint_t *num_mapped, size_t padding, cred_t *fcred)
{
        int error = 0;
        vattr_t vattr;
        struct lib_va *lvp;
        caddr_t start_addr;
        model_t model;

        /*
         * header has to be aligned to the native size of ulong_t in order
         * to avoid an unaligned access when dereferencing the header as
         * a ulong_t.  Thus we allocate our array on the stack of type
         * ulong_t and then have header, which we dereference later as a char
         * array point at lheader.
         */
        ulong_t lheader[(MAX_HEADER_SIZE / (sizeof (ulong_t))) + 1];
        caddr_t header = (caddr_t)&lheader;

        vattr.va_mask = AT_FSID | AT_NODEID | AT_CTIME | AT_MTIME | AT_SIZE;
        error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
        if (error) {
                return (error);
        }

        /*
         * Check lib_va to see if we already have a full description
         * for this library.  This is the fast path and only used for
         * ET_DYN ELF files (dynamic libraries).
         */
        if (padding == 0 && (lvp = lib_va_find(&vattr)) != NULL) {
                int num_segs;

                model = get_udatamodel();
                if ((model == DATAMODEL_ILP32 &&
                    lvp->lv_flags & LV_ELF64) ||
                    (model == DATAMODEL_LP64 &&
                    lvp->lv_flags & LV_ELF32)) {
                        lib_va_release(lvp);
                        MOBJ_STAT_ADD(fast_wrong_model);
                        return (ENOTSUP);
                }
                num_segs = lvp->lv_num_segs;
                if (*num_mapped < num_segs) {
                        *num_mapped = num_segs;
                        lib_va_release(lvp);
                        MOBJ_STAT_ADD(fast_e2big);
                        return (E2BIG);
                }

                /*
                 * Check to see if we have all the mappable program headers
                 * cached.
                 */
                if (num_segs <= LIBVA_CACHED_SEGS && num_segs != 0) {
                        MOBJ_STAT_ADD(fast);
                        start_addr = mmapobj_lookup_start_addr(lvp);
                        if (start_addr == NULL) {
                                lib_va_release(lvp);
                                return (ENOMEM);
                        }

                        bcopy(lvp->lv_mps, mrp,
                            num_segs * sizeof (mmapobj_result_t));

                        error = mmapobj_map_elf(vp, start_addr, mrp,
                            num_segs, fcred, ET_DYN);

                        lib_va_release(lvp);
                        if (error == 0) {
                                *num_mapped = num_segs;
                                MOBJ_STAT_ADD(fast_success);
                        }
                        return (error);
                }
                MOBJ_STAT_ADD(fast_not_now);

                /* Release it for now since we'll look it up below */
                lib_va_release(lvp);
        }

        /*
         * Time to see if this is a file we can interpret.  If it's smaller
         * than this, then we can't interpret it.
         */
        if (vattr.va_size < MAX_HEADER_SIZE) {
                MOBJ_STAT_ADD(small_file);
                return (ENOTSUP);
        }

        if ((error = vn_rdwr(UIO_READ, vp, header, MAX_HEADER_SIZE, 0,
            UIO_SYSSPACE, 0, (rlim64_t)0, fcred, NULL)) != 0) {
                MOBJ_STAT_ADD(read_error);
                return (error);
        }

        /* Verify file type */
        if (header[EI_MAG0] == ELFMAG0 && header[EI_MAG1] == ELFMAG1 &&
            header[EI_MAG2] == ELFMAG2 && header[EI_MAG3] == ELFMAG3) {
                return (doelfwork((Ehdr *)lheader, vp, mrp, num_mapped,
                    padding, fcred));
        }

#if defined(__sparc)
        /* On sparc, check for 4.X AOUT format */
        switch (((struct exec *)header)->a_magic) {
        case OMAGIC:
        case ZMAGIC:
        case NMAGIC:
                return (doaoutwork(vp, mrp, num_mapped,
                    (struct exec *)lheader, fcred));
        }
#endif

        /* Unsupported type */
        MOBJ_STAT_ADD(unsupported);
        return (ENOTSUP);
}

/*
 * Given a vnode, map it as either a flat file or interpret it and map
 * it according to the rules of the file type.
 * *num_mapped will contain the size of the mmapobj_result_t array passed in.
 * If padding is non-zero, the mappings will be padded by that amount
 * rounded up to the nearest pagesize.
 * If the mapping is successful, *num_mapped will contain the number of
 * distinct mappings created, and mrp will point to the array of
 * mmapobj_result_t's which describe these mappings.
 *
 * On error, -1 is returned and errno is set appropriately.
 * A special error case will set errno to E2BIG when there are more than
 * *num_mapped mappings to be created and *num_mapped will be set to the
 * number of mappings needed.
 */
int
mmapobj(vnode_t *vp, uint_t flags, mmapobj_result_t *mrp,
    uint_t *num_mapped, size_t padding, cred_t *fcred)
{
        int to_map;
        int error = 0;

        ASSERT((padding & PAGEOFFSET) == 0);
        ASSERT((flags & ~MMOBJ_ALL_FLAGS) == 0);
        ASSERT(num_mapped != NULL);
        ASSERT((flags & MMOBJ_PADDING) ? padding != 0 : padding == 0);

        if ((flags & MMOBJ_INTERPRET) == 0) {
                to_map = padding ? 3 : 1;
                if (*num_mapped < to_map) {
                        *num_mapped = to_map;
                        MOBJ_STAT_ADD(flat_e2big);
                        return (E2BIG);
                }
                error = mmapobj_map_flat(vp, mrp, padding, fcred);

                if (error) {
                        return (error);
                }
                *num_mapped = to_map;
                return (0);
        }

        error = mmapobj_map_interpret(vp, mrp, num_mapped, padding, fcred);
        return (error);
}