Merge commit '7e934d3acc051b7ee3ef0d11571fd1225800a607'

[unleashed.git] / kernel / os / grow.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/* Copyright 2013 OmniTI Computer Consulting, Inc. All rights reserved. */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/*        All Rights Reserved   */

#include <sys/types.h>
#include <sys/inttypes.h>
#include <sys/param.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/signal.h>
#include <sys/user.h>
#include <sys/errno.h>
#include <sys/var.h>
#include <sys/proc.h>
#include <sys/tuneable.h>
#include <sys/debug.h>
#include <sys/cmn_err.h>
#include <sys/cred.h>
#include <sys/vnode.h>
#include <sys/vfs.h>
#include <sys/vm.h>
#include <sys/file.h>
#include <sys/mman.h>
#include <sys/vmparam.h>
#include <sys/fcntl.h>
#include <sys/lwpchan_impl.h>
#include <sys/nbmlock.h>

#include <vm/hat.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <vm/seg_dev.h>
#include <vm/seg_vn.h>

int use_brk_lpg = 1;
int use_stk_lpg = 1;

/*
 * If set, we will not randomize mappings where the 'addr' argument is
 * non-NULL and not an alignment.
 */
int aslr_respect_mmap_hint = 1;

static int brk_lpg(caddr_t nva);
static int grow_lpg(caddr_t sp);

intptr_t
brk(caddr_t nva)
{
        int error;
        proc_t *p = curproc;

        /*
         * Serialize brk operations on an address space.
         * This also serves as the lock protecting p_brksize
         * and p_brkpageszc.
         */
        as_rangelock(p->p_as);

        /*
         * As a special case to aid the implementation of sbrk(3C), if given a
         * new brk of 0, return the current brk.  We'll hide this in brk(3C).
         */
        if (nva == 0) {
                intptr_t base = (intptr_t)(p->p_brkbase + p->p_brksize);
                as_rangeunlock(p->p_as);
                return (base);
        }

        if (use_brk_lpg && (p->p_flag & SAUTOLPG) != 0) {
                error = brk_lpg(nva);
        } else {
                error = brk_internal(nva, p->p_brkpageszc);
        }
        as_rangeunlock(p->p_as);
        return ((error != 0 ? set_errno(error) : 0));
}

/*
 * Algorithm: call arch-specific map_pgsz to get best page size to use,
 * then call brk_internal().
 * Returns 0 on success.
 */
static int
brk_lpg(caddr_t nva)
{
        struct proc *p = curproc;
        size_t pgsz, len;
        caddr_t addr, brkend;
        caddr_t bssbase = p->p_bssbase;
        caddr_t brkbase = p->p_brkbase;
        int oszc, szc;
        int err;

        oszc = p->p_brkpageszc;

        /*
         * If p_brkbase has not yet been set, the first call
         * to brk_internal() will initialize it.
         */
        if (brkbase == 0) {
                return (brk_internal(nva, oszc));
        }

        len = nva - bssbase;

        pgsz = map_pgsz(MAPPGSZ_HEAP, p, bssbase, len, 0);
        szc = page_szc(pgsz);

        /*
         * Covers two cases:
         * 1. page_szc() returns -1 for invalid page size, so we want to
         * ignore it in that case.
         * 2. By design we never decrease page size, as it is more stable.
         */
        if (szc <= oszc) {
                err = brk_internal(nva, oszc);
                /* If failed, back off to base page size. */
                if (err != 0 && oszc != 0) {
                        err = brk_internal(nva, 0);
                }
                return (err);
        }

        err = brk_internal(nva, szc);
        /* If using szc failed, map with base page size and return. */
        if (err != 0) {
                if (szc != 0) {
                        err = brk_internal(nva, 0);
                }
                return (err);
        }

        /*
         * Round up brk base to a large page boundary and remap
         * anything in the segment already faulted in beyond that
         * point.
         */
        addr = (caddr_t)P2ROUNDUP((uintptr_t)p->p_bssbase, pgsz);
        brkend = brkbase + p->p_brksize;
        len = brkend - addr;
        /* Check that len is not negative. Update page size code for heap. */
        if (addr >= p->p_bssbase && brkend > addr && IS_P2ALIGNED(len, pgsz)) {
                (void) as_setpagesize(p->p_as, addr, len, szc, B_FALSE);
                p->p_brkpageszc = szc;
        }

        ASSERT(err == 0);
        return (err);           /* should always be 0 */
}

/*
 * Returns 0 on success.
 */
int
brk_internal(caddr_t nva, uint_t brkszc)
{
        caddr_t ova;                    /* current break address */
        size_t size;
        int     error;
        struct proc *p = curproc;
        struct as *as = p->p_as;
        size_t pgsz;
        uint_t szc;
        rctl_qty_t as_rctl;

        /*
         * extend heap to brkszc alignment but use current p->p_brkpageszc
         * for the newly created segment. This allows the new extension
         * segment to be concatenated successfully with the existing brk
         * segment.
         */
        if ((szc = brkszc) != 0) {
                pgsz = page_get_pagesize(szc);
                ASSERT(pgsz > PAGESIZE);
        } else {
                pgsz = PAGESIZE;
        }

        mutex_enter(&p->p_lock);
        as_rctl = rctl_enforced_value(rctlproc_legacy[RLIMIT_DATA],
            p->p_rctls, p);
        mutex_exit(&p->p_lock);

        /*
         * If p_brkbase has not yet been set, the first call
         * to brk() will initialize it.
         */
        if (p->p_brkbase == 0)
                p->p_brkbase = nva;

        /*
         * Before multiple page size support existed p_brksize was the value
         * not rounded to the pagesize (i.e. it stored the exact user request
         * for heap size). If pgsz is greater than PAGESIZE calculate the
         * heap size as the real new heap size by rounding it up to pgsz.
         * This is useful since we may want to know where the heap ends
         * without knowing heap pagesize (e.g. some old code) and also if
         * heap pagesize changes we can update p_brkpageszc but delay adding
         * new mapping yet still know from p_brksize where the heap really
         * ends. The user requested heap end is stored in libc variable.
         */
        if (pgsz > PAGESIZE) {
                caddr_t tnva = (caddr_t)P2ROUNDUP((uintptr_t)nva, pgsz);
                size = tnva - p->p_brkbase;
                if (tnva < p->p_brkbase || (size > p->p_brksize &&
                    size > (size_t)as_rctl)) {
                        szc = 0;
                        pgsz = PAGESIZE;
                        size = nva - p->p_brkbase;
                }
        } else {
                size = nva - p->p_brkbase;
        }

        /*
         * use PAGESIZE to roundup ova because we want to know the real value
         * of the current heap end in case p_brkpageszc changes since the last
         * p_brksize was computed.
         */
        nva = (caddr_t)P2ROUNDUP((uintptr_t)nva, pgsz);
        ova = (caddr_t)P2ROUNDUP((uintptr_t)(p->p_brkbase + p->p_brksize),
            PAGESIZE);

        if ((nva < p->p_brkbase) || (size > p->p_brksize &&
            size > as_rctl)) {
                mutex_enter(&p->p_lock);
                (void) rctl_action(rctlproc_legacy[RLIMIT_DATA], p->p_rctls, p,
                    RCA_SAFE);
                mutex_exit(&p->p_lock);
                return (ENOMEM);
        }

        if (nva > ova) {
                struct segvn_crargs crargs =
                    SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);

                if (!(p->p_datprot & PROT_EXEC)) {
                        crargs.prot &= ~PROT_EXEC;
                }

                /*
                 * Add new zfod mapping to extend UNIX data segment
                 * AS_MAP_NO_LPOOB means use 0, and don't reapply OOB policies
                 * via map_pgszcvec(). Use AS_MAP_HEAP to get intermediate
                 * page sizes if ova is not aligned to szc's pgsz.
                 */
                if (szc > 0) {
                        caddr_t rbss;

                        rbss = (caddr_t)P2ROUNDUP((uintptr_t)p->p_bssbase,
                            pgsz);
                        if (IS_P2ALIGNED(p->p_bssbase, pgsz) || ova > rbss) {
                                crargs.szc = p->p_brkpageszc ? p->p_brkpageszc :
                                    AS_MAP_NO_LPOOB;
                        } else if (ova == rbss) {
                                crargs.szc = szc;
                        } else {
                                crargs.szc = AS_MAP_HEAP;
                        }
                } else {
                        crargs.szc = AS_MAP_NO_LPOOB;
                }
                crargs.lgrp_mem_policy_flags = LGRP_MP_FLAG_EXTEND_UP;
                error = as_map(as, ova, (size_t)(nva - ova), segvn_create,
                    &crargs);
                if (error) {
                        return (error);
                }

        } else if (nva < ova) {
                /*
                 * Release mapping to shrink UNIX data segment.
                 */
                (void) as_unmap(as, nva, (size_t)(ova - nva));
        }
        p->p_brksize = size;
        return (0);
}

/*
 * Grow the stack to include sp.  Return 1 if successful, 0 otherwise.
 * This routine assumes that the stack grows downward.
 */
int
grow(caddr_t sp)
{
        struct proc *p = curproc;
        struct as *as = p->p_as;
        size_t oldsize = p->p_stksize;
        size_t newsize;
        int err;

        /*
         * Serialize grow operations on an address space.
         * This also serves as the lock protecting p_stksize
         * and p_stkpageszc.
         */
        as_rangelock(as);
        if (use_stk_lpg && (p->p_flag & SAUTOLPG) != 0) {
                err = grow_lpg(sp);
        } else {
                err = grow_internal(sp, p->p_stkpageszc);
        }
        as_rangeunlock(as);

        if (err == 0 && (newsize = p->p_stksize) > oldsize) {
                ASSERT(IS_P2ALIGNED(oldsize, PAGESIZE));
                ASSERT(IS_P2ALIGNED(newsize, PAGESIZE));
                /*
                 * Set up translations so the process doesn't have to fault in
                 * the stack pages we just gave it.
                 */
                (void) as_fault(as->a_hat, as, p->p_usrstack - newsize,
                    newsize - oldsize, F_INVAL, S_WRITE);
        }
        return ((err == 0 ? 1 : 0));
}

/*
 * Algorithm: call arch-specific map_pgsz to get best page size to use,
 * then call grow_internal().
 * Returns 0 on success.
 */
static int
grow_lpg(caddr_t sp)
{
        struct proc *p = curproc;
        size_t pgsz;
        size_t len, newsize;
        caddr_t addr, saddr;
        caddr_t growend;
        int oszc, szc;
        int err;

        newsize = p->p_usrstack - sp;

        oszc = p->p_stkpageszc;
        pgsz = map_pgsz(MAPPGSZ_STK, p, sp, newsize, 0);
        szc = page_szc(pgsz);

        /*
         * Covers two cases:
         * 1. page_szc() returns -1 for invalid page size, so we want to
         * ignore it in that case.
         * 2. By design we never decrease page size, as it is more stable.
         * This shouldn't happen as the stack never shrinks.
         */
        if (szc <= oszc) {
                err = grow_internal(sp, oszc);
                /* failed, fall back to base page size */
                if (err != 0 && oszc != 0) {
                        err = grow_internal(sp, 0);
                }
                return (err);
        }

        /*
         * We've grown sufficiently to switch to a new page size.
         * So we are going to remap the whole segment with the new page size.
         */
        err = grow_internal(sp, szc);
        /* The grow with szc failed, so fall back to base page size. */
        if (err != 0) {
                if (szc != 0) {
                        err = grow_internal(sp, 0);
                }
                return (err);
        }

        /*
         * Round up stack pointer to a large page boundary and remap
         * any pgsz pages in the segment already faulted in beyond that
         * point.
         */
        saddr = p->p_usrstack - p->p_stksize;
        addr = (caddr_t)P2ROUNDUP((uintptr_t)saddr, pgsz);
        growend = (caddr_t)P2ALIGN((uintptr_t)p->p_usrstack, pgsz);
        len = growend - addr;
        /* Check that len is not negative. Update page size code for stack. */
        if (addr >= saddr && growend > addr && IS_P2ALIGNED(len, pgsz)) {
                (void) as_setpagesize(p->p_as, addr, len, szc, B_FALSE);
                p->p_stkpageszc = szc;
        }

        ASSERT(err == 0);
        return (err);           /* should always be 0 */
}

/*
 * This routine assumes that the stack grows downward.
 * Returns 0 on success, errno on failure.
 */
int
grow_internal(caddr_t sp, uint_t growszc)
{
        struct proc *p = curproc;
        size_t newsize;
        size_t oldsize;
        int    error;
        size_t pgsz;
        uint_t szc;
        struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);

        ASSERT(sp < p->p_usrstack);
        sp = (caddr_t)P2ALIGN((uintptr_t)sp, PAGESIZE);

        /*
         * grow to growszc alignment but use current p->p_stkpageszc for
         * the segvn_crargs szc passed to segvn_create. For memcntl to
         * increase the szc, this allows the new extension segment to be
         * concatenated successfully with the existing stack segment.
         */
        if ((szc = growszc) != 0) {
                pgsz = page_get_pagesize(szc);
                ASSERT(pgsz > PAGESIZE);
                newsize = p->p_usrstack - (caddr_t)P2ALIGN((uintptr_t)sp, pgsz);
                if (newsize > (size_t)p->p_stk_ctl) {
                        szc = 0;
                        pgsz = PAGESIZE;
                        newsize = p->p_usrstack - sp;
                }
        } else {
                pgsz = PAGESIZE;
                newsize = p->p_usrstack - sp;
        }

        if (newsize > (size_t)p->p_stk_ctl) {
                (void) rctl_action(rctlproc_legacy[RLIMIT_STACK], p->p_rctls, p,
                    RCA_UNSAFE_ALL);

                return (ENOMEM);
        }

        oldsize = p->p_stksize;
        ASSERT(P2PHASE(oldsize, PAGESIZE) == 0);

        if (newsize <= oldsize) {       /* prevent the stack from shrinking */
                return (0);
        }

        if (!(p->p_stkprot & PROT_EXEC)) {
                crargs.prot &= ~PROT_EXEC;
        }
        /*
         * extend stack with the proposed new growszc, which is different
         * than p_stkpageszc only on a memcntl to increase the stack pagesize.
         * AS_MAP_NO_LPOOB means use 0, and don't reapply OOB policies via
         * map_pgszcvec(). Use AS_MAP_STACK to get intermediate page sizes
         * if not aligned to szc's pgsz.
         */
        if (szc > 0) {
                caddr_t oldsp = p->p_usrstack - oldsize;
                caddr_t austk = (caddr_t)P2ALIGN((uintptr_t)p->p_usrstack,
                    pgsz);

                if (IS_P2ALIGNED(p->p_usrstack, pgsz) || oldsp < austk) {
                        crargs.szc = p->p_stkpageszc ? p->p_stkpageszc :
                            AS_MAP_NO_LPOOB;
                } else if (oldsp == austk) {
                        crargs.szc = szc;
                } else {
                        crargs.szc = AS_MAP_STACK;
                }
        } else {
                crargs.szc = AS_MAP_NO_LPOOB;
        }
        crargs.lgrp_mem_policy_flags = LGRP_MP_FLAG_EXTEND_DOWN;

        if ((error = as_map(p->p_as, p->p_usrstack - newsize, newsize - oldsize,
            segvn_create, &crargs)) != 0) {
                if (error == EAGAIN) {
                        cmn_err(CE_WARN, "Sorry, no swap space to grow stack "
                            "for pid %d (%s)", p->p_pid, PTOU(p)->u_comm);
                }
                return (error);
        }
        p->p_stksize = newsize;
        return (0);
}

/*
 * Find address for user to map.  If MAP_FIXED is not specified, we can pick
 * any address we want, but we will first try the value in *addrp if it is
 * non-NULL and _MAP_RANDOMIZE is not set.  Thus this is implementing a way to
 * try and get a preferred address.
 */
int
choose_addr(struct as *as, caddr_t *addrp, size_t len, offset_t off,
    int vacalign, uint_t flags)
{
        caddr_t basep = (caddr_t)(uintptr_t)((uintptr_t)*addrp & PAGEMASK);
        size_t lenp = len;

        ASSERT(AS_ISCLAIMGAP(as));      /* searches should be serialized */
        if (flags & MAP_FIXED) {
                (void) as_unmap(as, *addrp, len);
                return (0);
        } else if (basep != NULL &&
            ((flags & (MAP_ALIGN | _MAP_RANDOMIZE)) == 0) &&
            !as_gap(as, len, &basep, &lenp, 0, *addrp)) {
                /* User supplied address was available */
                *addrp = basep;
        } else {
                /*
                 * No user supplied address or the address supplied was not
                 * available.
                 */
                map_addr(addrp, len, off, vacalign, flags);
        }
        if (*addrp == NULL)
                return (ENOMEM);
        return (0);
}


/*
 * Used for MAP_ANON - fast way to get anonymous pages
 */
static int
zmap(struct as *as, caddr_t *addrp, size_t len, uint_t uprot, int flags,
    offset_t pos)
{
        struct segvn_crargs vn_a;
        int error;

        if (((PROT_ALL & uprot) != uprot))
                return (EACCES);

        if ((flags & MAP_FIXED) != 0) {
                caddr_t userlimit;

                /*
                 * Use the user address.  First verify that
                 * the address to be used is page aligned.
                 * Then make some simple bounds checks.
                 */
                if (((uintptr_t)*addrp & PAGEOFFSET) != 0)
                        return (EINVAL);

                userlimit = flags & _MAP_LOW32 ?
                    (caddr_t)USERLIMIT32 : as->a_userlimit;
                switch (valid_usr_range(*addrp, len, uprot, as, userlimit)) {
                case RANGE_OKAY:
                        break;
                case RANGE_BADPROT:
                        return (ENOTSUP);
                case RANGE_BADADDR:
                default:
                        return (ENOMEM);
                }
        }
        /*
         * No need to worry about vac alignment for anonymous
         * pages since this is a "clone" object that doesn't
         * yet exist.
         */
        error = choose_addr(as, addrp, len, pos, ADDR_NOVACALIGN, flags);
        if (error != 0) {
                return (error);
        }

        /*
         * Use the seg_vn segment driver; passing in the NULL amp
         * gives the desired "cloning" effect.
         */
        vn_a.vp = NULL;
        vn_a.offset = 0;
        vn_a.type = flags & MAP_TYPE;
        vn_a.prot = uprot;
        vn_a.maxprot = PROT_ALL;
        vn_a.flags = flags & ~MAP_TYPE;
        vn_a.cred = CRED();
        vn_a.amp = NULL;
        vn_a.szc = 0;
        vn_a.lgrp_mem_policy_flags = 0;

        return (as_map(as, *addrp, len, segvn_create, &vn_a));
}

#define RANDOMIZABLE_MAPPING(addr, flags) (((flags & MAP_FIXED) == 0) && \
        !(((flags & MAP_ALIGN) == 0) && (addr != 0) && aslr_respect_mmap_hint))

static int
smmap_common(caddr_t *addrp, size_t len,
    int prot, int flags, struct file *fp, offset_t pos)
{
        struct vnode *vp;
        struct as *as = curproc->p_as;
        uint_t uprot, maxprot, type;
        int error;
        int in_crit = 0;

        if ((flags & ~(MAP_SHARED | MAP_PRIVATE | MAP_FIXED |
            _MAP_LOW32 | MAP_NORESERVE | MAP_ANON | MAP_ALIGN |
            MAP_TEXT | MAP_INITDATA)) != 0) {
                /* | MAP_RENAME */      /* not implemented, let user know */
                return (EINVAL);
        }

        if ((flags & MAP_TEXT) && !(prot & PROT_EXEC)) {
                return (EINVAL);
        }

        if ((flags & (MAP_TEXT | MAP_INITDATA)) == (MAP_TEXT | MAP_INITDATA)) {
                return (EINVAL);
        }

        if ((flags & (MAP_FIXED | _MAP_RANDOMIZE)) ==
            (MAP_FIXED | _MAP_RANDOMIZE)) {
                return (EINVAL);
        }

        /*
         * If it's not a fixed allocation and mmap ASLR is enabled, randomize
         * it.
         */
        if (RANDOMIZABLE_MAPPING(*addrp, flags) &&
            secflag_enabled(curproc, PROC_SEC_ASLR))
                flags |= _MAP_RANDOMIZE;

        type = flags & MAP_TYPE;
        if (type != MAP_PRIVATE && type != MAP_SHARED)
                return (EINVAL);


        if (flags & MAP_ALIGN) {
                if (flags & MAP_FIXED)
                        return (EINVAL);

                /* alignment needs to be a power of 2 >= page size */
                if (((uintptr_t)*addrp < PAGESIZE && (uintptr_t)*addrp != 0) ||
                    !ISP2((uintptr_t)*addrp))
                        return (EINVAL);
        }
        /*
         * Check for bad lengths and file position.
         * We let the fop_map routine check for negative lengths
         * since on some vnode types this might be appropriate.
         */
        if (len == 0 || (pos & (uoff_t)PAGEOFFSET) != 0)
                return (EINVAL);

        maxprot = PROT_ALL;             /* start out allowing all accesses */
        uprot = prot | PROT_USER;

        if (fp == NULL) {
                ASSERT(flags & MAP_ANON);
                /* discard lwpchan mappings, like munmap() */
                if ((flags & MAP_FIXED) && curproc->p_lcp != NULL)
                        lwpchan_delete_mapping(curproc, *addrp, *addrp + len);
                as_rangelock(as);
                error = zmap(as, addrp, len, uprot, flags, pos);
                as_rangeunlock(as);
                /*
                 * Tell machine specific code that lwp has mapped shared memory
                 */
                if (error == 0 && (flags & MAP_SHARED)) {
                        /* EMPTY */
                        LWP_MMODEL_SHARED_AS(*addrp, len);
                }
                return (error);
        } else if ((flags & MAP_ANON) != 0)
                return (EINVAL);

        vp = fp->f_vnode;

        /* Can't execute code from "noexec" mounted filesystem. */
        if ((vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0)
                maxprot &= ~PROT_EXEC;

        /*
         * These checks were added as part of large files.
         *
         * Return ENXIO if the initial position is negative; return EOVERFLOW
         * if (offset + len) would overflow the maximum allowed offset for the
         * type of file descriptor being used.
         */
        if (vp->v_type == VREG) {
                if (pos < 0)
                        return (ENXIO);
                if ((offset_t)len > (OFFSET_MAX(fp) - pos))
                        return (EOVERFLOW);
        }

        if (type == MAP_SHARED && (fp->f_flag & FWRITE) == 0) {
                /* no write access allowed */
                maxprot &= ~PROT_WRITE;
        }

        /*
         * XXX - Do we also adjust maxprot based on protections
         * of the vnode?  E.g. if no execute permission is given
         * on the vnode for the current user, maxprot probably
         * should disallow PROT_EXEC also?  This is different
         * from the write access as this would be a per vnode
         * test as opposed to a per fd test for writability.
         */

        /*
         * Verify that the specified protections are not greater than
         * the maximum allowable protections.  Also test to make sure
         * that the file descriptor does allows for read access since
         * "write only" mappings are hard to do since normally we do
         * the read from the file before the page can be written.
         */
        if (((maxprot & uprot) != uprot) || (fp->f_flag & FREAD) == 0)
                return (EACCES);

        /*
         * If the user specified an address, do some simple checks here
         */
        if ((flags & MAP_FIXED) != 0) {
                caddr_t userlimit;

                /*
                 * Use the user address.  First verify that
                 * the address to be used is page aligned.
                 * Then make some simple bounds checks.
                 */
                if (((uintptr_t)*addrp & PAGEOFFSET) != 0)
                        return (EINVAL);

                userlimit = flags & _MAP_LOW32 ?
                    (caddr_t)USERLIMIT32 : as->a_userlimit;
                switch (valid_usr_range(*addrp, len, uprot, as, userlimit)) {
                case RANGE_OKAY:
                        break;
                case RANGE_BADPROT:
                        return (ENOTSUP);
                case RANGE_BADADDR:
                default:
                        return (ENOMEM);
                }
        }

        if ((prot & (PROT_READ | PROT_WRITE | PROT_EXEC)) &&
            nbl_need_check(vp)) {
                int svmand;
                nbl_op_t nop;

                nbl_start_crit(vp, RW_READER);
                in_crit = 1;
                error = nbl_svmand(vp, fp->f_cred, &svmand);
                if (error != 0)
                        goto done;
                if ((prot & PROT_WRITE) && (type == MAP_SHARED)) {
                        if (prot & (PROT_READ | PROT_EXEC)) {
                                nop = NBL_READWRITE;
                        } else {
                                nop = NBL_WRITE;
                        }
                } else {
                        nop = NBL_READ;
                }
                if (nbl_conflict(vp, nop, 0, LONG_MAX, svmand, NULL)) {
                        error = EACCES;
                        goto done;
                }
        }

        /* discard lwpchan mappings, like munmap() */
        if ((flags & MAP_FIXED) && curproc->p_lcp != NULL)
                lwpchan_delete_mapping(curproc, *addrp, *addrp + len);

        /*
         * Ok, now let the vnode map routine do its thing to set things up.
         */
        error = fop_map(vp, pos, as,
            addrp, len, uprot, maxprot, flags, fp->f_cred, NULL);

        if (error == 0) {
                /*
                 * Tell machine specific code that lwp has mapped shared memory
                 */
                if (flags & MAP_SHARED) {
                        /* EMPTY */
                        LWP_MMODEL_SHARED_AS(*addrp, len);
                }
                if (vp->v_type == VREG &&
                    (flags & (MAP_TEXT | MAP_INITDATA)) != 0) {
                        /*
                         * Mark this as an executable vnode
                         */
                        mutex_enter(&vp->v_lock);
                        vp->v_flag |= VVMEXEC;
                        mutex_exit(&vp->v_lock);
                }
        }

done:
        if (in_crit)
                nbl_end_crit(vp);
        return (error);
}

#ifdef _LP64
/*
 * LP64 mmap(2) system call: 64-bit offset, 64-bit address.
 *
 * The "large file" mmap routine mmap64(2) is also mapped to this routine
 * by the 64-bit version of libc.
 *
 * Eventually, this should be the only version, and have smmap_common()
 * folded back into it again.  Some day.
 */
caddr_t
smmap64(caddr_t addr, size_t len, int prot, int flags, int fd, off_t pos)
{
        struct file *fp;
        int error;

        if (fd == -1 && (flags & MAP_ANON) != 0)
                error = smmap_common(&addr, len, prot, flags,
                    NULL, (offset_t)pos);
        else if ((fp = getf(fd)) != NULL) {
                error = smmap_common(&addr, len, prot, flags,
                    fp, (offset_t)pos);
                releasef(fd);
        } else
                error = EBADF;

        return (error ? (caddr_t)(uintptr_t)set_errno(error) : addr);
}
#endif  /* _LP64 */

#if defined(_SYSCALL32_IMPL) || defined(_ILP32)

/*
 * ILP32 mmap(2) system call: 32-bit offset, 32-bit address.
 */
caddr_t
smmap32(caddr32_t addr, size32_t len, int prot, int flags, int fd, off32_t pos)
{
        struct file *fp;
        int error;
        caddr_t a = (caddr_t)(uintptr_t)addr;

        if (flags & _MAP_LOW32)
                error = EINVAL;
        else if (fd == -1 && (flags & MAP_ANON) != 0)
                error = smmap_common(&a, (size_t)len, prot,
                    flags | _MAP_LOW32, NULL, (offset_t)pos);
        else if ((fp = getf(fd)) != NULL) {
                error = smmap_common(&a, (size_t)len, prot,
                    flags | _MAP_LOW32, fp, (offset_t)pos);
                releasef(fd);
        } else
                error = EBADF;

        ASSERT(error != 0 || (uintptr_t)(a + len) < (uintptr_t)UINT32_MAX);

        return (error ? (caddr_t)(uintptr_t)set_errno(error) : a);
}

/*
 * ILP32 mmap64(2) system call: 64-bit offset, 32-bit address.
 *
 * Now things really get ugly because we can't use the C-style
 * calling convention for more than 6 args, and 64-bit parameter
 * passing on 32-bit systems is less than clean.
 */

struct mmaplf32a {
        caddr_t addr;
        size_t len;
#ifdef _LP64
        /*
         * 32-bit contents, 64-bit cells
         */
        uint64_t prot;
        uint64_t flags;
        uint64_t fd;
        uint64_t offhi;
        uint64_t offlo;
#else
        /*
         * 32-bit contents, 32-bit cells
         */
        uint32_t prot;
        uint32_t flags;
        uint32_t fd;
        uint32_t offhi;
        uint32_t offlo;
#endif
};

int
smmaplf32(struct mmaplf32a *uap, rval_t *rvp)
{
        struct file *fp;
        int error;
        caddr_t a = uap->addr;
        int flags = (int)uap->flags;
        int fd = (int)uap->fd;
#ifdef _BIG_ENDIAN
        offset_t off = ((uoff_t)uap->offhi << 32) | (uoff_t)uap->offlo;
#else
        offset_t off = ((uoff_t)uap->offlo << 32) | (uoff_t)uap->offhi;
#endif

        if (flags & _MAP_LOW32)
                error = EINVAL;
        else if (fd == -1 && (flags & MAP_ANON) != 0)
                error = smmap_common(&a, uap->len, (int)uap->prot,
                    flags | _MAP_LOW32, NULL, off);
        else if ((fp = getf(fd)) != NULL) {
                error = smmap_common(&a, uap->len, (int)uap->prot,
                    flags | _MAP_LOW32, fp, off);
                releasef(fd);
        } else
                error = EBADF;

        if (error == 0)
                rvp->r_val1 = (uintptr_t)a;
        return (error);
}

#endif  /* _SYSCALL32_IMPL || _ILP32 */

int
munmap(caddr_t addr, size_t len)
{
        struct proc *p = curproc;
        struct as *as = p->p_as;

        if (((uintptr_t)addr & PAGEOFFSET) != 0 || len == 0)
                return (set_errno(EINVAL));

        if (valid_usr_range(addr, len, 0, as, as->a_userlimit) != RANGE_OKAY)
                return (set_errno(EINVAL));

        /*
         * Discard lwpchan mappings.
         */
        if (p->p_lcp != NULL)
                lwpchan_delete_mapping(p, addr, addr + len);
        if (as_unmap(as, addr, len) != 0)
                return (set_errno(EINVAL));

        return (0);
}

int
mprotect(caddr_t addr, size_t len, int prot)
{
        struct as *as = curproc->p_as;
        uint_t uprot = prot | PROT_USER;
        int error;

        if (((uintptr_t)addr & PAGEOFFSET) != 0 || len == 0)
                return (set_errno(EINVAL));

        switch (valid_usr_range(addr, len, prot, as, as->a_userlimit)) {
        case RANGE_OKAY:
                break;
        case RANGE_BADPROT:
                return (set_errno(ENOTSUP));
        case RANGE_BADADDR:
        default:
                return (set_errno(ENOMEM));
        }

        error = as_setprot(as, addr, len, uprot);
        if (error)
                return (set_errno(error));
        return (0);
}

#define MC_CACHE        128                     /* internal result buffer */
#define MC_QUANTUM      (MC_CACHE * PAGESIZE)   /* addresses covered in loop */

int
mincore(caddr_t addr, size_t len, char *vecp)
{
        struct as *as = curproc->p_as;
        caddr_t ea;                     /* end address of loop */
        size_t rl;                      /* inner result length */
        char vec[MC_CACHE];             /* local vector cache */
        int error;
        model_t model;
        long    llen;

        model = get_udatamodel();
        /*
         * Validate form of address parameters.
         */
        if (model == DATAMODEL_NATIVE) {
                llen = (long)len;
        } else {
                llen = (int32_t)(size32_t)len;
        }
        if (((uintptr_t)addr & PAGEOFFSET) != 0 || llen <= 0)
                return (set_errno(EINVAL));

        if (valid_usr_range(addr, len, 0, as, as->a_userlimit) != RANGE_OKAY)
                return (set_errno(ENOMEM));

        /*
         * Loop over subranges of interval [addr : addr + len), recovering
         * results internally and then copying them out to caller.  Subrange
         * is based on the size of MC_CACHE, defined above.
         */
        for (ea = addr + len; addr < ea; addr += MC_QUANTUM) {
                error = as_incore(as, addr,
                    (size_t)MIN(MC_QUANTUM, ea - addr), vec, &rl);
                if (rl != 0) {
                        rl = (rl + PAGESIZE - 1) / PAGESIZE;
                        if (copyout(vec, vecp, rl) != 0)
                                return (set_errno(EFAULT));
                        vecp += rl;
                }
                if (error != 0)
                        return (set_errno(ENOMEM));
        }
        return (0);
}