5384 pvn_getpages may assert in valid scenarios

[illumos-gate.git] / usr / src / uts / common / vm / vm_pvn.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
 */

/*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/*        All Rights Reserved   */

/*
 * University Copyright- Copyright (c) 1982, 1986, 1988
 * The Regents of the University of California
 * All Rights Reserved
 *
 * University Acknowledgment- Portions of this document are derived from
 * software developed by the University of California, Berkeley, and its
 * contributors.
 */

/*
 * VM - paged vnode.
 *
 * This file supplies vm support for the vnode operations that deal with pages.
 */
#include <sys/types.h>
#include <sys/t_lock.h>
#include <sys/param.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/time.h>
#include <sys/buf.h>
#include <sys/vnode.h>
#include <sys/uio.h>
#include <sys/vmsystm.h>
#include <sys/mman.h>
#include <sys/vfs.h>
#include <sys/cred.h>
#include <sys/user.h>
#include <sys/kmem.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <sys/cpuvar.h>
#include <sys/vtrace.h>
#include <sys/tnf_probe.h>

#include <vm/hat.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <vm/rm.h>
#include <vm/pvn.h>
#include <vm/page.h>
#include <vm/seg_map.h>
#include <vm/seg_kmem.h>
#include <sys/fs/swapnode.h>

int pvn_nofodklust = 0;
int pvn_write_noklust = 0;

uint_t pvn_vmodsort_supported = 0;      /* set if HAT supports VMODSORT */
uint_t pvn_vmodsort_disable = 0;        /* set in /etc/system to disable HAT */
                                        /* support for vmodsort for testing */

static struct kmem_cache *marker_cache = NULL;

/*
 * Find the largest contiguous block which contains `addr' for file offset
 * `offset' in it while living within the file system block sizes (`vp_off'
 * and `vp_len') and the address space limits for which no pages currently
 * exist and which map to consecutive file offsets.
 */
page_t *
pvn_read_kluster(
        struct vnode *vp,
        u_offset_t off,
        struct seg *seg,
        caddr_t addr,
        u_offset_t *offp,                       /* return values */
        size_t *lenp,                           /* return values */
        u_offset_t vp_off,
        size_t vp_len,
        int isra)
{
        ssize_t deltaf, deltab;
        page_t *pp;
        page_t *plist = NULL;
        spgcnt_t pagesavail;
        u_offset_t vp_end;

        ASSERT(off >= vp_off && off < vp_off + vp_len);

        /*
         * We only want to do klustering/read ahead if there
         * is more than minfree pages currently available.
         */
        pagesavail = freemem - minfree;

        if (pagesavail <= 0)
                if (isra)
                        return ((page_t *)NULL);    /* ra case - give up */
                else
                        pagesavail = 1;             /* must return a page */

        /* We calculate in pages instead of bytes due to 32-bit overflows */
        if (pagesavail < (spgcnt_t)btopr(vp_len)) {
                /*
                 * Don't have enough free memory for the
                 * max request, try sizing down vp request.
                 */
                deltab = (ssize_t)(off - vp_off);
                vp_len -= deltab;
                vp_off += deltab;
                if (pagesavail < btopr(vp_len)) {
                        /*
                         * Still not enough memory, just settle for
                         * pagesavail which is at least 1.
                         */
                        vp_len = ptob(pagesavail);
                }
        }

        vp_end = vp_off + vp_len;
        ASSERT(off >= vp_off && off < vp_end);

        if (isra && SEGOP_KLUSTER(seg, addr, 0))
                return ((page_t *)NULL);        /* segment driver says no */

        if ((plist = page_create_va(vp, off,
            PAGESIZE, PG_EXCL | PG_WAIT, seg, addr)) == NULL)
                return ((page_t *)NULL);

        if (vp_len <= PAGESIZE || pvn_nofodklust) {
                *offp = off;
                *lenp = MIN(vp_len, PAGESIZE);
        } else {
                /*
                 * Scan back from front by incrementing "deltab" and
                 * comparing "off" with "vp_off + deltab" to avoid
                 * "signed" versus "unsigned" conversion problems.
                 */
                for (deltab = PAGESIZE; off >= vp_off + deltab;
                    deltab += PAGESIZE) {
                        /*
                         * Call back to the segment driver to verify that
                         * the klustering/read ahead operation makes sense.
                         */
                        if (SEGOP_KLUSTER(seg, addr, -deltab))
                                break;          /* page not eligible */
                        if ((pp = page_create_va(vp, off - deltab,
                            PAGESIZE, PG_EXCL, seg, addr - deltab))
                            == NULL)
                                break;          /* already have the page */
                        /*
                         * Add page to front of page list.
                         */
                        page_add(&plist, pp);
                }
                deltab -= PAGESIZE;

                /* scan forward from front */
                for (deltaf = PAGESIZE; off + deltaf < vp_end;
                    deltaf += PAGESIZE) {
                        /*
                         * Call back to the segment driver to verify that
                         * the klustering/read ahead operation makes sense.
                         */
                        if (SEGOP_KLUSTER(seg, addr, deltaf))
                                break;          /* page not file extension */
                        if ((pp = page_create_va(vp, off + deltaf,
                            PAGESIZE, PG_EXCL, seg, addr + deltaf))
                            == NULL)
                                break;          /* already have page */

                        /*
                         * Add page to end of page list.
                         */
                        page_add(&plist, pp);
                        plist = plist->p_next;
                }
                *offp = off = off - deltab;
                *lenp = deltab + deltaf;
                ASSERT(off >= vp_off);

                /*
                 * If we ended up getting more than was actually
                 * requested, retract the returned length to only
                 * reflect what was requested.  This might happen
                 * if we were allowed to kluster pages across a
                 * span of (say) 5 frags, and frag size is less
                 * than PAGESIZE.  We need a whole number of
                 * pages to contain those frags, but the returned
                 * size should only allow the returned range to
                 * extend as far as the end of the frags.
                 */
                if ((vp_off + vp_len) < (off + *lenp)) {
                        ASSERT(vp_end > off);
                        *lenp = vp_end - off;
                }
        }
        TRACE_3(TR_FAC_VM, TR_PVN_READ_KLUSTER,
            "pvn_read_kluster:seg %p addr %x isra %x",
            seg, addr, isra);
        return (plist);
}

/*
 * Handle pages for this vnode on either side of the page "pp"
 * which has been locked by the caller.  This routine will also
 * do klustering in the range [vp_off, vp_off + vp_len] up
 * until a page which is not found.  The offset and length
 * of pages included is returned in "*offp" and "*lenp".
 *
 * Returns a list of dirty locked pages all ready to be
 * written back.
 */
page_t *
pvn_write_kluster(
        struct vnode *vp,
        page_t *pp,
        u_offset_t *offp,               /* return values */
        size_t *lenp,                   /* return values */
        u_offset_t vp_off,
        size_t vp_len,
        int flags)
{
        u_offset_t off;
        page_t *dirty;
        size_t deltab, deltaf;
        se_t se;
        u_offset_t vp_end;

        off = pp->p_offset;

        /*
         * Kustering should not be done if we are invalidating
         * pages since we could destroy pages that belong to
         * some other process if this is a swap vnode.
         */
        if (pvn_write_noklust || ((flags & B_INVAL) && IS_SWAPVP(vp))) {
                *offp = off;
                *lenp = PAGESIZE;
                return (pp);
        }

        if (flags & (B_FREE | B_INVAL))
                se = SE_EXCL;
        else
                se = SE_SHARED;

        dirty = pp;
        /*
         * Scan backwards looking for pages to kluster by incrementing
         * "deltab" and comparing "off" with "vp_off + deltab" to
         * avoid "signed" versus "unsigned" conversion problems.
         */
        for (deltab = PAGESIZE; off >= vp_off + deltab; deltab += PAGESIZE) {
                pp = page_lookup_nowait(vp, off - deltab, se);
                if (pp == NULL)
                        break;          /* page not found */
                if (pvn_getdirty(pp, flags | B_DELWRI) == 0)
                        break;
                page_add(&dirty, pp);
        }
        deltab -= PAGESIZE;

        vp_end = vp_off + vp_len;
        /* now scan forwards looking for pages to kluster */
        for (deltaf = PAGESIZE; off + deltaf < vp_end; deltaf += PAGESIZE) {
                pp = page_lookup_nowait(vp, off + deltaf, se);
                if (pp == NULL)
                        break;          /* page not found */
                if (pvn_getdirty(pp, flags | B_DELWRI) == 0)
                        break;
                page_add(&dirty, pp);
                dirty = dirty->p_next;
        }

        *offp = off - deltab;
        *lenp = deltab + deltaf;
        return (dirty);
}

/*
 * Generic entry point used to release the "shared/exclusive" lock
 * and the "p_iolock" on pages after i/o is complete.
 */
void
pvn_io_done(page_t *plist)
{
        page_t *pp;

        while (plist != NULL) {
                pp = plist;
                page_sub(&plist, pp);
                page_io_unlock(pp);
                page_unlock(pp);
        }
}

/*
 * Entry point to be used by file system getpage subr's and
 * other such routines which either want to unlock pages (B_ASYNC
 * request) or destroy a list of pages if an error occurred.
 */
void
pvn_read_done(page_t *plist, int flags)
{
        page_t *pp;

        while (plist != NULL) {
                pp = plist;
                page_sub(&plist, pp);
                page_io_unlock(pp);
                if (flags & B_ERROR) {
                        /*LINTED: constant in conditional context*/
                        VN_DISPOSE(pp, B_INVAL, 0, kcred);
                } else {
                        (void) page_release(pp, 0);
                }
        }
}

/*
 * Automagic pageout.
 * When memory gets tight, start freeing pages popping out of the
 * write queue.
 */
int     write_free = 1;
pgcnt_t pages_before_pager = 200;       /* LMXXX */

/*
 * Routine to be called when page-out's complete.
 * The caller, typically VOP_PUTPAGE, has to explicity call this routine
 * after waiting for i/o to complete (biowait) to free the list of
 * pages associated with the buffer.  These pages must be locked
 * before i/o is initiated.
 *
 * If a write error occurs, the pages are marked as modified
 * so the write will be re-tried later.
 */

void
pvn_write_done(page_t *plist, int flags)
{
        int dfree = 0;
        int pgrec = 0;
        int pgout = 0;
        int pgpgout = 0;
        int anonpgout = 0;
        int anonfree = 0;
        int fspgout = 0;
        int fsfree = 0;
        int execpgout = 0;
        int execfree = 0;
        page_t *pp;
        struct cpu *cpup;
        struct vnode *vp = NULL;        /* for probe */
        uint_t ppattr;
        kmutex_t *vphm = NULL;

        ASSERT((flags & B_READ) == 0);

        /*
         * If we are about to start paging anyway, start freeing pages.
         */
        if (write_free && freemem < lotsfree + pages_before_pager &&
            (flags & B_ERROR) == 0) {
                flags |= B_FREE;
        }

        /*
         * Handle each page involved in the i/o operation.
         */
        while (plist != NULL) {
                pp = plist;
                ASSERT(PAGE_LOCKED(pp) && page_iolock_assert(pp));
                page_sub(&plist, pp);

                /* Kernel probe support */
                if (vp == NULL)
                        vp = pp->p_vnode;

                if (((flags & B_ERROR) == 0) && IS_VMODSORT(vp)) {
                        /*
                         * Move page to the top of the v_page list.
                         * Skip pages modified during IO.
                         */
                        vphm = page_vnode_mutex(vp);
                        mutex_enter(vphm);
                        if ((pp->p_vpnext != pp) && !hat_ismod(pp)) {
                                page_vpsub(&vp->v_pages, pp);
                                page_vpadd(&vp->v_pages, pp);
                        }
                        mutex_exit(vphm);
                }

                if (flags & B_ERROR) {
                        /*
                         * Write operation failed.  We don't want
                         * to destroy (or free) the page unless B_FORCE
                         * is set. We set the mod bit again and release
                         * all locks on the page so that it will get written
                         * back again later when things are hopefully
                         * better again.
                         * If B_INVAL and B_FORCE is set we really have
                         * to destroy the page.
                         */
                        if ((flags & (B_INVAL|B_FORCE)) == (B_INVAL|B_FORCE)) {
                                page_io_unlock(pp);
                                /*LINTED: constant in conditional context*/
                                VN_DISPOSE(pp, B_INVAL, 0, kcred);
                        } else {
                                hat_setmod_only(pp);
                                page_io_unlock(pp);
                                page_unlock(pp);
                        }
                } else if (flags & B_INVAL) {
                        /*
                         * XXX - Failed writes with B_INVAL set are
                         * not handled appropriately.
                         */
                        page_io_unlock(pp);
                        /*LINTED: constant in conditional context*/
                        VN_DISPOSE(pp, B_INVAL, 0, kcred);
                } else if (flags & B_FREE ||!hat_page_is_mapped(pp)) {
                        /*
                         * Update statistics for pages being paged out
                         */
                        if (pp->p_vnode) {
                                if (IS_SWAPFSVP(pp->p_vnode)) {
                                        anonpgout++;
                                } else {
                                        if (pp->p_vnode->v_flag & VVMEXEC) {
                                                execpgout++;
                                        } else {
                                                fspgout++;
                                        }
                                }
                        }
                        page_io_unlock(pp);
                        pgout = 1;
                        pgpgout++;
                        TRACE_1(TR_FAC_VM, TR_PAGE_WS_OUT,
                            "page_ws_out:pp %p", pp);

                        /*
                         * The page_struct_lock need not be acquired to
                         * examine "p_lckcnt" and "p_cowcnt" since we'll
                         * have an "exclusive" lock if the upgrade succeeds.
                         */
                        if (page_tryupgrade(pp) &&
                            pp->p_lckcnt == 0 && pp->p_cowcnt == 0) {
                                /*
                                 * Check if someone has reclaimed the
                                 * page.  If ref and mod are not set, no
                                 * one is using it so we can free it.
                                 * The rest of the system is careful
                                 * to use the NOSYNC flag to unload
                                 * translations set up for i/o w/o
                                 * affecting ref and mod bits.
                                 *
                                 * Obtain a copy of the real hardware
                                 * mod bit using hat_pagesync(pp, HAT_DONTZERO)
                                 * to avoid having to flush the cache.
                                 */
                                ppattr = hat_pagesync(pp, HAT_SYNC_DONTZERO |
                                    HAT_SYNC_STOPON_MOD);
                        ck_refmod:
                                if (!(ppattr & (P_REF | P_MOD))) {
                                        if (hat_page_is_mapped(pp)) {
                                                /*
                                                 * Doesn't look like the page
                                                 * was modified so now we
                                                 * really have to unload the
                                                 * translations.  Meanwhile
                                                 * another CPU could've
                                                 * modified it so we have to
                                                 * check again.  We don't loop
                                                 * forever here because now
                                                 * the translations are gone
                                                 * and no one can get a new one
                                                 * since we have the "exclusive"
                                                 * lock on the page.
                                                 */
                                                (void) hat_pageunload(pp,
                                                    HAT_FORCE_PGUNLOAD);
                                                ppattr = hat_page_getattr(pp,
                                                    P_REF | P_MOD);
                                                goto ck_refmod;
                                        }
                                        /*
                                         * Update statistics for pages being
                                         * freed
                                         */
                                        if (pp->p_vnode) {
                                                if (IS_SWAPFSVP(pp->p_vnode)) {
                                                        anonfree++;
                                                } else {
                                                        if (pp->p_vnode->v_flag
                                                            & VVMEXEC) {
                                                                execfree++;
                                                        } else {
                                                                fsfree++;
                                                        }
                                                }
                                        }
                                        /*LINTED: constant in conditional ctx*/
                                        VN_DISPOSE(pp, B_FREE,
                                            (flags & B_DONTNEED), kcred);
                                        dfree++;
                                } else {
                                        page_unlock(pp);
                                        pgrec++;
                                        TRACE_1(TR_FAC_VM, TR_PAGE_WS_FREE,
                                            "page_ws_free:pp %p", pp);
                                }
                        } else {
                                /*
                                 * Page is either `locked' in memory
                                 * or was reclaimed and now has a
                                 * "shared" lock, so release it.
                                 */
                                page_unlock(pp);
                        }
                } else {
                        /*
                         * Neither B_FREE nor B_INVAL nor B_ERROR.
                         * Just release locks.
                         */
                        page_io_unlock(pp);
                        page_unlock(pp);
                }
        }

        CPU_STATS_ENTER_K();
        cpup = CPU;             /* get cpup now that CPU cannot change */
        CPU_STATS_ADDQ(cpup, vm, dfree, dfree);
        CPU_STATS_ADDQ(cpup, vm, pgrec, pgrec);
        CPU_STATS_ADDQ(cpup, vm, pgout, pgout);
        CPU_STATS_ADDQ(cpup, vm, pgpgout, pgpgout);
        CPU_STATS_ADDQ(cpup, vm, anonpgout, anonpgout);
        CPU_STATS_ADDQ(cpup, vm, anonfree, anonfree);
        CPU_STATS_ADDQ(cpup, vm, fspgout, fspgout);
        CPU_STATS_ADDQ(cpup, vm, fsfree, fsfree);
        CPU_STATS_ADDQ(cpup, vm, execpgout, execpgout);
        CPU_STATS_ADDQ(cpup, vm, execfree, execfree);
        CPU_STATS_EXIT_K();

        /* Kernel probe */
        TNF_PROBE_4(pageout, "vm pageio io", /* CSTYLED */,
            tnf_opaque, vnode,                  vp,
            tnf_ulong,  pages_pageout,          pgpgout,
            tnf_ulong,  pages_freed,            dfree,
            tnf_ulong,  pages_reclaimed,        pgrec);
}

/*
 * Flags are composed of {B_ASYNC, B_INVAL, B_FREE, B_DONTNEED, B_DELWRI,
 * B_TRUNC, B_FORCE}.  B_DELWRI indicates that this page is part of a kluster
 * operation and is only to be considered if it doesn't involve any
 * waiting here.  B_TRUNC indicates that the file is being truncated
 * and so no i/o needs to be done. B_FORCE indicates that the page
 * must be destroyed so don't try wrting it out.
 *
 * The caller must ensure that the page is locked.  Returns 1, if
 * the page should be written back (the "iolock" is held in this
 * case), or 0 if the page has been dealt with or has been
 * unlocked.
 */
int
pvn_getdirty(page_t *pp, int flags)
{
        ASSERT((flags & (B_INVAL | B_FREE)) ?
            PAGE_EXCL(pp) : PAGE_SHARED(pp));
        ASSERT(PP_ISFREE(pp) == 0);

        /*
         * If trying to invalidate or free a logically `locked' page,
         * forget it.  Don't need page_struct_lock to check p_lckcnt and
         * p_cowcnt as the page is exclusively locked.
         */
        if ((flags & (B_INVAL | B_FREE)) && !(flags & (B_TRUNC|B_FORCE)) &&
            (pp->p_lckcnt != 0 || pp->p_cowcnt != 0)) {
                page_unlock(pp);
                return (0);
        }

        /*
         * Now acquire the i/o lock so we can add it to the dirty
         * list (if necessary).  We avoid blocking on the i/o lock
         * in the following cases:
         *
         *      If B_DELWRI is set, which implies that this request is
         *      due to a klustering operartion.
         *
         *      If this is an async (B_ASYNC) operation and we are not doing
         *      invalidation (B_INVAL) [The current i/o or fsflush will ensure
         *      that the the page is written out].
         */
        if ((flags & B_DELWRI) || ((flags & (B_INVAL | B_ASYNC)) == B_ASYNC)) {
                if (!page_io_trylock(pp)) {
                        page_unlock(pp);
                        return (0);
                }
        } else {
                page_io_lock(pp);
        }

        /*
         * If we want to free or invalidate the page then
         * we need to unload it so that anyone who wants
         * it will have to take a minor fault to get it.
         * Otherwise, we're just writing the page back so we
         * need to sync up the hardwre and software mod bit to
         * detect any future modifications.  We clear the
         * software mod bit when we put the page on the dirty
         * list.
         */
        if (flags & (B_INVAL | B_FREE)) {
                (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
        } else {
                (void) hat_pagesync(pp, HAT_SYNC_ZERORM);
        }

        if (!hat_ismod(pp) || (flags & B_TRUNC)) {
                /*
                 * Don't need to add it to the
                 * list after all.
                 */
                page_io_unlock(pp);
                if (flags & B_INVAL) {
                        /*LINTED: constant in conditional context*/
                        VN_DISPOSE(pp, B_INVAL, 0, kcred);
                } else if (flags & B_FREE) {
                        /*LINTED: constant in conditional context*/
                        VN_DISPOSE(pp, B_FREE, (flags & B_DONTNEED), kcred);
                } else {
                        /*
                         * This is advisory path for the callers
                         * of VOP_PUTPAGE() who prefer freeing the
                         * page _only_ if no one else is accessing it.
                         * E.g. segmap_release()
                         *
                         * The above hat_ismod() check is useless because:
                         * (1) we may not be holding SE_EXCL lock;
                         * (2) we've not unloaded _all_ translations
                         *
                         * Let page_release() do the heavy-lifting.
                         */
                        (void) page_release(pp, 1);
                }
                return (0);
        }

        /*
         * Page is dirty, get it ready for the write back
         * and add page to the dirty list.
         */
        hat_clrrefmod(pp);

        /*
         * If we're going to free the page when we're done
         * then we can let others try to use it starting now.
         * We'll detect the fact that they used it when the
         * i/o is done and avoid freeing the page.
         */
        if (flags & B_FREE)
                page_downgrade(pp);


        TRACE_1(TR_FAC_VM, TR_PVN_GETDIRTY, "pvn_getdirty:pp %p", pp);

        return (1);
}


/*ARGSUSED*/
static int
marker_constructor(void *buf, void *cdrarg, int kmflags)
{
        page_t *mark = buf;
        bzero(mark, sizeof (page_t));
        mark->p_hash = PVN_VPLIST_HASH_TAG;
        return (0);
}

void
pvn_init()
{
        if (pvn_vmodsort_disable == 0)
                pvn_vmodsort_supported = hat_supported(HAT_VMODSORT, NULL);
        marker_cache = kmem_cache_create("marker_cache",
            sizeof (page_t), 0, marker_constructor,
            NULL, NULL, NULL, NULL, 0);
}


/*
 * Process a vnode's page list for all pages whose offset is >= off.
 * Pages are to either be free'd, invalidated, or written back to disk.
 *
 * An "exclusive" lock is acquired for each page if B_INVAL or B_FREE
 * is specified, otherwise they are "shared" locked.
 *
 * Flags are {B_ASYNC, B_INVAL, B_FREE, B_DONTNEED, B_TRUNC}
 *
 * Special marker page_t's are inserted in the list in order
 * to keep track of where we are in the list when locks are dropped.
 *
 * Note the list is circular and insertions can happen only at the
 * head and tail of the list. The algorithm ensures visiting all pages
 * on the list in the following way:
 *
 *    Drop two marker pages at the end of the list.
 *
 *    Move one marker page backwards towards the start of the list until
 *    it is at the list head, processing the pages passed along the way.
 *
 *    Due to race conditions when the vphm mutex is dropped, additional pages
 *    can be added to either end of the list, so we'll continue to move
 *    the marker and process pages until it is up against the end marker.
 *
 * There is one special exit condition. If we are processing a VMODSORT
 * vnode and only writing back modified pages, we can stop as soon as
 * we run into an unmodified page.  This makes fsync(3) operations fast.
 */
int
pvn_vplist_dirty(
        vnode_t         *vp,
        u_offset_t      off,
        int             (*putapage)(vnode_t *, page_t *, u_offset_t *,
                        size_t *, int, cred_t *),
        int             flags,
        cred_t          *cred)
{
        page_t          *pp;
        page_t          *mark;          /* marker page that moves toward head */
        page_t          *end;           /* marker page at end of list */
        int             err = 0;
        int             error;
        kmutex_t        *vphm;
        se_t            se;
        page_t          **where_to_move;

        ASSERT(vp->v_type != VCHR);

        if (vp->v_pages == NULL)
                return (0);


        /*
         * Serialize vplist_dirty operations on this vnode by setting VVMLOCK.
         *
         * Don't block on VVMLOCK if B_ASYNC is set. This prevents sync()
         * from getting blocked while flushing pages to a dead NFS server.
         */
        mutex_enter(&vp->v_lock);
        if ((vp->v_flag & VVMLOCK) && (flags & B_ASYNC)) {
                mutex_exit(&vp->v_lock);
                return (EAGAIN);
        }

        while (vp->v_flag & VVMLOCK)
                cv_wait(&vp->v_cv, &vp->v_lock);

        if (vp->v_pages == NULL) {
                mutex_exit(&vp->v_lock);
                return (0);
        }

        vp->v_flag |= VVMLOCK;
        mutex_exit(&vp->v_lock);


        /*
         * Set up the marker pages used to walk the list
         */
        end = kmem_cache_alloc(marker_cache, KM_SLEEP);
        end->p_vnode = vp;
        end->p_offset = (u_offset_t)-2;
        mark = kmem_cache_alloc(marker_cache, KM_SLEEP);
        mark->p_vnode = vp;
        mark->p_offset = (u_offset_t)-1;

        /*
         * Grab the lock protecting the vnode's page list
         * note that this lock is dropped at times in the loop.
         */
        vphm = page_vnode_mutex(vp);
        mutex_enter(vphm);
        if (vp->v_pages == NULL)
                goto leave;

        /*
         * insert the markers and loop through the list of pages
         */
        page_vpadd(&vp->v_pages->p_vpprev->p_vpnext, mark);
        page_vpadd(&mark->p_vpnext, end);
        for (;;) {

                /*
                 * If only doing an async write back, then we can
                 * stop as soon as we get to start of the list.
                 */
                if (flags == B_ASYNC && vp->v_pages == mark)
                        break;

                /*
                 * otherwise stop when we've gone through all the pages
                 */
                if (mark->p_vpprev == end)
                        break;

                pp = mark->p_vpprev;
                if (vp->v_pages == pp)
                        where_to_move = &vp->v_pages;
                else
                        where_to_move = &pp->p_vpprev->p_vpnext;

                ASSERT(pp->p_vnode == vp);

                /*
                 * If just flushing dirty pages to disk and this vnode
                 * is using a sorted list of pages, we can stop processing
                 * as soon as we find an unmodified page. Since all the
                 * modified pages are visited first.
                 */
                if (IS_VMODSORT(vp) &&
                    !(flags & (B_INVAL | B_FREE | B_TRUNC))) {
                        if (!hat_ismod(pp) && !page_io_locked(pp)) {
#ifdef  DEBUG
                                /*
                                 * For debug kernels examine what should be
                                 * all the remaining clean pages, asserting
                                 * that they are not modified.
                                 */
                                page_t  *chk = pp;
                                int     attr;

                                page_vpsub(&vp->v_pages, mark);
                                page_vpadd(where_to_move, mark);
                                do {
                                        chk = chk->p_vpprev;
                                        ASSERT(chk != end);
                                        if (chk == mark)
                                                continue;
                                        attr = hat_page_getattr(chk, P_MOD |
                                            P_REF);
                                        if ((attr & P_MOD) == 0)
                                                continue;
                                        panic("v_pages list not all clean: "
                                            "page_t*=%p vnode=%p off=%lx "
                                            "attr=0x%x last clean page_t*=%p\n",
                                            (void *)chk, (void *)chk->p_vnode,
                                            (long)chk->p_offset, attr,
                                            (void *)pp);
                                } while (chk != vp->v_pages);
#endif
                                break;
                        } else if (!(flags & B_ASYNC) && !hat_ismod(pp)) {
                                /*
                                 * Couldn't get io lock, wait until IO is done.
                                 * Block only for sync IO since we don't want
                                 * to block async IO.
                                 */
                                mutex_exit(vphm);
                                page_io_wait(pp);
                                mutex_enter(vphm);
                                continue;
                        }
                }

                /*
                 * Skip this page if the offset is out of the desired range.
                 * Just move the marker and continue.
                 */
                if (pp->p_offset < off) {
                        page_vpsub(&vp->v_pages, mark);
                        page_vpadd(where_to_move, mark);
                        continue;
                }

                /*
                 * If we are supposed to invalidate or free this
                 * page, then we need an exclusive lock.
                 */
                se = (flags & (B_INVAL | B_FREE)) ? SE_EXCL : SE_SHARED;

                /*
                 * We must acquire the page lock for all synchronous
                 * operations (invalidate, free and write).
                 */
                if ((flags & B_INVAL) != 0 || (flags & B_ASYNC) == 0) {
                        /*
                         * If the page_lock() drops the mutex
                         * we must retry the loop.
                         */
                        if (!page_lock(pp, se, vphm, P_NO_RECLAIM))
                                continue;

                        /*
                         * It's ok to move the marker page now.
                         */
                        page_vpsub(&vp->v_pages, mark);
                        page_vpadd(where_to_move, mark);
                } else {

                        /*
                         * update the marker page for all remaining cases
                         */
                        page_vpsub(&vp->v_pages, mark);
                        page_vpadd(where_to_move, mark);

                        /*
                         * For write backs, If we can't lock the page, it's
                         * invalid or in the process of being destroyed.  Skip
                         * it, assuming someone else is writing it.
                         */
                        if (!page_trylock(pp, se))
                                continue;
                }

                ASSERT(pp->p_vnode == vp);

                /*
                 * Successfully locked the page, now figure out what to
                 * do with it. Free pages are easily dealt with, invalidate
                 * if desired or just go on to the next page.
                 */
                if (PP_ISFREE(pp)) {
                        if ((flags & B_INVAL) == 0) {
                                page_unlock(pp);
                                continue;
                        }

                        /*
                         * Invalidate (destroy) the page.
                         */
                        mutex_exit(vphm);
                        page_destroy_free(pp);
                        mutex_enter(vphm);
                        continue;
                }

                /*
                 * pvn_getdirty() figures out what do do with a dirty page.
                 * If the page is dirty, the putapage() routine will write it
                 * and will kluster any other adjacent dirty pages it can.
                 *
                 * pvn_getdirty() and `(*putapage)' unlock the page.
                 */
                mutex_exit(vphm);
                if (pvn_getdirty(pp, flags)) {
                        error = (*putapage)(vp, pp, NULL, NULL, flags, cred);
                        if (!err)
                                err = error;
                }
                mutex_enter(vphm);
        }
        page_vpsub(&vp->v_pages, mark);
        page_vpsub(&vp->v_pages, end);

leave:
        /*
         * Release v_pages mutex, also VVMLOCK and wakeup blocked thrds
         */
        mutex_exit(vphm);
        kmem_cache_free(marker_cache, mark);
        kmem_cache_free(marker_cache, end);
        mutex_enter(&vp->v_lock);
        vp->v_flag &= ~VVMLOCK;
        cv_broadcast(&vp->v_cv);
        mutex_exit(&vp->v_lock);
        return (err);
}

/*
 * Walk the vp->v_pages list, for every page call the callback function
 * pointed by *page_check. If page_check returns non-zero, then mark the
 * page as modified and if VMODSORT is set, move it to the end of v_pages
 * list. Moving makes sense only if we have at least two pages - this also
 * avoids having v_pages temporarily being NULL after calling page_vpsub()
 * if there was just one page.
 */
void
pvn_vplist_setdirty(vnode_t *vp, int (*page_check)(page_t *))
{
        page_t  *pp, *next, *end;
        kmutex_t        *vphm;
        int     shuffle;

        vphm = page_vnode_mutex(vp);
        mutex_enter(vphm);

        if (vp->v_pages == NULL) {
                mutex_exit(vphm);
                return;
        }

        end = vp->v_pages->p_vpprev;
        shuffle = IS_VMODSORT(vp) && (vp->v_pages != end);
        pp = vp->v_pages;

        for (;;) {
                next = pp->p_vpnext;
                if (pp->p_hash != PVN_VPLIST_HASH_TAG && page_check(pp)) {
                        /*
                         * hat_setmod_only() in contrast to hat_setmod() does
                         * not shuffle the pages and does not grab the mutex
                         * page_vnode_mutex. Exactly what we need.
                         */
                        hat_setmod_only(pp);
                        if (shuffle) {
                                page_vpsub(&vp->v_pages, pp);
                                ASSERT(vp->v_pages != NULL);
                                page_vpadd(&vp->v_pages->p_vpprev->p_vpnext,
                                    pp);
                        }
                }
                /* Stop if we have just processed the last page. */
                if (pp == end)
                        break;
                pp = next;
        }

        mutex_exit(vphm);
}

/*
 * Zero out zbytes worth of data. Caller should be aware that this
 * routine may enter back into the fs layer (xxx_getpage). Locks
 * that the xxx_getpage routine may need should not be held while
 * calling this.
 */
void
pvn_vpzero(struct vnode *vp, u_offset_t vplen, size_t zbytes)
{
        caddr_t addr;

        ASSERT(vp->v_type != VCHR);

        if (vp->v_pages == NULL)
                return;

        /*
         * zbytes may be zero but there still may be some portion of
         * a page which needs clearing (since zbytes is a function
         * of filesystem block size, not pagesize.)
         */
        if (zbytes == 0 && (PAGESIZE - (vplen & PAGEOFFSET)) == 0)
                return;

        /*
         * We get the last page and handle the partial
         * zeroing via kernel mappings.  This will make the page
         * dirty so that we know that when this page is written
         * back, the zeroed information will go out with it.  If
         * the page is not currently in memory, then the kzero
         * operation will cause it to be brought it.  We use kzero
         * instead of bzero so that if the page cannot be read in
         * for any reason, the system will not panic.  We need
         * to zero out a minimum of the fs given zbytes, but we
         * might also have to do more to get the entire last page.
         */

        if ((zbytes + (vplen & MAXBOFFSET)) > MAXBSIZE)
                panic("pvn_vptrunc zbytes");
        addr = segmap_getmapflt(segkmap, vp, vplen,
            MAX(zbytes, PAGESIZE - (vplen & PAGEOFFSET)), 1, S_WRITE);
        (void) kzero(addr + (vplen & MAXBOFFSET),
            MAX(zbytes, PAGESIZE - (vplen & PAGEOFFSET)));
        (void) segmap_release(segkmap, addr, SM_WRITE | SM_ASYNC);
}

/*
 * Handles common work of the VOP_GETPAGE routines by iterating page by page
 * calling the getpage helper for each.
 */
int
pvn_getpages(
        int (*getpage)(vnode_t *, u_offset_t, size_t, uint_t *, page_t *[],
                size_t, struct seg *, caddr_t, enum seg_rw, cred_t *),
        struct vnode *vp,
        u_offset_t off,
        size_t len,
        uint_t *protp,
        page_t *pl[],
        size_t plsz,
        struct seg *seg,
        caddr_t addr,
        enum seg_rw rw,
        struct cred *cred)
{
        page_t **ppp;
        u_offset_t o, eoff;
        size_t sz, xlen;
        int err;

        /* ensure that we have enough space */
        ASSERT(pl == NULL || plsz >= len);

        /*
         * Loop one page at a time and let getapage function fill
         * in the next page in array.  We only allow one page to be
         * returned at a time (except for the last page) so that we
         * don't have any problems with duplicates and other such
         * painful problems.  This is a very simple minded algorithm,
         * but it does the job correctly.  We hope that the cost of a
         * getapage call for a resident page that we might have been
         * able to get from an earlier call doesn't cost too much.
         */
        ppp = pl;
        sz = (pl != NULL) ? PAGESIZE : 0;
        eoff = off + len;
        xlen = len;
        for (o = off; o < eoff; o += PAGESIZE, addr += PAGESIZE,
            xlen -= PAGESIZE) {
                if (o + PAGESIZE >= eoff && pl != NULL) {
                        /*
                         * Last time through - allow the all of
                         * what's left of the pl[] array to be used.
                         */
                        sz = plsz - (o - off);
                }
                err = (*getpage)(vp, o, xlen, protp, ppp, sz, seg, addr,
                    rw, cred);
                if (err) {
                        /*
                         * Release any pages we already got.
                         */
                        if (o > off && pl != NULL) {
                                for (ppp = pl; *ppp != NULL; *ppp++ = NULL)
                                        (void) page_release(*ppp, 1);
                        }
                        break;
                }
                if (pl != NULL)
                        ppp++;
        }
        return (err);
}

/*
 * Initialize the page list array.
 */
/*ARGSUSED*/
void
pvn_plist_init(page_t *pp, page_t *pl[], size_t plsz,
    u_offset_t off, size_t io_len, enum seg_rw rw)
{
        ssize_t sz;
        page_t *ppcur, **ppp;

        /*
         * Set up to load plsz worth
         * starting at the needed page.
         */
        while (pp != NULL && pp->p_offset != off) {
                /*
                 * Remove page from the i/o list,
                 * release the i/o and the page lock.
                 */
                ppcur = pp;
                page_sub(&pp, ppcur);
                page_io_unlock(ppcur);
                (void) page_release(ppcur, 1);
        }

        if (pp == NULL) {
                pl[0] = NULL;
                return;
        }

        sz = plsz;

        /*
         * Initialize the page list array.
         */
        ppp = pl;
        do {
                ppcur = pp;
                *ppp++ = ppcur;
                page_sub(&pp, ppcur);
                page_io_unlock(ppcur);
                if (rw != S_CREATE)
                        page_downgrade(ppcur);
                sz -= PAGESIZE;
        } while (sz > 0 && pp != NULL);
        *ppp = NULL;            /* terminate list */

        /*
         * Now free the remaining pages that weren't
         * loaded in the page list.
         */
        while (pp != NULL) {
                ppcur = pp;
                page_sub(&pp, ppcur);
                page_io_unlock(ppcur);
                (void) page_release(ppcur, 1);
        }
}