Merge commit '7e934d3acc051b7ee3ef0d11571fd1225800a607'

[unleashed.git] / kernel / os / mem_config.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/cmn_err.h>
#include <sys/vmem.h>
#include <sys/kmem.h>
#include <sys/systm.h>
#include <sys/machsystm.h>      /* for page_freelist_coalesce() */
#include <sys/errno.h>
#include <sys/memnode.h>
#include <sys/memlist.h>
#include <sys/memlist_impl.h>
#include <sys/tuneable.h>
#include <sys/proc.h>
#include <sys/disp.h>
#include <sys/debug.h>
#include <sys/vm.h>
#include <sys/callb.h>
#include <sys/memlist_plat.h>   /* for installed_top_size() */
#include <sys/condvar_impl.h>   /* for CV_HAS_WAITERS() */
#include <sys/dumphdr.h>        /* for dump_resize() */
#include <sys/atomic.h>         /* for use in stats collection */
#include <sys/rwlock.h>
#include <sys/cpuvar.h>
#include <vm/seg_kmem.h>
#include <vm/seg_kpm.h>
#include <vm/page.h>
#include <vm/vm_dep.h>
#define SUNDDI_IMPL             /* so sunddi.h will not redefine splx() et al */
#include <sys/sunddi.h>
#include <sys/mem_config.h>
#include <sys/lgrp.h>
#include <sys/ddi.h>
#include <sys/modctl.h>

extern struct memlist *phys_avail;

extern uint_t page_ctrs_adjust(int);
void page_ctrs_cleanup(void);
static void kphysm_setup_post_add(pgcnt_t);
static int kphysm_setup_pre_del(pgcnt_t);
static void kphysm_setup_post_del(pgcnt_t, int);

static int kphysm_split_memseg(pfn_t base, pgcnt_t npgs);

static int delspan_reserve(pfn_t, pgcnt_t);
static void delspan_unreserve(pfn_t, pgcnt_t);

kmutex_t memseg_lists_lock;
struct memseg *memseg_va_avail;
struct memseg *memseg_alloc(void);
static struct memseg *memseg_delete_junk;
static struct memseg *memseg_edit_junk;
void memseg_remap_init(void);
static void memseg_remap_to_dummy(struct memseg *);
static void kphysm_addmem_error_undospan(pfn_t, pgcnt_t);
static struct memseg *memseg_reuse(pgcnt_t);

static struct kmem_cache *memseg_cache;

/*
 * Interfaces to manage externally allocated
 * page_t memory (metadata) for a memseg.
 */
#pragma weak    memseg_alloc_meta
#pragma weak    memseg_free_meta
#pragma weak    memseg_get_metapfn
#pragma weak    memseg_remap_meta

extern int ppvm_enable;
extern page_t *ppvm_base;
extern int memseg_alloc_meta(pfn_t, pgcnt_t, void **, pgcnt_t *);
extern void memseg_free_meta(void *, pgcnt_t);
extern pfn_t memseg_get_metapfn(void *, pgcnt_t);
extern void memseg_remap_meta(struct memseg *);
static int memseg_is_dynamic(struct memseg *);
static int memseg_includes_meta(struct memseg *);
pfn_t memseg_get_start(struct memseg *);
static void memseg_cpu_vm_flush(void);

int meta_alloc_enable;

#ifdef  DEBUG
static int memseg_debug;
#define MEMSEG_DEBUG(args...) if (memseg_debug) printf(args)
#else
#define MEMSEG_DEBUG(...)
#endif

/*
 * Add a chunk of memory to the system.
 * base: starting PAGESIZE page of new memory.
 * npgs: length in PAGESIZE pages.
 *
 * Adding mem this way doesn't increase the size of the hash tables;
 * growing them would be too hard.  This should be OK, but adding memory
 * dynamically most likely means more hash misses, since the tables will
 * be smaller than they otherwise would be.
 */
int
kphysm_add_memory_dynamic(pfn_t base, pgcnt_t npgs)
{
        page_t *pp;
        page_t          *opp, *oepp, *segpp;
        struct memseg   *seg;
        uint64_t        avmem;
        pfn_t           pfn;
        pfn_t           pt_base = base;
        pgcnt_t         tpgs = npgs;
        pgcnt_t         metapgs = 0;
        int             exhausted;
        pfn_t           pnum;
        int             mnode;
        caddr_t         vaddr;
        int             reuse;
        int             mlret;
        int             rv;
        int             flags;
        int             meta_alloc = 0;
        void            *mapva;
        void            *metabase = (void *)base;
        pgcnt_t         nkpmpgs = 0;
        offset_t        kpm_pages_off;

        cmn_err(CE_CONT,
            "?kphysm_add_memory_dynamic: adding %ldK at 0x%" PRIx64 "\n",
            npgs << (PAGESHIFT - 10), (uint64_t)base << PAGESHIFT);

        /*
         * Add this span in the delete list to prevent interactions.
         */
        if (!delspan_reserve(base, npgs)) {
                return (KPHYSM_ESPAN);
        }
        /*
         * Check to see if any of the memory span has been added
         * by trying an add to the installed memory list. This
         * forms the interlocking process for add.
         */

        memlist_write_lock();

        mlret = memlist_add_span((uint64_t)(pt_base) << PAGESHIFT,
            (uint64_t)(tpgs) << PAGESHIFT, &phys_install);

        if (mlret == MEML_SPANOP_OK)
                installed_top_size(phys_install, &physmax, &physinstalled);

        memlist_write_unlock();

        if (mlret != MEML_SPANOP_OK) {
                if (mlret == MEML_SPANOP_EALLOC) {
                        delspan_unreserve(pt_base, tpgs);
                        return (KPHYSM_ERESOURCE);
                } else if (mlret == MEML_SPANOP_ESPAN) {
                        delspan_unreserve(pt_base, tpgs);
                        return (KPHYSM_ESPAN);
                } else {
                        delspan_unreserve(pt_base, tpgs);
                        return (KPHYSM_ERESOURCE);
                }
        }

        if (meta_alloc_enable) {
                /*
                 * Allocate the page_t's from existing memory;
                 * if that fails, allocate from the incoming memory.
                 */
                rv = memseg_alloc_meta(base, npgs, &metabase, &metapgs);
                if (rv == KPHYSM_OK) {
                        ASSERT(metapgs);
                        ASSERT(btopr(npgs * sizeof (page_t)) <= metapgs);
                        meta_alloc = 1;
                        goto mapalloc;
                }
        }

        /*
         * We store the page_t's for this new memory in the first
         * few pages of the chunk. Here, we go and get'em ...
         */

        /*
         * The expression after the '-' gives the number of pages
         * that will fit in the new memory based on a requirement
         * of (PAGESIZE + sizeof (page_t)) bytes per page.
         */
        metapgs = npgs - (((uint64_t)(npgs) << PAGESHIFT) /
            (PAGESIZE + sizeof (page_t)));

        npgs -= metapgs;
        base += metapgs;

        ASSERT(btopr(npgs * sizeof (page_t)) <= metapgs);

        exhausted = (metapgs == 0 || npgs == 0);

        if (kpm_enable && !exhausted) {
                pgcnt_t start, end, nkpmpgs_prelim;
                size_t  ptsz;

                /*
                 * A viable kpm large page mapping must not overlap two
                 * dynamic memsegs. Therefore the total size is checked
                 * to be at least kpm_pgsz and also whether start and end
                 * points are at least kpm_pgsz aligned.
                 */
                if (ptokpmp(tpgs) < 1 || pmodkpmp(pt_base) ||
                    pmodkpmp(base + npgs)) {

                        kphysm_addmem_error_undospan(pt_base, tpgs);

                        /*
                         * There is no specific error code for violating
                         * kpm granularity constraints.
                         */
                        return (KPHYSM_ENOTVIABLE);
                }

                start = kpmptop(ptokpmp(base));
                end = kpmptop(ptokpmp(base + npgs));
                nkpmpgs_prelim = ptokpmp(end - start);
                ptsz = npgs * sizeof (page_t);
                metapgs = btopr(ptsz + nkpmpgs_prelim * KPMPAGE_T_SZ);
                exhausted = (tpgs <= metapgs);
                if (!exhausted) {
                        npgs = tpgs - metapgs;
                        base = pt_base + metapgs;

                        /* final nkpmpgs */
                        start = kpmptop(ptokpmp(base));
                        nkpmpgs = ptokpmp(end - start);
                        kpm_pages_off = ptsz +
                            (nkpmpgs_prelim - nkpmpgs) * KPMPAGE_T_SZ;
                }
        }

        /*
         * Is memory area supplied too small?
         */
        if (exhausted) {
                kphysm_addmem_error_undospan(pt_base, tpgs);
                /*
                 * There is no specific error code for 'too small'.
                 */
                return (KPHYSM_ERESOURCE);
        }

mapalloc:
        /*
         * We may re-use a previously allocated VA space for the page_ts
         * eventually, but we need to initialize and lock the pages first.
         */

        /*
         * Get an address in the kernel address map, map
         * the page_t pages and see if we can touch them.
         */

        mapva = vmem_alloc(heap_arena, ptob(metapgs), VM_NOSLEEP);
        if (mapva == NULL) {
                cmn_err(CE_WARN, "kphysm_add_memory_dynamic:"
                    " Can't allocate VA for page_ts");

                if (meta_alloc)
                        memseg_free_meta(metabase, metapgs);
                kphysm_addmem_error_undospan(pt_base, tpgs);

                return (KPHYSM_ERESOURCE);
        }
        pp = mapva;

        if (physmax < (pt_base + tpgs))
                physmax = (pt_base + tpgs);

        /*
         * In the remapping code we map one page at a time so we must do
         * the same here to match mapping sizes.
         */
        pfn = pt_base;
        vaddr = (caddr_t)pp;
        for (pnum = 0; pnum < metapgs; pnum++) {
                if (meta_alloc)
                        pfn = memseg_get_metapfn(metabase, (pgcnt_t)pnum);
                hat_devload(kas.a_hat, vaddr, ptob(1), pfn,
                    PROT_READ | PROT_WRITE,
                    HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
                pfn++;
                vaddr += ptob(1);
        }

        if (ddi_peek32((dev_info_t *)NULL,
            (int32_t *)pp, (int32_t *)0) == DDI_FAILURE) {

                cmn_err(CE_WARN, "kphysm_add_memory_dynamic:"
                    " Can't access pp array at 0x%p [phys 0x%lx]",
                    (void *)pp, pt_base);

                hat_unload(kas.a_hat, (caddr_t)pp, ptob(metapgs),
                    HAT_UNLOAD_UNMAP|HAT_UNLOAD_UNLOCK);

                vmem_free(heap_arena, mapva, ptob(metapgs));
                if (meta_alloc)
                        memseg_free_meta(metabase, metapgs);
                kphysm_addmem_error_undospan(pt_base, tpgs);

                return (KPHYSM_EFAULT);
        }

        /*
         * Add this memory slice to its memory node translation.
         *
         * Note that right now, each node may have only one slice;
         * this may change with COD or in larger SSM systems with
         * nested latency groups, so we must not assume that the
         * node does not yet exist.
         *
         * Note that there may be multiple memory nodes associated with
         * a single lgrp node on x86 systems.
         */
        pnum = pt_base + tpgs - 1;
        mem_node_add_range(pt_base, pnum);

        /*
         * Allocate or resize page counters as necessary to accommodate
         * the increase in memory pages.
         */
        mnode = PFN_2_MEM_NODE(pnum);
        PAGE_CTRS_ADJUST(base, npgs, rv);
        if (rv) {

                mem_node_del_range(pt_base, pnum);

                /* cleanup the  page counters */
                page_ctrs_cleanup();

                hat_unload(kas.a_hat, (caddr_t)pp, ptob(metapgs),
                    HAT_UNLOAD_UNMAP|HAT_UNLOAD_UNLOCK);

                vmem_free(heap_arena, mapva, ptob(metapgs));
                if (meta_alloc)
                        memseg_free_meta(metabase, metapgs);
                kphysm_addmem_error_undospan(pt_base, tpgs);

                return (KPHYSM_ERESOURCE);
        }

        /*
         * Update the phys_avail memory list.
         * The phys_install list was done at the start.
         */

        memlist_write_lock();

        mlret = memlist_add_span((uint64_t)(base) << PAGESHIFT,
            (uint64_t)(npgs) << PAGESHIFT, &phys_avail);
        ASSERT(mlret == MEML_SPANOP_OK);

        memlist_write_unlock();

        /* See if we can find a memseg to re-use. */
        if (meta_alloc) {
                seg = memseg_reuse(0);
                reuse = 1;      /* force unmapping of temp mapva */
                flags = MEMSEG_DYNAMIC | MEMSEG_META_ALLOC;
                /*
                 * There is a 1:1 fixed relationship between a pfn
                 * and a page_t VA.  The pfn is used as an index into
                 * the ppvm_base page_t table in order to calculate
                 * the page_t base address for a given pfn range.
                 */
                segpp = ppvm_base + base;
        } else {
                seg = memseg_reuse(metapgs);
                reuse = (seg != NULL);
                flags = MEMSEG_DYNAMIC | MEMSEG_META_INCL;
                segpp = pp;
        }

        /*
         * Initialize the memseg structure representing this memory
         * and add it to the existing list of memsegs. Do some basic
         * initialization and add the memory to the system.
         * In order to prevent lock deadlocks, the add_physmem()
         * code is repeated here, but split into several stages.
         *
         * If a memseg is reused, invalidate memseg pointers in
         * all cpu vm caches.  We need to do this this since the check
         *      pp >= seg->pages && pp < seg->epages
         * used in various places is not atomic and so the first compare
         * can happen before reuse and the second compare after reuse.
         * The invalidation ensures that a memseg is not deferenced while
         * it's page/pfn pointers are changing.
         */
        if (seg == NULL) {
                seg = memseg_alloc();
                ASSERT(seg != NULL);
                seg->msegflags = flags;
                MEMSEG_DEBUG("memseg_get: alloc seg=0x%p, pages=0x%p",
                    (void *)seg, (void *)(seg->pages));
                seg->pages = segpp;
        } else {
                ASSERT(seg->msegflags == flags);
                ASSERT(seg->pages_base == seg->pages_end);
                MEMSEG_DEBUG("memseg_get: reuse seg=0x%p, pages=0x%p",
                    (void *)seg, (void *)(seg->pages));
                if (meta_alloc) {
                        memseg_cpu_vm_flush();
                        seg->pages = segpp;
                }
        }

        seg->epages = seg->pages + npgs;
        seg->pages_base = base;
        seg->pages_end = base + npgs;

        /*
         * Initialize metadata. The page_ts are set to locked state
         * ready to be freed.
         */
        bzero((caddr_t)pp, ptob(metapgs));

        pfn = seg->pages_base;
        /* Save the original pp base in case we reuse a memseg. */
        opp = pp;
        oepp = opp + npgs;
        for (pp = opp; pp < oepp; pp++) {
                pp->p_pagenum = pfn;
                pfn++;
                page_iolock_init(pp);
                while (!page_lock(pp, SE_EXCL, NULL, P_RECLAIM))
                        continue;
                pp->p_offset = (uoff_t)-1;
        }

        if (reuse) {
                /* Remap our page_ts to the re-used memseg VA space. */
                pfn = pt_base;
                vaddr = (caddr_t)seg->pages;
                for (pnum = 0; pnum < metapgs; pnum++) {
                        if (meta_alloc)
                                pfn = memseg_get_metapfn(metabase,
                                    (pgcnt_t)pnum);
                        hat_devload(kas.a_hat, vaddr, ptob(1), pfn,
                            PROT_READ | PROT_WRITE,
                            HAT_LOAD_REMAP | HAT_LOAD | HAT_LOAD_NOCONSIST);
                        pfn++;
                        vaddr += ptob(1);
                }

                hat_unload(kas.a_hat, (caddr_t)opp, ptob(metapgs),
                    HAT_UNLOAD_UNMAP|HAT_UNLOAD_UNLOCK);

                vmem_free(heap_arena, mapva, ptob(metapgs));
        }

        hat_kpm_addmem_mseg_update(seg, nkpmpgs, kpm_pages_off);

        memsegs_lock(1);

        /*
         * The new memseg is inserted at the beginning of the list.
         * Not only does this save searching for the tail, but in the
         * case of a re-used memseg, it solves the problem of what
         * happens if some process has still got a pointer to the
         * memseg and follows the next pointer to continue traversing
         * the memsegs list.
         */

        hat_kpm_addmem_mseg_insert(seg);

        seg->next = memsegs;
        membar_producer();

        hat_kpm_addmem_memsegs_update(seg);

        memsegs = seg;

        build_pfn_hash();

        total_pages += npgs;

        /*
         * Recalculate the paging parameters now total_pages has changed.
         * This will also cause the clock hands to be reset before next use.
         */
        setupclock(1);

        memsegs_unlock(1);

        PLCNT_MODIFY_MAX(seg->pages_base, (long)npgs);

        /*
         * Free the pages outside the lock to avoid locking loops.
         */
        for (pp = seg->pages; pp < seg->epages; pp++) {
                page_free(pp, 1);
        }

        /*
         * Now that we've updated the appropriate memory lists we
         * need to reset a number of globals, since we've increased memory.
         * Several have already been updated for us as noted above. The
         * globals we're interested in at this point are:
         *   physmax - highest page frame number.
         *   physinstalled - number of pages currently installed (done earlier)
         *   maxmem - max free pages in the system
         *   physmem - physical memory pages available
         *   availrmem - real memory available
         */

        mutex_enter(&freemem_lock);
        maxmem += npgs;
        physmem += npgs;
        availrmem += npgs;
        availrmem_initial += npgs;

        mutex_exit(&freemem_lock);

        dump_resize();

        page_freelist_coalesce_all(mnode);

        kphysm_setup_post_add(npgs);

        cmn_err(CE_CONT, "?kphysm_add_memory_dynamic: mem = %ldK "
            "(0x%" PRIx64 ")\n",
            physinstalled << (PAGESHIFT - 10),
            (uint64_t)physinstalled << PAGESHIFT);

        avmem = (uint64_t)freemem << PAGESHIFT;
        cmn_err(CE_CONT, "?kphysm_add_memory_dynamic: "
            "avail mem = %" PRId64 "\n", avmem);

        /*
         * Update lgroup generation number on single lgroup systems
         */
        if (nlgrps == 1)
                lgrp_config(LGRP_CONFIG_GEN_UPDATE, 0, 0);

        /*
         * Inform DDI of update
         */
        ddi_mem_update((uint64_t)(pt_base) << PAGESHIFT,
            (uint64_t)(tpgs) << PAGESHIFT);

        delspan_unreserve(pt_base, tpgs);

        return (KPHYSM_OK);             /* Successfully added system memory */
}

/*
 * There are various error conditions in kphysm_add_memory_dynamic()
 * which require a rollback of already changed global state.
 */
static void
kphysm_addmem_error_undospan(pfn_t pt_base, pgcnt_t tpgs)
{
        int mlret;

        /* Unreserve memory span. */
        memlist_write_lock();

        mlret = memlist_delete_span(
            (uint64_t)(pt_base) << PAGESHIFT,
            (uint64_t)(tpgs) << PAGESHIFT, &phys_install);

        ASSERT(mlret == MEML_SPANOP_OK);
        phys_install_has_changed();
        installed_top_size(phys_install, &physmax, &physinstalled);

        memlist_write_unlock();
        delspan_unreserve(pt_base, tpgs);
}

/*
 * Only return an available memseg of exactly the right size
 * if size is required.
 * When the meta data area has it's own virtual address space
 * we will need to manage this more carefully and do best fit
 * allocations, possibly splitting an available area.
 */
struct memseg *
memseg_reuse(pgcnt_t metapgs)
{
        int type;
        struct memseg **segpp, *seg;

        mutex_enter(&memseg_lists_lock);

        segpp = &memseg_va_avail;
        for (; (seg = *segpp) != NULL; segpp = &seg->lnext) {
                caddr_t end;

                /*
                 * Make sure we are reusing the right segment type.
                 */
                type = metapgs ? MEMSEG_META_INCL : MEMSEG_META_ALLOC;

                if ((seg->msegflags & (MEMSEG_META_INCL | MEMSEG_META_ALLOC))
                    != type)
                        continue;

                if (kpm_enable)
                        end = hat_kpm_mseg_reuse(seg);
                else
                        end = (caddr_t)seg->epages;

                /*
                 * Check for the right size if it is provided.
                 */
                if (!metapgs || btopr(end - (caddr_t)seg->pages) == metapgs) {
                        *segpp = seg->lnext;
                        seg->lnext = NULL;
                        break;
                }
        }
        mutex_exit(&memseg_lists_lock);

        return (seg);
}

static uint_t handle_gen;

struct memdelspan {
        struct memdelspan *mds_next;
        pfn_t           mds_base;
        pgcnt_t         mds_npgs;
        uint_t          *mds_bitmap;
        uint_t          *mds_bitmap_retired;
};

#define NBPBMW          (sizeof (uint_t) * NBBY)
#define MDS_BITMAPBYTES(MDSP) \
        ((((MDSP)->mds_npgs + NBPBMW - 1) / NBPBMW) * sizeof (uint_t))

struct transit_list {
        struct transit_list     *trl_next;
        struct memdelspan       *trl_spans;
        int                     trl_collect;
};

struct transit_list_head {
        kmutex_t                trh_lock;
        struct transit_list     *trh_head;
};

static struct transit_list_head transit_list_head;

struct mem_handle;
static void transit_list_collect(struct mem_handle *, int);
static void transit_list_insert(struct transit_list *);
static void transit_list_remove(struct transit_list *);

#ifdef DEBUG
#define MEM_DEL_STATS
#endif /* DEBUG */

#ifdef MEM_DEL_STATS
static int mem_del_stat_print = 0;
struct mem_del_stat {
        uint_t  nloop;
        uint_t  need_free;
        uint_t  free_loop;
        uint_t  free_low;
        uint_t  free_failed;
        uint_t  ncheck;
        uint_t  nopaget;
        uint_t  lockfail;
        uint_t  nfree;
        uint_t  nreloc;
        uint_t  nrelocfail;
        uint_t  already_done;
        uint_t  first_notfree;
        uint_t  npplocked;
        uint_t  nlockreloc;
        uint_t  nnorepl;
        uint_t  nmodreloc;
        uint_t  ndestroy;
        uint_t  nputpage;
        uint_t  nnoreclaim;
        uint_t  ndelay;
        uint_t  demotefail;
        uint64_t nticks_total;
        uint64_t nticks_pgrp;
        uint_t  retired;
        uint_t  toxic;
        uint_t  failing;
        uint_t  modtoxic;
        uint_t  npplkdtoxic;
        uint_t  gptlmodfail;
        uint_t  gptllckfail;
};
/*
 * The stat values are only incremented in the delete thread
 * so no locking or atomic required.
 */
#define MDSTAT_INCR(MHP, FLD)   (MHP)->mh_delstat.FLD++
#define MDSTAT_TOTAL(MHP, ntck) ((MHP)->mh_delstat.nticks_total += (ntck))
#define MDSTAT_PGRP(MHP, ntck)  ((MHP)->mh_delstat.nticks_pgrp += (ntck))
static void mem_del_stat_print_func(struct mem_handle *);
#define MDSTAT_PRINT(MHP)       mem_del_stat_print_func((MHP))
#else /* MEM_DEL_STATS */
#define MDSTAT_INCR(MHP, FLD)
#define MDSTAT_TOTAL(MHP, ntck)
#define MDSTAT_PGRP(MHP, ntck)
#define MDSTAT_PRINT(MHP)
#endif /* MEM_DEL_STATS */

typedef enum mhnd_state {MHND_FREE = 0, MHND_INIT, MHND_STARTING,
        MHND_RUNNING, MHND_DONE, MHND_RELEASE} mhnd_state_t;

/*
 * mh_mutex must be taken to examine or change mh_exthandle and mh_state.
 * The mutex may not be required for other fields, dependent on mh_state.
 */
struct mem_handle {
        kmutex_t        mh_mutex;
        struct mem_handle *mh_next;
        memhandle_t     mh_exthandle;
        mhnd_state_t    mh_state;
        struct transit_list mh_transit;
        pgcnt_t         mh_phys_pages;
        pgcnt_t         mh_vm_pages;
        pgcnt_t         mh_hold_todo;
        void            (*mh_delete_complete)(void *, int error);
        void            *mh_delete_complete_arg;
        volatile uint_t mh_cancel;
        volatile uint_t mh_dr_aio_cleanup_cancel;
        volatile uint_t mh_aio_cleanup_done;
        kcondvar_t      mh_cv;
        kthread_id_t    mh_thread_id;
        page_t          *mh_deleted;    /* link through p_next */
#ifdef MEM_DEL_STATS
        struct mem_del_stat mh_delstat;
#endif /* MEM_DEL_STATS */
};

static struct mem_handle *mem_handle_head;
static kmutex_t mem_handle_list_mutex;

static struct mem_handle *
kphysm_allocate_mem_handle()
{
        struct mem_handle *mhp;

        mhp = kmem_zalloc(sizeof (struct mem_handle), KM_SLEEP);
        mutex_init(&mhp->mh_mutex, NULL, MUTEX_DEFAULT, NULL);
        mutex_enter(&mem_handle_list_mutex);
        mutex_enter(&mhp->mh_mutex);
        /* handle_gen is protected by list mutex. */
        mhp->mh_exthandle = (memhandle_t)(uintptr_t)(++handle_gen);
        mhp->mh_next = mem_handle_head;
        mem_handle_head = mhp;
        mutex_exit(&mem_handle_list_mutex);

        return (mhp);
}

static void
kphysm_free_mem_handle(struct mem_handle *mhp)
{
        struct mem_handle **mhpp;

        ASSERT(mutex_owned(&mhp->mh_mutex));
        ASSERT(mhp->mh_state == MHND_FREE);
        /*
         * Exit the mutex to preserve locking order. This is OK
         * here as once in the FREE state, the handle cannot
         * be found by a lookup.
         */
        mutex_exit(&mhp->mh_mutex);

        mutex_enter(&mem_handle_list_mutex);
        mhpp = &mem_handle_head;
        while (*mhpp != NULL && *mhpp != mhp)
                mhpp = &(*mhpp)->mh_next;
        ASSERT(*mhpp == mhp);
        /*
         * No need to lock the handle (mh_mutex) as only
         * mh_next changing and this is the only thread that
         * can be referncing mhp.
         */
        *mhpp = mhp->mh_next;
        mutex_exit(&mem_handle_list_mutex);

        mutex_destroy(&mhp->mh_mutex);
        kmem_free(mhp, sizeof (struct mem_handle));
}

/*
 * This function finds the internal mem_handle corresponding to an
 * external handle and returns it with the mh_mutex held.
 */
static struct mem_handle *
kphysm_lookup_mem_handle(memhandle_t handle)
{
        struct mem_handle *mhp;

        mutex_enter(&mem_handle_list_mutex);
        for (mhp = mem_handle_head; mhp != NULL; mhp = mhp->mh_next) {
                if (mhp->mh_exthandle == handle) {
                        mutex_enter(&mhp->mh_mutex);
                        /*
                         * The state of the handle could have been changed
                         * by kphysm_del_release() while waiting for mh_mutex.
                         */
                        if (mhp->mh_state == MHND_FREE) {
                                mutex_exit(&mhp->mh_mutex);
                                continue;
                        }
                        break;
                }
        }
        mutex_exit(&mem_handle_list_mutex);
        return (mhp);
}

int
kphysm_del_gethandle(memhandle_t *xmhp)
{
        struct mem_handle *mhp;

        mhp = kphysm_allocate_mem_handle();
        /*
         * The handle is allocated using KM_SLEEP, so cannot fail.
         * If the implementation is changed, the correct error to return
         * here would be KPHYSM_ENOHANDLES.
         */
        ASSERT(mhp->mh_state == MHND_FREE);
        mhp->mh_state = MHND_INIT;
        *xmhp = mhp->mh_exthandle;
        mutex_exit(&mhp->mh_mutex);
        return (KPHYSM_OK);
}

static int
overlapping(pfn_t b1, pgcnt_t l1, pfn_t b2, pgcnt_t l2)
{
        pfn_t e1, e2;

        e1 = b1 + l1;
        e2 = b2 + l2;

        return (!(b2 >= e1 || b1 >= e2));
}

static int can_remove_pgs(pgcnt_t);

static struct memdelspan *
span_to_install(pfn_t base, pgcnt_t npgs)
{
        struct memdelspan *mdsp;
        struct memdelspan *mdsp_new;
        uint64_t address, size, thislen;
        struct memlist *mlp;

        mdsp_new = NULL;

        address = (uint64_t)base << PAGESHIFT;
        size = (uint64_t)npgs << PAGESHIFT;
        while (size != 0) {
                memlist_read_lock();
                for (mlp = phys_install; mlp != NULL; mlp = mlp->ml_next) {
                        if (address >= (mlp->ml_address + mlp->ml_size))
                                continue;
                        if ((address + size) > mlp->ml_address)
                                break;
                }
                if (mlp == NULL) {
                        address += size;
                        size = 0;
                        thislen = 0;
                } else {
                        if (address < mlp->ml_address) {
                                size -= (mlp->ml_address - address);
                                address = mlp->ml_address;
                        }
                        ASSERT(address >= mlp->ml_address);
                        if ((address + size) >
                            (mlp->ml_address + mlp->ml_size)) {
                                thislen =
                                    mlp->ml_size - (address - mlp->ml_address);
                        } else {
                                thislen = size;
                        }
                }
                memlist_read_unlock();
                /* TODO: phys_install could change now */
                if (thislen == 0)
                        continue;
                mdsp = kmem_zalloc(sizeof (struct memdelspan), KM_SLEEP);
                mdsp->mds_base = btop(address);
                mdsp->mds_npgs = btop(thislen);
                mdsp->mds_next = mdsp_new;
                mdsp_new = mdsp;
                address += thislen;
                size -= thislen;
        }
        return (mdsp_new);
}

static void
free_delspans(struct memdelspan *mdsp)
{
        struct memdelspan *amdsp;

        while ((amdsp = mdsp) != NULL) {
                mdsp = amdsp->mds_next;
                kmem_free(amdsp, sizeof (struct memdelspan));
        }
}

/*
 * Concatenate lists. No list ordering is required.
 */

static void
delspan_concat(struct memdelspan **mdspp, struct memdelspan *mdsp)
{
        while (*mdspp != NULL)
                mdspp = &(*mdspp)->mds_next;

        *mdspp = mdsp;
}

/*
 * Given a new list of delspans, check there is no overlap with
 * all existing span activity (add or delete) and then concatenate
 * the new spans to the given list.
 * Return 1 for OK, 0 if overlapping.
 */
static int
delspan_insert(
        struct transit_list *my_tlp,
        struct memdelspan *mdsp_new)
{
        struct transit_list_head *trh;
        struct transit_list *tlp;
        int ret;

        trh = &transit_list_head;

        ASSERT(my_tlp != NULL);
        ASSERT(mdsp_new != NULL);

        ret = 1;
        mutex_enter(&trh->trh_lock);
        /* ASSERT(my_tlp->trl_spans == NULL || tlp_in_list(trh, my_tlp)); */
        for (tlp = trh->trh_head; tlp != NULL; tlp = tlp->trl_next) {
                struct memdelspan *mdsp;

                for (mdsp = tlp->trl_spans; mdsp != NULL;
                    mdsp = mdsp->mds_next) {
                        struct memdelspan *nmdsp;

                        for (nmdsp = mdsp_new; nmdsp != NULL;
                            nmdsp = nmdsp->mds_next) {
                                if (overlapping(mdsp->mds_base, mdsp->mds_npgs,
                                    nmdsp->mds_base, nmdsp->mds_npgs)) {
                                        ret = 0;
                                        goto done;
                                }
                        }
                }
        }
done:
        if (ret != 0) {
                if (my_tlp->trl_spans == NULL)
                        transit_list_insert(my_tlp);
                delspan_concat(&my_tlp->trl_spans, mdsp_new);
        }
        mutex_exit(&trh->trh_lock);
        return (ret);
}

static void
delspan_remove(
        struct transit_list *my_tlp,
        pfn_t base,
        pgcnt_t npgs)
{
        struct transit_list_head *trh;
        struct memdelspan *mdsp;

        trh = &transit_list_head;

        ASSERT(my_tlp != NULL);

        mutex_enter(&trh->trh_lock);
        if ((mdsp = my_tlp->trl_spans) != NULL) {
                if (npgs == 0) {
                        my_tlp->trl_spans = NULL;
                        free_delspans(mdsp);
                        transit_list_remove(my_tlp);
                } else {
                        struct memdelspan **prv;

                        prv = &my_tlp->trl_spans;
                        while (mdsp != NULL) {
                                pfn_t p_end;

                                p_end = mdsp->mds_base + mdsp->mds_npgs;
                                if (mdsp->mds_base >= base &&
                                    p_end <= (base + npgs)) {
                                        *prv = mdsp->mds_next;
                                        mdsp->mds_next = NULL;
                                        free_delspans(mdsp);
                                } else {
                                        prv = &mdsp->mds_next;
                                }
                                mdsp = *prv;
                        }
                        if (my_tlp->trl_spans == NULL)
                                transit_list_remove(my_tlp);
                }
        }
        mutex_exit(&trh->trh_lock);
}

/*
 * Reserve interface for add to stop delete before add finished.
 * This list is only accessed through the delspan_insert/remove
 * functions and so is fully protected by the mutex in struct transit_list.
 */

static struct transit_list reserve_transit;

static int
delspan_reserve(pfn_t base, pgcnt_t npgs)
{
        struct memdelspan *mdsp;
        int ret;

        mdsp = kmem_zalloc(sizeof (struct memdelspan), KM_SLEEP);
        mdsp->mds_base = base;
        mdsp->mds_npgs = npgs;
        if ((ret = delspan_insert(&reserve_transit, mdsp)) == 0) {
                free_delspans(mdsp);
        }
        return (ret);
}

static void
delspan_unreserve(pfn_t base, pgcnt_t npgs)
{
        delspan_remove(&reserve_transit, base, npgs);
}

/*
 * Return whether memseg was created by kphysm_add_memory_dynamic().
 */
static int
memseg_is_dynamic(struct memseg *seg)
{
        return (seg->msegflags & MEMSEG_DYNAMIC);
}

int
kphysm_del_span(
        memhandle_t handle,
        pfn_t base,
        pgcnt_t npgs)
{
        struct mem_handle *mhp;
        struct memseg *seg;
        struct memdelspan *mdsp;
        struct memdelspan *mdsp_new;
        pgcnt_t phys_pages, vm_pages;
        pfn_t p_end;
        page_t *pp;
        int ret;

        mhp = kphysm_lookup_mem_handle(handle);
        if (mhp == NULL) {
                return (KPHYSM_EHANDLE);
        }
        if (mhp->mh_state != MHND_INIT) {
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_ESEQUENCE);
        }

        /*
         * Intersect the span with the installed memory list (phys_install).
         */
        mdsp_new = span_to_install(base, npgs);
        if (mdsp_new == NULL) {
                /*
                 * No physical memory in this range. Is this an
                 * error? If an attempt to start the delete is made
                 * for OK returns from del_span such as this, start will
                 * return an error.
                 * Could return KPHYSM_ENOWORK.
                 */
                /*
                 * It is assumed that there are no error returns
                 * from span_to_install() due to kmem_alloc failure.
                 */
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_OK);
        }
        /*
         * Does this span overlap an existing span?
         */
        if (delspan_insert(&mhp->mh_transit, mdsp_new) == 0) {
                /*
                 * Differentiate between already on list for this handle
                 * (KPHYSM_EDUP) and busy elsewhere (KPHYSM_EBUSY).
                 */
                ret = KPHYSM_EBUSY;
                for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
                    mdsp = mdsp->mds_next) {
                        if (overlapping(mdsp->mds_base, mdsp->mds_npgs,
                            base, npgs)) {
                                ret = KPHYSM_EDUP;
                                break;
                        }
                }
                mutex_exit(&mhp->mh_mutex);
                free_delspans(mdsp_new);
                return (ret);
        }
        /*
         * At this point the spans in mdsp_new have been inserted into the
         * list of spans for this handle and thereby to the global list of
         * spans being processed. Each of these spans must now be checked
         * for relocatability. As a side-effect segments in the memseg list
         * may be split.
         *
         * Note that mdsp_new can no longer be used as it is now part of
         * a larger list. Select elements of this larger list based
         * on base and npgs.
         */
restart:
        phys_pages = 0;
        vm_pages = 0;
        ret = KPHYSM_OK;
        for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
            mdsp = mdsp->mds_next) {
                pgcnt_t pages_checked;

                if (!overlapping(mdsp->mds_base, mdsp->mds_npgs, base, npgs)) {
                        continue;
                }
                p_end = mdsp->mds_base + mdsp->mds_npgs;
                /*
                 * The pages_checked count is a hack. All pages should be
                 * checked for relocatability. Those not covered by memsegs
                 * should be tested with arch_kphysm_del_span_ok().
                 */
                pages_checked = 0;
                for (seg = memsegs; seg; seg = seg->next) {
                        pfn_t mseg_start;

                        if (seg->pages_base >= p_end ||
                            seg->pages_end <= mdsp->mds_base) {
                                /* Span and memseg don't overlap. */
                                continue;
                        }
                        mseg_start = memseg_get_start(seg);
                        /* Check that segment is suitable for delete. */
                        if (memseg_includes_meta(seg)) {
                                /*
                                 * Check that this segment is completely
                                 * within the span.
                                 */
                                if (mseg_start < mdsp->mds_base ||
                                    seg->pages_end > p_end) {
                                        ret = KPHYSM_EBUSY;
                                        break;
                                }
                                pages_checked += seg->pages_end - mseg_start;
                        } else {
                                /*
                                 * If this segment is larger than the span,
                                 * try to split it. After the split, it
                                 * is necessary to restart.
                                 */
                                if (seg->pages_base < mdsp->mds_base ||
                                    seg->pages_end > p_end) {
                                        pfn_t abase;
                                        pgcnt_t anpgs;
                                        int s_ret;

                                        /* Split required.  */
                                        if (mdsp->mds_base < seg->pages_base)
                                                abase = seg->pages_base;
                                        else
                                                abase = mdsp->mds_base;
                                        if (p_end > seg->pages_end)
                                                anpgs = seg->pages_end - abase;
                                        else
                                                anpgs = p_end - abase;
                                        s_ret = kphysm_split_memseg(abase,
                                            anpgs);
                                        if (s_ret == 0) {
                                                /* Split failed. */
                                                ret = KPHYSM_ERESOURCE;
                                                break;
                                        }
                                        goto restart;
                                }
                                pages_checked +=
                                    seg->pages_end - seg->pages_base;
                        }
                        /*
                         * The memseg is wholly within the delete span.
                         * The individual pages can now be checked.
                         */
                        /* Cage test. */
                        for (pp = seg->pages; pp < seg->epages; pp++) {
                                if (PP_ISNORELOC(pp)) {
                                        ret = KPHYSM_ENONRELOC;
                                        break;
                                }
                        }
                        if (ret != KPHYSM_OK) {
                                break;
                        }
                        phys_pages += (seg->pages_end - mseg_start);
                        vm_pages += MSEG_NPAGES(seg);
                }
                if (ret != KPHYSM_OK)
                        break;
                if (pages_checked != mdsp->mds_npgs) {
                        ret = KPHYSM_ENONRELOC;
                        break;
                }
        }

        if (ret == KPHYSM_OK) {
                mhp->mh_phys_pages += phys_pages;
                mhp->mh_vm_pages += vm_pages;
        } else {
                /*
                 * Keep holding the mh_mutex to prevent it going away.
                 */
                delspan_remove(&mhp->mh_transit, base, npgs);
        }
        mutex_exit(&mhp->mh_mutex);
        return (ret);
}

int
kphysm_del_span_query(
        pfn_t base,
        pgcnt_t npgs,
        memquery_t *mqp)
{
        struct memdelspan *mdsp;
        struct memdelspan *mdsp_new;
        int done_first_nonreloc;

        mqp->phys_pages = 0;
        mqp->managed = 0;
        mqp->nonrelocatable = 0;
        mqp->first_nonrelocatable = 0;
        mqp->last_nonrelocatable = 0;

        mdsp_new = span_to_install(base, npgs);
        /*
         * It is OK to proceed here if mdsp_new == NULL.
         */
        done_first_nonreloc = 0;
        for (mdsp = mdsp_new; mdsp != NULL; mdsp = mdsp->mds_next) {
                pfn_t sbase;
                pgcnt_t snpgs;

                mqp->phys_pages += mdsp->mds_npgs;
                sbase = mdsp->mds_base;
                snpgs = mdsp->mds_npgs;
                while (snpgs != 0) {
                        struct memseg *lseg, *seg;
                        pfn_t p_end;
                        page_t *pp;
                        pfn_t mseg_start;

                        p_end = sbase + snpgs;
                        /*
                         * Find the lowest addressed memseg that starts
                         * after sbase and account for it.
                         * This is to catch dynamic memsegs whose start
                         * is hidden.
                         */
                        seg = NULL;
                        for (lseg = memsegs; lseg != NULL; lseg = lseg->next) {
                                if ((lseg->pages_base >= sbase) ||
                                    (lseg->pages_base < p_end &&
                                    lseg->pages_end > sbase)) {
                                        if (seg == NULL ||
                                            seg->pages_base > lseg->pages_base)
                                                seg = lseg;
                                }
                        }
                        if (seg != NULL) {
                                mseg_start = memseg_get_start(seg);
                                /*
                                 * Now have the full extent of the memseg so
                                 * do the range check.
                                 */
                                if (mseg_start >= p_end ||
                                    seg->pages_end <= sbase) {
                                        /* Span does not overlap memseg. */
                                        seg = NULL;
                                }
                        }
                        /*
                         * Account for gap either before the segment if
                         * there is one or to the end of the span.
                         */
                        if (seg == NULL || mseg_start > sbase) {
                                pfn_t a_end;

                                a_end = (seg == NULL) ? p_end : mseg_start;
                                /*
                                 * Check with arch layer for relocatability.
                                 */
                                if (arch_kphysm_del_span_ok(sbase,
                                    (a_end - sbase))) {
                                        /*
                                         * No non-relocatble pages in this
                                         * area, avoid the fine-grained
                                         * test.
                                         */
                                        snpgs -= (a_end - sbase);
                                        sbase = a_end;
                                }
                                while (sbase < a_end) {
                                        if (!arch_kphysm_del_span_ok(sbase,
                                            1)) {
                                                mqp->nonrelocatable++;
                                                if (!done_first_nonreloc) {
                                                        mqp->
                                                            first_nonrelocatable
                                                            = sbase;
                                                        done_first_nonreloc = 1;
                                                }
                                                mqp->last_nonrelocatable =
                                                    sbase;
                                        }
                                        sbase++;
                                        snpgs--;
                                }
                        }
                        if (seg != NULL) {
                                ASSERT(mseg_start <= sbase);
                                if (seg->pages_base != mseg_start &&
                                    seg->pages_base > sbase) {
                                        pgcnt_t skip_pgs;

                                        /*
                                         * Skip the page_t area of a
                                         * dynamic memseg.
                                         */
                                        skip_pgs = seg->pages_base - sbase;
                                        if (snpgs <= skip_pgs) {
                                                sbase += snpgs;
                                                snpgs = 0;
                                                continue;
                                        }
                                        snpgs -= skip_pgs;
                                        sbase += skip_pgs;
                                }
                                ASSERT(snpgs != 0);
                                ASSERT(seg->pages_base <= sbase);
                                /*
                                 * The individual pages can now be checked.
                                 */
                                for (pp = seg->pages +
                                    (sbase - seg->pages_base);
                                    snpgs != 0 && pp < seg->epages; pp++) {
                                        mqp->managed++;
                                        if (PP_ISNORELOC(pp)) {
                                                mqp->nonrelocatable++;
                                                if (!done_first_nonreloc) {
                                                        mqp->
                                                            first_nonrelocatable
                                                            = sbase;
                                                        done_first_nonreloc = 1;
                                                }
                                                mqp->last_nonrelocatable =
                                                    sbase;
                                        }
                                        sbase++;
                                        snpgs--;
                                }
                        }
                }
        }

        free_delspans(mdsp_new);

        return (KPHYSM_OK);
}

/*
 * This release function can be called at any stage as follows:
 *      _gethandle only called
 *      _span(s) only called
 *      _start called but failed
 *      delete thread exited
 */
int
kphysm_del_release(memhandle_t handle)
{
        struct mem_handle *mhp;

        mhp = kphysm_lookup_mem_handle(handle);
        if (mhp == NULL) {
                return (KPHYSM_EHANDLE);
        }
        switch (mhp->mh_state) {
        case MHND_STARTING:
        case MHND_RUNNING:
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_ENOTFINISHED);
        case MHND_FREE:
                ASSERT(mhp->mh_state != MHND_FREE);
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_EHANDLE);
        case MHND_INIT:
                break;
        case MHND_DONE:
                break;
        case MHND_RELEASE:
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_ESEQUENCE);
        default:
#ifdef DEBUG
                cmn_err(CE_WARN, "kphysm_del_release(0x%p) state corrupt %d",
                    (void *)mhp, mhp->mh_state);
#endif /* DEBUG */
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_EHANDLE);
        }
        /*
         * Set state so that we can wait if necessary.
         * Also this means that we have read/write access to all
         * fields except mh_exthandle and mh_state.
         */
        mhp->mh_state = MHND_RELEASE;
        /*
         * The mem_handle cannot be de-allocated by any other operation
         * now, so no need to hold mh_mutex.
         */
        mutex_exit(&mhp->mh_mutex);

        delspan_remove(&mhp->mh_transit, 0, 0);
        mhp->mh_phys_pages = 0;
        mhp->mh_vm_pages = 0;
        mhp->mh_hold_todo = 0;
        mhp->mh_delete_complete = NULL;
        mhp->mh_delete_complete_arg = NULL;
        mhp->mh_cancel = 0;

        mutex_enter(&mhp->mh_mutex);
        ASSERT(mhp->mh_state == MHND_RELEASE);
        mhp->mh_state = MHND_FREE;

        kphysm_free_mem_handle(mhp);

        return (KPHYSM_OK);
}

/*
 * This cancel function can only be called with the thread running.
 */
int
kphysm_del_cancel(memhandle_t handle)
{
        struct mem_handle *mhp;

        mhp = kphysm_lookup_mem_handle(handle);
        if (mhp == NULL) {
                return (KPHYSM_EHANDLE);
        }
        if (mhp->mh_state != MHND_STARTING && mhp->mh_state != MHND_RUNNING) {
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_ENOTRUNNING);
        }
        /*
         * Set the cancel flag and wake the delete thread up.
         * The thread may be waiting on I/O, so the effect of the cancel
         * may be delayed.
         */
        if (mhp->mh_cancel == 0) {
                mhp->mh_cancel = KPHYSM_ECANCELLED;
                cv_signal(&mhp->mh_cv);
        }
        mutex_exit(&mhp->mh_mutex);
        return (KPHYSM_OK);
}

int
kphysm_del_status(
        memhandle_t handle,
        memdelstat_t *mdstp)
{
        struct mem_handle *mhp;

        mhp = kphysm_lookup_mem_handle(handle);
        if (mhp == NULL) {
                return (KPHYSM_EHANDLE);
        }
        /*
         * Calling kphysm_del_status() is allowed before the delete
         * is started to allow for status display.
         */
        if (mhp->mh_state != MHND_INIT && mhp->mh_state != MHND_STARTING &&
            mhp->mh_state != MHND_RUNNING) {
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_ENOTRUNNING);
        }
        mdstp->phys_pages = mhp->mh_phys_pages;
        mdstp->managed = mhp->mh_vm_pages;
        mdstp->collected = mhp->mh_vm_pages - mhp->mh_hold_todo;
        mutex_exit(&mhp->mh_mutex);
        return (KPHYSM_OK);
}

static int mem_delete_additional_pages = 100;

static int
can_remove_pgs(pgcnt_t npgs)
{
        /*
         * If all pageable pages were paged out, freemem would
         * equal availrmem.  There is a minimum requirement for
         * availrmem.
         */
        if ((availrmem - (tune.t_minarmem + mem_delete_additional_pages))
            < npgs)
                return (0);
        /* TODO: check swap space, etc. */
        return (1);
}

static int
get_availrmem(pgcnt_t npgs)
{
        int ret;

        mutex_enter(&freemem_lock);
        ret = can_remove_pgs(npgs);
        if (ret != 0)
                availrmem -= npgs;
        mutex_exit(&freemem_lock);
        return (ret);
}

static void
put_availrmem(pgcnt_t npgs)
{
        mutex_enter(&freemem_lock);
        availrmem += npgs;
        mutex_exit(&freemem_lock);
}

#define FREEMEM_INCR    100
static pgcnt_t freemem_incr = FREEMEM_INCR;
#define DEL_FREE_WAIT_FRAC      4
#define DEL_FREE_WAIT_TICKS     ((hz+DEL_FREE_WAIT_FRAC-1)/DEL_FREE_WAIT_FRAC)

#define DEL_BUSY_WAIT_FRAC      20
#define DEL_BUSY_WAIT_TICKS     ((hz+DEL_BUSY_WAIT_FRAC-1)/DEL_BUSY_WAIT_FRAC)

static void kphysm_del_cleanup(struct mem_handle *);

static void page_delete_collect(page_t *, struct mem_handle *);

static pgcnt_t
delthr_get_freemem(struct mem_handle *mhp)
{
        pgcnt_t free_get;
        int ret;

        ASSERT(MUTEX_HELD(&mhp->mh_mutex));

        MDSTAT_INCR(mhp, need_free);
        /*
         * Get up to freemem_incr pages.
         */
        free_get = freemem_incr;
        if (free_get > mhp->mh_hold_todo)
                free_get = mhp->mh_hold_todo;
        /*
         * Take free_get pages away from freemem,
         * waiting if necessary.
         */

        while (!mhp->mh_cancel) {
                mutex_exit(&mhp->mh_mutex);
                MDSTAT_INCR(mhp, free_loop);
                /*
                 * Duplicate test from page_create_throttle()
                 * but don't override with !PG_WAIT.
                 */
                if (freemem < (free_get + throttlefree)) {
                        MDSTAT_INCR(mhp, free_low);
                        ret = 0;
                } else {
                        ret = page_create_wait(free_get, 0);
                        if (ret == 0) {
                                /* EMPTY */
                                MDSTAT_INCR(mhp, free_failed);
                        }
                }
                if (ret != 0) {
                        mutex_enter(&mhp->mh_mutex);
                        return (free_get);
                }

                /*
                 * Put pressure on pageout.
                 */
                page_needfree(free_get);
                cv_signal(&proc_pageout->p_cv);

                mutex_enter(&mhp->mh_mutex);
                (void) cv_reltimedwait(&mhp->mh_cv, &mhp->mh_mutex,
                    DEL_FREE_WAIT_TICKS, TR_CLOCK_TICK);
                mutex_exit(&mhp->mh_mutex);
                page_needfree(-(spgcnt_t)free_get);

                mutex_enter(&mhp->mh_mutex);
        }
        return (0);
}

#define DR_AIO_CLEANUP_DELAY    25000   /* 0.025secs, in usec */
#define DR_AIO_CLEANUP_MAXLOOPS_NODELAY 100
/*
 * This function is run as a helper thread for delete_memory_thread.
 * It is needed in order to force kaio cleanup, so that pages used in kaio
 * will be unlocked and subsequently relocated by delete_memory_thread.
 * The address of the delete_memory_threads's mem_handle is passed in to
 * this thread function, and is used to set the mh_aio_cleanup_done member
 * prior to calling thread_exit().
 */
static void
dr_aio_cleanup_thread(caddr_t amhp)
{
        proc_t *procp;
        int (*aio_cleanup_dr_delete_memory)(proc_t *);
        int cleaned;
        int n = 0;
        struct mem_handle *mhp;
        volatile uint_t *pcancel;

        mhp = (struct mem_handle *)amhp;
        ASSERT(mhp != NULL);
        pcancel = &mhp->mh_dr_aio_cleanup_cancel;
        if (modload("sys", "kaio") == -1) {
                mhp->mh_aio_cleanup_done = 1;
                cmn_err(CE_WARN, "dr_aio_cleanup_thread: cannot load kaio");
                thread_exit();
        }
        aio_cleanup_dr_delete_memory = (int (*)(proc_t *))
            modgetsymvalue("aio_cleanup_dr_delete_memory", 0);
        if (aio_cleanup_dr_delete_memory == NULL) {
                mhp->mh_aio_cleanup_done = 1;
                cmn_err(CE_WARN,
            "aio_cleanup_dr_delete_memory not found in kaio");
                thread_exit();
        }
        do {
                cleaned = 0;
                mutex_enter(&pidlock);
                for (procp = practive; (*pcancel == 0) && (procp != NULL);
                    procp = procp->p_next) {
                        mutex_enter(&procp->p_lock);
                        if (procp->p_aio != NULL) {
                                /* cleanup proc's outstanding kaio */
                                cleaned +=
                                    (*aio_cleanup_dr_delete_memory)(procp);
                        }
                        mutex_exit(&procp->p_lock);
                }
                mutex_exit(&pidlock);
                if ((*pcancel == 0) &&
                    (!cleaned || (++n == DR_AIO_CLEANUP_MAXLOOPS_NODELAY))) {
                        /* delay a bit before retrying all procs again */
                        delay(drv_usectohz(DR_AIO_CLEANUP_DELAY));
                        n = 0;
                }
        } while (*pcancel == 0);
        mhp->mh_aio_cleanup_done = 1;
        thread_exit();
}

static void
delete_memory_thread(caddr_t amhp)
{
        struct mem_handle *mhp;
        struct memdelspan *mdsp;
        callb_cpr_t cprinfo;
        page_t *pp_targ;
        spgcnt_t freemem_left;
        void (*del_complete_funcp)(void *, int error);
        void *del_complete_arg;
        int comp_code;
        int ret;
        int first_scan;
        uint_t szc;
#ifdef MEM_DEL_STATS
        uint64_t start_total, ntick_total;
        uint64_t start_pgrp, ntick_pgrp;
#endif /* MEM_DEL_STATS */

        mhp = (struct mem_handle *)amhp;

#ifdef MEM_DEL_STATS
        start_total = ddi_get_lbolt();
#endif /* MEM_DEL_STATS */

        CALLB_CPR_INIT(&cprinfo, &mhp->mh_mutex,
            callb_generic_cpr, "memdel");

        mutex_enter(&mhp->mh_mutex);
        ASSERT(mhp->mh_state == MHND_STARTING);

        mhp->mh_state = MHND_RUNNING;
        mhp->mh_thread_id = curthread;

        mhp->mh_hold_todo = mhp->mh_vm_pages;
        mutex_exit(&mhp->mh_mutex);

        /* Allocate the remap pages now, if necessary. */
        memseg_remap_init();

        /*
         * Subtract from availrmem now if possible as availrmem
         * may not be available by the end of the delete.
         */
        if (!get_availrmem(mhp->mh_vm_pages)) {
                comp_code = KPHYSM_ENOTVIABLE;
                mutex_enter(&mhp->mh_mutex);
                goto early_exit;
        }

        ret = kphysm_setup_pre_del(mhp->mh_vm_pages);

        mutex_enter(&mhp->mh_mutex);

        if (ret != 0) {
                mhp->mh_cancel = KPHYSM_EREFUSED;
                goto refused;
        }

        transit_list_collect(mhp, 1);

        for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
            mdsp = mdsp->mds_next) {
                ASSERT(mdsp->mds_bitmap == NULL);
                mdsp->mds_bitmap = kmem_zalloc(MDS_BITMAPBYTES(mdsp), KM_SLEEP);
                mdsp->mds_bitmap_retired = kmem_zalloc(MDS_BITMAPBYTES(mdsp),
                    KM_SLEEP);
        }

        first_scan = 1;
        freemem_left = 0;
        /*
         * Start dr_aio_cleanup_thread, which periodically iterates
         * through the process list and invokes aio cleanup.  This
         * is needed in order to avoid a deadly embrace between the
         * delete_memory_thread (waiting on writer lock for page, with the
         * exclusive-wanted bit set), kaio read request threads (waiting for a
         * reader lock on the same page that is wanted by the
         * delete_memory_thread), and threads waiting for kaio completion
         * (blocked on spt_amp->lock).
         */
        mhp->mh_dr_aio_cleanup_cancel = 0;
        mhp->mh_aio_cleanup_done = 0;
        (void) thread_create(NULL, 0, dr_aio_cleanup_thread,
            (caddr_t)mhp, 0, &p0, TS_RUN, maxclsyspri - 1);
        while ((mhp->mh_hold_todo != 0) && (mhp->mh_cancel == 0)) {
                pgcnt_t collected;

                MDSTAT_INCR(mhp, nloop);
                collected = 0;
                for (mdsp = mhp->mh_transit.trl_spans; (mdsp != NULL) &&
                    (mhp->mh_cancel == 0); mdsp = mdsp->mds_next) {
                        pfn_t pfn, p_end;

                        p_end = mdsp->mds_base + mdsp->mds_npgs;
                        for (pfn = mdsp->mds_base; (pfn < p_end) &&
                            (mhp->mh_cancel == 0); pfn++) {
                                page_t *pp, *tpp, *tpp_targ;
                                pgcnt_t bit;
                                struct vnode *vp;
                                uoff_t offset;
                                int mod, result;
                                spgcnt_t pgcnt;

                                bit = pfn - mdsp->mds_base;
                                if ((mdsp->mds_bitmap[bit / NBPBMW] &
                                    (1 << (bit % NBPBMW))) != 0) {
                                        MDSTAT_INCR(mhp, already_done);
                                        continue;
                                }
                                if (freemem_left == 0) {
                                        freemem_left += delthr_get_freemem(mhp);
                                        if (freemem_left == 0)
                                                break;
                                }

                                /*
                                 * Release mh_mutex - some of this
                                 * stuff takes some time (eg PUTPAGE).
                                 */

                                mutex_exit(&mhp->mh_mutex);
                                MDSTAT_INCR(mhp, ncheck);

                                pp = page_numtopp_nolock(pfn);
                                if (pp == NULL) {
                                        /*
                                         * Not covered by a page_t - will
                                         * be dealt with elsewhere.
                                         */
                                        MDSTAT_INCR(mhp, nopaget);
                                        mutex_enter(&mhp->mh_mutex);
                                        mdsp->mds_bitmap[bit / NBPBMW] |=
                                            (1 << (bit % NBPBMW));
                                        continue;
                                }

                                if (!page_try_reclaim_lock(pp, SE_EXCL,
                                    SE_EXCL_WANTED | SE_RETIRED)) {
                                        /*
                                         * Page in use elsewhere.  Skip it.
                                         */
                                        MDSTAT_INCR(mhp, lockfail);
                                        mutex_enter(&mhp->mh_mutex);
                                        continue;
                                }
                                /*
                                 * See if the cage expanded into the delete.
                                 * This can happen as we have to allow the
                                 * cage to expand.
                                 */
                                if (PP_ISNORELOC(pp)) {
                                        page_unlock(pp);
                                        mutex_enter(&mhp->mh_mutex);
                                        mhp->mh_cancel = KPHYSM_ENONRELOC;
                                        break;
                                }
                                if (PP_RETIRED(pp)) {
                                        /*
                                         * Page has been retired and is
                                         * not part of the cage so we
                                         * can now do the accounting for
                                         * it.
                                         */
                                        MDSTAT_INCR(mhp, retired);
                                        mutex_enter(&mhp->mh_mutex);
                                        mdsp->mds_bitmap[bit / NBPBMW]
                                            |= (1 << (bit % NBPBMW));
                                        mdsp->mds_bitmap_retired[bit /
                                            NBPBMW] |=
                                            (1 << (bit % NBPBMW));
                                        mhp->mh_hold_todo--;
                                        continue;
                                }
                                ASSERT(freemem_left != 0);
                                if (PP_ISFREE(pp)) {
                                        /*
                                         * Like page_reclaim() only 'freemem'
                                         * processing is already done.
                                         */
                                        MDSTAT_INCR(mhp, nfree);
                                free_page_collect:
                                        if (PP_ISAGED(pp)) {
                                                page_list_sub(pp,
                                                    PG_FREE_LIST);
                                        } else {
                                                page_list_sub(pp,
                                                    PG_CACHE_LIST);
                                        }
                                        PP_CLRFREE(pp);
                                        PP_CLRAGED(pp);
                                        collected++;
                                        mutex_enter(&mhp->mh_mutex);
                                        page_delete_collect(pp, mhp);
                                        mdsp->mds_bitmap[bit / NBPBMW] |=
                                            (1 << (bit % NBPBMW));
                                        freemem_left--;
                                        continue;
                                }
                                VERIFY(pp->p_object != NULL);
                                ASSERT(pp->p_vnode != NULL);
                                if (first_scan) {
                                        MDSTAT_INCR(mhp, first_notfree);
                                        page_unlock(pp);
                                        mutex_enter(&mhp->mh_mutex);
                                        continue;
                                }
                                /*
                                 * Keep stats on pages encountered that
                                 * are marked for retirement.
                                 */
                                if (PP_TOXIC(pp)) {
                                        MDSTAT_INCR(mhp, toxic);
                                } else if (PP_PR_REQ(pp)) {
                                        MDSTAT_INCR(mhp, failing);
                                }
                                /*
                                 * In certain cases below, special exceptions
                                 * are made for pages that are toxic.  This
                                 * is because the current meaning of toxic
                                 * is that an uncorrectable error has been
                                 * previously associated with the page.
                                 */
                                if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
                                        if (!PP_TOXIC(pp)) {
                                                /*
                                                 * Must relocate locked in
                                                 * memory pages.
                                                 */
#ifdef MEM_DEL_STATS
                                                start_pgrp = ddi_get_lbolt();
#endif /* MEM_DEL_STATS */
                                                /*
                                                 * Lock all constituent pages
                                                 * of a large page to ensure
                                                 * that p_szc won't change.
                                                 */
                                                if (!group_page_trylock(pp,
                                                    SE_EXCL)) {
                                                        MDSTAT_INCR(mhp,
                                                            gptllckfail);
                                                        page_unlock(pp);
                                                        mutex_enter(
                                                            &mhp->mh_mutex);
                                                        continue;
                                                }
                                                MDSTAT_INCR(mhp, npplocked);
                                                pp_targ =
                                                    page_get_replacement_page(
                                                    pp, NULL, 0);
                                                if (pp_targ != NULL) {
#ifdef MEM_DEL_STATS
                                                        ntick_pgrp =
                                                            (uint64_t)
                                                            ddi_get_lbolt() -
                                                            start_pgrp;
#endif /* MEM_DEL_STATS */
                                                        MDSTAT_PGRP(mhp,
                                                            ntick_pgrp);
                                                        MDSTAT_INCR(mhp,
                                                            nlockreloc);
                                                        goto reloc;
                                                }
                                                group_page_unlock(pp);
                                                page_unlock(pp);
#ifdef MEM_DEL_STATS
                                                ntick_pgrp =
                                                    (uint64_t)ddi_get_lbolt() -
                                                    start_pgrp;
#endif /* MEM_DEL_STATS */
                                                MDSTAT_PGRP(mhp, ntick_pgrp);
                                                MDSTAT_INCR(mhp, nnorepl);
                                                mutex_enter(&mhp->mh_mutex);
                                                continue;
                                        } else {
                                                /*
                                                 * Cannot do anything about
                                                 * this page because it is
                                                 * toxic.
                                                 */
                                                MDSTAT_INCR(mhp, npplkdtoxic);
                                                page_unlock(pp);
                                                mutex_enter(&mhp->mh_mutex);
                                                continue;
                                        }
                                }
                                /*
                                 * Unload the mappings and check if mod bit
                                 * is set.
                                 */
                                ASSERT(!PP_ISKAS(pp));
                                (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
                                mod = hat_ismod(pp);

#ifdef MEM_DEL_STATS
                                start_pgrp = ddi_get_lbolt();
#endif /* MEM_DEL_STATS */
                                if (mod && !PP_TOXIC(pp)) {
                                        /*
                                         * Lock all constituent pages
                                         * of a large page to ensure
                                         * that p_szc won't change.
                                         */
                                        if (!group_page_trylock(pp, SE_EXCL)) {
                                                MDSTAT_INCR(mhp, gptlmodfail);
                                                page_unlock(pp);
                                                mutex_enter(&mhp->mh_mutex);
                                                continue;
                                        }
                                        pp_targ = page_get_replacement_page(pp,
                                            NULL, 0);
                                        if (pp_targ != NULL) {
                                                MDSTAT_INCR(mhp, nmodreloc);
#ifdef MEM_DEL_STATS
                                                ntick_pgrp =
                                                    (uint64_t)ddi_get_lbolt() -
                                                    start_pgrp;
#endif /* MEM_DEL_STATS */
                                                MDSTAT_PGRP(mhp, ntick_pgrp);
                                                goto reloc;
                                        }
                                        group_page_unlock(pp);
                                }

                                if (!page_try_demote_pages(pp)) {
                                        MDSTAT_INCR(mhp, demotefail);
                                        page_unlock(pp);
#ifdef MEM_DEL_STATS
                                        ntick_pgrp = (uint64_t)ddi_get_lbolt() -
                                            start_pgrp;
#endif /* MEM_DEL_STATS */
                                        MDSTAT_PGRP(mhp, ntick_pgrp);
                                        mutex_enter(&mhp->mh_mutex);
                                        continue;
                                }

                                /*
                                 * Regular 'page-out'.
                                 */
                                if (!mod) {
                                        MDSTAT_INCR(mhp, ndestroy);
                                        page_destroy(pp, 1);
                                        /*
                                         * page_destroy was called with
                                         * dontfree. As long as p_lckcnt
                                         * and p_cowcnt are both zero, the
                                         * only additional action of
                                         * page_destroy with !dontfree is to
                                         * call page_free, so we can collect
                                         * the page here.
                                         */
                                        collected++;
#ifdef MEM_DEL_STATS
                                        ntick_pgrp = (uint64_t)ddi_get_lbolt() -
                                            start_pgrp;
#endif /* MEM_DEL_STATS */
                                        MDSTAT_PGRP(mhp, ntick_pgrp);
                                        mutex_enter(&mhp->mh_mutex);
                                        page_delete_collect(pp, mhp);
                                        mdsp->mds_bitmap[bit / NBPBMW] |=
                                            (1 << (bit % NBPBMW));
                                        continue;
                                }
                                /*
                                 * The page is toxic and the mod bit is
                                 * set, we cannot do anything here to deal
                                 * with it.
                                 */
                                if (PP_TOXIC(pp)) {
                                        page_unlock(pp);
#ifdef MEM_DEL_STATS
                                        ntick_pgrp = (uint64_t)ddi_get_lbolt() -
                                            start_pgrp;
#endif /* MEM_DEL_STATS */
                                        MDSTAT_PGRP(mhp, ntick_pgrp);
                                        MDSTAT_INCR(mhp, modtoxic);
                                        mutex_enter(&mhp->mh_mutex);
                                        continue;
                                }
                                MDSTAT_INCR(mhp, nputpage);
                                vp = pp->p_vnode;
                                offset = pp->p_offset;
                                VN_HOLD(vp);
                                page_unlock(pp);
                                (void) fop_putpage(vp, offset, PAGESIZE,
                                    B_INVAL|B_FORCE, kcred, NULL);
                                VN_RELE(vp);
#ifdef MEM_DEL_STATS
                                ntick_pgrp = (uint64_t)ddi_get_lbolt() -
                                    start_pgrp;
#endif /* MEM_DEL_STATS */
                                MDSTAT_PGRP(mhp, ntick_pgrp);
                                /*
                                 * Try to get the page back immediately
                                 * so that it can be collected.
                                 */
                                pp = page_numtopp_nolock(pfn);
                                if (pp == NULL) {
                                        MDSTAT_INCR(mhp, nnoreclaim);
                                        /*
                                         * This should not happen as this
                                         * thread is deleting the page.
                                         * If this code is generalized, this
                                         * becomes a reality.
                                         */
#ifdef DEBUG
                                        cmn_err(CE_WARN,
                                            "delete_memory_thread(0x%p) "
                                            "pfn 0x%lx has no page_t",
                                            (void *)mhp, pfn);
#endif /* DEBUG */
                                        mutex_enter(&mhp->mh_mutex);
                                        continue;
                                }
                                if (page_try_reclaim_lock(pp, SE_EXCL,
                                    SE_EXCL_WANTED | SE_RETIRED)) {
                                        if (PP_ISFREE(pp)) {
                                                goto free_page_collect;
                                        }
                                        page_unlock(pp);
                                }
                                MDSTAT_INCR(mhp, nnoreclaim);
                                mutex_enter(&mhp->mh_mutex);
                                continue;

                        reloc:
                                /*
                                 * Got some freemem and a target
                                 * page, so move the data to avoid
                                 * I/O and lock problems.
                                 */
                                ASSERT(!page_iolock_assert(pp));
                                MDSTAT_INCR(mhp, nreloc);
                                /*
                                 * page_relocate() will return pgcnt: the
                                 * number of consecutive pages relocated.
                                 * If it is successful, pp will be a
                                 * linked list of the page structs that
                                 * were relocated. If page_relocate() is
                                 * unsuccessful, pp will be unmodified.
                                 */
#ifdef MEM_DEL_STATS
                                start_pgrp = ddi_get_lbolt();
#endif /* MEM_DEL_STATS */
                                result = page_relocate(&pp, &pp_targ, 0, 0,
                                    &pgcnt, NULL);
#ifdef MEM_DEL_STATS
                                ntick_pgrp = (uint64_t)ddi_get_lbolt() -
                                    start_pgrp;
#endif /* MEM_DEL_STATS */
                                MDSTAT_PGRP(mhp, ntick_pgrp);
                                if (result != 0) {
                                        MDSTAT_INCR(mhp, nrelocfail);
                                        /*
                                         * We did not succeed. We need
                                         * to give the pp_targ pages back.
                                         * page_free(pp_targ, 1) without
                                         * the freemem accounting.
                                         */
                                        group_page_unlock(pp);
                                        page_free_replacement_page(pp_targ);
                                        page_unlock(pp);
                                        mutex_enter(&mhp->mh_mutex);
                                        continue;
                                }

                                /*
                                 * We will then collect pgcnt pages.
                                 */
                                ASSERT(pgcnt > 0);
                                mutex_enter(&mhp->mh_mutex);
                                /*
                                 * We need to make sure freemem_left is
                                 * large enough.
                                 */
                                while ((freemem_left < pgcnt) &&
                                    (!mhp->mh_cancel)) {
                                        freemem_left +=
                                            delthr_get_freemem(mhp);
                                }

                                /*
                                 * Do not proceed if mh_cancel is set.
                                 */
                                if (mhp->mh_cancel) {
                                        while (pp_targ != NULL) {
                                                /*
                                                 * Unlink and unlock each page.
                                                 */
                                                tpp_targ = pp_targ;
                                                page_sub(&pp_targ, tpp_targ);
                                                page_unlock(tpp_targ);
                                        }
                                        /*
                                         * We need to give the pp pages back.
                                         * page_free(pp, 1) without the
                                         * freemem accounting.
                                         */
                                        page_free_replacement_page(pp);
                                        break;
                                }

                                /* Now remove pgcnt from freemem_left */
                                freemem_left -= pgcnt;
                                ASSERT(freemem_left >= 0);
                                szc = pp->p_szc;
                                while (pp != NULL) {
                                        /*
                                         * pp and pp_targ were passed back as
                                         * a linked list of pages.
                                         * Unlink and unlock each page.
                                         */
                                        tpp_targ = pp_targ;
                                        page_sub(&pp_targ, tpp_targ);
                                        page_unlock(tpp_targ);
                                        /*
                                         * The original page is now free
                                         * so remove it from the linked
                                         * list and collect it.
                                         */
                                        tpp = pp;
                                        page_sub(&pp, tpp);
                                        pfn = page_pptonum(tpp);
                                        collected++;
                                        ASSERT(PAGE_EXCL(tpp));
                                        VERIFY(tpp->p_object == NULL);
                                        ASSERT(tpp->p_vnode == NULL);
                                        ASSERT(!hat_page_is_mapped(tpp));
                                        ASSERT(tpp->p_szc == szc);
                                        tpp->p_szc = 0;
                                        page_delete_collect(tpp, mhp);
                                        bit = pfn - mdsp->mds_base;
                                        mdsp->mds_bitmap[bit / NBPBMW] |=
                                            (1 << (bit % NBPBMW));
                                }
                                ASSERT(pp_targ == NULL);
                        }
                }
                first_scan = 0;
                if ((mhp->mh_cancel == 0) && (mhp->mh_hold_todo != 0) &&
                    (collected == 0)) {
                        /*
                         * This code is needed as we cannot wait
                         * for a page to be locked OR the delete to
                         * be cancelled.  Also, we must delay so
                         * that other threads get a chance to run
                         * on our cpu, otherwise page locks may be
                         * held indefinitely by those threads.
                         */
                        MDSTAT_INCR(mhp, ndelay);
                        CALLB_CPR_SAFE_BEGIN(&cprinfo);
                        (void) cv_reltimedwait(&mhp->mh_cv, &mhp->mh_mutex,
                            DEL_BUSY_WAIT_TICKS, TR_CLOCK_TICK);
                        CALLB_CPR_SAFE_END(&cprinfo, &mhp->mh_mutex);
                }
        }
        /* stop the dr aio cleanup thread */
        mhp->mh_dr_aio_cleanup_cancel = 1;
        transit_list_collect(mhp, 0);
        if (freemem_left != 0) {
                /* Return any surplus. */
                page_create_putback(freemem_left);
                freemem_left = 0;
        }
#ifdef MEM_DEL_STATS
        ntick_total = (uint64_t)ddi_get_lbolt() - start_total;
#endif /* MEM_DEL_STATS */
        MDSTAT_TOTAL(mhp, ntick_total);
        MDSTAT_PRINT(mhp);

        /*
         * If the memory delete was cancelled, exclusive-wanted bits must
         * be cleared. If there are retired pages being deleted, they need
         * to be unretired.
         */
        for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
            mdsp = mdsp->mds_next) {
                pfn_t pfn, p_end;

                p_end = mdsp->mds_base + mdsp->mds_npgs;
                for (pfn = mdsp->mds_base; pfn < p_end; pfn++) {
                        page_t *pp;
                        pgcnt_t bit;

                        bit = pfn - mdsp->mds_base;
                        if (mhp->mh_cancel) {
                                pp = page_numtopp_nolock(pfn);
                                if (pp != NULL) {
                                        if ((mdsp->mds_bitmap[bit / NBPBMW] &
                                            (1 << (bit % NBPBMW))) == 0) {
                                                page_lock_clr_exclwanted(pp);
                                        }
                                }
                        } else {
                                pp = NULL;
                        }
                        if ((mdsp->mds_bitmap_retired[bit / NBPBMW] &
                            (1 << (bit % NBPBMW))) != 0) {
                                /* do we already have pp? */
                                if (pp == NULL) {
                                        pp = page_numtopp_nolock(pfn);
                                }
                                ASSERT(pp != NULL);
                                ASSERT(PP_RETIRED(pp));
                                if (mhp->mh_cancel != 0) {
                                        page_unlock(pp);
                                        /*
                                         * To satisfy ASSERT below in
                                         * cancel code.
                                         */
                                        mhp->mh_hold_todo++;
                                } else {
                                        (void) page_unretire_pp(pp,
                                            PR_UNR_CLEAN);
                                }
                        }
                }
        }
        /*
         * Free retired page bitmap and collected page bitmap
         */
        for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
            mdsp = mdsp->mds_next) {
                ASSERT(mdsp->mds_bitmap_retired != NULL);
                kmem_free(mdsp->mds_bitmap_retired, MDS_BITMAPBYTES(mdsp));
                mdsp->mds_bitmap_retired = NULL;        /* Paranoia. */
                ASSERT(mdsp->mds_bitmap != NULL);
                kmem_free(mdsp->mds_bitmap, MDS_BITMAPBYTES(mdsp));
                mdsp->mds_bitmap = NULL;        /* Paranoia. */
        }

        /* wait for our dr aio cancel thread to exit */
        while (!(mhp->mh_aio_cleanup_done)) {
                CALLB_CPR_SAFE_BEGIN(&cprinfo);
                delay(drv_usectohz(DR_AIO_CLEANUP_DELAY));
                CALLB_CPR_SAFE_END(&cprinfo, &mhp->mh_mutex);
        }
refused:
        if (mhp->mh_cancel != 0) {
                page_t *pp;

                comp_code = mhp->mh_cancel;
                /*
                 * Go through list of deleted pages (mh_deleted) freeing
                 * them.
                 */
                while ((pp = mhp->mh_deleted) != NULL) {
                        mhp->mh_deleted = pp->p_next;
                        mhp->mh_hold_todo++;
                        mutex_exit(&mhp->mh_mutex);
                        /* Restore p_next. */
                        pp->p_next = pp->p_prev;
                        if (PP_ISFREE(pp)) {
                                cmn_err(CE_PANIC,
                                    "page %p is free",
                                    (void *)pp);
                        }
                        page_free(pp, 1);
                        mutex_enter(&mhp->mh_mutex);
                }
                ASSERT(mhp->mh_hold_todo == mhp->mh_vm_pages);

                mutex_exit(&mhp->mh_mutex);
                put_availrmem(mhp->mh_vm_pages);
                mutex_enter(&mhp->mh_mutex);

                goto t_exit;
        }

        /*
         * All the pages are no longer in use and are exclusively locked.
         */

        mhp->mh_deleted = NULL;

        kphysm_del_cleanup(mhp);

        /*
         * mem_node_del_range needs to be after kphysm_del_cleanup so
         * that the mem_node_config[] will remain intact for the cleanup.
         */
        for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
            mdsp = mdsp->mds_next) {
                mem_node_del_range(mdsp->mds_base,
                    mdsp->mds_base + mdsp->mds_npgs - 1);
        }
        /* cleanup the page counters */
        page_ctrs_cleanup();

        comp_code = KPHYSM_OK;

t_exit:
        mutex_exit(&mhp->mh_mutex);
        kphysm_setup_post_del(mhp->mh_vm_pages,
            (comp_code == KPHYSM_OK) ? 0 : 1);
        mutex_enter(&mhp->mh_mutex);

early_exit:
        /* mhp->mh_mutex exited by CALLB_CPR_EXIT() */
        mhp->mh_state = MHND_DONE;
        del_complete_funcp = mhp->mh_delete_complete;
        del_complete_arg = mhp->mh_delete_complete_arg;
        CALLB_CPR_EXIT(&cprinfo);
        (*del_complete_funcp)(del_complete_arg, comp_code);
        thread_exit();
        /*NOTREACHED*/
}

/*
 * Start the delete of the memory from the system.
 */
int
kphysm_del_start(
        memhandle_t handle,
        void (*complete)(void *, int),
        void *complete_arg)
{
        struct mem_handle *mhp;

        mhp = kphysm_lookup_mem_handle(handle);
        if (mhp == NULL) {
                return (KPHYSM_EHANDLE);
        }
        switch (mhp->mh_state) {
        case MHND_FREE:
                ASSERT(mhp->mh_state != MHND_FREE);
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_EHANDLE);
        case MHND_INIT:
                break;
        case MHND_STARTING:
        case MHND_RUNNING:
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_ESEQUENCE);
        case MHND_DONE:
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_ESEQUENCE);
        case MHND_RELEASE:
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_ESEQUENCE);
        default:
#ifdef DEBUG
                cmn_err(CE_WARN, "kphysm_del_start(0x%p) state corrupt %d",
                    (void *)mhp, mhp->mh_state);
#endif /* DEBUG */
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_EHANDLE);
        }

        if (mhp->mh_transit.trl_spans == NULL) {
                mutex_exit(&mhp->mh_mutex);
                return (KPHYSM_ENOWORK);
        }

        ASSERT(complete != NULL);
        mhp->mh_delete_complete = complete;
        mhp->mh_delete_complete_arg = complete_arg;
        mhp->mh_state = MHND_STARTING;
        /*
         * Release the mutex in case thread_create sleeps.
         */
        mutex_exit(&mhp->mh_mutex);

        /*
         * The "obvious" process for this thread is pageout (proc_pageout)
         * but this gives the thread too much power over freemem
         * which results in freemem starvation.
         */
        (void) thread_create(NULL, 0, delete_memory_thread, mhp, 0, &p0,
            TS_RUN, maxclsyspri - 1);

        return (KPHYSM_OK);
}

static kmutex_t pp_dummy_lock;          /* Protects init. of pp_dummy. */
static caddr_t pp_dummy;
static pgcnt_t pp_dummy_npages;
static pfn_t *pp_dummy_pfn;     /* Array of dummy pfns. */

static void
memseg_remap_init_pages(page_t *pages, page_t *epages)
{
        page_t *pp;

        for (pp = pages; pp < epages; pp++) {
                pp->p_pagenum = PFN_INVALID;    /* XXXX */
                pp->p_offset = (uoff_t)-1;
                page_iolock_init(pp);
                while (!page_lock(pp, SE_EXCL, NULL, P_RECLAIM))
                        continue;
                page_lock_delete(pp);
        }
}

void
memseg_remap_init()
{
        mutex_enter(&pp_dummy_lock);
        if (pp_dummy == NULL) {
                uint_t dpages;
                int i;

                /*
                 * dpages starts off as the size of the structure and
                 * ends up as the minimum number of pages that will
                 * hold a whole number of page_t structures.
                 */
                dpages = sizeof (page_t);
                ASSERT(dpages != 0);
                ASSERT(dpages <= MMU_PAGESIZE);

                while ((dpages & 1) == 0)
                        dpages >>= 1;

                pp_dummy_npages = dpages;
                /*
                 * Allocate pp_dummy pages directly from static_arena,
                 * since these are whole page allocations and are
                 * referenced by physical address.  This also has the
                 * nice fringe benefit of hiding the memory from
                 * ::findleaks since it doesn't deal well with allocated
                 * kernel heap memory that doesn't have any mappings.
                 */
                pp_dummy = vmem_xalloc(static_arena, ptob(pp_dummy_npages),
                    PAGESIZE, 0, 0, NULL, NULL, VM_SLEEP);
                bzero(pp_dummy, ptob(pp_dummy_npages));
                ASSERT(((uintptr_t)pp_dummy & MMU_PAGEOFFSET) == 0);
                pp_dummy_pfn = kmem_alloc(sizeof (*pp_dummy_pfn) *
                    pp_dummy_npages, KM_SLEEP);
                for (i = 0; i < pp_dummy_npages; i++) {
                        pp_dummy_pfn[i] = hat_getpfnum(kas.a_hat,
                            &pp_dummy[MMU_PAGESIZE * i]);
                        ASSERT(pp_dummy_pfn[i] != PFN_INVALID);
                }
                /*
                 * Initialize the page_t's to a known 'deleted' state
                 * that matches the state of deleted pages.
                 */
                memseg_remap_init_pages((page_t *)pp_dummy,
                    (page_t *)(pp_dummy + ptob(pp_dummy_npages)));
                /* Remove kmem mappings for the pages for safety. */
                hat_unload(kas.a_hat, pp_dummy, ptob(pp_dummy_npages),
                    HAT_UNLOAD_UNLOCK);
                /* Leave pp_dummy pointer set as flag that init is done. */
        }
        mutex_exit(&pp_dummy_lock);
}

/*
 * Remap a page-aglined range of page_t's to dummy pages.
 */
void
remap_to_dummy(caddr_t va, pgcnt_t metapgs)
{
        int phase;

        ASSERT(IS_P2ALIGNED((uint64_t)(uintptr_t)va, PAGESIZE));

        /*
         * We may start remapping at a non-zero page offset
         * within the dummy pages since the low/high ends
         * of the outgoing pp's could be shared by other
         * memsegs (see memseg_remap_meta).
         */
        phase = btop((uint64_t)(uintptr_t)va) % pp_dummy_npages;
        /*CONSTCOND*/
        ASSERT(PAGESIZE % sizeof (page_t) || phase == 0);

        while (metapgs != 0) {
                pgcnt_t n;
                int i, j;

                n = pp_dummy_npages;
                if (n > metapgs)
                        n = metapgs;
                for (i = 0; i < n; i++) {
                        j = (i + phase) % pp_dummy_npages;
                        hat_devload(kas.a_hat, va, ptob(1), pp_dummy_pfn[j],
                            PROT_READ,
                            HAT_LOAD | HAT_LOAD_NOCONSIST |
                            HAT_LOAD_REMAP);
                        va += ptob(1);
                }
                metapgs -= n;
        }
}

static void
memseg_remap_to_dummy(struct memseg *seg)
{
        caddr_t pp;
        pgcnt_t metapgs;

        ASSERT(memseg_is_dynamic(seg));
        ASSERT(pp_dummy != NULL);


        if (!memseg_includes_meta(seg)) {
                memseg_remap_meta(seg);
                return;
        }

        pp = (caddr_t)seg->pages;
        metapgs = seg->pages_base - memseg_get_start(seg);
        ASSERT(metapgs != 0);

        seg->pages_end = seg->pages_base;

        remap_to_dummy(pp, metapgs);
}

/*
 * Transition all the deleted pages to the deleted state so that
 * page_lock will not wait. The page_lock_delete call will
 * also wake up any waiters.
 */
static void
memseg_lock_delete_all(struct memseg *seg)
{
        page_t *pp;

        for (pp = seg->pages; pp < seg->epages; pp++) {
                pp->p_pagenum = PFN_INVALID;    /* XXXX */
                page_lock_delete(pp);
        }
}

static void
kphysm_del_cleanup(struct mem_handle *mhp)
{
        struct memdelspan       *mdsp;
        struct memseg           *seg;
        struct memseg           **segpp;
        struct memseg           *seglist;
        pfn_t                   p_end;
        uint64_t                avmem;
        pgcnt_t                 avpgs;
        pgcnt_t                 npgs;

        avpgs = mhp->mh_vm_pages;

        memsegs_lock(1);

        /*
         * remove from main segment list.
         */
        npgs = 0;
        seglist = NULL;
        for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
            mdsp = mdsp->mds_next) {
                p_end = mdsp->mds_base + mdsp->mds_npgs;
                for (segpp = &memsegs; (seg = *segpp) != NULL; ) {
                        if (seg->pages_base >= p_end ||
                            seg->pages_end <= mdsp->mds_base) {
                                /* Span and memseg don't overlap. */
                                segpp = &((*segpp)->next);
                                continue;
                        }
                        ASSERT(seg->pages_base >= mdsp->mds_base);
                        ASSERT(seg->pages_end <= p_end);

                        PLCNT_MODIFY_MAX(seg->pages_base,
                            seg->pages_base - seg->pages_end);

                        /* Hide the memseg from future scans. */
                        hat_kpm_delmem_mseg_update(seg, segpp);
                        *segpp = seg->next;
                        membar_producer();      /* TODO: Needed? */
                        npgs += MSEG_NPAGES(seg);

                        /*
                         * Leave the deleted segment's next pointer intact
                         * in case a memsegs scanning loop is walking this
                         * segment concurrently.
                         */
                        seg->lnext = seglist;
                        seglist = seg;
                }
        }

        build_pfn_hash();

        ASSERT(npgs < total_pages);
        total_pages -= npgs;

        /*
         * Recalculate the paging parameters now total_pages has changed.
         * This will also cause the clock hands to be reset before next use.
         */
        setupclock(1);

        memsegs_unlock(1);

        mutex_exit(&mhp->mh_mutex);

        while ((seg = seglist) != NULL) {
                pfn_t mseg_start;
                pfn_t mseg_base, mseg_end;
                pgcnt_t mseg_npgs;
                int mlret;

                seglist = seg->lnext;

                /*
                 * Put the page_t's into the deleted state to stop
                 * cv_wait()s on the pages. When we remap, the dummy
                 * page_t's will be in the same state.
                 */
                memseg_lock_delete_all(seg);
                /*
                 * Collect up information based on pages_base and pages_end
                 * early so that we can flag early that the memseg has been
                 * deleted by setting pages_end == pages_base.
                 */
                mseg_base = seg->pages_base;
                mseg_end = seg->pages_end;
                mseg_npgs = MSEG_NPAGES(seg);
                mseg_start = memseg_get_start(seg);

                if (memseg_is_dynamic(seg)) {
                        /* Remap the meta data to our special dummy area. */
                        memseg_remap_to_dummy(seg);

                        mutex_enter(&memseg_lists_lock);
                        seg->lnext = memseg_va_avail;
                        memseg_va_avail = seg;
                        mutex_exit(&memseg_lists_lock);
                } else {
                        /*
                         * For memory whose page_ts were allocated
                         * at boot, we need to find a new use for
                         * the page_t memory.
                         * For the moment, just leak it.
                         * (It is held in the memseg_delete_junk list.)
                         */
                        seg->pages_end = seg->pages_base;

                        mutex_enter(&memseg_lists_lock);
                        seg->lnext = memseg_delete_junk;
                        memseg_delete_junk = seg;
                        mutex_exit(&memseg_lists_lock);
                }

                /* Must not use seg now as it could be re-used. */

                memlist_write_lock();

                mlret = memlist_delete_span(
                    (uint64_t)(mseg_base) << PAGESHIFT,
                    (uint64_t)(mseg_npgs) << PAGESHIFT,
                    &phys_avail);
                ASSERT(mlret == MEML_SPANOP_OK);

                mlret = memlist_delete_span(
                    (uint64_t)(mseg_start) << PAGESHIFT,
                    (uint64_t)(mseg_end - mseg_start) <<
                    PAGESHIFT,
                    &phys_install);
                ASSERT(mlret == MEML_SPANOP_OK);
                phys_install_has_changed();

                memlist_write_unlock();
        }

        memlist_read_lock();
        installed_top_size(phys_install, &physmax, &physinstalled);
        memlist_read_unlock();

        mutex_enter(&freemem_lock);
        maxmem -= avpgs;
        physmem -= avpgs;
        /* availrmem is adjusted during the delete. */
        availrmem_initial -= avpgs;

        mutex_exit(&freemem_lock);

        dump_resize();

        cmn_err(CE_CONT, "?kphysm_delete: mem = %ldK "
            "(0x%" PRIx64 ")\n",
            physinstalled << (PAGESHIFT - 10),
            (uint64_t)physinstalled << PAGESHIFT);

        avmem = (uint64_t)freemem << PAGESHIFT;
        cmn_err(CE_CONT, "?kphysm_delete: "
            "avail mem = %" PRId64 "\n", avmem);

        /*
         * Update lgroup generation number on single lgroup systems
         */
        if (nlgrps == 1)
                lgrp_config(LGRP_CONFIG_GEN_UPDATE, 0, 0);

        /* Successfully deleted system memory */
        mutex_enter(&mhp->mh_mutex);
}

static uint_t mdel_nullvp_waiter;

static void
page_delete_collect(
        page_t *pp,
        struct mem_handle *mhp)
{
        if (pp->p_vnode) {
                page_hashout(pp, false);
                /* do not do PP_SETAGED(pp); */
        } else {
                kmutex_t *sep;

                sep = page_se_mutex(pp);
                mutex_enter(sep);
                if (CV_HAS_WAITERS(&pp->p_cv)) {
                        mdel_nullvp_waiter++;
                        cv_broadcast(&pp->p_cv);
                }
                mutex_exit(sep);
        }
        ASSERT(pp->p_next == pp->p_prev);
        ASSERT(pp->p_next == NULL || pp->p_next == pp);
        pp->p_next = mhp->mh_deleted;
        mhp->mh_deleted = pp;
        ASSERT(mhp->mh_hold_todo != 0);
        mhp->mh_hold_todo--;
}

static void
transit_list_collect(struct mem_handle *mhp, int v)
{
        struct transit_list_head *trh;

        trh = &transit_list_head;
        mutex_enter(&trh->trh_lock);
        mhp->mh_transit.trl_collect = v;
        mutex_exit(&trh->trh_lock);
}

static void
transit_list_insert(struct transit_list *tlp)
{
        struct transit_list_head *trh;

        trh = &transit_list_head;
        ASSERT(MUTEX_HELD(&trh->trh_lock));
        tlp->trl_next = trh->trh_head;
        trh->trh_head = tlp;
}

static void
transit_list_remove(struct transit_list *tlp)
{
        struct transit_list_head *trh;
        struct transit_list **tlpp;

        trh = &transit_list_head;
        tlpp = &trh->trh_head;
        ASSERT(MUTEX_HELD(&trh->trh_lock));
        while (*tlpp != NULL && *tlpp != tlp)
                tlpp = &(*tlpp)->trl_next;
        ASSERT(*tlpp != NULL);
        if (*tlpp == tlp)
                *tlpp = tlp->trl_next;
        tlp->trl_next = NULL;
}

static struct transit_list *
pfnum_to_transit_list(struct transit_list_head *trh, pfn_t pfnum)
{
        struct transit_list *tlp;

        for (tlp = trh->trh_head; tlp != NULL; tlp = tlp->trl_next) {
                struct memdelspan *mdsp;

                for (mdsp = tlp->trl_spans; mdsp != NULL;
                    mdsp = mdsp->mds_next) {
                        if (pfnum >= mdsp->mds_base &&
                            pfnum < (mdsp->mds_base + mdsp->mds_npgs)) {
                                return (tlp);
                        }
                }
        }
        return (NULL);
}

int
pfn_is_being_deleted(pfn_t pfnum)
{
        struct transit_list_head *trh;
        struct transit_list *tlp;
        int ret;

        trh = &transit_list_head;
        if (trh->trh_head == NULL)
                return (0);

        mutex_enter(&trh->trh_lock);
        tlp = pfnum_to_transit_list(trh, pfnum);
        ret = (tlp != NULL && tlp->trl_collect);
        mutex_exit(&trh->trh_lock);

        return (ret);
}

#ifdef MEM_DEL_STATS
extern int hz;
static void
mem_del_stat_print_func(struct mem_handle *mhp)
{
        uint64_t tmp;

        if (mem_del_stat_print) {
                printf("memory delete loop %x/%x, statistics%s\n",
                    (uint_t)mhp->mh_transit.trl_spans->mds_base,
                    (uint_t)mhp->mh_transit.trl_spans->mds_npgs,
                    (mhp->mh_cancel ? " (cancelled)" : ""));
                printf("\t%8u nloop\n", mhp->mh_delstat.nloop);
                printf("\t%8u need_free\n", mhp->mh_delstat.need_free);
                printf("\t%8u free_loop\n", mhp->mh_delstat.free_loop);
                printf("\t%8u free_low\n", mhp->mh_delstat.free_low);
                printf("\t%8u free_failed\n", mhp->mh_delstat.free_failed);
                printf("\t%8u ncheck\n", mhp->mh_delstat.ncheck);
                printf("\t%8u nopaget\n", mhp->mh_delstat.nopaget);
                printf("\t%8u lockfail\n", mhp->mh_delstat.lockfail);
                printf("\t%8u nfree\n", mhp->mh_delstat.nfree);
                printf("\t%8u nreloc\n", mhp->mh_delstat.nreloc);
                printf("\t%8u nrelocfail\n", mhp->mh_delstat.nrelocfail);
                printf("\t%8u already_done\n", mhp->mh_delstat.already_done);
                printf("\t%8u first_notfree\n", mhp->mh_delstat.first_notfree);
                printf("\t%8u npplocked\n", mhp->mh_delstat.npplocked);
                printf("\t%8u nlockreloc\n", mhp->mh_delstat.nlockreloc);
                printf("\t%8u nnorepl\n", mhp->mh_delstat.nnorepl);
                printf("\t%8u nmodreloc\n", mhp->mh_delstat.nmodreloc);
                printf("\t%8u ndestroy\n", mhp->mh_delstat.ndestroy);
                printf("\t%8u nputpage\n", mhp->mh_delstat.nputpage);
                printf("\t%8u nnoreclaim\n", mhp->mh_delstat.nnoreclaim);
                printf("\t%8u ndelay\n", mhp->mh_delstat.ndelay);
                printf("\t%8u demotefail\n", mhp->mh_delstat.demotefail);
                printf("\t%8u retired\n", mhp->mh_delstat.retired);
                printf("\t%8u toxic\n", mhp->mh_delstat.toxic);
                printf("\t%8u failing\n", mhp->mh_delstat.failing);
                printf("\t%8u modtoxic\n", mhp->mh_delstat.modtoxic);
                printf("\t%8u npplkdtoxic\n", mhp->mh_delstat.npplkdtoxic);
                printf("\t%8u gptlmodfail\n", mhp->mh_delstat.gptlmodfail);
                printf("\t%8u gptllckfail\n", mhp->mh_delstat.gptllckfail);
                tmp = mhp->mh_delstat.nticks_total / hz;  /* seconds */
                printf(
                    "\t%"PRIu64" nticks_total - %"PRIu64" min %"PRIu64" sec\n",
                    mhp->mh_delstat.nticks_total, tmp / 60, tmp % 60);

                tmp = mhp->mh_delstat.nticks_pgrp / hz;  /* seconds */
                printf(
                    "\t%"PRIu64" nticks_pgrp - %"PRIu64" min %"PRIu64" sec\n",
                    mhp->mh_delstat.nticks_pgrp, tmp / 60, tmp % 60);
        }
}
#endif /* MEM_DEL_STATS */

struct mem_callback {
        kphysm_setup_vector_t   *vec;
        void                    *arg;
};

#define NMEMCALLBACKS           100

static struct mem_callback mem_callbacks[NMEMCALLBACKS];
static uint_t nmemcallbacks;
static krwlock_t mem_callback_rwlock;

int
kphysm_setup_func_register(kphysm_setup_vector_t *vec, void *arg)
{
        uint_t i, found;

        /*
         * This test will become more complicated when the version must
         * change.
         */
        if (vec->version != KPHYSM_SETUP_VECTOR_VERSION)
                return (EINVAL);

        if (vec->post_add == NULL || vec->pre_del == NULL ||
            vec->post_del == NULL)
                return (EINVAL);

        rw_enter(&mem_callback_rwlock, RW_WRITER);
        for (i = 0, found = 0; i < nmemcallbacks; i++) {
                if (mem_callbacks[i].vec == NULL && found == 0)
                        found = i + 1;
                if (mem_callbacks[i].vec == vec &&
                    mem_callbacks[i].arg == arg) {
#ifdef DEBUG
                        /* Catch this in DEBUG kernels. */
                        cmn_err(CE_WARN, "kphysm_setup_func_register"
                            "(0x%p, 0x%p) duplicate registration from 0x%p",
                            (void *)vec, arg, (void *)caller());
#endif /* DEBUG */
                        rw_exit(&mem_callback_rwlock);
                        return (EEXIST);
                }
        }
        if (found != 0) {
                i = found - 1;
        } else {
                ASSERT(nmemcallbacks < NMEMCALLBACKS);
                if (nmemcallbacks == NMEMCALLBACKS) {
                        rw_exit(&mem_callback_rwlock);
                        return (ENOMEM);
                }
                i = nmemcallbacks++;
        }
        mem_callbacks[i].vec = vec;
        mem_callbacks[i].arg = arg;
        rw_exit(&mem_callback_rwlock);
        return (0);
}

void
kphysm_setup_func_unregister(kphysm_setup_vector_t *vec, void *arg)
{
        uint_t i;

        rw_enter(&mem_callback_rwlock, RW_WRITER);
        for (i = 0; i < nmemcallbacks; i++) {
                if (mem_callbacks[i].vec == vec &&
                    mem_callbacks[i].arg == arg) {
                        mem_callbacks[i].vec = NULL;
                        mem_callbacks[i].arg = NULL;
                        if (i == (nmemcallbacks - 1))
                                nmemcallbacks--;
                        break;
                }
        }
        rw_exit(&mem_callback_rwlock);
}

static void
kphysm_setup_post_add(pgcnt_t delta_pages)
{
        uint_t i;

        rw_enter(&mem_callback_rwlock, RW_READER);
        for (i = 0; i < nmemcallbacks; i++) {
                if (mem_callbacks[i].vec != NULL) {
                        (*mem_callbacks[i].vec->post_add)
                            (mem_callbacks[i].arg, delta_pages);
                }
        }
        rw_exit(&mem_callback_rwlock);
}

/*
 * Note the locking between pre_del and post_del: The reader lock is held
 * between the two calls to stop the set of functions from changing.
 */

static int
kphysm_setup_pre_del(pgcnt_t delta_pages)
{
        uint_t i;
        int ret;
        int aret;

        ret = 0;
        rw_enter(&mem_callback_rwlock, RW_READER);
        for (i = 0; i < nmemcallbacks; i++) {
                if (mem_callbacks[i].vec != NULL) {
                        aret = (*mem_callbacks[i].vec->pre_del)
                            (mem_callbacks[i].arg, delta_pages);
                        ret |= aret;
                }
        }

        return (ret);
}

static void
kphysm_setup_post_del(pgcnt_t delta_pages, int cancelled)
{
        uint_t i;

        for (i = 0; i < nmemcallbacks; i++) {
                if (mem_callbacks[i].vec != NULL) {
                        (*mem_callbacks[i].vec->post_del)
                            (mem_callbacks[i].arg, delta_pages, cancelled);
                }
        }
        rw_exit(&mem_callback_rwlock);
}

static int
kphysm_split_memseg(
        pfn_t base,
        pgcnt_t npgs)
{
        struct memseg *seg;
        struct memseg **segpp;
        pgcnt_t size_low, size_high;
        struct memseg *seg_low, *seg_mid, *seg_high;

        /*
         * Lock the memsegs list against other updates now
         */
        memsegs_lock(1);

        /*
         * Find boot time memseg that wholly covers this area.
         */

        /* First find the memseg with page 'base' in it. */
        for (segpp = &memsegs; (seg = *segpp) != NULL;
            segpp = &((*segpp)->next)) {
                if (base >= seg->pages_base && base < seg->pages_end)
                        break;
        }
        if (seg == NULL) {
                memsegs_unlock(1);
                return (0);
        }
        if (memseg_includes_meta(seg)) {
                memsegs_unlock(1);
                return (0);
        }
        if ((base + npgs) > seg->pages_end) {
                memsegs_unlock(1);
                return (0);
        }

        /*
         * Work out the size of the two segments that will
         * surround the new segment, one for low address
         * and one for high.
         */
        ASSERT(base >= seg->pages_base);
        size_low = base - seg->pages_base;
        ASSERT(seg->pages_end >= (base + npgs));
        size_high = seg->pages_end - (base + npgs);

        /*
         * Sanity check.
         */
        if ((size_low + size_high) == 0) {
                memsegs_unlock(1);
                return (0);
        }

        /*
         * Allocate the new structures. The old memseg will not be freed
         * as there may be a reference to it.
         */
        seg_low = NULL;
        seg_high = NULL;

        if (size_low != 0)
                seg_low = memseg_alloc();

        seg_mid = memseg_alloc();

        if (size_high != 0)
                seg_high = memseg_alloc();

        /*
         * All allocation done now.
         */
        if (size_low != 0) {
                seg_low->pages = seg->pages;
                seg_low->epages = seg_low->pages + size_low;
                seg_low->pages_base = seg->pages_base;
                seg_low->pages_end = seg_low->pages_base + size_low;
                seg_low->next = seg_mid;
                seg_low->msegflags = seg->msegflags;
        }
        if (size_high != 0) {
                seg_high->pages = seg->epages - size_high;
                seg_high->epages = seg_high->pages + size_high;
                seg_high->pages_base = seg->pages_end - size_high;
                seg_high->pages_end = seg_high->pages_base + size_high;
                seg_high->next = seg->next;
                seg_high->msegflags = seg->msegflags;
        }

        seg_mid->pages = seg->pages + size_low;
        seg_mid->pages_base = seg->pages_base + size_low;
        seg_mid->epages = seg->epages - size_high;
        seg_mid->pages_end = seg->pages_end - size_high;
        seg_mid->next = (seg_high != NULL) ? seg_high : seg->next;
        seg_mid->msegflags = seg->msegflags;

        /*
         * Update hat_kpm specific info of all involved memsegs and
         * allow hat_kpm specific global chain updates.
         */
        hat_kpm_split_mseg_update(seg, segpp, seg_low, seg_mid, seg_high);

        /*
         * At this point we have two equivalent memseg sub-chains,
         * seg and seg_low/seg_mid/seg_high, which both chain on to
         * the same place in the global chain. By re-writing the pointer
         * in the previous element we switch atomically from using the old
         * (seg) to the new.
         */
        *segpp = (seg_low != NULL) ? seg_low : seg_mid;

        membar_enter();

        build_pfn_hash();
        memsegs_unlock(1);

        /*
         * We leave the old segment, 'seg', intact as there may be
         * references to it. Also, as the value of total_pages has not
         * changed and the memsegs list is effectively the same when
         * accessed via the old or the new pointer, we do not have to
         * cause pageout_scanner() to re-evaluate its hand pointers.
         *
         * We currently do not re-use or reclaim the page_t memory.
         * If we do, then this may have to change.
         */

        mutex_enter(&memseg_lists_lock);
        seg->lnext = memseg_edit_junk;
        memseg_edit_junk = seg;
        mutex_exit(&memseg_lists_lock);

        return (1);
}

/*
 * The sfmmu hat layer (e.g.) accesses some parts of the memseg
 * structure using physical addresses. Therefore a kmem_cache is
 * used with KMC_NOHASH to avoid page crossings within a memseg
 * structure. KMC_NOHASH requires that no external (outside of
 * slab) information is allowed. This, in turn, implies that the
 * cache's slabsize must be exactly a single page, since per-slab
 * information (e.g. the freelist for the slab) is kept at the
 * end of the slab, where it is easy to locate. Should be changed
 * when a more obvious kmem_cache interface/flag will become
 * available.
 */
void
mem_config_init()
{
        memseg_cache = kmem_cache_create("memseg_cache", sizeof (struct memseg),
            0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH);
}

struct memseg *
memseg_alloc()
{
        struct memseg *seg;

        seg = kmem_cache_alloc(memseg_cache, KM_SLEEP);
        bzero(seg, sizeof (struct memseg));

        return (seg);
}

/*
 * Return whether the page_t memory for this memseg
 * is included in the memseg itself.
 */
static int
memseg_includes_meta(struct memseg *seg)
{
        return (seg->msegflags & MEMSEG_META_INCL);
}

pfn_t
memseg_get_start(struct memseg *seg)
{
        pfn_t           pt_start;

        if (memseg_includes_meta(seg)) {
                pt_start = hat_getpfnum(kas.a_hat, (caddr_t)seg->pages);

                /* Meta data is required to be at the beginning */
                ASSERT(pt_start < seg->pages_base);
        } else
                pt_start = seg->pages_base;

        return (pt_start);
}

/*
 * Invalidate memseg pointers in cpu private vm data caches.
 */
static void
memseg_cpu_vm_flush()
{
        cpu_t *cp;
        vm_cpu_data_t *vc;

        mutex_enter(&cpu_lock);
        pause_cpus(NULL, NULL);

        cp = cpu_list;
        do {
                vc = cp->cpu_vm_data;
                vc->vc_pnum_memseg = NULL;
                vc->vc_pnext_memseg = NULL;

        } while ((cp = cp->cpu_next) != cpu_list);

        start_cpus();
        mutex_exit(&cpu_lock);
}