6514 AS_* lock macros simplification

[illumos-gate.git] / usr / src / uts / common / vm / hat_refmod.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * The following routines implement the hat layer's
 * recording of the referenced and modified bits.
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/debug.h>
#include <sys/kmem.h>

/*
 * Note, usage of cmn_err requires you not hold any hat layer locks.
 */
#include <sys/cmn_err.h>

#include <vm/as.h>
#include <vm/hat.h>

kmutex_t hat_statlock;          /* protects all hat statistics data */
struct hrmstat *hrm_memlist;    /* tracks memory alloced for hrm_blist blocks */
struct hrmstat **hrm_hashtab;   /* hash table for finding blocks quickly */
struct hrmstat *hrm_blist;
int hrm_blist_incr = HRM_BLIST_INCR;
int hrm_blist_lowater = HRM_BLIST_INCR/2;
int hrm_blist_num = 0;
int hrm_blist_total = 0;
int hrm_mlockinited = 0;
int hrm_allocfailmsg = 0;       /* print a message when allocations fail */
int hrm_allocfail = 0;

static struct hrmstat   *hrm_balloc(void);
static void     hrm_link(struct hrmstat *);
static void     hrm_setbits(struct hrmstat *, caddr_t, uint_t);
static void     hrm_hashout(struct hrmstat *);
static void     hrm_getblk(int);

#define hrm_hash(as, addr) \
        (HRM_HASHMASK & \
        (((uintptr_t)(addr) >> HRM_BASESHIFT) ^ ((uintptr_t)(as) >> 2)))

#define hrm_match(hrm, as, addr) \
        (((hrm)->hrm_as == (as) && \
        ((hrm)->hrm_base == ((uintptr_t)(addr) & HRM_BASEMASK))) ? 1 : 0)

/*
 * Called when an address space maps in more pages while stats are being
 * collected.
 */
/* ARGSUSED */
void
hat_resvstat(size_t chunk, struct as *as, caddr_t addr)
{
}

/*
 * Start the statistics gathering for an address space.
 * Return -1 if we can't do it, otherwise return an opaque
 * identifier to be used when querying for the gathered statistics.
 * The identifier is an unused bit in a_vbits.
 * Bit 0 is reserved for swsmon.
 */
int
hat_startstat(struct as *as)
{
        uint_t nbits;           /* number of bits */
        uint_t bn;              /* bit number */
        uint_t id;              /* new vbit, identifier */
        uint_t vbits;           /* used vbits of address space */
        size_t chunk;           /* mapped size for stats */

        /*
         * If the refmod saving memory allocator runs out, print
         * a warning message about how to fix it, see comment at
         * the beginning of hat_setstat.
         */
        if (hrm_allocfailmsg) {
                cmn_err(CE_WARN,
                    "hrm_balloc failures occured, increase hrm_blist_incr");
                hrm_allocfailmsg = 0;
        }

        /*
         * Verify that a buffer of statistics blocks exists
         * and allocate more, if needed.
         */

        chunk = hat_get_mapped_size(as->a_hat);
        chunk = (btop(chunk)/HRM_PAGES);
        if (chunk < HRM_BLIST_INCR)
                chunk = 0;

        hrm_getblk((int)chunk);

        /*
         * Find a unused id in the given address space.
         */
        hat_enter(as->a_hat);
        vbits = as->a_vbits;
        nbits = sizeof (as->a_vbits) * NBBY;
        for (bn = 1, id = 2; bn < (nbits - 1); bn++, id <<= 1)
                if ((id & vbits) == 0)
                        break;
        if (bn >= (nbits - 1)) {
                hat_exit(as->a_hat);
                return (-1);
        }
        as->a_vbits |= id;
        hat_exit(as->a_hat);
        (void) hat_stats_enable(as->a_hat);
        return (id);
}

/*
 * Record referenced and modified information for an address space.
 * Rmbits is a word containing the referenced bit in bit position 1
 * and the modified bit in bit position 0.
 *
 * For current informational uses, one can rerun any program using
 * this facility after modifying the hrm_blist_incr to be a larger
 * amount so that a larger buffer of blocks will be maintained.
 */
void
hat_setstat(struct as *as, caddr_t addr, size_t len, uint_t rmbits)
{
        struct hrmstat  *hrm;
        uint_t          vbits, newbits, nb;
        int             h;

        ASSERT(len == PAGESIZE);
        ASSERT((rmbits & ~(P_MOD|P_REF)) == 0);

        if (rmbits == 0)
                return;

        mutex_enter(&hat_statlock);

        /*
         * Search the hash list for the as and addr we are looking for
         * and set the ref and mod bits in every block that matches.
         */
        vbits = 0;
        h = hrm_hash(as, addr);
        for (hrm = hrm_hashtab[h]; hrm; hrm = hrm->hrm_hnext) {
                if (hrm_match(hrm, as, addr)) {
                        hrm_setbits(hrm, addr, rmbits);
                        vbits |= hrm->hrm_id;
                }
        }

        /*
         * If we didn't find a block for all of the enabled
         * vpages bits, then allocate and initialize a block
         * for each bit that was not found.
         */
        if (vbits != as->a_vbits) {
                newbits = (vbits ^ as->a_vbits) & as->a_vbits;
                while (newbits) {
                        if (ffs(newbits))
                                nb = 1 << (ffs(newbits)-1);
                        hrm = (struct hrmstat *)hrm_balloc();
                        if (hrm == NULL) {
                                hrm_allocfailmsg = 1;
                                hrm_allocfail++;
                                mutex_exit(&hat_statlock);
                                return;
                        }
                        hrm->hrm_as = as;
                        hrm->hrm_base = (uintptr_t)addr & HRM_BASEMASK;
                        hrm->hrm_id = nb;
                        hrm_link(hrm);
                        hrm_setbits(hrm, addr, rmbits);
                        newbits &= ~nb;
                }
        }
        mutex_exit(&hat_statlock);
}

/*
 * Free the resources used to maintain the referenced and modified
 * statistics for the virtual page view of an address space
 * identified by id.
 */
void
hat_freestat(struct as *as, int id)
{
        struct hrmstat *hrm;
        struct hrmstat *prev_ahrm;
        struct hrmstat *hrm_tmplist;
        struct hrmstat *hrm_next;

        hat_stats_disable(as->a_hat);   /* tell the hat layer to stop */
        hat_enter(as->a_hat);
        if (id == 0)
                as->a_vbits = 0;
        else
                as->a_vbits &= ~id;

        if ((hrm = as->a_hrm) == NULL) {
                hat_exit(as->a_hat);
                return;
        }
        hat_exit(as->a_hat);

        mutex_enter(&hat_statlock);

        for (prev_ahrm = NULL; hrm; hrm = hrm->hrm_anext) {
                if ((id == hrm->hrm_id) || (id == NULL)) {

                        hrm_hashout(hrm);
                        hrm->hrm_hnext = hrm_blist;
                        hrm_blist = hrm;
                        hrm_blist_num++;

                        if (prev_ahrm == NULL)
                                as->a_hrm = hrm->hrm_anext;
                        else
                                prev_ahrm->hrm_anext = hrm->hrm_anext;

                } else
                        prev_ahrm = hrm;
        }

        /*
         * If all statistics blocks are free,
         * return the memory to the system.
         */
        if (hrm_blist_num == hrm_blist_total) {
                /* zero the block list since we are giving back its memory */
                hrm_blist = NULL;
                hrm_blist_num = 0;
                hrm_blist_total = 0;
                hrm_tmplist = hrm_memlist;
                hrm_memlist = NULL;
        } else {
                hrm_tmplist = NULL;
        }

        mutex_exit(&hat_statlock);

        /*
         * If there are any hrmstat structures to be freed, this must only
         * be done after we've released hat_statlock.
         */
        while (hrm_tmplist != NULL) {
                hrm_next = hrm_tmplist->hrm_hnext;
                kmem_free(hrm_tmplist, hrm_tmplist->hrm_base);
                hrm_tmplist = hrm_next;
        }
}

/*
 * Grab memory for statistics gathering of the hat layer.
 */
static void
hrm_getblk(int chunk)
{
        struct hrmstat *hrm, *l;
        int i;
        int hrm_incr;

        mutex_enter(&hat_statlock);
        /*
         * XXX The whole private freelist management here really should be
         * overhauled.
         *
         * The freelist should have some knowledge of how much memory is
         * needed by a process and thus when hat_resvstat get's called, we can
         * increment the freelist needs for that process within this subsystem.
         * Thus there will be reservations for all processes which are being
         * watched which should be accurate, and consume less memory overall.
         *
         * For now, just make sure there's enough entries on the freelist to
         * handle the current chunk.
         */
        if ((hrm_blist == NULL) ||
            (hrm_blist_num <= hrm_blist_lowater) ||
            (chunk && (hrm_blist_num < chunk + hrm_blist_incr))) {
                mutex_exit(&hat_statlock);

                hrm_incr = chunk  + hrm_blist_incr;
                hrm = kmem_zalloc(sizeof (struct hrmstat) * hrm_incr, KM_SLEEP);
                hrm->hrm_base = sizeof (struct hrmstat) * hrm_incr;

                /*
                 * thread the allocated blocks onto a freelist
                 * using the first block to hold information for
                 * freeing them all later
                 */
                mutex_enter(&hat_statlock);
                hrm->hrm_hnext = hrm_memlist;
                hrm_memlist = hrm;

                hrm_blist_total += (hrm_incr - 1);
                for (i = 1; i < hrm_incr; i++) {
                        l = &hrm[i];
                        l->hrm_hnext = hrm_blist;
                        hrm_blist = l;
                        hrm_blist_num++;
                }
        }
        mutex_exit(&hat_statlock);
}

static void
hrm_hashin(struct hrmstat *hrm)
{
        int             h;

        ASSERT(MUTEX_HELD(&hat_statlock));
        h = hrm_hash(hrm->hrm_as, hrm->hrm_base);

        hrm->hrm_hnext = hrm_hashtab[h];
        hrm_hashtab[h] = hrm;
}

static void
hrm_hashout(struct hrmstat *hrm)
{
        struct hrmstat  *list, **prev_hrm;
        int             h;

        ASSERT(MUTEX_HELD(&hat_statlock));
        h = hrm_hash(hrm->hrm_as, hrm->hrm_base);
        list = hrm_hashtab[h];
        prev_hrm = &hrm_hashtab[h];

        while (list) {
                if (list == hrm) {
                        *prev_hrm = list->hrm_hnext;
                        return;
                }
                prev_hrm = &list->hrm_hnext;
                list = list->hrm_hnext;
        }
}


/*
 * Link a statistic block into an address space and also put it
 * on the hash list for future references.
 */
static void
hrm_link(struct hrmstat *hrm)
{
        struct as *as = hrm->hrm_as;

        ASSERT(MUTEX_HELD(&hat_statlock));
        hrm->hrm_anext = as->a_hrm;
        as->a_hrm = hrm;
        hrm_hashin(hrm);
}

/*
 * Allocate a block for statistics keeping.
 * Returns NULL if blocks are unavailable.
 */
static struct hrmstat *
hrm_balloc(void)
{
        struct hrmstat *hrm;

        ASSERT(MUTEX_HELD(&hat_statlock));

        hrm = hrm_blist;
        if (hrm != NULL) {
                hrm_blist = hrm->hrm_hnext;
                hrm_blist_num--;
                hrm->hrm_hnext = NULL;
        }
        return (hrm);
}

/*
 * Set the ref and mod bits for addr within statistics block hrm.
 */
static void
hrm_setbits(struct hrmstat *hrm, caddr_t addr, uint_t bits)
{
        uint_t po, bo, spb;
        uint_t nbits;

        po = ((uintptr_t)addr & HRM_BASEOFFSET) >> MMU_PAGESHIFT; /* pg off */
        bo = po / (NBBY / 2);                   /* which byte in bit array */
        spb = (3 - (po & 3)) * 2;               /* shift position within byte */
        nbits = bits << spb;                    /* bit mask */
        hrm->hrm_bits[bo] |= nbits;
}

/*
 * Return collected statistics about an address space.
 * If clearflag is set, atomically read and zero the bits.
 *
 * Fill in the data array supplied with the referenced and
 * modified bits collected for address range [addr ... addr + len]
 * in address space, as, uniquely identified by id.
 * The destination is a byte array.  We fill in three bits per byte:
 * referenced, modified, and hwmapped bits.
 * Kernel only interface, can't fault on destination data array.
 *
 */
void
hat_getstat(struct as *as, caddr_t addr, size_t len, uint_t id,
    caddr_t datap, int clearflag)
{
        size_t  np;             /* number of pages */
        caddr_t a;
        char    *dp;

        np = btop(len);
        bzero(datap, np);

        /* allocate enough statistics blocks to cover the len passed in */
        hrm_getblk(np / HRM_PAGES);

        hat_sync(as->a_hat, addr, len, clearflag);

        /* allocate more statistics blocks if needed */
        hrm_getblk(0);

        mutex_enter(&hat_statlock);
        if (hrm_hashtab == NULL) {
                /* can happen when victim process exits */
                mutex_exit(&hat_statlock);
                return;
        }
        dp = datap;
        a = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
        while (a < addr + len) {
                struct hrmstat  *hrm;
                size_t  n;              /* number of pages, temp */
                int     h;              /* hash index */
                uint_t  po;

                h = hrm_hash(as, a);
                n = (HRM_PAGES -
                    (((uintptr_t)a & HRM_PAGEMASK) >> MMU_PAGESHIFT));
                if (n > np)
                        n = np;
                po = ((uintptr_t)a & HRM_BASEOFFSET) >> MMU_PAGESHIFT;

                for (hrm = hrm_hashtab[h]; hrm; hrm = hrm->hrm_hnext) {
                        if (hrm->hrm_as == as &&
                            hrm->hrm_base == ((uintptr_t)a & HRM_BASEMASK) &&
                            id == hrm->hrm_id) {
                                int i, nr;
                                uint_t bo, spb;

                                /*
                                 * Extract leading unaligned bits.
                                 */
                                i = 0;
                                while (i < n && (po & 3)) {
                                        bo = po / (NBBY / 2);
                                        spb = (3 - (po & 3)) * 2;
                                        *dp++ |= (hrm->hrm_bits[bo] >> spb) & 3;
                                        if (clearflag)
                                                hrm->hrm_bits[bo] &= ~(3<<spb);
                                        po++;
                                        i++;
                                }
                                /*
                                 * Extract aligned bits.
                                 */
                                nr = n/4*4;
                                bo = po / (NBBY / 2);
                                while (i < nr) {
                                        int bits = hrm->hrm_bits[bo];
                                        *dp++ |= (bits >> 6) & 3;
                                        *dp++ |= (bits >> 4) & 3;
                                        *dp++ |= (bits >> 2) & 3;
                                        *dp++ |= (bits >> 0) & 3;
                                        if (clearflag)
                                                hrm->hrm_bits[bo] = 0;
                                        bo++;
                                        po += 4;
                                        i += 4;
                                }
                                /*
                                 * Extract trailing unaligned bits.
                                 */
                                while (i < n) {
                                        bo = po / (NBBY / 2);
                                        spb = (3 - (po & 3)) * 2;
                                        *dp++ |= (hrm->hrm_bits[bo] >> spb) & 3;
                                        if (clearflag)
                                                hrm->hrm_bits[bo] &= ~(3<<spb);
                                        po++;
                                        i++;
                                }

                                break;
                        }
                }
                if (hrm == NULL)
                        dp += n;
                np -= n;
                a += n * MMU_PAGESIZE;
        }
        mutex_exit(&hat_statlock);
}