5285 pass in cpu_pause_func via pause_cpus

[unleashed.git] / usr / src / uts / sun4 / os / prom_subr.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/cmn_err.h>
#include <sys/mutex.h>
#include <sys/systm.h>
#include <sys/sysmacros.h>
#include <sys/machsystm.h>
#include <sys/archsystm.h>
#include <sys/x_call.h>
#include <sys/promif.h>
#include <sys/prom_isa.h>
#include <sys/privregs.h>
#include <sys/vmem.h>
#include <sys/atomic.h>
#include <sys/panic.h>
#include <sys/rwlock.h>
#include <sys/reboot.h>
#include <sys/kdi.h>
#include <sys/kdi_machimpl.h>

/*
 * We are called with a pointer to a cell-sized argument array.
 * The service name (the first element of the argument array) is
 * the name of the callback being invoked.  When called, we are
 * running on the firmwares trap table as a trusted subroutine
 * of the firmware.
 *
 * We define entry points to allow callback handlers to be dynamically
 * added and removed, to support obpsym, which is a separate module
 * and can be dynamically loaded and unloaded and registers its
 * callback handlers dynamically.
 *
 * Note: The actual callback handler we register, is the assembly lang.
 * glue, callback_handler, which takes care of switching from a 64
 * bit stack and environment to a 32 bit stack and environment, and
 * back again, if the callback handler returns. callback_handler calls
 * vx_handler to process the callback.
 */

static kmutex_t vx_cmd_lock;    /* protect vx_cmd table */

#define VX_CMD_MAX      10
#define ENDADDR(a)      &a[sizeof (a) / sizeof (a[0])]
#define vx_cmd_end      ((struct vx_cmd *)(ENDADDR(vx_cmd)))

static struct vx_cmd {
        char    *service;       /* Service name */
        int     take_tba;       /* If Non-zero we take over the tba */
        void    (*func)(cell_t *argument_array);
} vx_cmd[VX_CMD_MAX+1];

void
init_vx_handler(void)
{
        extern int callback_handler(cell_t *arg_array);

        /*
         * initialize the lock protecting additions and deletions from
         * the vx_cmd table.  At callback time we don't need to grab
         * this lock.  Callback handlers do not need to modify the
         * callback handler table.
         */
        mutex_init(&vx_cmd_lock, NULL, MUTEX_DEFAULT, NULL);

        /*
         * Tell OBP about our callback handler.
         */
        (void) prom_set_callback((void *)callback_handler);
}

/*
 * Add a kernel callback handler to the kernel's list.
 * The table is static, so if you add a callback handler, increase
 * the value of VX_CMD_MAX. Find the first empty slot and use it.
 */
void
add_vx_handler(char *name, int flag, void (*func)(cell_t *))
{
        struct vx_cmd *vp;

        mutex_enter(&vx_cmd_lock);
        for (vp = vx_cmd; vp < vx_cmd_end; vp++) {
                if (vp->service == NULL) {
                        vp->service = name;
                        vp->take_tba = flag;
                        vp->func = func;
                        mutex_exit(&vx_cmd_lock);
                        return;
                }
        }
        mutex_exit(&vx_cmd_lock);

#ifdef  DEBUG

        /*
         * There must be enough entries to handle all callback entries.
         * Increase VX_CMD_MAX if this happens. This shouldn't happen.
         */
        cmn_err(CE_PANIC, "add_vx_handler <%s>", name);
        /* NOTREACHED */

#else   /* DEBUG */

        cmn_err(CE_WARN, "add_vx_handler: Can't add callback hander <%s>",
            name);

#endif  /* DEBUG */

}

/*
 * Remove a vx_handler function -- find the name string in the table,
 * and clear it.
 */
void
remove_vx_handler(char *name)
{
        struct vx_cmd *vp;

        mutex_enter(&vx_cmd_lock);
        for (vp = vx_cmd; vp < vx_cmd_end; vp++) {
                if (vp->service == NULL)
                        continue;
                if (strcmp(vp->service, name) != 0)
                        continue;
                vp->service = 0;
                vp->take_tba = 0;
                vp->func = 0;
                mutex_exit(&vx_cmd_lock);
                return;
        }
        mutex_exit(&vx_cmd_lock);
        cmn_err(CE_WARN, "remove_vx_handler: <%s> not found", name);
}

int
vx_handler(cell_t *argument_array)
{
        char *name;
        struct vx_cmd *vp;
        void *old_tba;

        name = p1275_cell2ptr(*argument_array);

        for (vp = vx_cmd; vp < vx_cmd_end; vp++) {
                if (vp->service == (char *)0)
                        continue;
                if (strcmp(vp->service, name) != 0)
                        continue;
                if (vp->take_tba != 0)  {
                        reestablish_curthread();
                        if (tba_taken_over != 0)
                                old_tba = set_tba((void *)&trap_table);
                }
                vp->func(argument_array);
                if ((vp->take_tba != 0) && (tba_taken_over != 0))
                        (void) set_tba(old_tba);
                return (0);     /* Service name was known */
        }

        return (-1);            /* Service name unknown */
}

/*
 * PROM Locking Primitives
 *
 * These routines are called immediately before and immediately after calling
 * into the firmware.  The firmware is single-threaded and assumes that the
 * kernel will implement locking to prevent simultaneous service calls.  In
 * addition, some service calls (particularly character rendering) can be
 * slow, so we would like to sleep if we cannot acquire the lock to allow the
 * caller's CPU to continue to perform useful work in the interim.  Service
 * routines may also be called early in boot as part of slave CPU startup
 * when mutexes and cvs are not yet available (i.e. they are still running on
 * the prom's TLB handlers and cannot touch curthread).  Therefore, these
 * routines must reduce to a simple compare-and-swap spin lock when necessary.
 * Finally, kernel code may wish to acquire the firmware lock before executing
 * a block of code that includes service calls, so we also allow the firmware
 * lock to be acquired recursively by the owning CPU after disabling preemption.
 *
 * To meet these constraints, the lock itself is implemented as a compare-and-
 * swap spin lock on the global prom_cpu pointer.  We implement recursion by
 * atomically incrementing the integer prom_holdcnt after acquiring the lock.
 * If the current CPU is an "adult" (determined by testing cpu_m.mutex_ready),
 * we disable preemption before acquiring the lock and leave it disabled once
 * the lock is held.  The kern_postprom() routine then enables preemption if
 * we drop the lock and prom_holdcnt returns to zero.  If the current CPU is
 * an adult and the lock is held by another adult CPU, we can safely sleep
 * until the lock is released.  To do so, we acquire the adaptive prom_mutex
 * and then sleep on prom_cv.  Therefore, service routines must not be called
 * from above LOCK_LEVEL on any adult CPU.  Finally, if recursive entry is
 * attempted on an adult CPU, we must also verify that curthread matches the
 * saved prom_thread (the original owner) to ensure that low-level interrupt
 * threads do not step on other threads running on the same CPU.
 */

static cpu_t *volatile prom_cpu;
static kthread_t *volatile prom_thread;
static uint32_t prom_holdcnt;
static kmutex_t prom_mutex;
static kcondvar_t prom_cv;

/*
 * The debugger uses PROM services, and is thus unable to run if any of the
 * CPUs on the system are executing in the PROM at the time of debugger entry.
 * If a CPU is determined to be in the PROM when the debugger is entered,
 * prom_return_enter_debugger will be set, thus triggering a programmed debugger
 * entry when the given CPU returns from the PROM.  That CPU is then released by
 * the debugger, and is allowed to complete PROM-related work.
 */
int prom_exit_enter_debugger;

void
kern_preprom(void)
{
        for (;;) {
                /*
                 * Load the current CPU pointer and examine the mutex_ready bit.
                 * It doesn't matter if we are preempted here because we are
                 * only trying to determine if we are in the *set* of mutex
                 * ready CPUs.  We cannot disable preemption until we confirm
                 * that we are running on a CPU in this set, since a call to
                 * kpreempt_disable() requires access to curthread.
                 */
                processorid_t cpuid = getprocessorid();
                cpu_t *cp = cpu[cpuid];
                cpu_t *prcp;

                if (panicstr)
                        return; /* just return if we are currently panicking */

                if (CPU_IN_SET(cpu_ready_set, cpuid) && cp->cpu_m.mutex_ready) {
                        /*
                         * Disable premption, and reload the current CPU.  We
                         * can't move from a mutex_ready cpu to a non-ready cpu
                         * so we don't need to re-check cp->cpu_m.mutex_ready.
                         */
                        kpreempt_disable();
                        cp = CPU;
                        ASSERT(cp->cpu_m.mutex_ready);

                        /*
                         * Try the lock.  If we don't get the lock, re-enable
                         * preemption and see if we should sleep.  If we are
                         * already the lock holder, remove the effect of the
                         * previous kpreempt_disable() before returning since
                         * preemption was disabled by an earlier kern_preprom.
                         */
                        prcp = atomic_cas_ptr((void *)&prom_cpu, NULL, cp);
                        if (prcp == NULL ||
                            (prcp == cp && prom_thread == curthread)) {
                                if (prcp == cp)
                                        kpreempt_enable();
                                break;
                        }

                        kpreempt_enable();

                        /*
                         * We have to be very careful here since both prom_cpu
                         * and prcp->cpu_m.mutex_ready can be changed at any
                         * time by a non mutex_ready cpu holding the lock.
                         * If the owner is mutex_ready, holding prom_mutex
                         * prevents kern_postprom() from completing.  If the
                         * owner isn't mutex_ready, we only know it will clear
                         * prom_cpu before changing cpu_m.mutex_ready, so we
                         * issue a membar after checking mutex_ready and then
                         * re-verify that prom_cpu is still held by the same
                         * cpu before actually proceeding to cv_wait().
                         */
                        mutex_enter(&prom_mutex);
                        prcp = prom_cpu;
                        if (prcp != NULL && prcp->cpu_m.mutex_ready != 0) {
                                membar_consumer();
                                if (prcp == prom_cpu)
                                        cv_wait(&prom_cv, &prom_mutex);
                        }
                        mutex_exit(&prom_mutex);

                } else {
                        /*
                         * If we are not yet mutex_ready, just attempt to grab
                         * the lock.  If we get it or already hold it, break.
                         */
                        ASSERT(getpil() == PIL_MAX);
                        prcp = atomic_cas_ptr((void *)&prom_cpu, NULL, cp);
                        if (prcp == NULL || prcp == cp)
                                break;
                }
        }

        /*
         * We now hold the prom_cpu lock.  Increment the hold count by one
         * and assert our current state before returning to the caller.
         */
        atomic_inc_32(&prom_holdcnt);
        ASSERT(prom_holdcnt >= 1);
        prom_thread = curthread;
}

/*
 * Drop the prom lock if it is held by the current CPU.  If the lock is held
 * recursively, return without clearing prom_cpu.  If the hold count is now
 * zero, clear prom_cpu and cv_signal any waiting CPU.
 */
void
kern_postprom(void)
{
        processorid_t cpuid = getprocessorid();
        cpu_t *cp = cpu[cpuid];

        if (panicstr)
                return; /* do not modify lock further if we have panicked */

        if (prom_cpu != cp)
                panic("kern_postprom: not owner, cp=%p owner=%p",
                    (void *)cp, (void *)prom_cpu);

        if (prom_holdcnt == 0)
                panic("kern_postprom: prom_holdcnt == 0, owner=%p",
                    (void *)prom_cpu);

        if (atomic_dec_32_nv(&prom_holdcnt) != 0)
                return; /* prom lock is held recursively by this CPU */

        if ((boothowto & RB_DEBUG) && prom_exit_enter_debugger)
                kmdb_enter();

        prom_thread = NULL;
        membar_producer();

        prom_cpu = NULL;
        membar_producer();

        if (CPU_IN_SET(cpu_ready_set, cpuid) && cp->cpu_m.mutex_ready) {
                mutex_enter(&prom_mutex);
                cv_signal(&prom_cv);
                mutex_exit(&prom_mutex);
                kpreempt_enable();
        }
}

/*
 * If the frame buffer device is busy, briefly capture the other CPUs so that
 * another CPU executing code to manipulate the device does not execute at the
 * same time we are rendering characters.  Refer to the comments and code in
 * common/os/console.c for more information on these callbacks.
 *
 * Notice that we explicitly acquire the PROM lock using kern_preprom() prior
 * to idling other CPUs.  The idling mechanism will cross-trap the other CPUs
 * and have them spin at MAX(%pil, XCALL_PIL), so we must be sure that none of
 * them are holding the PROM lock before we idle them and then call into the
 * PROM routines that render characters to the frame buffer.
 */
int
console_enter(int busy)
{
        int s = 0;

        if (busy && panicstr == NULL) {
                kern_preprom();
                s = splhi();
                idle_other_cpus();
        }

        return (s);
}

void
console_exit(int busy, int spl)
{
        if (busy && panicstr == NULL) {
                resume_other_cpus();
                splx(spl);
                kern_postprom();
        }
}

/*
 * This routine is a special form of pause_cpus().  It ensures that
 * prom functions are callable while the cpus are paused.
 */
void
promsafe_pause_cpus(void)
{
        pause_cpus(NULL, NULL);

        /* If some other cpu is entering or is in the prom, spin */
        while (prom_cpu || mutex_owner(&prom_mutex)) {

                start_cpus();
                mutex_enter(&prom_mutex);

                /* Wait for other cpu to exit prom */
                while (prom_cpu)
                        cv_wait(&prom_cv, &prom_mutex);

                mutex_exit(&prom_mutex);
                pause_cpus(NULL, NULL);
        }

        /* At this point all cpus are paused and none are in the prom */
}

/*
 * This routine is a special form of xc_attention().  It ensures that
 * prom functions are callable while the cpus are at attention.
 */
void
promsafe_xc_attention(cpuset_t cpuset)
{
        xc_attention(cpuset);

        /* If some other cpu is entering or is in the prom, spin */
        while (prom_cpu || mutex_owner(&prom_mutex)) {

                xc_dismissed(cpuset);
                mutex_enter(&prom_mutex);

                /* Wait for other cpu to exit prom */
                while (prom_cpu)
                        cv_wait(&prom_cv, &prom_mutex);

                mutex_exit(&prom_mutex);
                xc_attention(cpuset);
        }

        /* At this point all cpus are paused and none are in the prom */
}


#if defined(PROM_32BIT_ADDRS)

#include <sys/promimpl.h>
#include <vm/seg_kmem.h>
#include <sys/kmem.h>
#include <sys/bootconf.h>

/*
 * These routines are only used to workaround "poor feature interaction"
 * in OBP.  See bug 4115680 for details.
 *
 * Many of the promif routines need to allocate temporary buffers
 * with 32-bit addresses to pass in/out of the CIF.  The lifetime
 * of the buffers is extremely short, they are allocated and freed
 * around the CIF call.  We use vmem_alloc() to cache 32-bit memory.
 *
 * Note the code in promplat_free() to prevent exhausting the 32 bit
 * heap during boot.
 */
static void *promplat_last_free = NULL;
static size_t promplat_last_size;
static vmem_t *promplat_arena;
static kmutex_t promplat_lock;  /* protect arena, last_free, and last_size */

void *
promplat_alloc(size_t size)
{

        mutex_enter(&promplat_lock);
        if (promplat_arena == NULL) {
                promplat_arena = vmem_create("promplat", NULL, 0, 8,
                    segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
        }
        mutex_exit(&promplat_lock);

        return (vmem_alloc(promplat_arena, size, VM_NOSLEEP));
}

/*
 * Delaying the free() of small allocations gets more mileage
 * from pages during boot, otherwise a cycle of allocate/free
 * calls could burn through available heap32 space too quickly.
 */
void
promplat_free(void *p, size_t size)
{
        void *p2 = NULL;
        size_t s2;

        /*
         * If VM is initialized, clean up any delayed free().
         */
        if (kvseg.s_base != 0 && promplat_last_free != NULL) {
                mutex_enter(&promplat_lock);
                p2 = promplat_last_free;
                s2 = promplat_last_size;
                promplat_last_free = NULL;
                promplat_last_size = 0;
                mutex_exit(&promplat_lock);
                if (p2 != NULL) {
                        vmem_free(promplat_arena, p2, s2);
                        p2 = NULL;
                }
        }

        /*
         * Do the free if VM is initialized or it's a large allocation.
         */
        if (kvseg.s_base != 0 || size >= PAGESIZE) {
                vmem_free(promplat_arena, p, size);
                return;
        }

        /*
         * Otherwise, do the last free request and delay this one.
         */
        mutex_enter(&promplat_lock);
        if (promplat_last_free != NULL) {
                p2 = promplat_last_free;
                s2 = promplat_last_size;
        }
        promplat_last_free = p;
        promplat_last_size = size;
        mutex_exit(&promplat_lock);

        if (p2 != NULL)
                vmem_free(promplat_arena, p2, s2);
}

void
promplat_bcopy(const void *src, void *dst, size_t count)
{
        bcopy(src, dst, count);
}

#endif /* PROM_32BIT_ADDRS */

static prom_generation_cookie_t prom_tree_gen;
static krwlock_t prom_tree_lock;

int
prom_tree_access(int (*callback)(void *arg, int has_changed), void *arg,
    prom_generation_cookie_t *ckp)
{
        int chg, rv;

        rw_enter(&prom_tree_lock, RW_READER);
        /*
         * If the tree has changed since the caller last accessed it
         * pass 1 as the second argument to the callback function,
         * otherwise 0.
         */
        if (ckp != NULL && *ckp != prom_tree_gen) {
                *ckp = prom_tree_gen;
                chg = 1;
        } else
                chg = 0;
        rv = callback(arg, chg);
        rw_exit(&prom_tree_lock);
        return (rv);
}

int
prom_tree_update(int (*callback)(void *arg), void *arg)
{
        int rv;

        rw_enter(&prom_tree_lock, RW_WRITER);
        prom_tree_gen++;
        rv = callback(arg);
        rw_exit(&prom_tree_lock);
        return (rv);
}