Merge commit 'b1e7e97d3b60469b243b3b2e22c7d8cbd11c7c90'

[unleashed.git] / kernel / os / ftrace.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident   "%Z%%M% %I%     %E% SMI"

#include <sys/types.h>
#include <sys/kmem.h>
#include <sys/mutex.h>
#include <sys/cpuvar.h>
#include <sys/cmn_err.h>
#include <sys/systm.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/debug.h>
#include <sys/param.h>
#include <sys/atomic.h>
#include <sys/ftrace.h>

/*
 * Tunable parameters:
 *
 * ftrace_atboot        - whether to start fast tracing at boot.
 * ftrace_nent          - size of the per-CPU event ring buffer.
 */
int ftrace_atboot = 0;
int ftrace_nent = FTRACE_NENT;

/*
 * Global Tracing State:
 *
 *                NOTREADY(=0)
 *                  |
 *            ftrace_init()
 *                  |
 *                  |
 *                  v
 *      +-------->READY-------+
 *      |                     |
 *  ftrace_stop()         ftrace_start()
 *      |                     |
 *      +---(ENABLED|READY)<--+
 *
 * During boot, ftrace_init() is called and the state becomes
 * READY. If ftrace_atboot is set, ftrace_start() is called at
 * this time.
 *
 * If FTRACE_READY is set, then tracing can be enabled.
 * If FTRACE_ENABLED is set, tracing is enabled on the set of CPUs
 *   which are currently FTRACE_READY.
 */
static int ftrace_state = 0;

/*
 * Per-CPU Tracing State:
 *
 *     +-----------------READY<--------------+
 *     |                 ^   |               |
 *     |                 | ftrace_cpu_fini() |
 *     |                 |   |               |
 *     |   ftrace_cpu_init() |               |
 *     |                 |   v     ftrace_cpu_stop()
 *     |              NOTREADY(=0)           |
 *     |                   ^                 |
 * ftrace_cpu_start()      |                 |
 *     |              ftrace_cpu_fini()      |
 *     |                   |                 |
 *     +----------->(ENABLED|READY)----------+
 *
 */

/*
 * Locking :
 *
 * Trace context code does not use any lock. There is a per-cpu circular trace
 * buffer that has a head, a tail and a current pointer. Each record of this
 * buffer is of equal length. Before doing anything, trace context code checks
 * the per-cpu ENABLED bit. Trace buffer is allocated in non-trace context and
 * it sets this bit only after allocating and setting up the buffer. So trace
 * context code can't access the buffer till it is set up completely. The
 * buffer is freed also in non-trace context. The code that frees the buffer is
 * executed only after the corresponding cpu is powered off. So when this
 * happens, no trace context code can be running on it. We only need to make
 * sure that trace context code is not preempted from the cpu in the middle of
 * accessing the trace buffer. This can be achieved simply by disabling
 * interrupts temporarily. This approach makes the least assumption about the
 * state of the callers of tracing functions.
 *
 * A single global lock, ftrace_lock protects assignments to all global and
 * per-cpu trace variables. It does not protect reading of those in some cases.
 *
 * More specifically, it protects assignments to:
 *
 *   ftrace_state
 *   cpu[N]->cpu_ftrace.ftd_state
 *   cpu[N]->cpu_ftrace.ftd_first
 *   cpu[N]->cpu_ftrace.ftd_last
 *
 * Does _not_ protect reading of cpu[N]->cpu_ftrace.ftd_state
 * Does _not_ protect cpu[N]->cpu_ftrace.ftd_cur
 * Does _not_ protect reading of ftrace_state
 */
static kmutex_t ftrace_lock;

/*
 * Check whether a CPU is installed.
 */
#define IS_CPU(i) (cpu[i] != NULL)

static void
ftrace_cpu_init(int cpuid)
{
        ftrace_data_t *ftd;

        /*
         * This can be called with "cpu[cpuid]->cpu_flags & CPU_EXISTS"
         * being false - e.g. when a CPU is DR'ed in.
         */
        ASSERT(MUTEX_HELD(&ftrace_lock));
        ASSERT(IS_CPU(cpuid));

        ftd = &cpu[cpuid]->cpu_ftrace;
        if (ftd->ftd_state & FTRACE_READY)
                return;

        /*
         * We don't allocate the buffers until the first time
         *   ftrace_cpu_start() is called, so that they're not
         *   allocated if ftrace is never enabled.
         */
        ftd->ftd_state |= FTRACE_READY;
        ASSERT(!(ftd->ftd_state & FTRACE_ENABLED));
}

/*
 * Only called from cpu_unconfigure() (and cpu_configure() on error).
 * At this point, cpu[cpuid] is about to be freed and NULLed out,
 *   so we'd better clean up after ourselves.
 */
static void
ftrace_cpu_fini(int cpuid)
{
        ftrace_data_t *ftd;

        ASSERT(MUTEX_HELD(&ftrace_lock));
        ASSERT(IS_CPU(cpuid));
        ASSERT((cpu[cpuid]->cpu_flags & CPU_POWEROFF) != 0);

        ftd = &cpu[cpuid]->cpu_ftrace;
        if (!(ftd->ftd_state & FTRACE_READY))
                return;

        /*
         * This cpu is powered off and no code can be executing on it. So
         * we can simply finish our cleanup. There is no need for a xcall
         * to make sure that this cpu is out of trace context.
         *
         * The cpu structure will be cleared soon. But, for the sake of
         * debugging, clear our pointers and state.
         */
        if (ftd->ftd_first != NULL) {
                kmem_free(ftd->ftd_first,
                    ftrace_nent * sizeof (ftrace_record_t));
        }
        bzero(ftd, sizeof (ftrace_data_t));
}

static void
ftrace_cpu_start(int cpuid)
{
        ftrace_data_t *ftd;

        ASSERT(MUTEX_HELD(&ftrace_lock));
        ASSERT(IS_CPU(cpuid));
        ASSERT(ftrace_state & FTRACE_ENABLED);

        ftd = &cpu[cpuid]->cpu_ftrace;
        if (ftd->ftd_state & FTRACE_READY) {
                if (ftd->ftd_first == NULL) {
                        ftrace_record_t *ptrs;

                        mutex_exit(&ftrace_lock);
                        ptrs = kmem_zalloc(ftrace_nent *
                            sizeof (ftrace_record_t), KM_SLEEP);
                        mutex_enter(&ftrace_lock);
                        if (ftd->ftd_first != NULL) {
                                /*
                                 * Someone else beat us to it. The winner will
                                 * set up the pointers and the state.
                                 */
                                kmem_free(ptrs,
                                    ftrace_nent * sizeof (ftrace_record_t));
                                return;
                        }

                        ftd->ftd_first = ptrs;
                        ftd->ftd_last = ptrs + (ftrace_nent - 1);
                        ftd->ftd_cur = ptrs;
                        membar_producer();
                }
                ftd->ftd_state |= FTRACE_ENABLED;
        }
}

static void
ftrace_cpu_stop(int cpuid)
{
        ASSERT(MUTEX_HELD(&ftrace_lock));
        ASSERT(IS_CPU(cpuid));
        cpu[cpuid]->cpu_ftrace.ftd_state &= ~(FTRACE_ENABLED);
}

/*
 * Hook for DR.
 */
/*ARGSUSED*/
int
ftrace_cpu_setup(cpu_setup_t what, int id, void *arg)
{
        if (!(ftrace_state & FTRACE_READY))
                return (0);

        switch (what) {
        case CPU_CONFIG:
                mutex_enter(&ftrace_lock);
                ftrace_cpu_init(id);
                if (ftrace_state & FTRACE_ENABLED)
                        ftrace_cpu_start(id);
                mutex_exit(&ftrace_lock);
                break;

        case CPU_UNCONFIG:
                mutex_enter(&ftrace_lock);
                ftrace_cpu_fini(id);
                mutex_exit(&ftrace_lock);
                break;

        default:
                break;
        }
        return (0);
}

void
ftrace_init(void)
{
        int i;

        ASSERT(!(ftrace_state & FTRACE_READY));
        mutex_init(&ftrace_lock, NULL, MUTEX_DEFAULT, NULL);

        mutex_enter(&ftrace_lock);
        for (i = 0; i < NCPU; i++) {
                if (IS_CPU(i)) {
                        /* should have been kmem_zalloc()'ed */
                        ASSERT(cpu[i]->cpu_ftrace.ftd_state == 0);
                        ASSERT(cpu[i]->cpu_ftrace.ftd_first == NULL);
                        ASSERT(cpu[i]->cpu_ftrace.ftd_last == NULL);
                        ASSERT(cpu[i]->cpu_ftrace.ftd_cur == NULL);
                }
        }

        if (ftrace_nent < 1) {
                mutex_exit(&ftrace_lock);
                return;
        }

        for (i = 0; i < NCPU; i++)
                if (IS_CPU(i))
                        ftrace_cpu_init(i);

        ftrace_state |= FTRACE_READY;
        mutex_enter(&cpu_lock);
        register_cpu_setup_func(ftrace_cpu_setup, NULL);
        mutex_exit(&cpu_lock);
        mutex_exit(&ftrace_lock);

        if (ftrace_atboot)
                (void) ftrace_start();
}

/*
 * Called from uadmin ioctl, or via mp_init_table[] during boot.
 */
int
ftrace_start(void)
{
        int i, was_enabled = 0;

        if (ftrace_state & FTRACE_READY) {
                mutex_enter(&ftrace_lock);
                was_enabled = ((ftrace_state & FTRACE_ENABLED) != 0);
                ftrace_state |= FTRACE_ENABLED;
                for (i = 0; i < NCPU; i++)
                        if (IS_CPU(i))
                                ftrace_cpu_start(i);
                mutex_exit(&ftrace_lock);
        }

        return (was_enabled);
}

/*
 * Called from uadmin ioctl, to stop tracing.
 */
int
ftrace_stop(void)
{
        int i, was_enabled = 0;

        if (ftrace_state & FTRACE_READY) {
                mutex_enter(&ftrace_lock);
                if (ftrace_state & FTRACE_ENABLED) {
                        was_enabled = 1;
                        for (i = 0; i < NCPU; i++)
                                if (IS_CPU(i))
                                        ftrace_cpu_stop(i);
                        ftrace_state &= ~(FTRACE_ENABLED);
                }
                mutex_exit(&ftrace_lock);
        }
        return (was_enabled);
}

/*
 * ftrace_X() functions are called from trace context. All callers of ftrace_X()
 * tests FTRACE_ENABLED first. Although this is not very accurate, it keeps the
 * overhead very low when tracing is not enabled.
 *
 * gethrtime_unscaled() appears to be safe to be called in trace context. As an
 * added precaution, we call these before we disable interrupts on this cpu.
 */

void
ftrace_0(char *str, caddr_t caller)
{
        ftrace_record_t *r;
        struct cpu *cp;
        ftrace_data_t *ftd;
        ftrace_icookie_t cookie;
        hrtime_t  timestamp;

        timestamp = gethrtime_unscaled();

        cookie = ftrace_interrupt_disable();

        cp = CPU;
        ftd = &cp->cpu_ftrace;

        if (!(ftd->ftd_state & FTRACE_ENABLED)) {
                ftrace_interrupt_enable(cookie);
                return;
        }

        r = ftd->ftd_cur;
        r->ftr_event = str;
        r->ftr_thread = curthread;
        r->ftr_tick = timestamp;
        r->ftr_caller = caller;

        if (r++ == ftd->ftd_last)
                r = ftd->ftd_first;
        ftd->ftd_cur = r;

        ftrace_interrupt_enable(cookie);
}

void
ftrace_1(char *str, ulong_t arg1, caddr_t caller)
{
        ftrace_record_t *r;
        struct cpu *cp;
        ftrace_data_t *ftd;
        ftrace_icookie_t cookie;
        hrtime_t  timestamp;

        timestamp = gethrtime_unscaled();

        cookie = ftrace_interrupt_disable();

        cp = CPU;
        ftd = &cp->cpu_ftrace;

        if (!(ftd->ftd_state & FTRACE_ENABLED)) {
                ftrace_interrupt_enable(cookie);
                return;
        }

        r = ftd->ftd_cur;
        r->ftr_event = str;
        r->ftr_thread = curthread;
        r->ftr_tick = timestamp;
        r->ftr_caller = caller;
        r->ftr_data1 = arg1;

        if (r++ == ftd->ftd_last)
                r = ftd->ftd_first;
        ftd->ftd_cur = r;

        ftrace_interrupt_enable(cookie);
}

void
ftrace_2(char *str, ulong_t arg1, ulong_t arg2, caddr_t caller)
{
        ftrace_record_t *r;
        struct cpu *cp;
        ftrace_data_t *ftd;
        ftrace_icookie_t cookie;
        hrtime_t  timestamp;

        timestamp = gethrtime_unscaled();

        cookie = ftrace_interrupt_disable();

        cp = CPU;
        ftd = &cp->cpu_ftrace;

        if (!(ftd->ftd_state & FTRACE_ENABLED)) {
                ftrace_interrupt_enable(cookie);
                return;
        }

        r = ftd->ftd_cur;
        r->ftr_event = str;
        r->ftr_thread = curthread;
        r->ftr_tick = timestamp;
        r->ftr_caller = caller;
        r->ftr_data1 = arg1;
        r->ftr_data2 = arg2;

        if (r++ == ftd->ftd_last)
                r = ftd->ftd_first;
        ftd->ftd_cur = r;

        ftrace_interrupt_enable(cookie);
}

void
ftrace_3(char *str, ulong_t arg1, ulong_t arg2, ulong_t arg3, caddr_t caller)
{
        ftrace_record_t *r;
        struct cpu *cp;
        ftrace_data_t *ftd;
        ftrace_icookie_t cookie;
        hrtime_t  timestamp;

        timestamp = gethrtime_unscaled();

        cookie = ftrace_interrupt_disable();

        cp = CPU;
        ftd = &cp->cpu_ftrace;

        if (!(ftd->ftd_state & FTRACE_ENABLED)) {
                ftrace_interrupt_enable(cookie);
                return;
        }

        r = ftd->ftd_cur;
        r->ftr_event = str;
        r->ftr_thread = curthread;
        r->ftr_tick = timestamp;
        r->ftr_caller = caller;
        r->ftr_data1 = arg1;
        r->ftr_data2 = arg2;
        r->ftr_data3 = arg3;

        if (r++ == ftd->ftd_last)
                r = ftd->ftd_first;
        ftd->ftd_cur = r;

        ftrace_interrupt_enable(cookie);
}

void
ftrace_3_notick(char *str, ulong_t arg1, ulong_t arg2,
    ulong_t arg3, caddr_t caller)
{
        ftrace_record_t *r;
        struct cpu *cp;
        ftrace_data_t *ftd;
        ftrace_icookie_t cookie;

        cookie = ftrace_interrupt_disable();

        cp = CPU;
        ftd = &cp->cpu_ftrace;

        if (!(ftd->ftd_state & FTRACE_ENABLED)) {
                ftrace_interrupt_enable(cookie);
                return;
        }

        r = ftd->ftd_cur;
        r->ftr_event = str;
        r->ftr_thread = curthread;
        r->ftr_tick = 0;
        r->ftr_caller = caller;
        r->ftr_data1 = arg1;
        r->ftr_data2 = arg2;
        r->ftr_data3 = arg3;

        if (r++ == ftd->ftd_last)
                r = ftd->ftd_first;
        ftd->ftd_cur = r;

        ftrace_interrupt_enable(cookie);
}