if_iwm - Use chan list from ieee80211_scan_state for scan, not ic_channels.

[dragonfly.git] / sys / dev / disk / nvme / nvme_admin.c

/*
 * Copyright (c) 2016 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * Administration thread
 *
 * - Handles resetting, features, iteration of namespaces, and disk
 *   attachments.  Most admin operations are serialized by the admin thread.
 *
 * - Ioctls as well as any BIOs which require more sophisticated processing
 *   are handed to this thread as well.
 *
 * - Can freeze/resume other queues for various purposes.
 */

#include "nvme.h"

static void nvme_admin_thread(void *arg);
static int nvme_admin_state_identify_ctlr(nvme_softc_t *sc);
static int nvme_admin_state_make_queues(nvme_softc_t *sc);
static int nvme_admin_state_identify_ns(nvme_softc_t *sc);
static int nvme_admin_state_operating(nvme_softc_t *sc);
static int nvme_admin_state_failed(nvme_softc_t *sc);

/*
 * Start the admin thread and block until it says it is running.
 */
int
nvme_start_admin_thread(nvme_softc_t *sc)
{
        int error, intr_flags;

        lockinit(&sc->admin_lk, "admlk", 0, 0);
        lockinit(&sc->ioctl_lk, "nvioc", 0, 0);
        sc->admin_signal = 0;

        intr_flags = INTR_MPSAFE;
        if (sc->nirqs == 1) {
                /* This interrupt processes data CQs too */
                intr_flags |= INTR_HIFREQ;
        }

        error = bus_setup_intr(sc->dev, sc->irq[0], intr_flags,
                               nvme_intr, &sc->comqueues[0],
                               &sc->irq_handle[0], NULL);
        if (error) {
                device_printf(sc->dev, "unable to install interrupt\n");
                return error;
        }
        lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
        kthread_create(nvme_admin_thread, sc, &sc->admintd, "nvme_admin");
        while ((sc->admin_signal & ADMIN_SIG_RUNNING) == 0)
                lksleep(&sc->admin_signal, &sc->admin_lk, 0, "nvwbeg", 0);
        lockmgr(&sc->admin_lk, LK_RELEASE);

        return 0;
}

/*
 * Stop the admin thread and block until it says it is done.
 */
void
nvme_stop_admin_thread(nvme_softc_t *sc)
{
        uint32_t i;

        atomic_set_int(&sc->admin_signal, ADMIN_SIG_STOP);

        /*
         * We have to wait for the admin thread to finish its probe
         * before shutting it down.  Break out if the admin thread
         * never managed to even start.
         */
        lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
        while ((sc->admin_signal & ADMIN_SIG_PROBED) == 0) {
                if ((sc->admin_signal & ADMIN_SIG_RUNNING) == 0)
                        break;
                lksleep(&sc->admin_signal, &sc->admin_lk, 0, "nvwend", 0);
        }
        lockmgr(&sc->admin_lk, LK_RELEASE);

        /*
         * Disconnect our disks while the admin thread is still running,
         * ensuring that the poll works even if interrupts are broken.
         * Otherwise we could deadlock in the devfs core.
         */
        for (i = 0; i < NVME_MAX_NAMESPACES; ++i) {
                nvme_softns_t *nsc;

                if ((nsc = sc->nscary[i]) != NULL) {
                        nvme_disk_detach(nsc);

                        kfree(nsc, M_NVME);
                        sc->nscary[i] = NULL;
                }
        }

        /*
         * Ask the admin thread to shut-down.
         */
        lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
        wakeup(&sc->admin_signal);
        while (sc->admin_signal & ADMIN_SIG_RUNNING)
                lksleep(&sc->admin_signal, &sc->admin_lk, 0, "nvwend", 0);
        lockmgr(&sc->admin_lk, LK_RELEASE);
        if (sc->irq_handle[0]) {
                bus_teardown_intr(sc->dev, sc->irq[0], sc->irq_handle[0]);
                sc->irq_handle[0] = NULL;
        }
        lockuninit(&sc->ioctl_lk);
        lockuninit(&sc->admin_lk);

        /*
         * Thread might be running on another cpu, give it time to actually
         * exit before returning in case the caller is about to unload the
         * module.  Otherwise we don't need this.
         */
        nvme_os_sleep(1);
}

static
void
nvme_admin_thread(void *arg)
{
        nvme_softc_t *sc = arg;
        uint32_t i;

        lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
        atomic_set_int(&sc->admin_signal, ADMIN_SIG_RUNNING);
        wakeup(&sc->admin_signal);

        sc->admin_func = nvme_admin_state_identify_ctlr;

        while ((sc->admin_signal & ADMIN_SIG_STOP) == 0) {
                for (i = 0; i <= sc->niocomqs; ++i) {
                        nvme_comqueue_t *comq = &sc->comqueues[i];

                        if (comq->nqe == 0)     /* not configured */
                                continue;

                        lockmgr(&comq->lk, LK_EXCLUSIVE);
                        nvme_poll_completions(comq, &comq->lk);
                        lockmgr(&comq->lk, LK_RELEASE);
                }
                if (sc->admin_signal & ADMIN_SIG_REQUEUE) {
                        atomic_clear_int(&sc->admin_signal, ADMIN_SIG_REQUEUE);
                        nvme_disk_requeues(sc);
                }
                if (sc->admin_func(sc) == 0 &&
                    (sc->admin_signal & ADMIN_SIG_RUN_MASK) == 0) {
                        lksleep(&sc->admin_signal, &sc->admin_lk, 0,
                                "nvidle", hz);
                }
        }

        /*
         * Cleanup state.
         *
         * Note that we actually issue delete queue commands here.  The NVME
         * spec says that for a normal shutdown the I/O queues should be
         * deleted prior to issuing the shutdown in the CONFIG register.
         */
        for (i = 1; i <= sc->niosubqs; ++i) {
                nvme_delete_subqueue(sc, i);
                nvme_free_subqueue(sc, i);
        }
        for (i = 1; i <= sc->niocomqs; ++i) {
                nvme_delete_comqueue(sc, i);
                nvme_free_comqueue(sc, i);
        }

        /*
         * Signal that we are done.
         */
        atomic_clear_int(&sc->admin_signal, ADMIN_SIG_RUNNING);
        wakeup(&sc->admin_signal);
        lockmgr(&sc->admin_lk, LK_RELEASE);
}

/*
 * Identify the controller
 */
static
int
nvme_admin_state_identify_ctlr(nvme_softc_t *sc)
{
        nvme_request_t *req;
        nvme_ident_ctlr_data_t *rp;
        int status;
        uint64_t mempgsize;
        char serial[20+16];
        char model[40+16];

        /*
         * Identify Controller
         */
        mempgsize = NVME_CAP_MEMPG_MIN_GET(sc->cap);

        req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
        req->cmd.identify.cns = NVME_CNS_CTLR;
        req->cmd.identify.cntid = 0;
        bzero(req->info, sizeof(*req->info));
        nvme_submit_request(req);
        status = nvme_wait_request(req, hz);
        /* XXX handle status */

        sc->idctlr = req->info->idctlr;
        nvme_put_request(req);

        rp = &sc->idctlr;

        KKASSERT(sizeof(sc->idctlr.serialno) == 20);
        KKASSERT(sizeof(sc->idctlr.modelno) == 40);
        bzero(serial, sizeof(serial));
        bzero(model, sizeof(model));
        bcopy(rp->serialno, serial, sizeof(rp->serialno));
        bcopy(rp->modelno, model, sizeof(rp->modelno));
        string_cleanup(serial, 0);
        string_cleanup(model, 0);

        device_printf(sc->dev, "Model %s BaseSerial %s nscount=%d\n",
                      model, serial, rp->ns_count);

        sc->admin_func = nvme_admin_state_make_queues;

        return 1;
}

#define COMQFIXUP(msix, ncomqs) ((((msix) - 1) % ncomqs) + 1)

/*
 * Request and create the I/O queues.  Figure out CPU mapping optimizations.
 */
static
int
nvme_admin_state_make_queues(nvme_softc_t *sc)
{
        nvme_request_t *req;
        uint16_t niosubqs;
        uint16_t niocomqs;
        uint32_t i;
        uint16_t qno;
        int status;
        int error;

        /*
         * Calculate how many I/O queues (non-inclusive of admin queue)
         * we want to have, up to 65535.  dw0 in the response returns the
         * number of queues the controller gives us.  Submission and
         * Completion queues are specified separately.
         *
         * This driver runs optimally with 4 submission queues and one
         * completion queue per cpu (rdhipri, rdlopri, wrhipri, wrlopri),
         *
         * +1 for dumps                 XXX future
         * +1 for async events          XXX future
         */
        req = nvme_get_admin_request(sc, NVME_OP_SET_FEATURES);

        niosubqs = ncpus * 2 + 0;
        niocomqs = ncpus + 0;
        if (niosubqs > NVME_MAX_QUEUES)
                niosubqs = NVME_MAX_QUEUES;
        if (niocomqs > NVME_MAX_QUEUES)
                niocomqs = NVME_MAX_QUEUES;
        device_printf(sc->dev, "Request %u/%u queues, ", niosubqs, niocomqs);

        req->cmd.setfeat.flags = NVME_FID_NUMQUEUES;
        req->cmd.setfeat.numqs.nsqr = niosubqs - 1;     /* 0's based 0=1 */
        req->cmd.setfeat.numqs.ncqr = niocomqs - 1;     /* 0's based 0=1 */

        nvme_submit_request(req);

        /*
         * Get response and set our operations mode.
         */
        status = nvme_wait_request(req, hz);
        /* XXX handle status */

        if (status == 0) {
                sc->niosubqs = 1 + (req->res.setfeat.dw0 & 0xFFFFU);
                sc->niocomqs = 1 + ((req->res.setfeat.dw0 >> 16) & 0xFFFFU);
        } else {
                sc->niosubqs = 0;
                sc->niocomqs = 0;
        }
        kprintf("Returns %u/%u queues, ", sc->niosubqs, sc->niocomqs);

        nvme_put_request(req);

        sc->dumpqno = 0;
        sc->eventqno = 0;

        if (sc->niosubqs >= ncpus * 2 + 0 && sc->niocomqs >= ncpus + 0) {
                /*
                 * If we got all the queues we wanted do a full-bore setup of
                 * qmap[cpu][type].
                 *
                 * Remember that subq 0 / comq 0 is the admin queue.
                 */
                kprintf("optimal map\n");
                qno = 1;
                for (i = 0; i < ncpus; ++i) {
                        int cpuqno = COMQFIXUP(sc->cputovect[i], ncpus);

                        KKASSERT(cpuqno != 0);
                        sc->qmap[i][0] = qno + 0;
                        sc->qmap[i][1] = qno + 1;
                        sc->subqueues[qno + 0].comqid = cpuqno;
                        sc->subqueues[qno + 1].comqid = cpuqno;
                        qno += 2;
                }
                sc->niosubqs = ncpus * 2 + 0;
                sc->niocomqs = ncpus + 0;
        } else if (sc->niosubqs >= ncpus && sc->niocomqs >= ncpus) {
                /*
                 * We have enough to give each cpu its own submission
                 * and completion queue.
                 *
                 * leave dumpqno and eventqno set to the admin queue.
                 */
                kprintf("nominal map 1:1 cpu\n");
                for (i = 0; i < ncpus; ++i) {
                        qno = sc->cputovect[i];
                        KKASSERT(qno != 0);
                        sc->qmap[i][0] = qno;
                        sc->qmap[i][1] = qno;
                        sc->subqueues[qno].comqid = COMQFIXUP(qno, ncpus);
                }
                sc->niosubqs = ncpus;
                sc->niocomqs = ncpus;
        } else if (sc->niosubqs >= 2 && sc->niocomqs >= 2) {
                /*
                 * prioritize trying to distribute available queues to
                 * cpus, don't separate read and write.
                 *
                 * leave dumpqno and eventqno set to the admin queue.
                 */
                kprintf("rw-sep map (%d, %d)\n", sc->niosubqs, sc->niocomqs);
                for (i = 0; i < ncpus; ++i) {
                        int cpuqno = COMQFIXUP(sc->cputovect[i], sc->niocomqs);
                        int qno = COMQFIXUP((i + 1), sc->niosubqs);

                        KKASSERT(qno != 0);
                        sc->qmap[i][0] = qno;           /* read */
                        sc->qmap[i][1] = qno;           /* write */
                        sc->subqueues[qno].comqid = cpuqno;
                        /* do not increment qno */
                }
#if 0
                sc->niosubqs = 2;
                sc->niocomqs = 2;
#endif
        } else if (sc->niosubqs >= 2) {
                /*
                 * We have enough to have separate read and write queues.
                 */
                kprintf("basic map\n");
                qno = 1;
                for (i = 0; i < ncpus; ++i) {
                        int cpuqno = COMQFIXUP(sc->cputovect[i], 1);

                        KKASSERT(qno != 0);
                        sc->qmap[i][0] = qno + 0;       /* read */
                        sc->qmap[i][1] = qno + 1;       /* write */
                        if (i <= 0)
                                sc->subqueues[qno + 0].comqid = cpuqno;
                        if (i <= 1)
                                sc->subqueues[qno + 1].comqid = cpuqno;
                }
                sc->niosubqs = 2;
                sc->niocomqs = 1;
        } else {
                /*
                 * Minimal configuration, all cpus and I/O types use the
                 * same queue.  Sad day.
                 */
                kprintf("minimal map\n");
                sc->dumpqno = 0;
                sc->eventqno = 0;
                for (i = 0; i < ncpus; ++i) {
                        sc->qmap[i][0] = 1;
                        sc->qmap[i][1] = 1;
                }
                sc->subqueues[1].comqid = 1;
                sc->niosubqs = 1;
                sc->niocomqs = 1;
        }

        /*
         * Create all I/O submission and completion queues.  The I/O
         * queues start at 1 and are inclusive of niosubqs and niocomqs.
         *
         * NOTE: Completion queues must be created before submission queues.
         *       That is, the completion queue specified when creating a
         *       submission queue must already exist.
         */
        error = 0;
        for (i = 1; i <= sc->niocomqs; ++i) {
                error += nvme_alloc_comqueue(sc, i);
                if (error) {
                        device_printf(sc->dev, "Unable to allocate comqs\n");
                        break;
                }
                error += nvme_create_comqueue(sc, i);
        }
        for (i = 1; i <= sc->niosubqs; ++i) {
                error += nvme_alloc_subqueue(sc, i);
                if (error) {
                        device_printf(sc->dev, "Unable to allocate subqs\n");
                        break;
                }
                error += nvme_create_subqueue(sc, i);
        }

        if (error) {
                device_printf(sc->dev, "Failed to initialize device!\n");
                sc->admin_func = nvme_admin_state_failed;
        } else {
                sc->admin_func = nvme_admin_state_identify_ns;
        }

        /*
         * Basically interrupt coalescing is worthless if we care about
         * performance, at least on the Intel 750.  Setting the threshold
         * has no effect if time is set to 0.  The smallest time that can
         * be set is a value of 1 (== 100uS), which is much too long.  That
         * is only 10,000 interrupts/sec/cpu and on the Intel 750 it totally
         * destroys sequential performance.
         */
        req = nvme_get_admin_request(sc, NVME_OP_SET_FEATURES);

        device_printf(sc->dev, "Interrupt Coalesce: 100uS / 4 qentries\n");

        req->cmd.setfeat.flags = NVME_FID_INTCOALESCE;
        req->cmd.setfeat.intcoal.thr = 0;
        req->cmd.setfeat.intcoal.time = 0;

        nvme_submit_request(req);
        status = nvme_wait_request(req, hz);
        if (status) {
                device_printf(sc->dev,
                              "Interrupt coalesce failed status=%d\n",
                              status);
        }
        nvme_put_request(req);

        return 1;
}

/*
 * Identify available namespaces, iterate, and attach to disks.
 */
static
int
nvme_admin_state_identify_ns(nvme_softc_t *sc)
{
        nvme_request_t *req;
        nvme_ident_ns_list_t *rp;
        int status;
        uint32_t i;
        uint32_t j;

        if (bootverbose) {
                if (sc->idctlr.admin_cap & NVME_ADMIN_NSMANAGE)
                        device_printf(sc->dev,
                                      "Namespace management supported\n");
                else
                        device_printf(sc->dev,
                                      "Namespace management not supported\n");
        }
#if 0
        /*
         * Identify Controllers         TODO TODO TODO
         */
        if (sc->idctlr.admin_cap & NVME_ADMIN_NSMANAGE) {
                req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
                req->cmd.identify.cns = NVME_CNS_ANY_CTLR_LIST;
                req->cmd.identify.cntid = 0;
                bzero(req->info, sizeof(*req->info));
                nvme_submit_request(req);
                status = nvme_wait_request(req, hz);
                kprintf("nsquery status %08x\n", status);

#if 0
                for (i = 0; i < req->info->ctlrlist.idcount; ++i) {
                        kprintf("CTLR %04x\n", req->info->ctlrlist.ctlrids[i]);
                }
#endif
                nvme_put_request(req);
        }
#endif

        rp = kmalloc(sizeof(*rp), M_NVME, M_WAITOK | M_ZERO);
        if (sc->idctlr.admin_cap & NVME_ADMIN_NSMANAGE) {
                /*
                 * Namespace management supported, query active namespaces.
                 */
                req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
                req->cmd.identify.cns = NVME_CNS_ACT_NSLIST;
                req->cmd.identify.cntid = 0;
                bzero(req->info, sizeof(*req->info));
                nvme_submit_request(req);
                status = nvme_wait_request(req, hz);
                kprintf("nsquery status %08x\n", status);
                /* XXX handle status */

                cpu_lfence();
                *rp = req->info->nslist;
                nvme_put_request(req);
        } else {
                /*
                 * Namespace management not supported, assume nsids 1..N.
                 */
                for (i = 1; i <= sc->idctlr.ns_count && i <= 1024; ++i)
                        rp->nsids[i-1] = i;
        }

        /*
         * Identify each Namespace
         */
        for (i = 0; i < 1024; ++i) {
                nvme_softns_t *nsc;
                nvme_lba_fmt_data_t *lbafmt;

                if (rp->nsids[i] == 0)
                        continue;
                req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
                req->cmd.identify.cns = NVME_CNS_ACT_NS;
                req->cmd.identify.cntid = 0;
                req->cmd.identify.head.nsid = rp->nsids[i];
                bzero(req->info, sizeof(*req->info));
                nvme_submit_request(req);
                status = nvme_wait_request(req, hz);
                if (status != 0) {
                        kprintf("NS FAILED %08x\n", status);
                        continue;
                }

                for (j = 0; j < NVME_MAX_NAMESPACES; ++j) {
                        if (sc->nscary[j] &&
                            sc->nscary[j]->nsid == rp->nsids[i])
                                break;
                }
                if (j == NVME_MAX_NAMESPACES) {
                        j = i;
                        if (sc->nscary[j] != NULL) {
                                for (j = NVME_MAX_NAMESPACES - 1; j >= 0; --j) {
                                        if (sc->nscary[j] == NULL)
                                                break;
                                }
                        }
                }
                if (j < 0) {
                        device_printf(sc->dev, "not enough room in nscary for "
                                               "namespace %08x\n", rp->nsids[i]);
                        nvme_put_request(req);
                        continue;
                }
                nsc = sc->nscary[j];
                if (nsc == NULL) {
                        nsc = kmalloc(sizeof(*nsc), M_NVME, M_WAITOK | M_ZERO);
                        nsc->unit = nvme_alloc_disk_unit();
                        sc->nscary[j] = nsc;
                }
                if (sc->nscmax <= j)
                        sc->nscmax = j + 1;
                nsc->sc = sc;
                nsc->nsid = rp->nsids[i];
                nsc->state = NVME_NSC_STATE_UNATTACHED;
                nsc->idns = req->info->idns;
                bioq_init(&nsc->bioq);
                lockinit(&nsc->lk, "nvnsc", 0, 0);

                nvme_put_request(req);

                j = NVME_FLBAS_SEL_GET(nsc->idns.flbas);
                lbafmt = &nsc->idns.lba_fmt[j];
                nsc->blksize = 1 << lbafmt->sect_size;

                /*
                 * Attach the namespace
                 */
                nvme_disk_attach(nsc);
        }
        kfree(rp, M_NVME);

        sc->admin_func = nvme_admin_state_operating;
        return 1;
}

static
int
nvme_admin_state_operating(nvme_softc_t *sc)
{
        if ((sc->admin_signal & ADMIN_SIG_PROBED) == 0) {
                atomic_set_int(&sc->admin_signal, ADMIN_SIG_PROBED);
                wakeup(&sc->admin_signal);
        }

        return 0;
}

static
int
nvme_admin_state_failed(nvme_softc_t *sc)
{
        if ((sc->admin_signal & ADMIN_SIG_PROBED) == 0) {
                atomic_set_int(&sc->admin_signal, ADMIN_SIG_PROBED);
                wakeup(&sc->admin_signal);
        }

        return 0;
}