block/nvme.c

   1 /*
   2  * NVMe block driver based on vfio
   3  *
   4  * Copyright 2016 - 2018 Red Hat, Inc.
   5  *
   6  * Authors:
   7  *   Fam Zheng <famz@redhat.com>
   8  *   Paolo Bonzini <pbonzini@redhat.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  */
  13
  14 #include "qemu/osdep.h"
  15 #include <linux/vfio.h>
  16 #include "qapi/error.h"
  17 #include "qapi/qmp/qdict.h"
  18 #include "qapi/qmp/qstring.h"
  19 #include "qemu/error-report.h"
  20 #include "qemu/main-loop.h"
  21 #include "qemu/module.h"
  22 #include "qemu/cutils.h"
  23 #include "qemu/option.h"
  24 #include "qemu/vfio-helpers.h"
  25 #include "block/block_int.h"
  26 #include "sysemu/replay.h"
  27 #include "trace.h"
  28
  29 #include "block/nvme.h"
  30
  31 #define NVME_SQ_ENTRY_BYTES 64
  32 #define NVME_CQ_ENTRY_BYTES 16
  33 #define NVME_QUEUE_SIZE 128
  34 #define NVME_DOORBELL_SIZE 4096
  35
  36 /*
  37  * We have to leave one slot empty as that is the full queue case where
  38  * head == tail + 1.
  39  */
  40 #define NVME_NUM_REQS (NVME_QUEUE_SIZE - 1)
  41
  42 typedef struct BDRVNVMeState BDRVNVMeState;
  43
  44 typedef struct {
  45     int32_t  head, tail;
  46     uint8_t  *queue;
  47     uint64_t iova;
  48     /* Hardware MMIO register */
  49     volatile uint32_t *doorbell;
  50 } NVMeQueue;
  51
  52 typedef struct {
  53     BlockCompletionFunc *cb;
  54     void *opaque;
  55     int cid;
  56     void *prp_list_page;
  57     uint64_t prp_list_iova;
  58     int free_req_next; /* q->reqs[] index of next free req */
  59 } NVMeRequest;
  60
  61 typedef struct {
  62     QemuMutex   lock;
  63
  64     /* Read from I/O code path, initialized under BQL */
  65     BDRVNVMeState   *s;
  66     int             index;
  67
  68     /* Fields protected by BQL */
  69     uint8_t     *prp_list_pages;
  70
  71     /* Fields protected by @lock */
  72     CoQueue     free_req_queue;
  73     NVMeQueue   sq, cq;
  74     int         cq_phase;
  75     int         free_req_head;
  76     NVMeRequest reqs[NVME_NUM_REQS];
  77     int         need_kick;
  78     int         inflight;
  79
  80     /* Thread-safe, no lock necessary */
  81     QEMUBH      *completion_bh;
  82 } NVMeQueuePair;
  83
  84 #define INDEX_ADMIN     0
  85 #define INDEX_IO(n)     (1 + n)
  86
  87 /* This driver shares a single MSIX IRQ for the admin and I/O queues */
  88 enum {
  89     MSIX_SHARED_IRQ_IDX = 0,
  90     MSIX_IRQ_COUNT = 1
  91 };
  92
  93 struct BDRVNVMeState {
  94     AioContext *aio_context;
  95     QEMUVFIOState *vfio;
  96     /* Memory mapped registers */
  97     volatile struct {
  98         uint32_t sq_tail;
  99         uint32_t cq_head;
 100     } *doorbells;
 101     /* The submission/completion queue pairs.
 102      * [0]: admin queue.
 103      * [1..]: io queues.
 104      */
 105     NVMeQueuePair **queues;
 106     int nr_queues;
 107     size_t page_size;
 108     /* How many uint32_t elements does each doorbell entry take. */
 109     size_t doorbell_scale;
 110     bool write_cache_supported;
 111     EventNotifier irq_notifier[MSIX_IRQ_COUNT];
 112
 113     uint64_t nsze; /* Namespace size reported by identify command */
 114     int nsid;      /* The namespace id to read/write data. */
 115     int blkshift;
 116
 117     uint64_t max_transfer;
 118     bool plugged;
 119
 120     bool supports_write_zeroes;
 121     bool supports_discard;
 122
 123     CoMutex dma_map_lock;
 124     CoQueue dma_flush_queue;
 125
 126     /* Total size of mapped qiov, accessed under dma_map_lock */
 127     int dma_map_count;
 128
 129     /* PCI address (required for nvme_refresh_filename()) */
 130     char *device;
 131 };
 132
 133 #define NVME_BLOCK_OPT_DEVICE "device"
 134 #define NVME_BLOCK_OPT_NAMESPACE "namespace"
 135
 136 static void nvme_process_completion_bh(void *opaque);
 137
 138 static QemuOptsList runtime_opts = {
 139     .name = "nvme",
 140     .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
 141     .desc = {
 142         {
 143             .name = NVME_BLOCK_OPT_DEVICE,
 144             .type = QEMU_OPT_STRING,
 145             .help = "NVMe PCI device address",
 146         },
 147         {
 148             .name = NVME_BLOCK_OPT_NAMESPACE,
 149             .type = QEMU_OPT_NUMBER,
 150             .help = "NVMe namespace",
 151         },
 152         { /* end of list */ }
 153     },
 154 };
 155
 156 static void nvme_init_queue(BDRVNVMeState *s, NVMeQueue *q,
 157                             int nentries, int entry_bytes, Error **errp)
 158 {
 159     size_t bytes;
 160     int r;
 161
 162     bytes = ROUND_UP(nentries * entry_bytes, s->page_size);
 163     q->head = q->tail = 0;
 164     q->queue = qemu_try_memalign(s->page_size, bytes);
 165     if (!q->queue) {
 166         error_setg(errp, "Cannot allocate queue");
 167         return;
 168     }
 169     memset(q->queue, 0, bytes);
 170     r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova);
 171     if (r) {
 172         error_setg(errp, "Cannot map queue");
 173     }
 174 }
 175
 176 static void nvme_free_queue_pair(NVMeQueuePair *q)
 177 {
 178     if (q->completion_bh) {
 179         qemu_bh_delete(q->completion_bh);
 180     }
 181     qemu_vfree(q->prp_list_pages);
 182     qemu_vfree(q->sq.queue);
 183     qemu_vfree(q->cq.queue);
 184     qemu_mutex_destroy(&q->lock);
 185     g_free(q);
 186 }
 187
 188 static void nvme_free_req_queue_cb(void *opaque)
 189 {
 190     NVMeQueuePair *q = opaque;
 191
 192     qemu_mutex_lock(&q->lock);
 193     while (qemu_co_enter_next(&q->free_req_queue, &q->lock)) {
 194         /* Retry all pending requests */
 195     }
 196     qemu_mutex_unlock(&q->lock);
 197 }
 198
 199 static NVMeQueuePair *nvme_create_queue_pair(BDRVNVMeState *s,
 200                                              AioContext *aio_context,
 201                                              int idx, int size,
 202                                              Error **errp)
 203 {
 204     int i, r;
 205     Error *local_err = NULL;
 206     NVMeQueuePair *q;
 207     uint64_t prp_list_iova;
 208
 209     q = g_try_new0(NVMeQueuePair, 1);
 210     if (!q) {
 211         return NULL;
 212     }
 213     q->prp_list_pages = qemu_try_memalign(s->page_size,
 214                                           s->page_size * NVME_NUM_REQS);
 215     if (!q->prp_list_pages) {
 216         goto fail;
 217     }
 218     memset(q->prp_list_pages, 0, s->page_size * NVME_NUM_REQS);
 219     qemu_mutex_init(&q->lock);
 220     q->s = s;
 221     q->index = idx;
 222     qemu_co_queue_init(&q->free_req_queue);
 223     q->completion_bh = aio_bh_new(aio_context, nvme_process_completion_bh, q);
 224     r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages,
 225                           s->page_size * NVME_NUM_REQS,
 226                           false, &prp_list_iova);
 227     if (r) {
 228         goto fail;
 229     }
 230     q->free_req_head = -1;
 231     for (i = 0; i < NVME_NUM_REQS; i++) {
 232         NVMeRequest *req = &q->reqs[i];
 233         req->cid = i + 1;
 234         req->free_req_next = q->free_req_head;
 235         q->free_req_head = i;
 236         req->prp_list_page = q->prp_list_pages + i * s->page_size;
 237         req->prp_list_iova = prp_list_iova + i * s->page_size;
 238     }
 239
 240     nvme_init_queue(s, &q->sq, size, NVME_SQ_ENTRY_BYTES, &local_err);
 241     if (local_err) {
 242         error_propagate(errp, local_err);
 243         goto fail;
 244     }
 245     q->sq.doorbell = &s->doorbells[idx * s->doorbell_scale].sq_tail;
 246
 247     nvme_init_queue(s, &q->cq, size, NVME_CQ_ENTRY_BYTES, &local_err);
 248     if (local_err) {
 249         error_propagate(errp, local_err);
 250         goto fail;
 251     }
 252     q->cq.doorbell = &s->doorbells[idx * s->doorbell_scale].cq_head;
 253
 254     return q;
 255 fail:
 256     nvme_free_queue_pair(q);
 257     return NULL;
 258 }
 259
 260 /* With q->lock */
 261 static void nvme_kick(NVMeQueuePair *q)
 262 {
 263     BDRVNVMeState *s = q->s;
 264
 265     if (s->plugged || !q->need_kick) {
 266         return;
 267     }
 268     trace_nvme_kick(s, q->index);
 269     assert(!(q->sq.tail & 0xFF00));
 270     /* Fence the write to submission queue entry before notifying the device. */
 271     smp_wmb();
 272     *q->sq.doorbell = cpu_to_le32(q->sq.tail);
 273     q->inflight += q->need_kick;
 274     q->need_kick = 0;
 275 }
 276
 277 /* Find a free request element if any, otherwise:
 278  * a) if in coroutine context, try to wait for one to become available;
 279  * b) if not in coroutine, return NULL;
 280  */
 281 static NVMeRequest *nvme_get_free_req(NVMeQueuePair *q)
 282 {
 283     NVMeRequest *req;
 284
 285     qemu_mutex_lock(&q->lock);
 286
 287     while (q->free_req_head == -1) {
 288         if (qemu_in_coroutine()) {
 289             trace_nvme_free_req_queue_wait(q);
 290             qemu_co_queue_wait(&q->free_req_queue, &q->lock);
 291         } else {
 292             qemu_mutex_unlock(&q->lock);
 293             return NULL;
 294         }
 295     }
 296
 297     req = &q->reqs[q->free_req_head];
 298     q->free_req_head = req->free_req_next;
 299     req->free_req_next = -1;
 300
 301     qemu_mutex_unlock(&q->lock);
 302     return req;
 303 }
 304
 305 /* With q->lock */
 306 static void nvme_put_free_req_locked(NVMeQueuePair *q, NVMeRequest *req)
 307 {
 308     req->free_req_next = q->free_req_head;
 309     q->free_req_head = req - q->reqs;
 310 }
 311
 312 /* With q->lock */
 313 static void nvme_wake_free_req_locked(NVMeQueuePair *q)
 314 {
 315     if (!qemu_co_queue_empty(&q->free_req_queue)) {
 316         replay_bh_schedule_oneshot_event(q->s->aio_context,
 317                 nvme_free_req_queue_cb, q);
 318     }
 319 }
 320
 321 /* Insert a request in the freelist and wake waiters */
 322 static void nvme_put_free_req_and_wake(NVMeQueuePair *q, NVMeRequest *req)
 323 {
 324     qemu_mutex_lock(&q->lock);
 325     nvme_put_free_req_locked(q, req);
 326     nvme_wake_free_req_locked(q);
 327     qemu_mutex_unlock(&q->lock);
 328 }
 329
 330 static inline int nvme_translate_error(const NvmeCqe *c)
 331 {
 332     uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF;
 333     if (status) {
 334         trace_nvme_error(le32_to_cpu(c->result),
 335                          le16_to_cpu(c->sq_head),
 336                          le16_to_cpu(c->sq_id),
 337                          le16_to_cpu(c->cid),
 338                          le16_to_cpu(status));
 339     }
 340     switch (status) {
 341     case 0:
 342         return 0;
 343     case 1:
 344         return -ENOSYS;
 345     case 2:
 346         return -EINVAL;
 347     default:
 348         return -EIO;
 349     }
 350 }
 351
 352 /* With q->lock */
 353 static bool nvme_process_completion(NVMeQueuePair *q)
 354 {
 355     BDRVNVMeState *s = q->s;
 356     bool progress = false;
 357     NVMeRequest *preq;
 358     NVMeRequest req;
 359     NvmeCqe *c;
 360
 361     trace_nvme_process_completion(s, q->index, q->inflight);
 362     if (s->plugged) {
 363         trace_nvme_process_completion_queue_plugged(s, q->index);
 364         return false;
 365     }
 366
 367     /*
 368      * Support re-entrancy when a request cb() function invokes aio_poll().
 369      * Pending completions must be visible to aio_poll() so that a cb()
 370      * function can wait for the completion of another request.
 371      *
 372      * The aio_poll() loop will execute our BH and we'll resume completion
 373      * processing there.
 374      */
 375     qemu_bh_schedule(q->completion_bh);
 376
 377     assert(q->inflight >= 0);
 378     while (q->inflight) {
 379         int ret;
 380         int16_t cid;
 381
 382         c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES];
 383         if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) {
 384             break;
 385         }
 386         ret = nvme_translate_error(c);
 387         q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE;
 388         if (!q->cq.head) {
 389             q->cq_phase = !q->cq_phase;
 390         }
 391         cid = le16_to_cpu(c->cid);
 392         if (cid == 0 || cid > NVME_QUEUE_SIZE) {
 393             fprintf(stderr, "Unexpected CID in completion queue: %" PRIu32 "\n",
 394                     cid);
 395             continue;
 396         }
 397         trace_nvme_complete_command(s, q->index, cid);
 398         preq = &q->reqs[cid - 1];
 399         req = *preq;
 400         assert(req.cid == cid);
 401         assert(req.cb);
 402         nvme_put_free_req_locked(q, preq);
 403         preq->cb = preq->opaque = NULL;
 404         q->inflight--;
 405         qemu_mutex_unlock(&q->lock);
 406         req.cb(req.opaque, ret);
 407         qemu_mutex_lock(&q->lock);
 408         progress = true;
 409     }
 410     if (progress) {
 411         /* Notify the device so it can post more completions. */
 412         smp_mb_release();
 413         *q->cq.doorbell = cpu_to_le32(q->cq.head);
 414         nvme_wake_free_req_locked(q);
 415     }
 416
 417     qemu_bh_cancel(q->completion_bh);
 418
 419     return progress;
 420 }
 421
 422 static void nvme_process_completion_bh(void *opaque)
 423 {
 424     NVMeQueuePair *q = opaque;
 425
 426     /*
 427      * We're being invoked because a nvme_process_completion() cb() function
 428      * called aio_poll(). The callback may be waiting for further completions
 429      * so notify the device that it has space to fill in more completions now.
 430      */
 431     smp_mb_release();
 432     *q->cq.doorbell = cpu_to_le32(q->cq.head);
 433     nvme_wake_free_req_locked(q);
 434
 435     nvme_process_completion(q);
 436 }
 437
 438 static void nvme_trace_command(const NvmeCmd *cmd)
 439 {
 440     int i;
 441
 442     if (!trace_event_get_state_backends(TRACE_NVME_SUBMIT_COMMAND_RAW)) {
 443         return;
 444     }
 445     for (i = 0; i < 8; ++i) {
 446         uint8_t *cmdp = (uint8_t *)cmd + i * 8;
 447         trace_nvme_submit_command_raw(cmdp[0], cmdp[1], cmdp[2], cmdp[3],
 448                                       cmdp[4], cmdp[5], cmdp[6], cmdp[7]);
 449     }
 450 }
 451
 452 static void nvme_submit_command(NVMeQueuePair *q, NVMeRequest *req,
 453                                 NvmeCmd *cmd, BlockCompletionFunc cb,
 454                                 void *opaque)
 455 {
 456     assert(!req->cb);
 457     req->cb = cb;
 458     req->opaque = opaque;
 459     cmd->cid = cpu_to_le32(req->cid);
 460
 461     trace_nvme_submit_command(q->s, q->index, req->cid);
 462     nvme_trace_command(cmd);
 463     qemu_mutex_lock(&q->lock);
 464     memcpy((uint8_t *)q->sq.queue +
 465            q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd));
 466     q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE;
 467     q->need_kick++;
 468     nvme_kick(q);
 469     nvme_process_completion(q);
 470     qemu_mutex_unlock(&q->lock);
 471 }
 472
 473 static void nvme_cmd_sync_cb(void *opaque, int ret)
 474 {
 475     int *pret = opaque;
 476     *pret = ret;
 477     aio_wait_kick();
 478 }
 479
 480 static int nvme_cmd_sync(BlockDriverState *bs, NVMeQueuePair *q,
 481                          NvmeCmd *cmd)
 482 {
 483     AioContext *aio_context = bdrv_get_aio_context(bs);
 484     NVMeRequest *req;
 485     int ret = -EINPROGRESS;
 486     req = nvme_get_free_req(q);
 487     if (!req) {
 488         return -EBUSY;
 489     }
 490     nvme_submit_command(q, req, cmd, nvme_cmd_sync_cb, &ret);
 491
 492     AIO_WAIT_WHILE(aio_context, ret == -EINPROGRESS);
 493     return ret;
 494 }
 495
 496 static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
 497 {
 498     BDRVNVMeState *s = bs->opaque;
 499     union {
 500         NvmeIdCtrl ctrl;
 501         NvmeIdNs ns;
 502     } *id;
 503     NvmeLBAF *lbaf;
 504     uint16_t oncs;
 505     int r;
 506     uint64_t iova;
 507     NvmeCmd cmd = {
 508         .opcode = NVME_ADM_CMD_IDENTIFY,
 509         .cdw10 = cpu_to_le32(0x1),
 510     };
 511
 512     id = qemu_try_memalign(s->page_size, sizeof(*id));
 513     if (!id) {
 514         error_setg(errp, "Cannot allocate buffer for identify response");
 515         goto out;
 516     }
 517     r = qemu_vfio_dma_map(s->vfio, id, sizeof(*id), true, &iova);
 518     if (r) {
 519         error_setg(errp, "Cannot map buffer for DMA");
 520         goto out;
 521     }
 522
 523     memset(id, 0, sizeof(*id));
 524     cmd.dptr.prp1 = cpu_to_le64(iova);
 525     if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
 526         error_setg(errp, "Failed to identify controller");
 527         goto out;
 528     }
 529
 530     if (le32_to_cpu(id->ctrl.nn) < namespace) {
 531         error_setg(errp, "Invalid namespace");
 532         goto out;
 533     }
 534     s->write_cache_supported = le32_to_cpu(id->ctrl.vwc) & 0x1;
 535     s->max_transfer = (id->ctrl.mdts ? 1 << id->ctrl.mdts : 0) * s->page_size;
 536     /* For now the page list buffer per command is one page, to hold at most
 537      * s->page_size / sizeof(uint64_t) entries. */
 538     s->max_transfer = MIN_NON_ZERO(s->max_transfer,
 539                           s->page_size / sizeof(uint64_t) * s->page_size);
 540
 541     oncs = le16_to_cpu(id->ctrl.oncs);
 542     s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROES);
 543     s->supports_discard = !!(oncs & NVME_ONCS_DSM);
 544
 545     memset(id, 0, sizeof(*id));
 546     cmd.cdw10 = 0;
 547     cmd.nsid = cpu_to_le32(namespace);
 548     if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
 549         error_setg(errp, "Failed to identify namespace");
 550         goto out;
 551     }
 552
 553     s->nsze = le64_to_cpu(id->ns.nsze);
 554     lbaf = &id->ns.lbaf[NVME_ID_NS_FLBAS_INDEX(id->ns.flbas)];
 555
 556     if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(id->ns.dlfeat) &&
 557             NVME_ID_NS_DLFEAT_READ_BEHAVIOR(id->ns.dlfeat) ==
 558                     NVME_ID_NS_DLFEAT_READ_BEHAVIOR_ZEROES) {
 559         bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP;
 560     }
 561
 562     if (lbaf->ms) {
 563         error_setg(errp, "Namespaces with metadata are not yet supported");
 564         goto out;
 565     }
 566
 567     if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 ||
 568         (1 << lbaf->ds) > s->page_size)
 569     {
 570         error_setg(errp, "Namespace has unsupported block size (2^%d)",
 571                    lbaf->ds);
 572         goto out;
 573     }
 574
 575     s->blkshift = lbaf->ds;
 576 out:
 577     qemu_vfio_dma_unmap(s->vfio, id);
 578     qemu_vfree(id);
 579 }
 580
 581 static bool nvme_poll_queue(NVMeQueuePair *q)
 582 {
 583     bool progress = false;
 584
 585     const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES;
 586     NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset];
 587
 588     /*
 589      * Do an early check for completions. q->lock isn't needed because
 590      * nvme_process_completion() only runs in the event loop thread and
 591      * cannot race with itself.
 592      */
 593     if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) {
 594         return false;
 595     }
 596
 597     qemu_mutex_lock(&q->lock);
 598     while (nvme_process_completion(q)) {
 599         /* Keep polling */
 600         progress = true;
 601     }
 602     qemu_mutex_unlock(&q->lock);
 603
 604     return progress;
 605 }
 606
 607 static bool nvme_poll_queues(BDRVNVMeState *s)
 608 {
 609     bool progress = false;
 610     int i;
 611
 612     for (i = 0; i < s->nr_queues; i++) {
 613         if (nvme_poll_queue(s->queues[i])) {
 614             progress = true;
 615         }
 616     }
 617     return progress;
 618 }
 619
 620 static void nvme_handle_event(EventNotifier *n)
 621 {
 622     BDRVNVMeState *s = container_of(n, BDRVNVMeState,
 623                                     irq_notifier[MSIX_SHARED_IRQ_IDX]);
 624
 625     trace_nvme_handle_event(s);
 626     event_notifier_test_and_clear(n);
 627     nvme_poll_queues(s);
 628 }
 629
 630 static bool nvme_add_io_queue(BlockDriverState *bs, Error **errp)
 631 {
 632     BDRVNVMeState *s = bs->opaque;
 633     int n = s->nr_queues;
 634     NVMeQueuePair *q;
 635     NvmeCmd cmd;
 636     int queue_size = NVME_QUEUE_SIZE;
 637
 638     q = nvme_create_queue_pair(s, bdrv_get_aio_context(bs),
 639                                n, queue_size, errp);
 640     if (!q) {
 641         return false;
 642     }
 643     cmd = (NvmeCmd) {
 644         .opcode = NVME_ADM_CMD_CREATE_CQ,
 645         .dptr.prp1 = cpu_to_le64(q->cq.iova),
 646         .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
 647         .cdw11 = cpu_to_le32(0x3),
 648     };
 649     if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
 650         error_setg(errp, "Failed to create CQ io queue [%d]", n);
 651         goto out_error;
 652     }
 653     cmd = (NvmeCmd) {
 654         .opcode = NVME_ADM_CMD_CREATE_SQ,
 655         .dptr.prp1 = cpu_to_le64(q->sq.iova),
 656         .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
 657         .cdw11 = cpu_to_le32(0x1 | (n << 16)),
 658     };
 659     if (nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd)) {
 660         error_setg(errp, "Failed to create SQ io queue [%d]", n);
 661         goto out_error;
 662     }
 663     s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1);
 664     s->queues[n] = q;
 665     s->nr_queues++;
 666     return true;
 667 out_error:
 668     nvme_free_queue_pair(q);
 669     return false;
 670 }
 671
 672 static bool nvme_poll_cb(void *opaque)
 673 {
 674     EventNotifier *e = opaque;
 675     BDRVNVMeState *s = container_of(e, BDRVNVMeState,
 676                                     irq_notifier[MSIX_SHARED_IRQ_IDX]);
 677
 678     trace_nvme_poll_cb(s);
 679     return nvme_poll_queues(s);
 680 }
 681
 682 static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
 683                      Error **errp)
 684 {
 685     BDRVNVMeState *s = bs->opaque;
 686     AioContext *aio_context = bdrv_get_aio_context(bs);
 687     int ret;
 688     uint64_t cap;
 689     uint64_t timeout_ms;
 690     uint64_t deadline, now;
 691     Error *local_err = NULL;
 692     volatile NvmeBar *regs = NULL;
 693
 694     qemu_co_mutex_init(&s->dma_map_lock);
 695     qemu_co_queue_init(&s->dma_flush_queue);
 696     s->device = g_strdup(device);
 697     s->nsid = namespace;
 698     s->aio_context = bdrv_get_aio_context(bs);
 699     ret = event_notifier_init(&s->irq_notifier[MSIX_SHARED_IRQ_IDX], 0);
 700     if (ret) {
 701         error_setg(errp, "Failed to init event notifier");
 702         return ret;
 703     }
 704
 705     s->vfio = qemu_vfio_open_pci(device, errp);
 706     if (!s->vfio) {
 707         ret = -EINVAL;
 708         goto out;
 709     }
 710
 711     regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, sizeof(NvmeBar),
 712                                  PROT_READ | PROT_WRITE, errp);
 713     if (!regs) {
 714         ret = -EINVAL;
 715         goto out;
 716     }
 717     /* Perform initialize sequence as described in NVMe spec "7.6.1
 718      * Initialization". */
 719
 720     cap = le64_to_cpu(regs->cap);
 721     if (!NVME_CAP_CSS(cap)) {
 722         error_setg(errp, "Device doesn't support NVMe command set");
 723         ret = -EINVAL;
 724         goto out;
 725     }
 726
 727     s->page_size = MAX(4096, 1 << NVME_CAP_MPSMIN(cap));
 728     s->doorbell_scale = (4 << NVME_CAP_DSTRD(cap)) / sizeof(uint32_t);
 729     bs->bl.opt_mem_alignment = s->page_size;
 730     timeout_ms = MIN(500 * NVME_CAP_TO(cap), 30000);
 731
 732     /* Reset device to get a clean state. */
 733     regs->cc = cpu_to_le32(le32_to_cpu(regs->cc) & 0xFE);
 734     /* Wait for CSTS.RDY = 0. */
 735     deadline = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ms * SCALE_MS;
 736     while (NVME_CSTS_RDY(le32_to_cpu(regs->csts))) {
 737         if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
 738             error_setg(errp, "Timeout while waiting for device to reset (%"
 739                              PRId64 " ms)",
 740                        timeout_ms);
 741             ret = -ETIMEDOUT;
 742             goto out;
 743         }
 744     }
 745
 746     s->doorbells = qemu_vfio_pci_map_bar(s->vfio, 0, sizeof(NvmeBar),
 747                                          NVME_DOORBELL_SIZE, PROT_WRITE, errp);
 748     if (!s->doorbells) {
 749         ret = -EINVAL;
 750         goto out;
 751     }
 752
 753     /* Set up admin queue. */
 754     s->queues = g_new(NVMeQueuePair *, 1);
 755     s->queues[INDEX_ADMIN] = nvme_create_queue_pair(s, aio_context, 0,
 756                                                           NVME_QUEUE_SIZE,
 757                                                           errp);
 758     if (!s->queues[INDEX_ADMIN]) {
 759         ret = -EINVAL;
 760         goto out;
 761     }
 762     s->nr_queues = 1;
 763     QEMU_BUILD_BUG_ON(NVME_QUEUE_SIZE & 0xF000);
 764     regs->aqa = cpu_to_le32((NVME_QUEUE_SIZE << AQA_ACQS_SHIFT) |
 765                             (NVME_QUEUE_SIZE << AQA_ASQS_SHIFT));
 766     regs->asq = cpu_to_le64(s->queues[INDEX_ADMIN]->sq.iova);
 767     regs->acq = cpu_to_le64(s->queues[INDEX_ADMIN]->cq.iova);
 768
 769     /* After setting up all control registers we can enable device now. */
 770     regs->cc = cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES) << CC_IOCQES_SHIFT) |
 771                            (ctz32(NVME_SQ_ENTRY_BYTES) << CC_IOSQES_SHIFT) |
 772                            CC_EN_MASK);
 773     /* Wait for CSTS.RDY = 1. */
 774     now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
 775     deadline = now + timeout_ms * SCALE_MS;
 776     while (!NVME_CSTS_RDY(le32_to_cpu(regs->csts))) {
 777         if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
 778             error_setg(errp, "Timeout while waiting for device to start (%"
 779                              PRId64 " ms)",
 780                        timeout_ms);
 781             ret = -ETIMEDOUT;
 782             goto out;
 783         }
 784     }
 785
 786     ret = qemu_vfio_pci_init_irq(s->vfio, s->irq_notifier,
 787                                  VFIO_PCI_MSIX_IRQ_INDEX, errp);
 788     if (ret) {
 789         goto out;
 790     }
 791     aio_set_event_notifier(bdrv_get_aio_context(bs),
 792                            &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
 793                            false, nvme_handle_event, nvme_poll_cb);
 794
 795     nvme_identify(bs, namespace, &local_err);
 796     if (local_err) {
 797         error_propagate(errp, local_err);
 798         ret = -EIO;
 799         goto out;
 800     }
 801
 802     /* Set up command queues. */
 803     if (!nvme_add_io_queue(bs, errp)) {
 804         ret = -EIO;
 805     }
 806 out:
 807     if (regs) {
 808         qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)regs, 0, sizeof(NvmeBar));
 809     }
 810
 811     /* Cleaning up is done in nvme_file_open() upon error. */
 812     return ret;
 813 }
 814
 815 /* Parse a filename in the format of nvme://XXXX:XX:XX.X/X. Example:
 816  *
 817  *     nvme://0000:44:00.0/1
 818  *
 819  * where the "nvme://" is a fixed form of the protocol prefix, the middle part
 820  * is the PCI address, and the last part is the namespace number starting from
 821  * 1 according to the NVMe spec. */
 822 static void nvme_parse_filename(const char *filename, QDict *options,
 823                                 Error **errp)
 824 {
 825     int pref = strlen("nvme://");
 826
 827     if (strlen(filename) > pref && !strncmp(filename, "nvme://", pref)) {
 828         const char *tmp = filename + pref;
 829         char *device;
 830         const char *namespace;
 831         unsigned long ns;
 832         const char *slash = strchr(tmp, '/');
 833         if (!slash) {
 834             qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, tmp);
 835             return;
 836         }
 837         device = g_strndup(tmp, slash - tmp);
 838         qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, device);
 839         g_free(device);
 840         namespace = slash + 1;
 841         if (*namespace && qemu_strtoul(namespace, NULL, 10, &ns)) {
 842             error_setg(errp, "Invalid namespace '%s', positive number expected",
 843                        namespace);
 844             return;
 845         }
 846         qdict_put_str(options, NVME_BLOCK_OPT_NAMESPACE,
 847                       *namespace ? namespace : "1");
 848     }
 849 }
 850
 851 static int nvme_enable_disable_write_cache(BlockDriverState *bs, bool enable,
 852                                            Error **errp)
 853 {
 854     int ret;
 855     BDRVNVMeState *s = bs->opaque;
 856     NvmeCmd cmd = {
 857         .opcode = NVME_ADM_CMD_SET_FEATURES,
 858         .nsid = cpu_to_le32(s->nsid),
 859         .cdw10 = cpu_to_le32(0x06),
 860         .cdw11 = cpu_to_le32(enable ? 0x01 : 0x00),
 861     };
 862
 863     ret = nvme_cmd_sync(bs, s->queues[INDEX_ADMIN], &cmd);
 864     if (ret) {
 865         error_setg(errp, "Failed to configure NVMe write cache");
 866     }
 867     return ret;
 868 }
 869
 870 static void nvme_close(BlockDriverState *bs)
 871 {
 872     int i;
 873     BDRVNVMeState *s = bs->opaque;
 874
 875     for (i = 0; i < s->nr_queues; ++i) {
 876         nvme_free_queue_pair(s->queues[i]);
 877     }
 878     g_free(s->queues);
 879     aio_set_event_notifier(bdrv_get_aio_context(bs),
 880                            &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
 881                            false, NULL, NULL);
 882     event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]);
 883     qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->doorbells,
 884                             sizeof(NvmeBar), NVME_DOORBELL_SIZE);
 885     qemu_vfio_close(s->vfio);
 886
 887     g_free(s->device);
 888 }
 889
 890 static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
 891                           Error **errp)
 892 {
 893     const char *device;
 894     QemuOpts *opts;
 895     int namespace;
 896     int ret;
 897     BDRVNVMeState *s = bs->opaque;
 898
 899     bs->supported_write_flags = BDRV_REQ_FUA;
 900
 901     opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
 902     qemu_opts_absorb_qdict(opts, options, &error_abort);
 903     device = qemu_opt_get(opts, NVME_BLOCK_OPT_DEVICE);
 904     if (!device) {
 905         error_setg(errp, "'" NVME_BLOCK_OPT_DEVICE "' option is required");
 906         qemu_opts_del(opts);
 907         return -EINVAL;
 908     }
 909
 910     namespace = qemu_opt_get_number(opts, NVME_BLOCK_OPT_NAMESPACE, 1);
 911     ret = nvme_init(bs, device, namespace, errp);
 912     qemu_opts_del(opts);
 913     if (ret) {
 914         goto fail;
 915     }
 916     if (flags & BDRV_O_NOCACHE) {
 917         if (!s->write_cache_supported) {
 918             error_setg(errp,
 919                        "NVMe controller doesn't support write cache configuration");
 920             ret = -EINVAL;
 921         } else {
 922             ret = nvme_enable_disable_write_cache(bs, !(flags & BDRV_O_NOCACHE),
 923                                                   errp);
 924         }
 925         if (ret) {
 926             goto fail;
 927         }
 928     }
 929     return 0;
 930 fail:
 931     nvme_close(bs);
 932     return ret;
 933 }
 934
 935 static int64_t nvme_getlength(BlockDriverState *bs)
 936 {
 937     BDRVNVMeState *s = bs->opaque;
 938     return s->nsze << s->blkshift;
 939 }
 940
 941 static uint32_t nvme_get_blocksize(BlockDriverState *bs)
 942 {
 943     BDRVNVMeState *s = bs->opaque;
 944     assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12);
 945     return UINT32_C(1) << s->blkshift;
 946 }
 947
 948 static int nvme_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
 949 {
 950     uint32_t blocksize = nvme_get_blocksize(bs);
 951     bsz->phys = blocksize;
 952     bsz->log = blocksize;
 953     return 0;
 954 }
 955
 956 /* Called with s->dma_map_lock */
 957 static coroutine_fn int nvme_cmd_unmap_qiov(BlockDriverState *bs,
 958                                             QEMUIOVector *qiov)
 959 {
 960     int r = 0;
 961     BDRVNVMeState *s = bs->opaque;
 962
 963     s->dma_map_count -= qiov->size;
 964     if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) {
 965         r = qemu_vfio_dma_reset_temporary(s->vfio);
 966         if (!r) {
 967             qemu_co_queue_restart_all(&s->dma_flush_queue);
 968         }
 969     }
 970     return r;
 971 }
 972
 973 /* Called with s->dma_map_lock */
 974 static coroutine_fn int nvme_cmd_map_qiov(BlockDriverState *bs, NvmeCmd *cmd,
 975                                           NVMeRequest *req, QEMUIOVector *qiov)
 976 {
 977     BDRVNVMeState *s = bs->opaque;
 978     uint64_t *pagelist = req->prp_list_page;
 979     int i, j, r;
 980     int entries = 0;
 981
 982     assert(qiov->size);
 983     assert(QEMU_IS_ALIGNED(qiov->size, s->page_size));
 984     assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t));
 985     for (i = 0; i < qiov->niov; ++i) {
 986         bool retry = true;
 987         uint64_t iova;
 988 try_map:
 989         r = qemu_vfio_dma_map(s->vfio,
 990                               qiov->iov[i].iov_base,
 991                               qiov->iov[i].iov_len,
 992                               true, &iova);
 993         if (r == -ENOMEM && retry) {
 994             retry = false;
 995             trace_nvme_dma_flush_queue_wait(s);
 996             if (s->dma_map_count) {
 997                 trace_nvme_dma_map_flush(s);
 998                 qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock);
 999             } else {
1000                 r = qemu_vfio_dma_reset_temporary(s->vfio);
1001                 if (r) {
1002                     goto fail;
1003                 }
1004             }
1005             goto try_map;
1006         }
1007         if (r) {
1008             goto fail;
1009         }
1010
1011         for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) {
1012             pagelist[entries++] = cpu_to_le64(iova + j * s->page_size);
1013         }
1014         trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base,
1015                                     qiov->iov[i].iov_len / s->page_size);
1016     }
1017
1018     s->dma_map_count += qiov->size;
1019
1020     assert(entries <= s->page_size / sizeof(uint64_t));
1021     switch (entries) {
1022     case 0:
1023         abort();
1024     case 1:
1025         cmd->dptr.prp1 = pagelist[0];
1026         cmd->dptr.prp2 = 0;
1027         break;
1028     case 2:
1029         cmd->dptr.prp1 = pagelist[0];
1030         cmd->dptr.prp2 = pagelist[1];
1031         break;
1032     default:
1033         cmd->dptr.prp1 = pagelist[0];
1034         cmd->dptr.prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t));
1035         break;
1036     }
1037     trace_nvme_cmd_map_qiov(s, cmd, req, qiov, entries);
1038     for (i = 0; i < entries; ++i) {
1039         trace_nvme_cmd_map_qiov_pages(s, i, pagelist[i]);
1040     }
1041     return 0;
1042 fail:
1043     /* No need to unmap [0 - i) iovs even if we've failed, since we don't
1044      * increment s->dma_map_count. This is okay for fixed mapping memory areas
1045      * because they are already mapped before calling this function; for
1046      * temporary mappings, a later nvme_cmd_(un)map_qiov will reclaim by
1047      * calling qemu_vfio_dma_reset_temporary when necessary. */
1048     return r;
1049 }
1050
1051 typedef struct {
1052     Coroutine *co;
1053     int ret;
1054     AioContext *ctx;
1055 } NVMeCoData;
1056
1057 static void nvme_rw_cb_bh(void *opaque)
1058 {
1059     NVMeCoData *data = opaque;
1060     qemu_coroutine_enter(data->co);
1061 }
1062
1063 static void nvme_rw_cb(void *opaque, int ret)
1064 {
1065     NVMeCoData *data = opaque;
1066     data->ret = ret;
1067     if (!data->co) {
1068         /* The rw coroutine hasn't yielded, don't try to enter. */
1069         return;
1070     }
1071     replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data);
1072 }
1073
1074 static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs,
1075                                             uint64_t offset, uint64_t bytes,
1076                                             QEMUIOVector *qiov,
1077                                             bool is_write,
1078                                             int flags)
1079 {
1080     int r;
1081     BDRVNVMeState *s = bs->opaque;
1082     NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1083     NVMeRequest *req;
1084
1085     uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) |
1086                        (flags & BDRV_REQ_FUA ? 1 << 30 : 0);
1087     NvmeCmd cmd = {
1088         .opcode = is_write ? NVME_CMD_WRITE : NVME_CMD_READ,
1089         .nsid = cpu_to_le32(s->nsid),
1090         .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1091         .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1092         .cdw12 = cpu_to_le32(cdw12),
1093     };
1094     NVMeCoData data = {
1095         .ctx = bdrv_get_aio_context(bs),
1096         .ret = -EINPROGRESS,
1097     };
1098
1099     trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov);
1100     assert(s->nr_queues > 1);
1101     req = nvme_get_free_req(ioq);
1102     assert(req);
1103
1104     qemu_co_mutex_lock(&s->dma_map_lock);
1105     r = nvme_cmd_map_qiov(bs, &cmd, req, qiov);
1106     qemu_co_mutex_unlock(&s->dma_map_lock);
1107     if (r) {
1108         nvme_put_free_req_and_wake(ioq, req);
1109         return r;
1110     }
1111     nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1112
1113     data.co = qemu_coroutine_self();
1114     while (data.ret == -EINPROGRESS) {
1115         qemu_coroutine_yield();
1116     }
1117
1118     qemu_co_mutex_lock(&s->dma_map_lock);
1119     r = nvme_cmd_unmap_qiov(bs, qiov);
1120     qemu_co_mutex_unlock(&s->dma_map_lock);
1121     if (r) {
1122         return r;
1123     }
1124
1125     trace_nvme_rw_done(s, is_write, offset, bytes, data.ret);
1126     return data.ret;
1127 }
1128
1129 static inline bool nvme_qiov_aligned(BlockDriverState *bs,
1130                                      const QEMUIOVector *qiov)
1131 {
1132     int i;
1133     BDRVNVMeState *s = bs->opaque;
1134
1135     for (i = 0; i < qiov->niov; ++i) {
1136         if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base, s->page_size) ||
1137             !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, s->page_size)) {
1138             trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base,
1139                                       qiov->iov[i].iov_len, s->page_size);
1140             return false;
1141         }
1142     }
1143     return true;
1144 }
1145
1146 static int nvme_co_prw(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
1147                        QEMUIOVector *qiov, bool is_write, int flags)
1148 {
1149     BDRVNVMeState *s = bs->opaque;
1150     int r;
1151     uint8_t *buf = NULL;
1152     QEMUIOVector local_qiov;
1153
1154     assert(QEMU_IS_ALIGNED(offset, s->page_size));
1155     assert(QEMU_IS_ALIGNED(bytes, s->page_size));
1156     assert(bytes <= s->max_transfer);
1157     if (nvme_qiov_aligned(bs, qiov)) {
1158         return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags);
1159     }
1160     trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write);
1161     buf = qemu_try_memalign(s->page_size, bytes);
1162
1163     if (!buf) {
1164         return -ENOMEM;
1165     }
1166     qemu_iovec_init(&local_qiov, 1);
1167     if (is_write) {
1168         qemu_iovec_to_buf(qiov, 0, buf, bytes);
1169     }
1170     qemu_iovec_add(&local_qiov, buf, bytes);
1171     r = nvme_co_prw_aligned(bs, offset, bytes, &local_qiov, is_write, flags);
1172     qemu_iovec_destroy(&local_qiov);
1173     if (!r && !is_write) {
1174         qemu_iovec_from_buf(qiov, 0, buf, bytes);
1175     }
1176     qemu_vfree(buf);
1177     return r;
1178 }
1179
1180 static coroutine_fn int nvme_co_preadv(BlockDriverState *bs,
1181                                        uint64_t offset, uint64_t bytes,
1182                                        QEMUIOVector *qiov, int flags)
1183 {
1184     return nvme_co_prw(bs, offset, bytes, qiov, false, flags);
1185 }
1186
1187 static coroutine_fn int nvme_co_pwritev(BlockDriverState *bs,
1188                                         uint64_t offset, uint64_t bytes,
1189                                         QEMUIOVector *qiov, int flags)
1190 {
1191     return nvme_co_prw(bs, offset, bytes, qiov, true, flags);
1192 }
1193
1194 static coroutine_fn int nvme_co_flush(BlockDriverState *bs)
1195 {
1196     BDRVNVMeState *s = bs->opaque;
1197     NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1198     NVMeRequest *req;
1199     NvmeCmd cmd = {
1200         .opcode = NVME_CMD_FLUSH,
1201         .nsid = cpu_to_le32(s->nsid),
1202     };
1203     NVMeCoData data = {
1204         .ctx = bdrv_get_aio_context(bs),
1205         .ret = -EINPROGRESS,
1206     };
1207
1208     assert(s->nr_queues > 1);
1209     req = nvme_get_free_req(ioq);
1210     assert(req);
1211     nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1212
1213     data.co = qemu_coroutine_self();
1214     if (data.ret == -EINPROGRESS) {
1215         qemu_coroutine_yield();
1216     }
1217
1218     return data.ret;
1219 }
1220
1221
1222 static coroutine_fn int nvme_co_pwrite_zeroes(BlockDriverState *bs,
1223                                               int64_t offset,
1224                                               int bytes,
1225                                               BdrvRequestFlags flags)
1226 {
1227     BDRVNVMeState *s = bs->opaque;
1228     NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1229     NVMeRequest *req;
1230
1231     uint32_t cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF;
1232
1233     if (!s->supports_write_zeroes) {
1234         return -ENOTSUP;
1235     }
1236
1237     NvmeCmd cmd = {
1238         .opcode = NVME_CMD_WRITE_ZEROES,
1239         .nsid = cpu_to_le32(s->nsid),
1240         .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1241         .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1242     };
1243
1244     NVMeCoData data = {
1245         .ctx = bdrv_get_aio_context(bs),
1246         .ret = -EINPROGRESS,
1247     };
1248
1249     if (flags & BDRV_REQ_MAY_UNMAP) {
1250         cdw12 |= (1 << 25);
1251     }
1252
1253     if (flags & BDRV_REQ_FUA) {
1254         cdw12 |= (1 << 30);
1255     }
1256
1257     cmd.cdw12 = cpu_to_le32(cdw12);
1258
1259     trace_nvme_write_zeroes(s, offset, bytes, flags);
1260     assert(s->nr_queues > 1);
1261     req = nvme_get_free_req(ioq);
1262     assert(req);
1263
1264     nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1265
1266     data.co = qemu_coroutine_self();
1267     while (data.ret == -EINPROGRESS) {
1268         qemu_coroutine_yield();
1269     }
1270
1271     trace_nvme_rw_done(s, true, offset, bytes, data.ret);
1272     return data.ret;
1273 }
1274
1275
1276 static int coroutine_fn nvme_co_pdiscard(BlockDriverState *bs,
1277                                          int64_t offset,
1278                                          int bytes)
1279 {
1280     BDRVNVMeState *s = bs->opaque;
1281     NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1282     NVMeRequest *req;
1283     NvmeDsmRange *buf;
1284     QEMUIOVector local_qiov;
1285     int ret;
1286
1287     NvmeCmd cmd = {
1288         .opcode = NVME_CMD_DSM,
1289         .nsid = cpu_to_le32(s->nsid),
1290         .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/
1291         .cdw11 = cpu_to_le32(1 << 2), /*deallocate bit*/
1292     };
1293
1294     NVMeCoData data = {
1295         .ctx = bdrv_get_aio_context(bs),
1296         .ret = -EINPROGRESS,
1297     };
1298
1299     if (!s->supports_discard) {
1300         return -ENOTSUP;
1301     }
1302
1303     assert(s->nr_queues > 1);
1304
1305     buf = qemu_try_memalign(s->page_size, s->page_size);
1306     if (!buf) {
1307         return -ENOMEM;
1308     }
1309     memset(buf, 0, s->page_size);
1310     buf->nlb = cpu_to_le32(bytes >> s->blkshift);
1311     buf->slba = cpu_to_le64(offset >> s->blkshift);
1312     buf->cattr = 0;
1313
1314     qemu_iovec_init(&local_qiov, 1);
1315     qemu_iovec_add(&local_qiov, buf, 4096);
1316
1317     req = nvme_get_free_req(ioq);
1318     assert(req);
1319
1320     qemu_co_mutex_lock(&s->dma_map_lock);
1321     ret = nvme_cmd_map_qiov(bs, &cmd, req, &local_qiov);
1322     qemu_co_mutex_unlock(&s->dma_map_lock);
1323
1324     if (ret) {
1325         nvme_put_free_req_and_wake(ioq, req);
1326         goto out;
1327     }
1328
1329     trace_nvme_dsm(s, offset, bytes);
1330
1331     nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1332
1333     data.co = qemu_coroutine_self();
1334     while (data.ret == -EINPROGRESS) {
1335         qemu_coroutine_yield();
1336     }
1337
1338     qemu_co_mutex_lock(&s->dma_map_lock);
1339     ret = nvme_cmd_unmap_qiov(bs, &local_qiov);
1340     qemu_co_mutex_unlock(&s->dma_map_lock);
1341
1342     if (ret) {
1343         goto out;
1344     }
1345
1346     ret = data.ret;
1347     trace_nvme_dsm_done(s, offset, bytes, ret);
1348 out:
1349     qemu_iovec_destroy(&local_qiov);
1350     qemu_vfree(buf);
1351     return ret;
1352
1353 }
1354
1355
1356 static int nvme_reopen_prepare(BDRVReopenState *reopen_state,
1357                                BlockReopenQueue *queue, Error **errp)
1358 {
1359     return 0;
1360 }
1361
1362 static void nvme_refresh_filename(BlockDriverState *bs)
1363 {
1364     BDRVNVMeState *s = bs->opaque;
1365
1366     snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i",
1367              s->device, s->nsid);
1368 }
1369
1370 static void nvme_refresh_limits(BlockDriverState *bs, Error **errp)
1371 {
1372     BDRVNVMeState *s = bs->opaque;
1373
1374     bs->bl.opt_mem_alignment = s->page_size;
1375     bs->bl.request_alignment = s->page_size;
1376     bs->bl.max_transfer = s->max_transfer;
1377 }
1378
1379 static void nvme_detach_aio_context(BlockDriverState *bs)
1380 {
1381     BDRVNVMeState *s = bs->opaque;
1382
1383     for (int i = 0; i < s->nr_queues; i++) {
1384         NVMeQueuePair *q = s->queues[i];
1385
1386         qemu_bh_delete(q->completion_bh);
1387         q->completion_bh = NULL;
1388     }
1389
1390     aio_set_event_notifier(bdrv_get_aio_context(bs),
1391                            &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1392                            false, NULL, NULL);
1393 }
1394
1395 static void nvme_attach_aio_context(BlockDriverState *bs,
1396                                     AioContext *new_context)
1397 {
1398     BDRVNVMeState *s = bs->opaque;
1399
1400     s->aio_context = new_context;
1401     aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1402                            false, nvme_handle_event, nvme_poll_cb);
1403
1404     for (int i = 0; i < s->nr_queues; i++) {
1405         NVMeQueuePair *q = s->queues[i];
1406
1407         q->completion_bh =
1408             aio_bh_new(new_context, nvme_process_completion_bh, q);
1409     }
1410 }
1411
1412 static void nvme_aio_plug(BlockDriverState *bs)
1413 {
1414     BDRVNVMeState *s = bs->opaque;
1415     assert(!s->plugged);
1416     s->plugged = true;
1417 }
1418
1419 static void nvme_aio_unplug(BlockDriverState *bs)
1420 {
1421     int i;
1422     BDRVNVMeState *s = bs->opaque;
1423     assert(s->plugged);
1424     s->plugged = false;
1425     for (i = INDEX_IO(0); i < s->nr_queues; i++) {
1426         NVMeQueuePair *q = s->queues[i];
1427         qemu_mutex_lock(&q->lock);
1428         nvme_kick(q);
1429         nvme_process_completion(q);
1430         qemu_mutex_unlock(&q->lock);
1431     }
1432 }
1433
1434 static void nvme_register_buf(BlockDriverState *bs, void *host, size_t size)
1435 {
1436     int ret;
1437     BDRVNVMeState *s = bs->opaque;
1438
1439     ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL);
1440     if (ret) {
1441         /* FIXME: we may run out of IOVA addresses after repeated
1442          * bdrv_register_buf/bdrv_unregister_buf, because nvme_vfio_dma_unmap
1443          * doesn't reclaim addresses for fixed mappings. */
1444         error_report("nvme_register_buf failed: %s", strerror(-ret));
1445     }
1446 }
1447
1448 static void nvme_unregister_buf(BlockDriverState *bs, void *host)
1449 {
1450     BDRVNVMeState *s = bs->opaque;
1451
1452     qemu_vfio_dma_unmap(s->vfio, host);
1453 }
1454
1455 static const char *const nvme_strong_runtime_opts[] = {
1456     NVME_BLOCK_OPT_DEVICE,
1457     NVME_BLOCK_OPT_NAMESPACE,
1458
1459     NULL
1460 };
1461
1462 static BlockDriver bdrv_nvme = {
1463     .format_name              = "nvme",
1464     .protocol_name            = "nvme",
1465     .instance_size            = sizeof(BDRVNVMeState),
1466
1467     .bdrv_co_create_opts      = bdrv_co_create_opts_simple,
1468     .create_opts              = &bdrv_create_opts_simple,
1469
1470     .bdrv_parse_filename      = nvme_parse_filename,
1471     .bdrv_file_open           = nvme_file_open,
1472     .bdrv_close               = nvme_close,
1473     .bdrv_getlength           = nvme_getlength,
1474     .bdrv_probe_blocksizes    = nvme_probe_blocksizes,
1475
1476     .bdrv_co_preadv           = nvme_co_preadv,
1477     .bdrv_co_pwritev          = nvme_co_pwritev,
1478
1479     .bdrv_co_pwrite_zeroes    = nvme_co_pwrite_zeroes,
1480     .bdrv_co_pdiscard         = nvme_co_pdiscard,
1481
1482     .bdrv_co_flush_to_disk    = nvme_co_flush,
1483     .bdrv_reopen_prepare      = nvme_reopen_prepare,
1484
1485     .bdrv_refresh_filename    = nvme_refresh_filename,
1486     .bdrv_refresh_limits      = nvme_refresh_limits,
1487     .strong_runtime_opts      = nvme_strong_runtime_opts,
1488
1489     .bdrv_detach_aio_context  = nvme_detach_aio_context,
1490     .bdrv_attach_aio_context  = nvme_attach_aio_context,
1491
1492     .bdrv_io_plug             = nvme_aio_plug,
1493     .bdrv_io_unplug           = nvme_aio_unplug,
1494
1495     .bdrv_register_buf        = nvme_register_buf,
1496     .bdrv_unregister_buf      = nvme_unregister_buf,
1497 };
1498
1499 static void bdrv_nvme_init(void)
1500 {
1501     bdrv_register(&bdrv_nvme);
1502 }
1503
1504 block_init(bdrv_nvme_init);