2 * NVMe block driver based on vfio
4 * Copyright 2016 - 2018 Red Hat, Inc.
7 * Fam Zheng <famz@redhat.com>
8 * Paolo Bonzini <pbonzini@redhat.com>
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
15 #include <linux/vfio.h>
16 #include "qapi/error.h"
17 #include "qapi/qmp/qdict.h"
18 #include "qapi/qmp/qstring.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "qemu/cutils.h"
23 #include "qemu/option.h"
24 #include "qemu/memalign.h"
25 #include "qemu/vfio-helpers.h"
26 #include "block/block_int.h"
27 #include "sysemu/replay.h"
30 #include "block/nvme.h"
32 #define NVME_SQ_ENTRY_BYTES 64
33 #define NVME_CQ_ENTRY_BYTES 16
34 #define NVME_QUEUE_SIZE 128
35 #define NVME_DOORBELL_SIZE 4096
38 * We have to leave one slot empty as that is the full queue case where
41 #define NVME_NUM_REQS (NVME_QUEUE_SIZE - 1)
43 typedef struct BDRVNVMeState BDRVNVMeState
;
45 /* Same index is used for queues and IRQs */
47 #define INDEX_IO(n) (1 + n)
49 /* This driver shares a single MSIX IRQ for the admin and I/O queues */
51 MSIX_SHARED_IRQ_IDX
= 0,
59 /* Hardware MMIO register */
60 volatile uint32_t *doorbell
;
64 BlockCompletionFunc
*cb
;
68 uint64_t prp_list_iova
;
69 int free_req_next
; /* q->reqs[] index of next free req */
75 /* Read from I/O code path, initialized under BQL */
79 /* Fields protected by BQL */
80 uint8_t *prp_list_pages
;
82 /* Fields protected by @lock */
83 CoQueue free_req_queue
;
87 NVMeRequest reqs
[NVME_NUM_REQS
];
91 /* Thread-safe, no lock necessary */
92 QEMUBH
*completion_bh
;
95 struct BDRVNVMeState
{
96 AioContext
*aio_context
;
99 /* Memory mapped registers */
104 /* The submission/completion queue pairs.
108 NVMeQueuePair
**queues
;
109 unsigned queue_count
;
111 /* How many uint32_t elements does each doorbell entry take. */
112 size_t doorbell_scale
;
113 bool write_cache_supported
;
114 EventNotifier irq_notifier
[MSIX_IRQ_COUNT
];
116 uint64_t nsze
; /* Namespace size reported by identify command */
117 int nsid
; /* The namespace id to read/write data. */
120 uint64_t max_transfer
;
123 bool supports_write_zeroes
;
124 bool supports_discard
;
126 CoMutex dma_map_lock
;
127 CoQueue dma_flush_queue
;
129 /* Total size of mapped qiov, accessed under dma_map_lock */
132 /* PCI address (required for nvme_refresh_filename()) */
136 uint64_t completion_errors
;
137 uint64_t aligned_accesses
;
138 uint64_t unaligned_accesses
;
142 #define NVME_BLOCK_OPT_DEVICE "device"
143 #define NVME_BLOCK_OPT_NAMESPACE "namespace"
145 static void nvme_process_completion_bh(void *opaque
);
147 static QemuOptsList runtime_opts
= {
149 .head
= QTAILQ_HEAD_INITIALIZER(runtime_opts
.head
),
152 .name
= NVME_BLOCK_OPT_DEVICE
,
153 .type
= QEMU_OPT_STRING
,
154 .help
= "NVMe PCI device address",
157 .name
= NVME_BLOCK_OPT_NAMESPACE
,
158 .type
= QEMU_OPT_NUMBER
,
159 .help
= "NVMe namespace",
161 { /* end of list */ }
165 /* Returns true on success, false on failure. */
166 static bool nvme_init_queue(BDRVNVMeState
*s
, NVMeQueue
*q
,
167 unsigned nentries
, size_t entry_bytes
, Error
**errp
)
172 bytes
= ROUND_UP(nentries
* entry_bytes
, qemu_real_host_page_size());
173 q
->head
= q
->tail
= 0;
174 q
->queue
= qemu_try_memalign(qemu_real_host_page_size(), bytes
);
176 error_setg(errp
, "Cannot allocate queue");
179 memset(q
->queue
, 0, bytes
);
180 r
= qemu_vfio_dma_map(s
->vfio
, q
->queue
, bytes
, false, &q
->iova
, errp
);
182 error_prepend(errp
, "Cannot map queue: ");
187 static void nvme_free_queue(NVMeQueue
*q
)
189 qemu_vfree(q
->queue
);
192 static void nvme_free_queue_pair(NVMeQueuePair
*q
)
194 trace_nvme_free_queue_pair(q
->index
, q
, &q
->cq
, &q
->sq
);
195 if (q
->completion_bh
) {
196 qemu_bh_delete(q
->completion_bh
);
198 nvme_free_queue(&q
->sq
);
199 nvme_free_queue(&q
->cq
);
200 qemu_vfree(q
->prp_list_pages
);
201 qemu_mutex_destroy(&q
->lock
);
205 static void nvme_free_req_queue_cb(void *opaque
)
207 NVMeQueuePair
*q
= opaque
;
209 qemu_mutex_lock(&q
->lock
);
210 while (q
->free_req_head
!= -1 &&
211 qemu_co_enter_next(&q
->free_req_queue
, &q
->lock
)) {
212 /* Retry waiting requests */
214 qemu_mutex_unlock(&q
->lock
);
217 static NVMeQueuePair
*nvme_create_queue_pair(BDRVNVMeState
*s
,
218 AioContext
*aio_context
,
219 unsigned idx
, size_t size
,
224 uint64_t prp_list_iova
;
227 q
= g_try_new0(NVMeQueuePair
, 1);
229 error_setg(errp
, "Cannot allocate queue pair");
232 trace_nvme_create_queue_pair(idx
, q
, size
, aio_context
,
233 event_notifier_get_fd(s
->irq_notifier
));
234 bytes
= QEMU_ALIGN_UP(s
->page_size
* NVME_NUM_REQS
,
235 qemu_real_host_page_size());
236 q
->prp_list_pages
= qemu_try_memalign(qemu_real_host_page_size(), bytes
);
237 if (!q
->prp_list_pages
) {
238 error_setg(errp
, "Cannot allocate PRP page list");
241 memset(q
->prp_list_pages
, 0, bytes
);
242 qemu_mutex_init(&q
->lock
);
245 qemu_co_queue_init(&q
->free_req_queue
);
246 q
->completion_bh
= aio_bh_new(aio_context
, nvme_process_completion_bh
, q
);
247 r
= qemu_vfio_dma_map(s
->vfio
, q
->prp_list_pages
, bytes
,
248 false, &prp_list_iova
, errp
);
250 error_prepend(errp
, "Cannot map buffer for DMA: ");
253 q
->free_req_head
= -1;
254 for (i
= 0; i
< NVME_NUM_REQS
; i
++) {
255 NVMeRequest
*req
= &q
->reqs
[i
];
257 req
->free_req_next
= q
->free_req_head
;
258 q
->free_req_head
= i
;
259 req
->prp_list_page
= q
->prp_list_pages
+ i
* s
->page_size
;
260 req
->prp_list_iova
= prp_list_iova
+ i
* s
->page_size
;
263 if (!nvme_init_queue(s
, &q
->sq
, size
, NVME_SQ_ENTRY_BYTES
, errp
)) {
266 q
->sq
.doorbell
= &s
->doorbells
[idx
* s
->doorbell_scale
].sq_tail
;
268 if (!nvme_init_queue(s
, &q
->cq
, size
, NVME_CQ_ENTRY_BYTES
, errp
)) {
271 q
->cq
.doorbell
= &s
->doorbells
[idx
* s
->doorbell_scale
].cq_head
;
275 nvme_free_queue_pair(q
);
280 static void nvme_kick(NVMeQueuePair
*q
)
282 BDRVNVMeState
*s
= q
->s
;
284 if (s
->plugged
|| !q
->need_kick
) {
287 trace_nvme_kick(s
, q
->index
);
288 assert(!(q
->sq
.tail
& 0xFF00));
289 /* Fence the write to submission queue entry before notifying the device. */
291 *q
->sq
.doorbell
= cpu_to_le32(q
->sq
.tail
);
292 q
->inflight
+= q
->need_kick
;
296 static NVMeRequest
*nvme_get_free_req_nofail_locked(NVMeQueuePair
*q
)
300 req
= &q
->reqs
[q
->free_req_head
];
301 q
->free_req_head
= req
->free_req_next
;
302 req
->free_req_next
= -1;
306 /* Return a free request element if any, otherwise return NULL. */
307 static NVMeRequest
*nvme_get_free_req_nowait(NVMeQueuePair
*q
)
309 QEMU_LOCK_GUARD(&q
->lock
);
310 if (q
->free_req_head
== -1) {
313 return nvme_get_free_req_nofail_locked(q
);
317 * Wait for a free request to become available if necessary, then
320 static coroutine_fn NVMeRequest
*nvme_get_free_req(NVMeQueuePair
*q
)
322 QEMU_LOCK_GUARD(&q
->lock
);
324 while (q
->free_req_head
== -1) {
325 trace_nvme_free_req_queue_wait(q
->s
, q
->index
);
326 qemu_co_queue_wait(&q
->free_req_queue
, &q
->lock
);
329 return nvme_get_free_req_nofail_locked(q
);
333 static void nvme_put_free_req_locked(NVMeQueuePair
*q
, NVMeRequest
*req
)
335 req
->free_req_next
= q
->free_req_head
;
336 q
->free_req_head
= req
- q
->reqs
;
340 static void nvme_wake_free_req_locked(NVMeQueuePair
*q
)
342 if (!qemu_co_queue_empty(&q
->free_req_queue
)) {
343 replay_bh_schedule_oneshot_event(q
->s
->aio_context
,
344 nvme_free_req_queue_cb
, q
);
348 /* Insert a request in the freelist and wake waiters */
349 static void nvme_put_free_req_and_wake(NVMeQueuePair
*q
, NVMeRequest
*req
)
351 qemu_mutex_lock(&q
->lock
);
352 nvme_put_free_req_locked(q
, req
);
353 nvme_wake_free_req_locked(q
);
354 qemu_mutex_unlock(&q
->lock
);
357 static inline int nvme_translate_error(const NvmeCqe
*c
)
359 uint16_t status
= (le16_to_cpu(c
->status
) >> 1) & 0xFF;
361 trace_nvme_error(le32_to_cpu(c
->result
),
362 le16_to_cpu(c
->sq_head
),
363 le16_to_cpu(c
->sq_id
),
365 le16_to_cpu(status
));
380 static bool nvme_process_completion(NVMeQueuePair
*q
)
382 BDRVNVMeState
*s
= q
->s
;
383 bool progress
= false;
388 trace_nvme_process_completion(s
, q
->index
, q
->inflight
);
390 trace_nvme_process_completion_queue_plugged(s
, q
->index
);
395 * Support re-entrancy when a request cb() function invokes aio_poll().
396 * Pending completions must be visible to aio_poll() so that a cb()
397 * function can wait for the completion of another request.
399 * The aio_poll() loop will execute our BH and we'll resume completion
402 qemu_bh_schedule(q
->completion_bh
);
404 assert(q
->inflight
>= 0);
405 while (q
->inflight
) {
409 c
= (NvmeCqe
*)&q
->cq
.queue
[q
->cq
.head
* NVME_CQ_ENTRY_BYTES
];
410 if ((le16_to_cpu(c
->status
) & 0x1) == q
->cq_phase
) {
413 ret
= nvme_translate_error(c
);
415 s
->stats
.completion_errors
++;
417 q
->cq
.head
= (q
->cq
.head
+ 1) % NVME_QUEUE_SIZE
;
419 q
->cq_phase
= !q
->cq_phase
;
421 cid
= le16_to_cpu(c
->cid
);
422 if (cid
== 0 || cid
> NVME_QUEUE_SIZE
) {
423 warn_report("NVMe: Unexpected CID in completion queue: %"PRIu32
", "
424 "queue size: %u", cid
, NVME_QUEUE_SIZE
);
427 trace_nvme_complete_command(s
, q
->index
, cid
);
428 preq
= &q
->reqs
[cid
- 1];
430 assert(req
.cid
== cid
);
432 nvme_put_free_req_locked(q
, preq
);
433 preq
->cb
= preq
->opaque
= NULL
;
435 qemu_mutex_unlock(&q
->lock
);
436 req
.cb(req
.opaque
, ret
);
437 qemu_mutex_lock(&q
->lock
);
441 /* Notify the device so it can post more completions. */
443 *q
->cq
.doorbell
= cpu_to_le32(q
->cq
.head
);
444 nvme_wake_free_req_locked(q
);
447 qemu_bh_cancel(q
->completion_bh
);
452 static void nvme_process_completion_bh(void *opaque
)
454 NVMeQueuePair
*q
= opaque
;
457 * We're being invoked because a nvme_process_completion() cb() function
458 * called aio_poll(). The callback may be waiting for further completions
459 * so notify the device that it has space to fill in more completions now.
462 *q
->cq
.doorbell
= cpu_to_le32(q
->cq
.head
);
463 nvme_wake_free_req_locked(q
);
465 nvme_process_completion(q
);
468 static void nvme_trace_command(const NvmeCmd
*cmd
)
472 if (!trace_event_get_state_backends(TRACE_NVME_SUBMIT_COMMAND_RAW
)) {
475 for (i
= 0; i
< 8; ++i
) {
476 uint8_t *cmdp
= (uint8_t *)cmd
+ i
* 8;
477 trace_nvme_submit_command_raw(cmdp
[0], cmdp
[1], cmdp
[2], cmdp
[3],
478 cmdp
[4], cmdp
[5], cmdp
[6], cmdp
[7]);
482 static void nvme_submit_command(NVMeQueuePair
*q
, NVMeRequest
*req
,
483 NvmeCmd
*cmd
, BlockCompletionFunc cb
,
488 req
->opaque
= opaque
;
489 cmd
->cid
= cpu_to_le16(req
->cid
);
491 trace_nvme_submit_command(q
->s
, q
->index
, req
->cid
);
492 nvme_trace_command(cmd
);
493 qemu_mutex_lock(&q
->lock
);
494 memcpy((uint8_t *)q
->sq
.queue
+
495 q
->sq
.tail
* NVME_SQ_ENTRY_BYTES
, cmd
, sizeof(*cmd
));
496 q
->sq
.tail
= (q
->sq
.tail
+ 1) % NVME_QUEUE_SIZE
;
499 nvme_process_completion(q
);
500 qemu_mutex_unlock(&q
->lock
);
503 static void nvme_admin_cmd_sync_cb(void *opaque
, int ret
)
510 static int nvme_admin_cmd_sync(BlockDriverState
*bs
, NvmeCmd
*cmd
)
512 BDRVNVMeState
*s
= bs
->opaque
;
513 NVMeQueuePair
*q
= s
->queues
[INDEX_ADMIN
];
514 AioContext
*aio_context
= bdrv_get_aio_context(bs
);
516 int ret
= -EINPROGRESS
;
517 req
= nvme_get_free_req_nowait(q
);
521 nvme_submit_command(q
, req
, cmd
, nvme_admin_cmd_sync_cb
, &ret
);
523 AIO_WAIT_WHILE(aio_context
, ret
== -EINPROGRESS
);
527 /* Returns true on success, false on failure. */
528 static bool nvme_identify(BlockDriverState
*bs
, int namespace, Error
**errp
)
530 BDRVNVMeState
*s
= bs
->opaque
;
532 QEMU_AUTO_VFREE
union {
541 .opcode
= NVME_ADM_CMD_IDENTIFY
,
542 .cdw10
= cpu_to_le32(0x1),
544 size_t id_size
= QEMU_ALIGN_UP(sizeof(*id
), qemu_real_host_page_size());
546 id
= qemu_try_memalign(qemu_real_host_page_size(), id_size
);
548 error_setg(errp
, "Cannot allocate buffer for identify response");
551 r
= qemu_vfio_dma_map(s
->vfio
, id
, id_size
, true, &iova
, errp
);
553 error_prepend(errp
, "Cannot map buffer for DMA: ");
557 memset(id
, 0, id_size
);
558 cmd
.dptr
.prp1
= cpu_to_le64(iova
);
559 if (nvme_admin_cmd_sync(bs
, &cmd
)) {
560 error_setg(errp
, "Failed to identify controller");
564 if (le32_to_cpu(id
->ctrl
.nn
) < namespace) {
565 error_setg(errp
, "Invalid namespace");
568 s
->write_cache_supported
= le32_to_cpu(id
->ctrl
.vwc
) & 0x1;
569 s
->max_transfer
= (id
->ctrl
.mdts
? 1 << id
->ctrl
.mdts
: 0) * s
->page_size
;
570 /* For now the page list buffer per command is one page, to hold at most
571 * s->page_size / sizeof(uint64_t) entries. */
572 s
->max_transfer
= MIN_NON_ZERO(s
->max_transfer
,
573 s
->page_size
/ sizeof(uint64_t) * s
->page_size
);
575 oncs
= le16_to_cpu(id
->ctrl
.oncs
);
576 s
->supports_write_zeroes
= !!(oncs
& NVME_ONCS_WRITE_ZEROES
);
577 s
->supports_discard
= !!(oncs
& NVME_ONCS_DSM
);
579 memset(id
, 0, id_size
);
581 cmd
.nsid
= cpu_to_le32(namespace);
582 if (nvme_admin_cmd_sync(bs
, &cmd
)) {
583 error_setg(errp
, "Failed to identify namespace");
587 s
->nsze
= le64_to_cpu(id
->ns
.nsze
);
588 lbaf
= &id
->ns
.lbaf
[NVME_ID_NS_FLBAS_INDEX(id
->ns
.flbas
)];
590 if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(id
->ns
.dlfeat
) &&
591 NVME_ID_NS_DLFEAT_READ_BEHAVIOR(id
->ns
.dlfeat
) ==
592 NVME_ID_NS_DLFEAT_READ_BEHAVIOR_ZEROES
) {
593 bs
->supported_write_flags
|= BDRV_REQ_MAY_UNMAP
;
597 error_setg(errp
, "Namespaces with metadata are not yet supported");
601 if (lbaf
->ds
< BDRV_SECTOR_BITS
|| lbaf
->ds
> 12 ||
602 (1 << lbaf
->ds
) > s
->page_size
)
604 error_setg(errp
, "Namespace has unsupported block size (2^%d)",
610 s
->blkshift
= lbaf
->ds
;
612 qemu_vfio_dma_unmap(s
->vfio
, id
);
617 static void nvme_poll_queue(NVMeQueuePair
*q
)
619 const size_t cqe_offset
= q
->cq
.head
* NVME_CQ_ENTRY_BYTES
;
620 NvmeCqe
*cqe
= (NvmeCqe
*)&q
->cq
.queue
[cqe_offset
];
622 trace_nvme_poll_queue(q
->s
, q
->index
);
624 * Do an early check for completions. q->lock isn't needed because
625 * nvme_process_completion() only runs in the event loop thread and
626 * cannot race with itself.
628 if ((le16_to_cpu(cqe
->status
) & 0x1) == q
->cq_phase
) {
632 qemu_mutex_lock(&q
->lock
);
633 while (nvme_process_completion(q
)) {
636 qemu_mutex_unlock(&q
->lock
);
639 static void nvme_poll_queues(BDRVNVMeState
*s
)
643 for (i
= 0; i
< s
->queue_count
; i
++) {
644 nvme_poll_queue(s
->queues
[i
]);
648 static void nvme_handle_event(EventNotifier
*n
)
650 BDRVNVMeState
*s
= container_of(n
, BDRVNVMeState
,
651 irq_notifier
[MSIX_SHARED_IRQ_IDX
]);
653 trace_nvme_handle_event(s
);
654 event_notifier_test_and_clear(n
);
658 static bool nvme_add_io_queue(BlockDriverState
*bs
, Error
**errp
)
660 BDRVNVMeState
*s
= bs
->opaque
;
661 unsigned n
= s
->queue_count
;
664 unsigned queue_size
= NVME_QUEUE_SIZE
;
666 assert(n
<= UINT16_MAX
);
667 q
= nvme_create_queue_pair(s
, bdrv_get_aio_context(bs
),
668 n
, queue_size
, errp
);
673 .opcode
= NVME_ADM_CMD_CREATE_CQ
,
674 .dptr
.prp1
= cpu_to_le64(q
->cq
.iova
),
675 .cdw10
= cpu_to_le32(((queue_size
- 1) << 16) | n
),
676 .cdw11
= cpu_to_le32(NVME_CQ_IEN
| NVME_CQ_PC
),
678 if (nvme_admin_cmd_sync(bs
, &cmd
)) {
679 error_setg(errp
, "Failed to create CQ io queue [%u]", n
);
683 .opcode
= NVME_ADM_CMD_CREATE_SQ
,
684 .dptr
.prp1
= cpu_to_le64(q
->sq
.iova
),
685 .cdw10
= cpu_to_le32(((queue_size
- 1) << 16) | n
),
686 .cdw11
= cpu_to_le32(NVME_SQ_PC
| (n
<< 16)),
688 if (nvme_admin_cmd_sync(bs
, &cmd
)) {
689 error_setg(errp
, "Failed to create SQ io queue [%u]", n
);
692 s
->queues
= g_renew(NVMeQueuePair
*, s
->queues
, n
+ 1);
697 nvme_free_queue_pair(q
);
701 static bool nvme_poll_cb(void *opaque
)
703 EventNotifier
*e
= opaque
;
704 BDRVNVMeState
*s
= container_of(e
, BDRVNVMeState
,
705 irq_notifier
[MSIX_SHARED_IRQ_IDX
]);
708 for (i
= 0; i
< s
->queue_count
; i
++) {
709 NVMeQueuePair
*q
= s
->queues
[i
];
710 const size_t cqe_offset
= q
->cq
.head
* NVME_CQ_ENTRY_BYTES
;
711 NvmeCqe
*cqe
= (NvmeCqe
*)&q
->cq
.queue
[cqe_offset
];
714 * q->lock isn't needed because nvme_process_completion() only runs in
715 * the event loop thread and cannot race with itself.
717 if ((le16_to_cpu(cqe
->status
) & 0x1) != q
->cq_phase
) {
724 static void nvme_poll_ready(EventNotifier
*e
)
726 BDRVNVMeState
*s
= container_of(e
, BDRVNVMeState
,
727 irq_notifier
[MSIX_SHARED_IRQ_IDX
]);
732 static int nvme_init(BlockDriverState
*bs
, const char *device
, int namespace,
735 BDRVNVMeState
*s
= bs
->opaque
;
737 AioContext
*aio_context
= bdrv_get_aio_context(bs
);
742 uint64_t deadline
, now
;
743 volatile NvmeBar
*regs
= NULL
;
745 qemu_co_mutex_init(&s
->dma_map_lock
);
746 qemu_co_queue_init(&s
->dma_flush_queue
);
747 s
->device
= g_strdup(device
);
749 s
->aio_context
= bdrv_get_aio_context(bs
);
750 ret
= event_notifier_init(&s
->irq_notifier
[MSIX_SHARED_IRQ_IDX
], 0);
752 error_setg(errp
, "Failed to init event notifier");
756 s
->vfio
= qemu_vfio_open_pci(device
, errp
);
762 regs
= qemu_vfio_pci_map_bar(s
->vfio
, 0, 0, sizeof(NvmeBar
),
763 PROT_READ
| PROT_WRITE
, errp
);
768 /* Perform initialize sequence as described in NVMe spec "7.6.1
769 * Initialization". */
771 cap
= le64_to_cpu(regs
->cap
);
772 trace_nvme_controller_capability_raw(cap
);
773 trace_nvme_controller_capability("Maximum Queue Entries Supported",
774 1 + NVME_CAP_MQES(cap
));
775 trace_nvme_controller_capability("Contiguous Queues Required",
777 trace_nvme_controller_capability("Doorbell Stride",
778 1 << (2 + NVME_CAP_DSTRD(cap
)));
779 trace_nvme_controller_capability("Subsystem Reset Supported",
780 NVME_CAP_NSSRS(cap
));
781 trace_nvme_controller_capability("Memory Page Size Minimum",
782 1 << (12 + NVME_CAP_MPSMIN(cap
)));
783 trace_nvme_controller_capability("Memory Page Size Maximum",
784 1 << (12 + NVME_CAP_MPSMAX(cap
)));
785 if (!NVME_CAP_CSS(cap
)) {
786 error_setg(errp
, "Device doesn't support NVMe command set");
791 s
->page_size
= 1u << (12 + NVME_CAP_MPSMIN(cap
));
792 s
->doorbell_scale
= (4 << NVME_CAP_DSTRD(cap
)) / sizeof(uint32_t);
793 bs
->bl
.opt_mem_alignment
= s
->page_size
;
794 bs
->bl
.request_alignment
= s
->page_size
;
795 timeout_ms
= MIN(500 * NVME_CAP_TO(cap
), 30000);
797 ver
= le32_to_cpu(regs
->vs
);
798 trace_nvme_controller_spec_version(extract32(ver
, 16, 16),
799 extract32(ver
, 8, 8),
800 extract32(ver
, 0, 8));
802 /* Reset device to get a clean state. */
803 regs
->cc
= cpu_to_le32(le32_to_cpu(regs
->cc
) & 0xFE);
804 /* Wait for CSTS.RDY = 0. */
805 deadline
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) + timeout_ms
* SCALE_MS
;
806 while (NVME_CSTS_RDY(le32_to_cpu(regs
->csts
))) {
807 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) > deadline
) {
808 error_setg(errp
, "Timeout while waiting for device to reset (%"
816 s
->bar0_wo_map
= qemu_vfio_pci_map_bar(s
->vfio
, 0, 0,
817 sizeof(NvmeBar
) + NVME_DOORBELL_SIZE
,
819 s
->doorbells
= (void *)((uintptr_t)s
->bar0_wo_map
+ sizeof(NvmeBar
));
825 /* Set up admin queue. */
826 s
->queues
= g_new(NVMeQueuePair
*, 1);
827 q
= nvme_create_queue_pair(s
, aio_context
, 0, NVME_QUEUE_SIZE
, errp
);
832 s
->queues
[INDEX_ADMIN
] = q
;
834 QEMU_BUILD_BUG_ON((NVME_QUEUE_SIZE
- 1) & 0xF000);
835 regs
->aqa
= cpu_to_le32(((NVME_QUEUE_SIZE
- 1) << AQA_ACQS_SHIFT
) |
836 ((NVME_QUEUE_SIZE
- 1) << AQA_ASQS_SHIFT
));
837 regs
->asq
= cpu_to_le64(q
->sq
.iova
);
838 regs
->acq
= cpu_to_le64(q
->cq
.iova
);
840 /* After setting up all control registers we can enable device now. */
841 regs
->cc
= cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES
) << CC_IOCQES_SHIFT
) |
842 (ctz32(NVME_SQ_ENTRY_BYTES
) << CC_IOSQES_SHIFT
) |
844 /* Wait for CSTS.RDY = 1. */
845 now
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
846 deadline
= now
+ timeout_ms
* SCALE_MS
;
847 while (!NVME_CSTS_RDY(le32_to_cpu(regs
->csts
))) {
848 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) > deadline
) {
849 error_setg(errp
, "Timeout while waiting for device to start (%"
857 ret
= qemu_vfio_pci_init_irq(s
->vfio
, s
->irq_notifier
,
858 VFIO_PCI_MSIX_IRQ_INDEX
, errp
);
862 aio_set_event_notifier(bdrv_get_aio_context(bs
),
863 &s
->irq_notifier
[MSIX_SHARED_IRQ_IDX
],
864 false, nvme_handle_event
, nvme_poll_cb
,
867 if (!nvme_identify(bs
, namespace, errp
)) {
872 /* Set up command queues. */
873 if (!nvme_add_io_queue(bs
, errp
)) {
878 qemu_vfio_pci_unmap_bar(s
->vfio
, 0, (void *)regs
, 0, sizeof(NvmeBar
));
881 /* Cleaning up is done in nvme_file_open() upon error. */
885 /* Parse a filename in the format of nvme://XXXX:XX:XX.X/X. Example:
887 * nvme://0000:44:00.0/1
889 * where the "nvme://" is a fixed form of the protocol prefix, the middle part
890 * is the PCI address, and the last part is the namespace number starting from
891 * 1 according to the NVMe spec. */
892 static void nvme_parse_filename(const char *filename
, QDict
*options
,
895 int pref
= strlen("nvme://");
897 if (strlen(filename
) > pref
&& !strncmp(filename
, "nvme://", pref
)) {
898 const char *tmp
= filename
+ pref
;
900 const char *namespace;
902 const char *slash
= strchr(tmp
, '/');
904 qdict_put_str(options
, NVME_BLOCK_OPT_DEVICE
, tmp
);
907 device
= g_strndup(tmp
, slash
- tmp
);
908 qdict_put_str(options
, NVME_BLOCK_OPT_DEVICE
, device
);
910 namespace = slash
+ 1;
911 if (*namespace && qemu_strtoul(namespace, NULL
, 10, &ns
)) {
912 error_setg(errp
, "Invalid namespace '%s', positive number expected",
916 qdict_put_str(options
, NVME_BLOCK_OPT_NAMESPACE
,
917 *namespace ? namespace : "1");
921 static int nvme_enable_disable_write_cache(BlockDriverState
*bs
, bool enable
,
925 BDRVNVMeState
*s
= bs
->opaque
;
927 .opcode
= NVME_ADM_CMD_SET_FEATURES
,
928 .nsid
= cpu_to_le32(s
->nsid
),
929 .cdw10
= cpu_to_le32(0x06),
930 .cdw11
= cpu_to_le32(enable
? 0x01 : 0x00),
933 ret
= nvme_admin_cmd_sync(bs
, &cmd
);
935 error_setg(errp
, "Failed to configure NVMe write cache");
940 static void nvme_close(BlockDriverState
*bs
)
942 BDRVNVMeState
*s
= bs
->opaque
;
944 for (unsigned i
= 0; i
< s
->queue_count
; ++i
) {
945 nvme_free_queue_pair(s
->queues
[i
]);
948 aio_set_event_notifier(bdrv_get_aio_context(bs
),
949 &s
->irq_notifier
[MSIX_SHARED_IRQ_IDX
],
950 false, NULL
, NULL
, NULL
);
951 event_notifier_cleanup(&s
->irq_notifier
[MSIX_SHARED_IRQ_IDX
]);
952 qemu_vfio_pci_unmap_bar(s
->vfio
, 0, s
->bar0_wo_map
,
953 0, sizeof(NvmeBar
) + NVME_DOORBELL_SIZE
);
954 qemu_vfio_close(s
->vfio
);
959 static int nvme_file_open(BlockDriverState
*bs
, QDict
*options
, int flags
,
966 BDRVNVMeState
*s
= bs
->opaque
;
968 bs
->supported_write_flags
= BDRV_REQ_FUA
;
970 opts
= qemu_opts_create(&runtime_opts
, NULL
, 0, &error_abort
);
971 qemu_opts_absorb_qdict(opts
, options
, &error_abort
);
972 device
= qemu_opt_get(opts
, NVME_BLOCK_OPT_DEVICE
);
974 error_setg(errp
, "'" NVME_BLOCK_OPT_DEVICE
"' option is required");
979 namespace = qemu_opt_get_number(opts
, NVME_BLOCK_OPT_NAMESPACE
, 1);
980 ret
= nvme_init(bs
, device
, namespace, errp
);
985 if (flags
& BDRV_O_NOCACHE
) {
986 if (!s
->write_cache_supported
) {
988 "NVMe controller doesn't support write cache configuration");
991 ret
= nvme_enable_disable_write_cache(bs
, !(flags
& BDRV_O_NOCACHE
),
1004 static int64_t nvme_getlength(BlockDriverState
*bs
)
1006 BDRVNVMeState
*s
= bs
->opaque
;
1007 return s
->nsze
<< s
->blkshift
;
1010 static uint32_t nvme_get_blocksize(BlockDriverState
*bs
)
1012 BDRVNVMeState
*s
= bs
->opaque
;
1013 assert(s
->blkshift
>= BDRV_SECTOR_BITS
&& s
->blkshift
<= 12);
1014 return UINT32_C(1) << s
->blkshift
;
1017 static int nvme_probe_blocksizes(BlockDriverState
*bs
, BlockSizes
*bsz
)
1019 uint32_t blocksize
= nvme_get_blocksize(bs
);
1020 bsz
->phys
= blocksize
;
1021 bsz
->log
= blocksize
;
1025 /* Called with s->dma_map_lock */
1026 static coroutine_fn
int nvme_cmd_unmap_qiov(BlockDriverState
*bs
,
1030 BDRVNVMeState
*s
= bs
->opaque
;
1032 s
->dma_map_count
-= qiov
->size
;
1033 if (!s
->dma_map_count
&& !qemu_co_queue_empty(&s
->dma_flush_queue
)) {
1034 r
= qemu_vfio_dma_reset_temporary(s
->vfio
);
1036 qemu_co_queue_restart_all(&s
->dma_flush_queue
);
1042 /* Called with s->dma_map_lock */
1043 static coroutine_fn
int nvme_cmd_map_qiov(BlockDriverState
*bs
, NvmeCmd
*cmd
,
1044 NVMeRequest
*req
, QEMUIOVector
*qiov
)
1046 BDRVNVMeState
*s
= bs
->opaque
;
1047 uint64_t *pagelist
= req
->prp_list_page
;
1050 Error
*local_err
= NULL
, **errp
= NULL
;
1053 assert(QEMU_IS_ALIGNED(qiov
->size
, s
->page_size
));
1054 assert(qiov
->size
/ s
->page_size
<= s
->page_size
/ sizeof(uint64_t));
1055 for (i
= 0; i
< qiov
->niov
; ++i
) {
1058 size_t len
= QEMU_ALIGN_UP(qiov
->iov
[i
].iov_len
,
1059 qemu_real_host_page_size());
1061 r
= qemu_vfio_dma_map(s
->vfio
,
1062 qiov
->iov
[i
].iov_base
,
1063 len
, true, &iova
, errp
);
1066 * In addition to the -ENOMEM error, the VFIO_IOMMU_MAP_DMA
1067 * ioctl returns -ENOSPC to signal the user exhausted the DMA
1068 * mappings available for a container since Linux kernel commit
1069 * 492855939bdb ("vfio/type1: Limit DMA mappings per container",
1070 * April 2019, see CVE-2019-3882).
1072 * This block driver already handles this error path by checking
1073 * for the -ENOMEM error, so we directly replace -ENOSPC by
1074 * -ENOMEM. Beside, -ENOSPC has a specific meaning for blockdev
1075 * coroutines: it triggers BLOCKDEV_ON_ERROR_ENOSPC and
1076 * BLOCK_ERROR_ACTION_STOP which stops the VM, asking the operator
1077 * to add more storage to the blockdev. Not something we can do
1078 * easily with an IOMMU :)
1082 if (r
== -ENOMEM
&& retry
) {
1084 * We exhausted the DMA mappings available for our container:
1085 * recycle the volatile IOVA mappings.
1088 trace_nvme_dma_flush_queue_wait(s
);
1089 if (s
->dma_map_count
) {
1090 trace_nvme_dma_map_flush(s
);
1091 qemu_co_queue_wait(&s
->dma_flush_queue
, &s
->dma_map_lock
);
1093 r
= qemu_vfio_dma_reset_temporary(s
->vfio
);
1106 for (j
= 0; j
< qiov
->iov
[i
].iov_len
/ s
->page_size
; j
++) {
1107 pagelist
[entries
++] = cpu_to_le64(iova
+ j
* s
->page_size
);
1109 trace_nvme_cmd_map_qiov_iov(s
, i
, qiov
->iov
[i
].iov_base
,
1110 qiov
->iov
[i
].iov_len
/ s
->page_size
);
1113 s
->dma_map_count
+= qiov
->size
;
1115 assert(entries
<= s
->page_size
/ sizeof(uint64_t));
1120 cmd
->dptr
.prp1
= pagelist
[0];
1124 cmd
->dptr
.prp1
= pagelist
[0];
1125 cmd
->dptr
.prp2
= pagelist
[1];
1128 cmd
->dptr
.prp1
= pagelist
[0];
1129 cmd
->dptr
.prp2
= cpu_to_le64(req
->prp_list_iova
+ sizeof(uint64_t));
1132 trace_nvme_cmd_map_qiov(s
, cmd
, req
, qiov
, entries
);
1133 for (i
= 0; i
< entries
; ++i
) {
1134 trace_nvme_cmd_map_qiov_pages(s
, i
, pagelist
[i
]);
1138 /* No need to unmap [0 - i) iovs even if we've failed, since we don't
1139 * increment s->dma_map_count. This is okay for fixed mapping memory areas
1140 * because they are already mapped before calling this function; for
1141 * temporary mappings, a later nvme_cmd_(un)map_qiov will reclaim by
1142 * calling qemu_vfio_dma_reset_temporary when necessary. */
1144 error_reportf_err(local_err
, "Cannot map buffer for DMA: ");
1155 static void nvme_rw_cb_bh(void *opaque
)
1157 NVMeCoData
*data
= opaque
;
1158 qemu_coroutine_enter(data
->co
);
1161 static void nvme_rw_cb(void *opaque
, int ret
)
1163 NVMeCoData
*data
= opaque
;
1166 /* The rw coroutine hasn't yielded, don't try to enter. */
1169 replay_bh_schedule_oneshot_event(data
->ctx
, nvme_rw_cb_bh
, data
);
1172 static coroutine_fn
int nvme_co_prw_aligned(BlockDriverState
*bs
,
1173 uint64_t offset
, uint64_t bytes
,
1179 BDRVNVMeState
*s
= bs
->opaque
;
1180 NVMeQueuePair
*ioq
= s
->queues
[INDEX_IO(0)];
1183 uint32_t cdw12
= (((bytes
>> s
->blkshift
) - 1) & 0xFFFF) |
1184 (flags
& BDRV_REQ_FUA
? 1 << 30 : 0);
1186 .opcode
= is_write
? NVME_CMD_WRITE
: NVME_CMD_READ
,
1187 .nsid
= cpu_to_le32(s
->nsid
),
1188 .cdw10
= cpu_to_le32((offset
>> s
->blkshift
) & 0xFFFFFFFF),
1189 .cdw11
= cpu_to_le32(((offset
>> s
->blkshift
) >> 32) & 0xFFFFFFFF),
1190 .cdw12
= cpu_to_le32(cdw12
),
1193 .ctx
= bdrv_get_aio_context(bs
),
1194 .ret
= -EINPROGRESS
,
1197 trace_nvme_prw_aligned(s
, is_write
, offset
, bytes
, flags
, qiov
->niov
);
1198 assert(s
->queue_count
> 1);
1199 req
= nvme_get_free_req(ioq
);
1202 qemu_co_mutex_lock(&s
->dma_map_lock
);
1203 r
= nvme_cmd_map_qiov(bs
, &cmd
, req
, qiov
);
1204 qemu_co_mutex_unlock(&s
->dma_map_lock
);
1206 nvme_put_free_req_and_wake(ioq
, req
);
1209 nvme_submit_command(ioq
, req
, &cmd
, nvme_rw_cb
, &data
);
1211 data
.co
= qemu_coroutine_self();
1212 while (data
.ret
== -EINPROGRESS
) {
1213 qemu_coroutine_yield();
1216 qemu_co_mutex_lock(&s
->dma_map_lock
);
1217 r
= nvme_cmd_unmap_qiov(bs
, qiov
);
1218 qemu_co_mutex_unlock(&s
->dma_map_lock
);
1223 trace_nvme_rw_done(s
, is_write
, offset
, bytes
, data
.ret
);
1227 static inline bool nvme_qiov_aligned(BlockDriverState
*bs
,
1228 const QEMUIOVector
*qiov
)
1231 BDRVNVMeState
*s
= bs
->opaque
;
1233 for (i
= 0; i
< qiov
->niov
; ++i
) {
1234 if (!QEMU_PTR_IS_ALIGNED(qiov
->iov
[i
].iov_base
,
1235 qemu_real_host_page_size()) ||
1236 !QEMU_IS_ALIGNED(qiov
->iov
[i
].iov_len
, qemu_real_host_page_size())) {
1237 trace_nvme_qiov_unaligned(qiov
, i
, qiov
->iov
[i
].iov_base
,
1238 qiov
->iov
[i
].iov_len
, s
->page_size
);
1245 static coroutine_fn
int nvme_co_prw(BlockDriverState
*bs
,
1246 uint64_t offset
, uint64_t bytes
,
1247 QEMUIOVector
*qiov
, bool is_write
,
1250 BDRVNVMeState
*s
= bs
->opaque
;
1252 QEMU_AUTO_VFREE
uint8_t *buf
= NULL
;
1253 QEMUIOVector local_qiov
;
1254 size_t len
= QEMU_ALIGN_UP(bytes
, qemu_real_host_page_size());
1255 assert(QEMU_IS_ALIGNED(offset
, s
->page_size
));
1256 assert(QEMU_IS_ALIGNED(bytes
, s
->page_size
));
1257 assert(bytes
<= s
->max_transfer
);
1258 if (nvme_qiov_aligned(bs
, qiov
)) {
1259 s
->stats
.aligned_accesses
++;
1260 return nvme_co_prw_aligned(bs
, offset
, bytes
, qiov
, is_write
, flags
);
1262 s
->stats
.unaligned_accesses
++;
1263 trace_nvme_prw_buffered(s
, offset
, bytes
, qiov
->niov
, is_write
);
1264 buf
= qemu_try_memalign(qemu_real_host_page_size(), len
);
1269 qemu_iovec_init(&local_qiov
, 1);
1271 qemu_iovec_to_buf(qiov
, 0, buf
, bytes
);
1273 qemu_iovec_add(&local_qiov
, buf
, bytes
);
1274 r
= nvme_co_prw_aligned(bs
, offset
, bytes
, &local_qiov
, is_write
, flags
);
1275 qemu_iovec_destroy(&local_qiov
);
1276 if (!r
&& !is_write
) {
1277 qemu_iovec_from_buf(qiov
, 0, buf
, bytes
);
1282 static coroutine_fn
int nvme_co_preadv(BlockDriverState
*bs
,
1283 int64_t offset
, int64_t bytes
,
1285 BdrvRequestFlags flags
)
1287 return nvme_co_prw(bs
, offset
, bytes
, qiov
, false, flags
);
1290 static coroutine_fn
int nvme_co_pwritev(BlockDriverState
*bs
,
1291 int64_t offset
, int64_t bytes
,
1293 BdrvRequestFlags flags
)
1295 return nvme_co_prw(bs
, offset
, bytes
, qiov
, true, flags
);
1298 static coroutine_fn
int nvme_co_flush(BlockDriverState
*bs
)
1300 BDRVNVMeState
*s
= bs
->opaque
;
1301 NVMeQueuePair
*ioq
= s
->queues
[INDEX_IO(0)];
1304 .opcode
= NVME_CMD_FLUSH
,
1305 .nsid
= cpu_to_le32(s
->nsid
),
1308 .ctx
= bdrv_get_aio_context(bs
),
1309 .ret
= -EINPROGRESS
,
1312 assert(s
->queue_count
> 1);
1313 req
= nvme_get_free_req(ioq
);
1315 nvme_submit_command(ioq
, req
, &cmd
, nvme_rw_cb
, &data
);
1317 data
.co
= qemu_coroutine_self();
1318 if (data
.ret
== -EINPROGRESS
) {
1319 qemu_coroutine_yield();
1326 static coroutine_fn
int nvme_co_pwrite_zeroes(BlockDriverState
*bs
,
1329 BdrvRequestFlags flags
)
1331 BDRVNVMeState
*s
= bs
->opaque
;
1332 NVMeQueuePair
*ioq
= s
->queues
[INDEX_IO(0)];
1336 if (!s
->supports_write_zeroes
) {
1344 cdw12
= ((bytes
>> s
->blkshift
) - 1) & 0xFFFF;
1346 * We should not lose information. pwrite_zeroes_alignment and
1347 * max_pwrite_zeroes guarantees it.
1349 assert(((cdw12
+ 1) << s
->blkshift
) == bytes
);
1352 .opcode
= NVME_CMD_WRITE_ZEROES
,
1353 .nsid
= cpu_to_le32(s
->nsid
),
1354 .cdw10
= cpu_to_le32((offset
>> s
->blkshift
) & 0xFFFFFFFF),
1355 .cdw11
= cpu_to_le32(((offset
>> s
->blkshift
) >> 32) & 0xFFFFFFFF),
1359 .ctx
= bdrv_get_aio_context(bs
),
1360 .ret
= -EINPROGRESS
,
1363 if (flags
& BDRV_REQ_MAY_UNMAP
) {
1367 if (flags
& BDRV_REQ_FUA
) {
1371 cmd
.cdw12
= cpu_to_le32(cdw12
);
1373 trace_nvme_write_zeroes(s
, offset
, bytes
, flags
);
1374 assert(s
->queue_count
> 1);
1375 req
= nvme_get_free_req(ioq
);
1378 nvme_submit_command(ioq
, req
, &cmd
, nvme_rw_cb
, &data
);
1380 data
.co
= qemu_coroutine_self();
1381 while (data
.ret
== -EINPROGRESS
) {
1382 qemu_coroutine_yield();
1385 trace_nvme_rw_done(s
, true, offset
, bytes
, data
.ret
);
1390 static int coroutine_fn
nvme_co_pdiscard(BlockDriverState
*bs
,
1394 BDRVNVMeState
*s
= bs
->opaque
;
1395 NVMeQueuePair
*ioq
= s
->queues
[INDEX_IO(0)];
1397 QEMU_AUTO_VFREE NvmeDsmRange
*buf
= NULL
;
1398 QEMUIOVector local_qiov
;
1402 .opcode
= NVME_CMD_DSM
,
1403 .nsid
= cpu_to_le32(s
->nsid
),
1404 .cdw10
= cpu_to_le32(0), /*number of ranges - 0 based*/
1405 .cdw11
= cpu_to_le32(1 << 2), /*deallocate bit*/
1409 .ctx
= bdrv_get_aio_context(bs
),
1410 .ret
= -EINPROGRESS
,
1413 if (!s
->supports_discard
) {
1417 assert(s
->queue_count
> 1);
1420 * Filling the @buf requires @offset and @bytes to satisfy restrictions
1421 * defined in nvme_refresh_limits().
1423 assert(QEMU_IS_ALIGNED(bytes
, 1UL << s
->blkshift
));
1424 assert(QEMU_IS_ALIGNED(offset
, 1UL << s
->blkshift
));
1425 assert((bytes
>> s
->blkshift
) <= UINT32_MAX
);
1427 buf
= qemu_try_memalign(s
->page_size
, s
->page_size
);
1431 memset(buf
, 0, s
->page_size
);
1432 buf
->nlb
= cpu_to_le32(bytes
>> s
->blkshift
);
1433 buf
->slba
= cpu_to_le64(offset
>> s
->blkshift
);
1436 qemu_iovec_init(&local_qiov
, 1);
1437 qemu_iovec_add(&local_qiov
, buf
, 4096);
1439 req
= nvme_get_free_req(ioq
);
1442 qemu_co_mutex_lock(&s
->dma_map_lock
);
1443 ret
= nvme_cmd_map_qiov(bs
, &cmd
, req
, &local_qiov
);
1444 qemu_co_mutex_unlock(&s
->dma_map_lock
);
1447 nvme_put_free_req_and_wake(ioq
, req
);
1451 trace_nvme_dsm(s
, offset
, bytes
);
1453 nvme_submit_command(ioq
, req
, &cmd
, nvme_rw_cb
, &data
);
1455 data
.co
= qemu_coroutine_self();
1456 while (data
.ret
== -EINPROGRESS
) {
1457 qemu_coroutine_yield();
1460 qemu_co_mutex_lock(&s
->dma_map_lock
);
1461 ret
= nvme_cmd_unmap_qiov(bs
, &local_qiov
);
1462 qemu_co_mutex_unlock(&s
->dma_map_lock
);
1469 trace_nvme_dsm_done(s
, offset
, bytes
, ret
);
1471 qemu_iovec_destroy(&local_qiov
);
1476 static int coroutine_fn
nvme_co_truncate(BlockDriverState
*bs
, int64_t offset
,
1477 bool exact
, PreallocMode prealloc
,
1478 BdrvRequestFlags flags
, Error
**errp
)
1482 if (prealloc
!= PREALLOC_MODE_OFF
) {
1483 error_setg(errp
, "Unsupported preallocation mode '%s'",
1484 PreallocMode_str(prealloc
));
1488 cur_length
= nvme_getlength(bs
);
1489 if (offset
!= cur_length
&& exact
) {
1490 error_setg(errp
, "Cannot resize NVMe devices");
1492 } else if (offset
> cur_length
) {
1493 error_setg(errp
, "Cannot grow NVMe devices");
1500 static int nvme_reopen_prepare(BDRVReopenState
*reopen_state
,
1501 BlockReopenQueue
*queue
, Error
**errp
)
1506 static void nvme_refresh_filename(BlockDriverState
*bs
)
1508 BDRVNVMeState
*s
= bs
->opaque
;
1510 snprintf(bs
->exact_filename
, sizeof(bs
->exact_filename
), "nvme://%s/%i",
1511 s
->device
, s
->nsid
);
1514 static void nvme_refresh_limits(BlockDriverState
*bs
, Error
**errp
)
1516 BDRVNVMeState
*s
= bs
->opaque
;
1518 bs
->bl
.opt_mem_alignment
= s
->page_size
;
1519 bs
->bl
.request_alignment
= s
->page_size
;
1520 bs
->bl
.max_transfer
= s
->max_transfer
;
1523 * Look at nvme_co_pwrite_zeroes: after shift and decrement we should get
1526 bs
->bl
.max_pwrite_zeroes
= 1ULL << (s
->blkshift
+ 16);
1527 bs
->bl
.pwrite_zeroes_alignment
= MAX(bs
->bl
.request_alignment
,
1528 1UL << s
->blkshift
);
1530 bs
->bl
.max_pdiscard
= (uint64_t)UINT32_MAX
<< s
->blkshift
;
1531 bs
->bl
.pdiscard_alignment
= MAX(bs
->bl
.request_alignment
,
1532 1UL << s
->blkshift
);
1535 static void nvme_detach_aio_context(BlockDriverState
*bs
)
1537 BDRVNVMeState
*s
= bs
->opaque
;
1539 for (unsigned i
= 0; i
< s
->queue_count
; i
++) {
1540 NVMeQueuePair
*q
= s
->queues
[i
];
1542 qemu_bh_delete(q
->completion_bh
);
1543 q
->completion_bh
= NULL
;
1546 aio_set_event_notifier(bdrv_get_aio_context(bs
),
1547 &s
->irq_notifier
[MSIX_SHARED_IRQ_IDX
],
1548 false, NULL
, NULL
, NULL
);
1551 static void nvme_attach_aio_context(BlockDriverState
*bs
,
1552 AioContext
*new_context
)
1554 BDRVNVMeState
*s
= bs
->opaque
;
1556 s
->aio_context
= new_context
;
1557 aio_set_event_notifier(new_context
, &s
->irq_notifier
[MSIX_SHARED_IRQ_IDX
],
1558 false, nvme_handle_event
, nvme_poll_cb
,
1561 for (unsigned i
= 0; i
< s
->queue_count
; i
++) {
1562 NVMeQueuePair
*q
= s
->queues
[i
];
1565 aio_bh_new(new_context
, nvme_process_completion_bh
, q
);
1569 static void nvme_aio_plug(BlockDriverState
*bs
)
1571 BDRVNVMeState
*s
= bs
->opaque
;
1572 assert(!s
->plugged
);
1576 static void nvme_aio_unplug(BlockDriverState
*bs
)
1578 BDRVNVMeState
*s
= bs
->opaque
;
1581 for (unsigned i
= INDEX_IO(0); i
< s
->queue_count
; i
++) {
1582 NVMeQueuePair
*q
= s
->queues
[i
];
1583 qemu_mutex_lock(&q
->lock
);
1585 nvme_process_completion(q
);
1586 qemu_mutex_unlock(&q
->lock
);
1590 static bool nvme_register_buf(BlockDriverState
*bs
, void *host
, size_t size
,
1594 BDRVNVMeState
*s
= bs
->opaque
;
1597 * FIXME: we may run out of IOVA addresses after repeated
1598 * bdrv_register_buf/bdrv_unregister_buf, because nvme_vfio_dma_unmap
1599 * doesn't reclaim addresses for fixed mappings.
1601 ret
= qemu_vfio_dma_map(s
->vfio
, host
, size
, false, NULL
, errp
);
1605 static void nvme_unregister_buf(BlockDriverState
*bs
, void *host
, size_t size
)
1607 BDRVNVMeState
*s
= bs
->opaque
;
1609 qemu_vfio_dma_unmap(s
->vfio
, host
);
1612 static BlockStatsSpecific
*nvme_get_specific_stats(BlockDriverState
*bs
)
1614 BlockStatsSpecific
*stats
= g_new(BlockStatsSpecific
, 1);
1615 BDRVNVMeState
*s
= bs
->opaque
;
1617 stats
->driver
= BLOCKDEV_DRIVER_NVME
;
1618 stats
->u
.nvme
= (BlockStatsSpecificNvme
) {
1619 .completion_errors
= s
->stats
.completion_errors
,
1620 .aligned_accesses
= s
->stats
.aligned_accesses
,
1621 .unaligned_accesses
= s
->stats
.unaligned_accesses
,
1627 static const char *const nvme_strong_runtime_opts
[] = {
1628 NVME_BLOCK_OPT_DEVICE
,
1629 NVME_BLOCK_OPT_NAMESPACE
,
1634 static BlockDriver bdrv_nvme
= {
1635 .format_name
= "nvme",
1636 .protocol_name
= "nvme",
1637 .instance_size
= sizeof(BDRVNVMeState
),
1639 .bdrv_co_create_opts
= bdrv_co_create_opts_simple
,
1640 .create_opts
= &bdrv_create_opts_simple
,
1642 .bdrv_parse_filename
= nvme_parse_filename
,
1643 .bdrv_file_open
= nvme_file_open
,
1644 .bdrv_close
= nvme_close
,
1645 .bdrv_getlength
= nvme_getlength
,
1646 .bdrv_probe_blocksizes
= nvme_probe_blocksizes
,
1647 .bdrv_co_truncate
= nvme_co_truncate
,
1649 .bdrv_co_preadv
= nvme_co_preadv
,
1650 .bdrv_co_pwritev
= nvme_co_pwritev
,
1652 .bdrv_co_pwrite_zeroes
= nvme_co_pwrite_zeroes
,
1653 .bdrv_co_pdiscard
= nvme_co_pdiscard
,
1655 .bdrv_co_flush_to_disk
= nvme_co_flush
,
1656 .bdrv_reopen_prepare
= nvme_reopen_prepare
,
1658 .bdrv_refresh_filename
= nvme_refresh_filename
,
1659 .bdrv_refresh_limits
= nvme_refresh_limits
,
1660 .strong_runtime_opts
= nvme_strong_runtime_opts
,
1661 .bdrv_get_specific_stats
= nvme_get_specific_stats
,
1663 .bdrv_detach_aio_context
= nvme_detach_aio_context
,
1664 .bdrv_attach_aio_context
= nvme_attach_aio_context
,
1666 .bdrv_io_plug
= nvme_aio_plug
,
1667 .bdrv_io_unplug
= nvme_aio_unplug
,
1669 .bdrv_register_buf
= nvme_register_buf
,
1670 .bdrv_unregister_buf
= nvme_unregister_buf
,
1673 static void bdrv_nvme_init(void)
1675 bdrv_register(&bdrv_nvme
);
1678 block_init(bdrv_nvme_init
);