2 * QEMU NVM Express Controller
4 * Copyright (c) 2012, Intel Corporation
6 * Written by Keith Busch <keith.busch@intel.com>
8 * This code is licensed under the GNU GPL v2 or later.
12 * Reference Specs: http://www.nvmexpress.org, 1.2, 1.1, 1.0e
14 * https://nvmexpress.org/developers/nvme-specification/
19 * -drive file=<file>,if=none,id=<drive_id>
20 * -device nvme,serial=<serial>,id=<bus_name>, \
21 * cmb_size_mb=<cmb_size_mb[optional]>, \
22 * [pmrdev=<mem_backend_file_id>,] \
23 * max_ioqpairs=<N[optional]>, \
24 * aerl=<N[optional]>, aer_max_queued=<N[optional]>, \
26 * -device nvme-ns,drive=<drive_id>,bus=bus_name,nsid=<nsid>
28 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
29 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
31 * cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation
32 * in available BAR's. cmb_size_mb= will take precedence over pmrdev= when
34 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
36 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
37 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
40 * nvme device parameters
41 * ~~~~~~~~~~~~~~~~~~~~~~
43 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
44 * of concurrently outstanding Asynchronous Event Request commands suppoert
45 * by the controller. This is a 0's based value.
48 * This is the maximum number of events that the device will enqueue for
49 * completion when there are no oustanding AERs. When the maximum number of
50 * enqueued events are reached, subsequent events will be dropped.
54 #include "qemu/osdep.h"
55 #include "qemu/units.h"
56 #include "qemu/error-report.h"
57 #include "hw/block/block.h"
58 #include "hw/pci/msix.h"
59 #include "hw/pci/pci.h"
60 #include "hw/qdev-properties.h"
61 #include "migration/vmstate.h"
62 #include "sysemu/sysemu.h"
63 #include "qapi/error.h"
64 #include "qapi/visitor.h"
65 #include "sysemu/hostmem.h"
66 #include "sysemu/block-backend.h"
67 #include "exec/memory.h"
69 #include "qemu/module.h"
70 #include "qemu/cutils.h"
75 #define NVME_MAX_IOQPAIRS 0xffff
76 #define NVME_DB_SIZE 4
77 #define NVME_SPEC_VER 0x00010300
78 #define NVME_CMB_BIR 2
79 #define NVME_PMR_BIR 2
80 #define NVME_TEMPERATURE 0x143
81 #define NVME_TEMPERATURE_WARNING 0x157
82 #define NVME_TEMPERATURE_CRITICAL 0x175
83 #define NVME_NUM_FW_SLOTS 1
85 #define NVME_GUEST_ERR(trace, fmt, ...) \
87 (trace_##trace)(__VA_ARGS__); \
88 qemu_log_mask(LOG_GUEST_ERROR, #trace \
89 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
92 static const bool nvme_feature_support
[NVME_FID_MAX
] = {
93 [NVME_ARBITRATION
] = true,
94 [NVME_POWER_MANAGEMENT
] = true,
95 [NVME_TEMPERATURE_THRESHOLD
] = true,
96 [NVME_ERROR_RECOVERY
] = true,
97 [NVME_VOLATILE_WRITE_CACHE
] = true,
98 [NVME_NUMBER_OF_QUEUES
] = true,
99 [NVME_INTERRUPT_COALESCING
] = true,
100 [NVME_INTERRUPT_VECTOR_CONF
] = true,
101 [NVME_WRITE_ATOMICITY
] = true,
102 [NVME_ASYNCHRONOUS_EVENT_CONF
] = true,
103 [NVME_TIMESTAMP
] = true,
106 static const uint32_t nvme_feature_cap
[NVME_FID_MAX
] = {
107 [NVME_TEMPERATURE_THRESHOLD
] = NVME_FEAT_CAP_CHANGE
,
108 [NVME_ERROR_RECOVERY
] = NVME_FEAT_CAP_CHANGE
| NVME_FEAT_CAP_NS
,
109 [NVME_VOLATILE_WRITE_CACHE
] = NVME_FEAT_CAP_CHANGE
,
110 [NVME_NUMBER_OF_QUEUES
] = NVME_FEAT_CAP_CHANGE
,
111 [NVME_ASYNCHRONOUS_EVENT_CONF
] = NVME_FEAT_CAP_CHANGE
,
112 [NVME_TIMESTAMP
] = NVME_FEAT_CAP_CHANGE
,
115 static const uint32_t nvme_cse_acs
[256] = {
116 [NVME_ADM_CMD_DELETE_SQ
] = NVME_CMD_EFF_CSUPP
,
117 [NVME_ADM_CMD_CREATE_SQ
] = NVME_CMD_EFF_CSUPP
,
118 [NVME_ADM_CMD_GET_LOG_PAGE
] = NVME_CMD_EFF_CSUPP
,
119 [NVME_ADM_CMD_DELETE_CQ
] = NVME_CMD_EFF_CSUPP
,
120 [NVME_ADM_CMD_CREATE_CQ
] = NVME_CMD_EFF_CSUPP
,
121 [NVME_ADM_CMD_IDENTIFY
] = NVME_CMD_EFF_CSUPP
,
122 [NVME_ADM_CMD_ABORT
] = NVME_CMD_EFF_CSUPP
,
123 [NVME_ADM_CMD_SET_FEATURES
] = NVME_CMD_EFF_CSUPP
,
124 [NVME_ADM_CMD_GET_FEATURES
] = NVME_CMD_EFF_CSUPP
,
125 [NVME_ADM_CMD_ASYNC_EV_REQ
] = NVME_CMD_EFF_CSUPP
,
128 static const uint32_t nvme_cse_iocs_none
[256];
130 static const uint32_t nvme_cse_iocs_nvm
[256] = {
131 [NVME_CMD_FLUSH
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
132 [NVME_CMD_WRITE_ZEROES
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
133 [NVME_CMD_WRITE
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
134 [NVME_CMD_READ
] = NVME_CMD_EFF_CSUPP
,
135 [NVME_CMD_DSM
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
136 [NVME_CMD_COMPARE
] = NVME_CMD_EFF_CSUPP
,
139 static void nvme_process_sq(void *opaque
);
141 static uint16_t nvme_cid(NvmeRequest
*req
)
147 return le16_to_cpu(req
->cqe
.cid
);
150 static uint16_t nvme_sqid(NvmeRequest
*req
)
152 return le16_to_cpu(req
->sq
->sqid
);
155 static bool nvme_addr_is_cmb(NvmeCtrl
*n
, hwaddr addr
)
157 hwaddr low
= n
->ctrl_mem
.addr
;
158 hwaddr hi
= n
->ctrl_mem
.addr
+ int128_get64(n
->ctrl_mem
.size
);
160 return addr
>= low
&& addr
< hi
;
163 static inline void *nvme_addr_to_cmb(NvmeCtrl
*n
, hwaddr addr
)
165 assert(nvme_addr_is_cmb(n
, addr
));
167 return &n
->cmbuf
[addr
- n
->ctrl_mem
.addr
];
170 static int nvme_addr_read(NvmeCtrl
*n
, hwaddr addr
, void *buf
, int size
)
172 hwaddr hi
= addr
+ size
- 1;
177 if (n
->bar
.cmbsz
&& nvme_addr_is_cmb(n
, addr
) && nvme_addr_is_cmb(n
, hi
)) {
178 memcpy(buf
, nvme_addr_to_cmb(n
, addr
), size
);
182 return pci_dma_read(&n
->parent_obj
, addr
, buf
, size
);
185 static bool nvme_nsid_valid(NvmeCtrl
*n
, uint32_t nsid
)
187 return nsid
&& (nsid
== NVME_NSID_BROADCAST
|| nsid
<= n
->num_namespaces
);
190 static int nvme_check_sqid(NvmeCtrl
*n
, uint16_t sqid
)
192 return sqid
< n
->params
.max_ioqpairs
+ 1 && n
->sq
[sqid
] != NULL
? 0 : -1;
195 static int nvme_check_cqid(NvmeCtrl
*n
, uint16_t cqid
)
197 return cqid
< n
->params
.max_ioqpairs
+ 1 && n
->cq
[cqid
] != NULL
? 0 : -1;
200 static void nvme_inc_cq_tail(NvmeCQueue
*cq
)
203 if (cq
->tail
>= cq
->size
) {
205 cq
->phase
= !cq
->phase
;
209 static void nvme_inc_sq_head(NvmeSQueue
*sq
)
211 sq
->head
= (sq
->head
+ 1) % sq
->size
;
214 static uint8_t nvme_cq_full(NvmeCQueue
*cq
)
216 return (cq
->tail
+ 1) % cq
->size
== cq
->head
;
219 static uint8_t nvme_sq_empty(NvmeSQueue
*sq
)
221 return sq
->head
== sq
->tail
;
224 static void nvme_irq_check(NvmeCtrl
*n
)
226 if (msix_enabled(&(n
->parent_obj
))) {
229 if (~n
->bar
.intms
& n
->irq_status
) {
230 pci_irq_assert(&n
->parent_obj
);
232 pci_irq_deassert(&n
->parent_obj
);
236 static void nvme_irq_assert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
238 if (cq
->irq_enabled
) {
239 if (msix_enabled(&(n
->parent_obj
))) {
240 trace_pci_nvme_irq_msix(cq
->vector
);
241 msix_notify(&(n
->parent_obj
), cq
->vector
);
243 trace_pci_nvme_irq_pin();
244 assert(cq
->vector
< 32);
245 n
->irq_status
|= 1 << cq
->vector
;
249 trace_pci_nvme_irq_masked();
253 static void nvme_irq_deassert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
255 if (cq
->irq_enabled
) {
256 if (msix_enabled(&(n
->parent_obj
))) {
259 assert(cq
->vector
< 32);
260 n
->irq_status
&= ~(1 << cq
->vector
);
266 static void nvme_req_clear(NvmeRequest
*req
)
270 memset(&req
->cqe
, 0x0, sizeof(req
->cqe
));
271 req
->status
= NVME_SUCCESS
;
274 static void nvme_req_exit(NvmeRequest
*req
)
277 qemu_sglist_destroy(&req
->qsg
);
281 qemu_iovec_destroy(&req
->iov
);
285 static uint16_t nvme_map_addr_cmb(NvmeCtrl
*n
, QEMUIOVector
*iov
, hwaddr addr
,
292 trace_pci_nvme_map_addr_cmb(addr
, len
);
294 if (!nvme_addr_is_cmb(n
, addr
) || !nvme_addr_is_cmb(n
, addr
+ len
- 1)) {
295 return NVME_DATA_TRAS_ERROR
;
298 qemu_iovec_add(iov
, nvme_addr_to_cmb(n
, addr
), len
);
303 static uint16_t nvme_map_addr(NvmeCtrl
*n
, QEMUSGList
*qsg
, QEMUIOVector
*iov
,
304 hwaddr addr
, size_t len
)
310 trace_pci_nvme_map_addr(addr
, len
);
312 if (nvme_addr_is_cmb(n
, addr
)) {
313 if (qsg
&& qsg
->sg
) {
314 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
320 qemu_iovec_init(iov
, 1);
323 return nvme_map_addr_cmb(n
, iov
, addr
, len
);
326 if (iov
&& iov
->iov
) {
327 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
333 pci_dma_sglist_init(qsg
, &n
->parent_obj
, 1);
336 qemu_sglist_add(qsg
, addr
, len
);
341 static uint16_t nvme_map_prp(NvmeCtrl
*n
, uint64_t prp1
, uint64_t prp2
,
342 uint32_t len
, NvmeRequest
*req
)
344 hwaddr trans_len
= n
->page_size
- (prp1
% n
->page_size
);
345 trans_len
= MIN(len
, trans_len
);
346 int num_prps
= (len
>> n
->page_bits
) + 1;
348 bool prp_list_in_cmb
= false;
351 QEMUSGList
*qsg
= &req
->qsg
;
352 QEMUIOVector
*iov
= &req
->iov
;
354 trace_pci_nvme_map_prp(trans_len
, len
, prp1
, prp2
, num_prps
);
356 if (nvme_addr_is_cmb(n
, prp1
)) {
357 qemu_iovec_init(iov
, num_prps
);
359 pci_dma_sglist_init(qsg
, &n
->parent_obj
, num_prps
);
362 status
= nvme_map_addr(n
, qsg
, iov
, prp1
, trans_len
);
369 if (len
> n
->page_size
) {
370 uint64_t prp_list
[n
->max_prp_ents
];
371 uint32_t nents
, prp_trans
;
374 if (nvme_addr_is_cmb(n
, prp2
)) {
375 prp_list_in_cmb
= true;
378 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
379 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
380 ret
= nvme_addr_read(n
, prp2
, (void *)prp_list
, prp_trans
);
382 trace_pci_nvme_err_addr_read(prp2
);
383 return NVME_DATA_TRAS_ERROR
;
386 uint64_t prp_ent
= le64_to_cpu(prp_list
[i
]);
388 if (i
== n
->max_prp_ents
- 1 && len
> n
->page_size
) {
389 if (unlikely(prp_ent
& (n
->page_size
- 1))) {
390 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
391 return NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
394 if (prp_list_in_cmb
!= nvme_addr_is_cmb(n
, prp_ent
)) {
395 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
399 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
400 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
401 ret
= nvme_addr_read(n
, prp_ent
, (void *)prp_list
,
404 trace_pci_nvme_err_addr_read(prp_ent
);
405 return NVME_DATA_TRAS_ERROR
;
407 prp_ent
= le64_to_cpu(prp_list
[i
]);
410 if (unlikely(prp_ent
& (n
->page_size
- 1))) {
411 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
412 return NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
415 trans_len
= MIN(len
, n
->page_size
);
416 status
= nvme_map_addr(n
, qsg
, iov
, prp_ent
, trans_len
);
425 if (unlikely(prp2
& (n
->page_size
- 1))) {
426 trace_pci_nvme_err_invalid_prp2_align(prp2
);
427 return NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
429 status
= nvme_map_addr(n
, qsg
, iov
, prp2
, len
);
440 * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
441 * number of bytes mapped in len.
443 static uint16_t nvme_map_sgl_data(NvmeCtrl
*n
, QEMUSGList
*qsg
,
445 NvmeSglDescriptor
*segment
, uint64_t nsgld
,
446 size_t *len
, NvmeRequest
*req
)
448 dma_addr_t addr
, trans_len
;
452 for (int i
= 0; i
< nsgld
; i
++) {
453 uint8_t type
= NVME_SGL_TYPE(segment
[i
].type
);
456 case NVME_SGL_DESCR_TYPE_BIT_BUCKET
:
457 if (req
->cmd
.opcode
== NVME_CMD_WRITE
) {
460 case NVME_SGL_DESCR_TYPE_DATA_BLOCK
:
462 case NVME_SGL_DESCR_TYPE_SEGMENT
:
463 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT
:
464 return NVME_INVALID_NUM_SGL_DESCRS
| NVME_DNR
;
466 return NVME_SGL_DESCR_TYPE_INVALID
| NVME_DNR
;
469 dlen
= le32_to_cpu(segment
[i
].len
);
477 * All data has been mapped, but the SGL contains additional
478 * segments and/or descriptors. The controller might accept
479 * ignoring the rest of the SGL.
481 uint32_t sgls
= le32_to_cpu(n
->id_ctrl
.sgls
);
482 if (sgls
& NVME_CTRL_SGLS_EXCESS_LENGTH
) {
486 trace_pci_nvme_err_invalid_sgl_excess_length(nvme_cid(req
));
487 return NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
490 trans_len
= MIN(*len
, dlen
);
492 if (type
== NVME_SGL_DESCR_TYPE_BIT_BUCKET
) {
496 addr
= le64_to_cpu(segment
[i
].addr
);
498 if (UINT64_MAX
- addr
< dlen
) {
499 return NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
502 status
= nvme_map_addr(n
, qsg
, iov
, addr
, trans_len
);
514 static uint16_t nvme_map_sgl(NvmeCtrl
*n
, QEMUSGList
*qsg
, QEMUIOVector
*iov
,
515 NvmeSglDescriptor sgl
, size_t len
,
519 * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
520 * dynamically allocating a potentially huge SGL. The spec allows the SGL
521 * to be larger (as in number of bytes required to describe the SGL
522 * descriptors and segment chain) than the command transfer size, so it is
523 * not bounded by MDTS.
525 const int SEG_CHUNK_SIZE
= 256;
527 NvmeSglDescriptor segment
[SEG_CHUNK_SIZE
], *sgld
, *last_sgld
;
531 bool sgl_in_cmb
= false;
536 addr
= le64_to_cpu(sgl
.addr
);
538 trace_pci_nvme_map_sgl(nvme_cid(req
), NVME_SGL_TYPE(sgl
.type
), len
);
541 * If the entire transfer can be described with a single data block it can
542 * be mapped directly.
544 if (NVME_SGL_TYPE(sgl
.type
) == NVME_SGL_DESCR_TYPE_DATA_BLOCK
) {
545 status
= nvme_map_sgl_data(n
, qsg
, iov
, sgld
, 1, &len
, req
);
554 * If the segment is located in the CMB, the submission queue of the
555 * request must also reside there.
557 if (nvme_addr_is_cmb(n
, addr
)) {
558 if (!nvme_addr_is_cmb(n
, req
->sq
->dma_addr
)) {
559 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
566 switch (NVME_SGL_TYPE(sgld
->type
)) {
567 case NVME_SGL_DESCR_TYPE_SEGMENT
:
568 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT
:
571 return NVME_INVALID_SGL_SEG_DESCR
| NVME_DNR
;
574 seg_len
= le32_to_cpu(sgld
->len
);
576 /* check the length of the (Last) Segment descriptor */
577 if ((!seg_len
|| seg_len
& 0xf) &&
578 (NVME_SGL_TYPE(sgld
->type
) != NVME_SGL_DESCR_TYPE_BIT_BUCKET
)) {
579 return NVME_INVALID_SGL_SEG_DESCR
| NVME_DNR
;
582 if (UINT64_MAX
- addr
< seg_len
) {
583 return NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
586 nsgld
= seg_len
/ sizeof(NvmeSglDescriptor
);
588 while (nsgld
> SEG_CHUNK_SIZE
) {
589 if (nvme_addr_read(n
, addr
, segment
, sizeof(segment
))) {
590 trace_pci_nvme_err_addr_read(addr
);
591 status
= NVME_DATA_TRAS_ERROR
;
595 status
= nvme_map_sgl_data(n
, qsg
, iov
, segment
, SEG_CHUNK_SIZE
,
601 nsgld
-= SEG_CHUNK_SIZE
;
602 addr
+= SEG_CHUNK_SIZE
* sizeof(NvmeSglDescriptor
);
605 ret
= nvme_addr_read(n
, addr
, segment
, nsgld
*
606 sizeof(NvmeSglDescriptor
));
608 trace_pci_nvme_err_addr_read(addr
);
609 status
= NVME_DATA_TRAS_ERROR
;
613 last_sgld
= &segment
[nsgld
- 1];
616 * If the segment ends with a Data Block or Bit Bucket Descriptor Type,
619 switch (NVME_SGL_TYPE(last_sgld
->type
)) {
620 case NVME_SGL_DESCR_TYPE_DATA_BLOCK
:
621 case NVME_SGL_DESCR_TYPE_BIT_BUCKET
:
622 status
= nvme_map_sgl_data(n
, qsg
, iov
, segment
, nsgld
, &len
, req
);
634 * If the last descriptor was not a Data Block or Bit Bucket, then the
635 * current segment must not be a Last Segment.
637 if (NVME_SGL_TYPE(sgld
->type
) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT
) {
638 status
= NVME_INVALID_SGL_SEG_DESCR
| NVME_DNR
;
643 addr
= le64_to_cpu(sgld
->addr
);
646 * Do not map the last descriptor; it will be a Segment or Last Segment
647 * descriptor and is handled by the next iteration.
649 status
= nvme_map_sgl_data(n
, qsg
, iov
, segment
, nsgld
- 1, &len
, req
);
655 * If the next segment is in the CMB, make sure that the sgl was
656 * already located there.
658 if (sgl_in_cmb
!= nvme_addr_is_cmb(n
, addr
)) {
659 status
= NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
665 /* if there is any residual left in len, the SGL was too short */
667 status
= NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
675 qemu_iovec_destroy(iov
);
679 qemu_sglist_destroy(qsg
);
685 static uint16_t nvme_map_dptr(NvmeCtrl
*n
, size_t len
, NvmeRequest
*req
)
689 switch (NVME_CMD_FLAGS_PSDT(req
->cmd
.flags
)) {
691 prp1
= le64_to_cpu(req
->cmd
.dptr
.prp1
);
692 prp2
= le64_to_cpu(req
->cmd
.dptr
.prp2
);
694 return nvme_map_prp(n
, prp1
, prp2
, len
, req
);
695 case NVME_PSDT_SGL_MPTR_CONTIGUOUS
:
696 case NVME_PSDT_SGL_MPTR_SGL
:
697 /* SGLs shall not be used for Admin commands in NVMe over PCIe */
698 if (!req
->sq
->sqid
) {
699 return NVME_INVALID_FIELD
| NVME_DNR
;
702 return nvme_map_sgl(n
, &req
->qsg
, &req
->iov
, req
->cmd
.dptr
.sgl
, len
,
705 return NVME_INVALID_FIELD
;
709 static uint16_t nvme_dma(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
710 DMADirection dir
, NvmeRequest
*req
)
712 uint16_t status
= NVME_SUCCESS
;
714 status
= nvme_map_dptr(n
, len
, req
);
719 /* assert that only one of qsg and iov carries data */
720 assert((req
->qsg
.nsg
> 0) != (req
->iov
.niov
> 0));
722 if (req
->qsg
.nsg
> 0) {
725 if (dir
== DMA_DIRECTION_TO_DEVICE
) {
726 residual
= dma_buf_write(ptr
, len
, &req
->qsg
);
728 residual
= dma_buf_read(ptr
, len
, &req
->qsg
);
731 if (unlikely(residual
)) {
732 trace_pci_nvme_err_invalid_dma();
733 status
= NVME_INVALID_FIELD
| NVME_DNR
;
738 if (dir
== DMA_DIRECTION_TO_DEVICE
) {
739 bytes
= qemu_iovec_to_buf(&req
->iov
, 0, ptr
, len
);
741 bytes
= qemu_iovec_from_buf(&req
->iov
, 0, ptr
, len
);
744 if (unlikely(bytes
!= len
)) {
745 trace_pci_nvme_err_invalid_dma();
746 status
= NVME_INVALID_FIELD
| NVME_DNR
;
753 static void nvme_post_cqes(void *opaque
)
755 NvmeCQueue
*cq
= opaque
;
756 NvmeCtrl
*n
= cq
->ctrl
;
757 NvmeRequest
*req
, *next
;
760 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
764 if (nvme_cq_full(cq
)) {
769 req
->cqe
.status
= cpu_to_le16((req
->status
<< 1) | cq
->phase
);
770 req
->cqe
.sq_id
= cpu_to_le16(sq
->sqid
);
771 req
->cqe
.sq_head
= cpu_to_le16(sq
->head
);
772 addr
= cq
->dma_addr
+ cq
->tail
* n
->cqe_size
;
773 ret
= pci_dma_write(&n
->parent_obj
, addr
, (void *)&req
->cqe
,
776 trace_pci_nvme_err_addr_write(addr
);
777 trace_pci_nvme_err_cfs();
778 n
->bar
.csts
= NVME_CSTS_FAILED
;
781 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
782 nvme_inc_cq_tail(cq
);
784 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
786 if (cq
->tail
!= cq
->head
) {
787 nvme_irq_assert(n
, cq
);
791 static void nvme_enqueue_req_completion(NvmeCQueue
*cq
, NvmeRequest
*req
)
793 assert(cq
->cqid
== req
->sq
->cqid
);
794 trace_pci_nvme_enqueue_req_completion(nvme_cid(req
), cq
->cqid
,
798 trace_pci_nvme_err_req_status(nvme_cid(req
), nvme_nsid(req
->ns
),
799 req
->status
, req
->cmd
.opcode
);
802 QTAILQ_REMOVE(&req
->sq
->out_req_list
, req
, entry
);
803 QTAILQ_INSERT_TAIL(&cq
->req_list
, req
, entry
);
804 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
807 static void nvme_process_aers(void *opaque
)
809 NvmeCtrl
*n
= opaque
;
810 NvmeAsyncEvent
*event
, *next
;
812 trace_pci_nvme_process_aers(n
->aer_queued
);
814 QTAILQ_FOREACH_SAFE(event
, &n
->aer_queue
, entry
, next
) {
816 NvmeAerResult
*result
;
818 /* can't post cqe if there is nothing to complete */
819 if (!n
->outstanding_aers
) {
820 trace_pci_nvme_no_outstanding_aers();
824 /* ignore if masked (cqe posted, but event not cleared) */
825 if (n
->aer_mask
& (1 << event
->result
.event_type
)) {
826 trace_pci_nvme_aer_masked(event
->result
.event_type
, n
->aer_mask
);
830 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
833 n
->aer_mask
|= 1 << event
->result
.event_type
;
834 n
->outstanding_aers
--;
836 req
= n
->aer_reqs
[n
->outstanding_aers
];
838 result
= (NvmeAerResult
*) &req
->cqe
.result
;
839 result
->event_type
= event
->result
.event_type
;
840 result
->event_info
= event
->result
.event_info
;
841 result
->log_page
= event
->result
.log_page
;
844 trace_pci_nvme_aer_post_cqe(result
->event_type
, result
->event_info
,
847 nvme_enqueue_req_completion(&n
->admin_cq
, req
);
851 static void nvme_enqueue_event(NvmeCtrl
*n
, uint8_t event_type
,
852 uint8_t event_info
, uint8_t log_page
)
854 NvmeAsyncEvent
*event
;
856 trace_pci_nvme_enqueue_event(event_type
, event_info
, log_page
);
858 if (n
->aer_queued
== n
->params
.aer_max_queued
) {
859 trace_pci_nvme_enqueue_event_noqueue(n
->aer_queued
);
863 event
= g_new(NvmeAsyncEvent
, 1);
864 event
->result
= (NvmeAerResult
) {
865 .event_type
= event_type
,
866 .event_info
= event_info
,
867 .log_page
= log_page
,
870 QTAILQ_INSERT_TAIL(&n
->aer_queue
, event
, entry
);
873 nvme_process_aers(n
);
876 static void nvme_clear_events(NvmeCtrl
*n
, uint8_t event_type
)
878 n
->aer_mask
&= ~(1 << event_type
);
879 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
880 nvme_process_aers(n
);
884 static inline uint16_t nvme_check_mdts(NvmeCtrl
*n
, size_t len
)
886 uint8_t mdts
= n
->params
.mdts
;
888 if (mdts
&& len
> n
->page_size
<< mdts
) {
889 return NVME_INVALID_FIELD
| NVME_DNR
;
895 static inline uint16_t nvme_check_bounds(NvmeNamespace
*ns
, uint64_t slba
,
898 uint64_t nsze
= le64_to_cpu(ns
->id_ns
.nsze
);
900 if (unlikely(UINT64_MAX
- slba
< nlb
|| slba
+ nlb
> nsze
)) {
901 return NVME_LBA_RANGE
| NVME_DNR
;
907 static uint16_t nvme_check_dulbe(NvmeNamespace
*ns
, uint64_t slba
,
910 BlockDriverState
*bs
= blk_bs(ns
->blkconf
.blk
);
912 int64_t pnum
= 0, bytes
= nvme_l2b(ns
, nlb
);
913 int64_t offset
= nvme_l2b(ns
, slba
);
917 Error
*local_err
= NULL
;
920 * `pnum` holds the number of bytes after offset that shares the same
921 * allocation status as the byte at offset. If `pnum` is different from
922 * `bytes`, we should check the allocation status of the next range and
923 * continue this until all bytes have been checked.
928 ret
= bdrv_block_status(bs
, offset
, bytes
, &pnum
, NULL
, NULL
);
930 error_setg_errno(&local_err
, -ret
, "unable to get block status");
931 error_report_err(local_err
);
933 return NVME_INTERNAL_DEV_ERROR
;
936 zeroed
= !!(ret
& BDRV_BLOCK_ZERO
);
938 trace_pci_nvme_block_status(offset
, bytes
, pnum
, ret
, zeroed
);
945 } while (pnum
!= bytes
);
950 static void nvme_aio_err(NvmeRequest
*req
, int ret
)
952 uint16_t status
= NVME_SUCCESS
;
953 Error
*local_err
= NULL
;
955 switch (req
->cmd
.opcode
) {
957 status
= NVME_UNRECOVERED_READ
;
961 case NVME_CMD_WRITE_ZEROES
:
962 status
= NVME_WRITE_FAULT
;
965 status
= NVME_INTERNAL_DEV_ERROR
;
969 trace_pci_nvme_err_aio(nvme_cid(req
), strerror(ret
), status
);
971 error_setg_errno(&local_err
, -ret
, "aio failed");
972 error_report_err(local_err
);
975 * Set the command status code to the first encountered error but allow a
976 * subsequent Internal Device Error to trump it.
978 if (req
->status
&& status
!= NVME_INTERNAL_DEV_ERROR
) {
982 req
->status
= status
;
985 static void nvme_rw_cb(void *opaque
, int ret
)
987 NvmeRequest
*req
= opaque
;
988 NvmeNamespace
*ns
= req
->ns
;
990 BlockBackend
*blk
= ns
->blkconf
.blk
;
991 BlockAcctCookie
*acct
= &req
->acct
;
992 BlockAcctStats
*stats
= blk_get_stats(blk
);
994 trace_pci_nvme_rw_cb(nvme_cid(req
), blk_name(blk
));
997 block_acct_done(stats
, acct
);
999 block_acct_failed(stats
, acct
);
1000 nvme_aio_err(req
, ret
);
1003 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1006 static void nvme_aio_discard_cb(void *opaque
, int ret
)
1008 NvmeRequest
*req
= opaque
;
1009 uintptr_t *discards
= (uintptr_t *)&req
->opaque
;
1011 trace_pci_nvme_aio_discard_cb(nvme_cid(req
));
1014 nvme_aio_err(req
, ret
);
1023 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1026 struct nvme_compare_ctx
{
1032 static void nvme_compare_cb(void *opaque
, int ret
)
1034 NvmeRequest
*req
= opaque
;
1035 NvmeNamespace
*ns
= req
->ns
;
1036 struct nvme_compare_ctx
*ctx
= req
->opaque
;
1037 g_autofree
uint8_t *buf
= NULL
;
1040 trace_pci_nvme_compare_cb(nvme_cid(req
));
1043 block_acct_done(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
);
1045 block_acct_failed(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
);
1046 nvme_aio_err(req
, ret
);
1050 buf
= g_malloc(ctx
->len
);
1052 status
= nvme_dma(nvme_ctrl(req
), buf
, ctx
->len
, DMA_DIRECTION_TO_DEVICE
,
1055 req
->status
= status
;
1059 if (memcmp(buf
, ctx
->bounce
, ctx
->len
)) {
1060 req
->status
= NVME_CMP_FAILURE
;
1064 qemu_iovec_destroy(&ctx
->iov
);
1065 g_free(ctx
->bounce
);
1068 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1071 static uint16_t nvme_dsm(NvmeCtrl
*n
, NvmeRequest
*req
)
1073 NvmeNamespace
*ns
= req
->ns
;
1074 NvmeDsmCmd
*dsm
= (NvmeDsmCmd
*) &req
->cmd
;
1076 uint32_t attr
= le32_to_cpu(dsm
->attributes
);
1077 uint32_t nr
= (le32_to_cpu(dsm
->nr
) & 0xff) + 1;
1079 uint16_t status
= NVME_SUCCESS
;
1081 trace_pci_nvme_dsm(nvme_cid(req
), nvme_nsid(ns
), nr
, attr
);
1083 if (attr
& NVME_DSMGMT_AD
) {
1086 NvmeDsmRange range
[nr
];
1087 uintptr_t *discards
= (uintptr_t *)&req
->opaque
;
1089 status
= nvme_dma(n
, (uint8_t *)range
, sizeof(range
),
1090 DMA_DIRECTION_TO_DEVICE
, req
);
1096 * AIO callbacks may be called immediately, so initialize discards to 1
1097 * to make sure the the callback does not complete the request before
1098 * all discards have been issued.
1102 for (int i
= 0; i
< nr
; i
++) {
1103 uint64_t slba
= le64_to_cpu(range
[i
].slba
);
1104 uint32_t nlb
= le32_to_cpu(range
[i
].nlb
);
1106 if (nvme_check_bounds(ns
, slba
, nlb
)) {
1107 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
,
1112 trace_pci_nvme_dsm_deallocate(nvme_cid(req
), nvme_nsid(ns
), slba
,
1115 offset
= nvme_l2b(ns
, slba
);
1116 len
= nvme_l2b(ns
, nlb
);
1119 size_t bytes
= MIN(BDRV_REQUEST_MAX_BYTES
, len
);
1123 blk_aio_pdiscard(ns
->blkconf
.blk
, offset
, bytes
,
1124 nvme_aio_discard_cb
, req
);
1131 /* account for the 1-initialization */
1135 status
= NVME_NO_COMPLETE
;
1137 status
= req
->status
;
1144 static uint16_t nvme_compare(NvmeCtrl
*n
, NvmeRequest
*req
)
1146 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1147 NvmeNamespace
*ns
= req
->ns
;
1148 BlockBackend
*blk
= ns
->blkconf
.blk
;
1149 uint64_t slba
= le64_to_cpu(rw
->slba
);
1150 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
1151 size_t len
= nvme_l2b(ns
, nlb
);
1152 int64_t offset
= nvme_l2b(ns
, slba
);
1153 uint8_t *bounce
= NULL
;
1154 struct nvme_compare_ctx
*ctx
= NULL
;
1157 trace_pci_nvme_compare(nvme_cid(req
), nvme_nsid(ns
), slba
, nlb
);
1159 status
= nvme_check_mdts(n
, len
);
1161 trace_pci_nvme_err_mdts(nvme_cid(req
), len
);
1165 status
= nvme_check_bounds(ns
, slba
, nlb
);
1167 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
1171 if (NVME_ERR_REC_DULBE(ns
->features
.err_rec
)) {
1172 status
= nvme_check_dulbe(ns
, slba
, nlb
);
1178 bounce
= g_malloc(len
);
1180 ctx
= g_new(struct nvme_compare_ctx
, 1);
1181 ctx
->bounce
= bounce
;
1186 qemu_iovec_init(&ctx
->iov
, 1);
1187 qemu_iovec_add(&ctx
->iov
, bounce
, len
);
1189 block_acct_start(blk_get_stats(blk
), &req
->acct
, len
, BLOCK_ACCT_READ
);
1190 blk_aio_preadv(blk
, offset
, &ctx
->iov
, 0, nvme_compare_cb
, req
);
1192 return NVME_NO_COMPLETE
;
1195 static uint16_t nvme_flush(NvmeCtrl
*n
, NvmeRequest
*req
)
1197 block_acct_start(blk_get_stats(req
->ns
->blkconf
.blk
), &req
->acct
, 0,
1199 req
->aiocb
= blk_aio_flush(req
->ns
->blkconf
.blk
, nvme_rw_cb
, req
);
1200 return NVME_NO_COMPLETE
;
1203 static uint16_t nvme_read(NvmeCtrl
*n
, NvmeRequest
*req
)
1205 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1206 NvmeNamespace
*ns
= req
->ns
;
1207 uint64_t slba
= le64_to_cpu(rw
->slba
);
1208 uint32_t nlb
= (uint32_t)le16_to_cpu(rw
->nlb
) + 1;
1209 uint64_t data_size
= nvme_l2b(ns
, nlb
);
1210 uint64_t data_offset
;
1211 BlockBackend
*blk
= ns
->blkconf
.blk
;
1214 trace_pci_nvme_read(nvme_cid(req
), nvme_nsid(ns
), nlb
, data_size
, slba
);
1216 status
= nvme_check_mdts(n
, data_size
);
1218 trace_pci_nvme_err_mdts(nvme_cid(req
), data_size
);
1222 status
= nvme_check_bounds(ns
, slba
, nlb
);
1224 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
1228 status
= nvme_map_dptr(n
, data_size
, req
);
1233 if (NVME_ERR_REC_DULBE(ns
->features
.err_rec
)) {
1234 status
= nvme_check_dulbe(ns
, slba
, nlb
);
1240 data_offset
= nvme_l2b(ns
, slba
);
1242 block_acct_start(blk_get_stats(blk
), &req
->acct
, data_size
,
1245 req
->aiocb
= dma_blk_read(blk
, &req
->qsg
, data_offset
,
1246 BDRV_SECTOR_SIZE
, nvme_rw_cb
, req
);
1248 req
->aiocb
= blk_aio_preadv(blk
, data_offset
, &req
->iov
, 0,
1251 return NVME_NO_COMPLETE
;
1254 block_acct_invalid(blk_get_stats(blk
), BLOCK_ACCT_READ
);
1255 return status
| NVME_DNR
;
1258 static uint16_t nvme_do_write(NvmeCtrl
*n
, NvmeRequest
*req
, bool wrz
)
1260 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1261 NvmeNamespace
*ns
= req
->ns
;
1262 uint64_t slba
= le64_to_cpu(rw
->slba
);
1263 uint32_t nlb
= (uint32_t)le16_to_cpu(rw
->nlb
) + 1;
1264 uint64_t data_size
= nvme_l2b(ns
, nlb
);
1265 uint64_t data_offset
;
1266 BlockBackend
*blk
= ns
->blkconf
.blk
;
1269 trace_pci_nvme_write(nvme_cid(req
), nvme_io_opc_str(rw
->opcode
),
1270 nvme_nsid(ns
), nlb
, data_size
, slba
);
1273 status
= nvme_check_mdts(n
, data_size
);
1275 trace_pci_nvme_err_mdts(nvme_cid(req
), data_size
);
1280 status
= nvme_check_bounds(ns
, slba
, nlb
);
1282 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
1286 data_offset
= nvme_l2b(ns
, slba
);
1289 status
= nvme_map_dptr(n
, data_size
, req
);
1294 block_acct_start(blk_get_stats(blk
), &req
->acct
, data_size
,
1297 req
->aiocb
= dma_blk_write(blk
, &req
->qsg
, data_offset
,
1298 BDRV_SECTOR_SIZE
, nvme_rw_cb
, req
);
1300 req
->aiocb
= blk_aio_pwritev(blk
, data_offset
, &req
->iov
, 0,
1304 block_acct_start(blk_get_stats(blk
), &req
->acct
, 0, BLOCK_ACCT_WRITE
);
1305 req
->aiocb
= blk_aio_pwrite_zeroes(blk
, data_offset
, data_size
,
1306 BDRV_REQ_MAY_UNMAP
, nvme_rw_cb
,
1309 return NVME_NO_COMPLETE
;
1312 block_acct_invalid(blk_get_stats(blk
), BLOCK_ACCT_WRITE
);
1313 return status
| NVME_DNR
;
1316 static inline uint16_t nvme_write(NvmeCtrl
*n
, NvmeRequest
*req
)
1318 return nvme_do_write(n
, req
, false);
1321 static inline uint16_t nvme_write_zeroes(NvmeCtrl
*n
, NvmeRequest
*req
)
1323 return nvme_do_write(n
, req
, true);
1326 static uint16_t nvme_io_cmd(NvmeCtrl
*n
, NvmeRequest
*req
)
1328 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
1330 trace_pci_nvme_io_cmd(nvme_cid(req
), nsid
, nvme_sqid(req
),
1331 req
->cmd
.opcode
, nvme_io_opc_str(req
->cmd
.opcode
));
1333 if (!nvme_nsid_valid(n
, nsid
)) {
1334 return NVME_INVALID_NSID
| NVME_DNR
;
1337 req
->ns
= nvme_ns(n
, nsid
);
1338 if (unlikely(!req
->ns
)) {
1339 return NVME_INVALID_FIELD
| NVME_DNR
;
1342 if (!(req
->ns
->iocs
[req
->cmd
.opcode
] & NVME_CMD_EFF_CSUPP
)) {
1343 trace_pci_nvme_err_invalid_opc(req
->cmd
.opcode
);
1344 return NVME_INVALID_OPCODE
| NVME_DNR
;
1347 switch (req
->cmd
.opcode
) {
1348 case NVME_CMD_FLUSH
:
1349 return nvme_flush(n
, req
);
1350 case NVME_CMD_WRITE_ZEROES
:
1351 return nvme_write_zeroes(n
, req
);
1352 case NVME_CMD_WRITE
:
1353 return nvme_write(n
, req
);
1355 return nvme_read(n
, req
);
1356 case NVME_CMD_COMPARE
:
1357 return nvme_compare(n
, req
);
1359 return nvme_dsm(n
, req
);
1364 return NVME_INVALID_OPCODE
| NVME_DNR
;
1367 static void nvme_free_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
)
1369 n
->sq
[sq
->sqid
] = NULL
;
1370 timer_free(sq
->timer
);
1377 static uint16_t nvme_del_sq(NvmeCtrl
*n
, NvmeRequest
*req
)
1379 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)&req
->cmd
;
1380 NvmeRequest
*r
, *next
;
1383 uint16_t qid
= le16_to_cpu(c
->qid
);
1385 if (unlikely(!qid
|| nvme_check_sqid(n
, qid
))) {
1386 trace_pci_nvme_err_invalid_del_sq(qid
);
1387 return NVME_INVALID_QID
| NVME_DNR
;
1390 trace_pci_nvme_del_sq(qid
);
1393 while (!QTAILQ_EMPTY(&sq
->out_req_list
)) {
1394 r
= QTAILQ_FIRST(&sq
->out_req_list
);
1396 blk_aio_cancel(r
->aiocb
);
1398 if (!nvme_check_cqid(n
, sq
->cqid
)) {
1399 cq
= n
->cq
[sq
->cqid
];
1400 QTAILQ_REMOVE(&cq
->sq_list
, sq
, entry
);
1403 QTAILQ_FOREACH_SAFE(r
, &cq
->req_list
, entry
, next
) {
1405 QTAILQ_REMOVE(&cq
->req_list
, r
, entry
);
1406 QTAILQ_INSERT_TAIL(&sq
->req_list
, r
, entry
);
1411 nvme_free_sq(sq
, n
);
1412 return NVME_SUCCESS
;
1415 static void nvme_init_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
, uint64_t dma_addr
,
1416 uint16_t sqid
, uint16_t cqid
, uint16_t size
)
1422 sq
->dma_addr
= dma_addr
;
1426 sq
->head
= sq
->tail
= 0;
1427 sq
->io_req
= g_new0(NvmeRequest
, sq
->size
);
1429 QTAILQ_INIT(&sq
->req_list
);
1430 QTAILQ_INIT(&sq
->out_req_list
);
1431 for (i
= 0; i
< sq
->size
; i
++) {
1432 sq
->io_req
[i
].sq
= sq
;
1433 QTAILQ_INSERT_TAIL(&(sq
->req_list
), &sq
->io_req
[i
], entry
);
1435 sq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_process_sq
, sq
);
1437 assert(n
->cq
[cqid
]);
1439 QTAILQ_INSERT_TAIL(&(cq
->sq_list
), sq
, entry
);
1443 static uint16_t nvme_create_sq(NvmeCtrl
*n
, NvmeRequest
*req
)
1446 NvmeCreateSq
*c
= (NvmeCreateSq
*)&req
->cmd
;
1448 uint16_t cqid
= le16_to_cpu(c
->cqid
);
1449 uint16_t sqid
= le16_to_cpu(c
->sqid
);
1450 uint16_t qsize
= le16_to_cpu(c
->qsize
);
1451 uint16_t qflags
= le16_to_cpu(c
->sq_flags
);
1452 uint64_t prp1
= le64_to_cpu(c
->prp1
);
1454 trace_pci_nvme_create_sq(prp1
, sqid
, cqid
, qsize
, qflags
);
1456 if (unlikely(!cqid
|| nvme_check_cqid(n
, cqid
))) {
1457 trace_pci_nvme_err_invalid_create_sq_cqid(cqid
);
1458 return NVME_INVALID_CQID
| NVME_DNR
;
1460 if (unlikely(!sqid
|| sqid
> n
->params
.max_ioqpairs
||
1461 n
->sq
[sqid
] != NULL
)) {
1462 trace_pci_nvme_err_invalid_create_sq_sqid(sqid
);
1463 return NVME_INVALID_QID
| NVME_DNR
;
1465 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
1466 trace_pci_nvme_err_invalid_create_sq_size(qsize
);
1467 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
1469 if (unlikely(prp1
& (n
->page_size
- 1))) {
1470 trace_pci_nvme_err_invalid_create_sq_addr(prp1
);
1471 return NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
1473 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags
)))) {
1474 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags
));
1475 return NVME_INVALID_FIELD
| NVME_DNR
;
1477 sq
= g_malloc0(sizeof(*sq
));
1478 nvme_init_sq(sq
, n
, prp1
, sqid
, cqid
, qsize
+ 1);
1479 return NVME_SUCCESS
;
1483 uint64_t units_read
;
1484 uint64_t units_written
;
1485 uint64_t read_commands
;
1486 uint64_t write_commands
;
1489 static void nvme_set_blk_stats(NvmeNamespace
*ns
, struct nvme_stats
*stats
)
1491 BlockAcctStats
*s
= blk_get_stats(ns
->blkconf
.blk
);
1493 stats
->units_read
+= s
->nr_bytes
[BLOCK_ACCT_READ
] >> BDRV_SECTOR_BITS
;
1494 stats
->units_written
+= s
->nr_bytes
[BLOCK_ACCT_WRITE
] >> BDRV_SECTOR_BITS
;
1495 stats
->read_commands
+= s
->nr_ops
[BLOCK_ACCT_READ
];
1496 stats
->write_commands
+= s
->nr_ops
[BLOCK_ACCT_WRITE
];
1499 static uint16_t nvme_smart_info(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
1500 uint64_t off
, NvmeRequest
*req
)
1502 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
1503 struct nvme_stats stats
= { 0 };
1504 NvmeSmartLog smart
= { 0 };
1509 if (off
>= sizeof(smart
)) {
1510 return NVME_INVALID_FIELD
| NVME_DNR
;
1513 if (nsid
!= 0xffffffff) {
1514 ns
= nvme_ns(n
, nsid
);
1516 return NVME_INVALID_NSID
| NVME_DNR
;
1518 nvme_set_blk_stats(ns
, &stats
);
1522 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
1527 nvme_set_blk_stats(ns
, &stats
);
1531 trans_len
= MIN(sizeof(smart
) - off
, buf_len
);
1533 smart
.data_units_read
[0] = cpu_to_le64(DIV_ROUND_UP(stats
.units_read
,
1535 smart
.data_units_written
[0] = cpu_to_le64(DIV_ROUND_UP(stats
.units_written
,
1537 smart
.host_read_commands
[0] = cpu_to_le64(stats
.read_commands
);
1538 smart
.host_write_commands
[0] = cpu_to_le64(stats
.write_commands
);
1540 smart
.temperature
= cpu_to_le16(n
->temperature
);
1542 if ((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
1543 (n
->temperature
<= n
->features
.temp_thresh_low
)) {
1544 smart
.critical_warning
|= NVME_SMART_TEMPERATURE
;
1547 current_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
1548 smart
.power_on_hours
[0] =
1549 cpu_to_le64((((current_ms
- n
->starttime_ms
) / 1000) / 60) / 60);
1552 nvme_clear_events(n
, NVME_AER_TYPE_SMART
);
1555 return nvme_dma(n
, (uint8_t *) &smart
+ off
, trans_len
,
1556 DMA_DIRECTION_FROM_DEVICE
, req
);
1559 static uint16_t nvme_fw_log_info(NvmeCtrl
*n
, uint32_t buf_len
, uint64_t off
,
1563 NvmeFwSlotInfoLog fw_log
= {
1567 if (off
>= sizeof(fw_log
)) {
1568 return NVME_INVALID_FIELD
| NVME_DNR
;
1571 strpadcpy((char *)&fw_log
.frs1
, sizeof(fw_log
.frs1
), "1.0", ' ');
1572 trans_len
= MIN(sizeof(fw_log
) - off
, buf_len
);
1574 return nvme_dma(n
, (uint8_t *) &fw_log
+ off
, trans_len
,
1575 DMA_DIRECTION_FROM_DEVICE
, req
);
1578 static uint16_t nvme_error_info(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
1579 uint64_t off
, NvmeRequest
*req
)
1582 NvmeErrorLog errlog
;
1584 if (off
>= sizeof(errlog
)) {
1585 return NVME_INVALID_FIELD
| NVME_DNR
;
1589 nvme_clear_events(n
, NVME_AER_TYPE_ERROR
);
1592 memset(&errlog
, 0x0, sizeof(errlog
));
1593 trans_len
= MIN(sizeof(errlog
) - off
, buf_len
);
1595 return nvme_dma(n
, (uint8_t *)&errlog
, trans_len
,
1596 DMA_DIRECTION_FROM_DEVICE
, req
);
1599 static uint16_t nvme_cmd_effects(NvmeCtrl
*n
, uint8_t csi
, uint32_t buf_len
,
1600 uint64_t off
, NvmeRequest
*req
)
1602 NvmeEffectsLog log
= {};
1603 const uint32_t *src_iocs
= NULL
;
1606 if (off
>= sizeof(log
)) {
1607 trace_pci_nvme_err_invalid_log_page_offset(off
, sizeof(log
));
1608 return NVME_INVALID_FIELD
| NVME_DNR
;
1611 switch (NVME_CC_CSS(n
->bar
.cc
)) {
1612 case NVME_CC_CSS_NVM
:
1613 src_iocs
= nvme_cse_iocs_nvm
;
1615 case NVME_CC_CSS_ADMIN_ONLY
:
1617 case NVME_CC_CSS_CSI
:
1620 src_iocs
= nvme_cse_iocs_nvm
;
1625 memcpy(log
.acs
, nvme_cse_acs
, sizeof(nvme_cse_acs
));
1628 memcpy(log
.iocs
, src_iocs
, sizeof(log
.iocs
));
1631 trans_len
= MIN(sizeof(log
) - off
, buf_len
);
1633 return nvme_dma(n
, ((uint8_t *)&log
) + off
, trans_len
,
1634 DMA_DIRECTION_FROM_DEVICE
, req
);
1637 static uint16_t nvme_get_log(NvmeCtrl
*n
, NvmeRequest
*req
)
1639 NvmeCmd
*cmd
= &req
->cmd
;
1641 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
1642 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
1643 uint32_t dw12
= le32_to_cpu(cmd
->cdw12
);
1644 uint32_t dw13
= le32_to_cpu(cmd
->cdw13
);
1645 uint8_t lid
= dw10
& 0xff;
1646 uint8_t lsp
= (dw10
>> 8) & 0xf;
1647 uint8_t rae
= (dw10
>> 15) & 0x1;
1648 uint8_t csi
= le32_to_cpu(cmd
->cdw14
) >> 24;
1649 uint32_t numdl
, numdu
;
1650 uint64_t off
, lpol
, lpou
;
1654 numdl
= (dw10
>> 16);
1655 numdu
= (dw11
& 0xffff);
1659 len
= (((numdu
<< 16) | numdl
) + 1) << 2;
1660 off
= (lpou
<< 32ULL) | lpol
;
1663 return NVME_INVALID_FIELD
| NVME_DNR
;
1666 trace_pci_nvme_get_log(nvme_cid(req
), lid
, lsp
, rae
, len
, off
);
1668 status
= nvme_check_mdts(n
, len
);
1670 trace_pci_nvme_err_mdts(nvme_cid(req
), len
);
1675 case NVME_LOG_ERROR_INFO
:
1676 return nvme_error_info(n
, rae
, len
, off
, req
);
1677 case NVME_LOG_SMART_INFO
:
1678 return nvme_smart_info(n
, rae
, len
, off
, req
);
1679 case NVME_LOG_FW_SLOT_INFO
:
1680 return nvme_fw_log_info(n
, len
, off
, req
);
1681 case NVME_LOG_CMD_EFFECTS
:
1682 return nvme_cmd_effects(n
, csi
, len
, off
, req
);
1684 trace_pci_nvme_err_invalid_log_page(nvme_cid(req
), lid
);
1685 return NVME_INVALID_FIELD
| NVME_DNR
;
1689 static void nvme_free_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
)
1691 n
->cq
[cq
->cqid
] = NULL
;
1692 timer_free(cq
->timer
);
1693 msix_vector_unuse(&n
->parent_obj
, cq
->vector
);
1699 static uint16_t nvme_del_cq(NvmeCtrl
*n
, NvmeRequest
*req
)
1701 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)&req
->cmd
;
1703 uint16_t qid
= le16_to_cpu(c
->qid
);
1705 if (unlikely(!qid
|| nvme_check_cqid(n
, qid
))) {
1706 trace_pci_nvme_err_invalid_del_cq_cqid(qid
);
1707 return NVME_INVALID_CQID
| NVME_DNR
;
1711 if (unlikely(!QTAILQ_EMPTY(&cq
->sq_list
))) {
1712 trace_pci_nvme_err_invalid_del_cq_notempty(qid
);
1713 return NVME_INVALID_QUEUE_DEL
;
1715 nvme_irq_deassert(n
, cq
);
1716 trace_pci_nvme_del_cq(qid
);
1717 nvme_free_cq(cq
, n
);
1718 return NVME_SUCCESS
;
1721 static void nvme_init_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
, uint64_t dma_addr
,
1722 uint16_t cqid
, uint16_t vector
, uint16_t size
,
1723 uint16_t irq_enabled
)
1727 ret
= msix_vector_use(&n
->parent_obj
, vector
);
1732 cq
->dma_addr
= dma_addr
;
1734 cq
->irq_enabled
= irq_enabled
;
1735 cq
->vector
= vector
;
1736 cq
->head
= cq
->tail
= 0;
1737 QTAILQ_INIT(&cq
->req_list
);
1738 QTAILQ_INIT(&cq
->sq_list
);
1740 cq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_post_cqes
, cq
);
1743 static uint16_t nvme_create_cq(NvmeCtrl
*n
, NvmeRequest
*req
)
1746 NvmeCreateCq
*c
= (NvmeCreateCq
*)&req
->cmd
;
1747 uint16_t cqid
= le16_to_cpu(c
->cqid
);
1748 uint16_t vector
= le16_to_cpu(c
->irq_vector
);
1749 uint16_t qsize
= le16_to_cpu(c
->qsize
);
1750 uint16_t qflags
= le16_to_cpu(c
->cq_flags
);
1751 uint64_t prp1
= le64_to_cpu(c
->prp1
);
1753 trace_pci_nvme_create_cq(prp1
, cqid
, vector
, qsize
, qflags
,
1754 NVME_CQ_FLAGS_IEN(qflags
) != 0);
1756 if (unlikely(!cqid
|| cqid
> n
->params
.max_ioqpairs
||
1757 n
->cq
[cqid
] != NULL
)) {
1758 trace_pci_nvme_err_invalid_create_cq_cqid(cqid
);
1759 return NVME_INVALID_QID
| NVME_DNR
;
1761 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
1762 trace_pci_nvme_err_invalid_create_cq_size(qsize
);
1763 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
1765 if (unlikely(prp1
& (n
->page_size
- 1))) {
1766 trace_pci_nvme_err_invalid_create_cq_addr(prp1
);
1767 return NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
1769 if (unlikely(!msix_enabled(&n
->parent_obj
) && vector
)) {
1770 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
1771 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
1773 if (unlikely(vector
>= n
->params
.msix_qsize
)) {
1774 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
1775 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
1777 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags
)))) {
1778 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags
));
1779 return NVME_INVALID_FIELD
| NVME_DNR
;
1782 cq
= g_malloc0(sizeof(*cq
));
1783 nvme_init_cq(cq
, n
, prp1
, cqid
, vector
, qsize
+ 1,
1784 NVME_CQ_FLAGS_IEN(qflags
));
1787 * It is only required to set qs_created when creating a completion queue;
1788 * creating a submission queue without a matching completion queue will
1791 n
->qs_created
= true;
1792 return NVME_SUCCESS
;
1795 static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl
*n
, NvmeRequest
*req
)
1797 uint8_t id
[NVME_IDENTIFY_DATA_SIZE
] = {};
1799 return nvme_dma(n
, id
, sizeof(id
), DMA_DIRECTION_FROM_DEVICE
, req
);
1802 static uint16_t nvme_identify_ctrl(NvmeCtrl
*n
, NvmeRequest
*req
)
1804 trace_pci_nvme_identify_ctrl();
1806 return nvme_dma(n
, (uint8_t *)&n
->id_ctrl
, sizeof(n
->id_ctrl
),
1807 DMA_DIRECTION_FROM_DEVICE
, req
);
1810 static uint16_t nvme_identify_ctrl_csi(NvmeCtrl
*n
, NvmeRequest
*req
)
1812 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1814 trace_pci_nvme_identify_ctrl_csi(c
->csi
);
1816 if (c
->csi
== NVME_CSI_NVM
) {
1817 return nvme_rpt_empty_id_struct(n
, req
);
1820 return NVME_INVALID_FIELD
| NVME_DNR
;
1823 static uint16_t nvme_identify_ns(NvmeCtrl
*n
, NvmeRequest
*req
)
1826 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1827 uint32_t nsid
= le32_to_cpu(c
->nsid
);
1829 trace_pci_nvme_identify_ns(nsid
);
1831 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
1832 return NVME_INVALID_NSID
| NVME_DNR
;
1835 ns
= nvme_ns(n
, nsid
);
1836 if (unlikely(!ns
)) {
1837 return nvme_rpt_empty_id_struct(n
, req
);
1840 return nvme_dma(n
, (uint8_t *)&ns
->id_ns
, sizeof(NvmeIdNs
),
1841 DMA_DIRECTION_FROM_DEVICE
, req
);
1844 static uint16_t nvme_identify_ns_csi(NvmeCtrl
*n
, NvmeRequest
*req
)
1847 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1848 uint32_t nsid
= le32_to_cpu(c
->nsid
);
1850 trace_pci_nvme_identify_ns_csi(nsid
, c
->csi
);
1852 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
1853 return NVME_INVALID_NSID
| NVME_DNR
;
1856 ns
= nvme_ns(n
, nsid
);
1857 if (unlikely(!ns
)) {
1858 return nvme_rpt_empty_id_struct(n
, req
);
1861 if (c
->csi
== NVME_CSI_NVM
) {
1862 return nvme_rpt_empty_id_struct(n
, req
);
1865 return NVME_INVALID_FIELD
| NVME_DNR
;
1868 static uint16_t nvme_identify_nslist(NvmeCtrl
*n
, NvmeRequest
*req
)
1871 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1872 uint32_t min_nsid
= le32_to_cpu(c
->nsid
);
1873 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
1874 static const int data_len
= sizeof(list
);
1875 uint32_t *list_ptr
= (uint32_t *)list
;
1878 trace_pci_nvme_identify_nslist(min_nsid
);
1881 * Both 0xffffffff (NVME_NSID_BROADCAST) and 0xfffffffe are invalid values
1882 * since the Active Namespace ID List should return namespaces with ids
1883 * *higher* than the NSID specified in the command. This is also specified
1884 * in the spec (NVM Express v1.3d, Section 5.15.4).
1886 if (min_nsid
>= NVME_NSID_BROADCAST
- 1) {
1887 return NVME_INVALID_NSID
| NVME_DNR
;
1890 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
1895 if (ns
->params
.nsid
<= min_nsid
) {
1898 list_ptr
[j
++] = cpu_to_le32(ns
->params
.nsid
);
1899 if (j
== data_len
/ sizeof(uint32_t)) {
1904 return nvme_dma(n
, list
, data_len
, DMA_DIRECTION_FROM_DEVICE
, req
);
1907 static uint16_t nvme_identify_nslist_csi(NvmeCtrl
*n
, NvmeRequest
*req
)
1910 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1911 uint32_t min_nsid
= le32_to_cpu(c
->nsid
);
1912 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
1913 static const int data_len
= sizeof(list
);
1914 uint32_t *list_ptr
= (uint32_t *)list
;
1917 trace_pci_nvme_identify_nslist_csi(min_nsid
, c
->csi
);
1920 * Same as in nvme_identify_nslist(), 0xffffffff/0xfffffffe are invalid.
1922 if (min_nsid
>= NVME_NSID_BROADCAST
- 1) {
1923 return NVME_INVALID_NSID
| NVME_DNR
;
1926 if (c
->csi
!= NVME_CSI_NVM
) {
1927 return NVME_INVALID_FIELD
| NVME_DNR
;
1930 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
1935 if (ns
->params
.nsid
<= min_nsid
) {
1938 list_ptr
[j
++] = cpu_to_le32(ns
->params
.nsid
);
1939 if (j
== data_len
/ sizeof(uint32_t)) {
1944 return nvme_dma(n
, list
, data_len
, DMA_DIRECTION_FROM_DEVICE
, req
);
1947 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl
*n
, NvmeRequest
*req
)
1950 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1951 uint32_t nsid
= le32_to_cpu(c
->nsid
);
1952 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
1957 uint8_t v
[NVME_NIDL_UUID
];
1965 struct data
*ns_descrs
= (struct data
*)list
;
1967 trace_pci_nvme_identify_ns_descr_list(nsid
);
1969 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
1970 return NVME_INVALID_NSID
| NVME_DNR
;
1973 ns
= nvme_ns(n
, nsid
);
1974 if (unlikely(!ns
)) {
1975 return NVME_INVALID_FIELD
| NVME_DNR
;
1979 * Because the NGUID and EUI64 fields are 0 in the Identify Namespace data
1980 * structure, a Namespace UUID (nidt = 0x3) must be reported in the
1981 * Namespace Identification Descriptor. Add the namespace UUID here.
1983 ns_descrs
->uuid
.hdr
.nidt
= NVME_NIDT_UUID
;
1984 ns_descrs
->uuid
.hdr
.nidl
= NVME_NIDL_UUID
;
1985 memcpy(&ns_descrs
->uuid
.v
, ns
->params
.uuid
.data
, NVME_NIDL_UUID
);
1987 ns_descrs
->csi
.hdr
.nidt
= NVME_NIDT_CSI
;
1988 ns_descrs
->csi
.hdr
.nidl
= NVME_NIDL_CSI
;
1989 ns_descrs
->csi
.v
= ns
->csi
;
1991 return nvme_dma(n
, list
, sizeof(list
), DMA_DIRECTION_FROM_DEVICE
, req
);
1994 static uint16_t nvme_identify_cmd_set(NvmeCtrl
*n
, NvmeRequest
*req
)
1996 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
1997 static const int data_len
= sizeof(list
);
1999 trace_pci_nvme_identify_cmd_set();
2001 NVME_SET_CSI(*list
, NVME_CSI_NVM
);
2002 return nvme_dma(n
, list
, data_len
, DMA_DIRECTION_FROM_DEVICE
, req
);
2005 static uint16_t nvme_identify(NvmeCtrl
*n
, NvmeRequest
*req
)
2007 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
2009 switch (le32_to_cpu(c
->cns
)) {
2010 case NVME_ID_CNS_NS
:
2011 return nvme_identify_ns(n
, req
);
2012 case NVME_ID_CNS_CS_NS
:
2013 return nvme_identify_ns_csi(n
, req
);
2014 case NVME_ID_CNS_CTRL
:
2015 return nvme_identify_ctrl(n
, req
);
2016 case NVME_ID_CNS_CS_CTRL
:
2017 return nvme_identify_ctrl_csi(n
, req
);
2018 case NVME_ID_CNS_NS_ACTIVE_LIST
:
2019 return nvme_identify_nslist(n
, req
);
2020 case NVME_ID_CNS_CS_NS_ACTIVE_LIST
:
2021 return nvme_identify_nslist_csi(n
, req
);
2022 case NVME_ID_CNS_NS_DESCR_LIST
:
2023 return nvme_identify_ns_descr_list(n
, req
);
2024 case NVME_ID_CNS_IO_COMMAND_SET
:
2025 return nvme_identify_cmd_set(n
, req
);
2027 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c
->cns
));
2028 return NVME_INVALID_FIELD
| NVME_DNR
;
2032 static uint16_t nvme_abort(NvmeCtrl
*n
, NvmeRequest
*req
)
2034 uint16_t sqid
= le32_to_cpu(req
->cmd
.cdw10
) & 0xffff;
2036 req
->cqe
.result
= 1;
2037 if (nvme_check_sqid(n
, sqid
)) {
2038 return NVME_INVALID_FIELD
| NVME_DNR
;
2041 return NVME_SUCCESS
;
2044 static inline void nvme_set_timestamp(NvmeCtrl
*n
, uint64_t ts
)
2046 trace_pci_nvme_setfeat_timestamp(ts
);
2048 n
->host_timestamp
= le64_to_cpu(ts
);
2049 n
->timestamp_set_qemu_clock_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
2052 static inline uint64_t nvme_get_timestamp(const NvmeCtrl
*n
)
2054 uint64_t current_time
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
2055 uint64_t elapsed_time
= current_time
- n
->timestamp_set_qemu_clock_ms
;
2057 union nvme_timestamp
{
2059 uint64_t timestamp
:48;
2067 union nvme_timestamp ts
;
2069 ts
.timestamp
= n
->host_timestamp
+ elapsed_time
;
2071 /* If the host timestamp is non-zero, set the timestamp origin */
2072 ts
.origin
= n
->host_timestamp
? 0x01 : 0x00;
2074 trace_pci_nvme_getfeat_timestamp(ts
.all
);
2076 return cpu_to_le64(ts
.all
);
2079 static uint16_t nvme_get_feature_timestamp(NvmeCtrl
*n
, NvmeRequest
*req
)
2081 uint64_t timestamp
= nvme_get_timestamp(n
);
2083 return nvme_dma(n
, (uint8_t *)×tamp
, sizeof(timestamp
),
2084 DMA_DIRECTION_FROM_DEVICE
, req
);
2087 static uint16_t nvme_get_feature(NvmeCtrl
*n
, NvmeRequest
*req
)
2089 NvmeCmd
*cmd
= &req
->cmd
;
2090 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
2091 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
2092 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
2094 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
2095 NvmeGetFeatureSelect sel
= NVME_GETFEAT_SELECT(dw10
);
2099 static const uint32_t nvme_feature_default
[NVME_FID_MAX
] = {
2100 [NVME_ARBITRATION
] = NVME_ARB_AB_NOLIMIT
,
2103 trace_pci_nvme_getfeat(nvme_cid(req
), nsid
, fid
, sel
, dw11
);
2105 if (!nvme_feature_support
[fid
]) {
2106 return NVME_INVALID_FIELD
| NVME_DNR
;
2109 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
2110 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
2112 * The Reservation Notification Mask and Reservation Persistence
2113 * features require a status code of Invalid Field in Command when
2114 * NSID is 0xFFFFFFFF. Since the device does not support those
2115 * features we can always return Invalid Namespace or Format as we
2116 * should do for all other features.
2118 return NVME_INVALID_NSID
| NVME_DNR
;
2121 if (!nvme_ns(n
, nsid
)) {
2122 return NVME_INVALID_FIELD
| NVME_DNR
;
2127 case NVME_GETFEAT_SELECT_CURRENT
:
2129 case NVME_GETFEAT_SELECT_SAVED
:
2130 /* no features are saveable by the controller; fallthrough */
2131 case NVME_GETFEAT_SELECT_DEFAULT
:
2133 case NVME_GETFEAT_SELECT_CAP
:
2134 result
= nvme_feature_cap
[fid
];
2139 case NVME_TEMPERATURE_THRESHOLD
:
2143 * The controller only implements the Composite Temperature sensor, so
2144 * return 0 for all other sensors.
2146 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
2150 switch (NVME_TEMP_THSEL(dw11
)) {
2151 case NVME_TEMP_THSEL_OVER
:
2152 result
= n
->features
.temp_thresh_hi
;
2154 case NVME_TEMP_THSEL_UNDER
:
2155 result
= n
->features
.temp_thresh_low
;
2159 return NVME_INVALID_FIELD
| NVME_DNR
;
2160 case NVME_ERROR_RECOVERY
:
2161 if (!nvme_nsid_valid(n
, nsid
)) {
2162 return NVME_INVALID_NSID
| NVME_DNR
;
2165 ns
= nvme_ns(n
, nsid
);
2166 if (unlikely(!ns
)) {
2167 return NVME_INVALID_FIELD
| NVME_DNR
;
2170 result
= ns
->features
.err_rec
;
2172 case NVME_VOLATILE_WRITE_CACHE
:
2173 result
= n
->features
.vwc
;
2174 trace_pci_nvme_getfeat_vwcache(result
? "enabled" : "disabled");
2176 case NVME_ASYNCHRONOUS_EVENT_CONF
:
2177 result
= n
->features
.async_config
;
2179 case NVME_TIMESTAMP
:
2180 return nvme_get_feature_timestamp(n
, req
);
2187 case NVME_TEMPERATURE_THRESHOLD
:
2190 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
2194 if (NVME_TEMP_THSEL(dw11
) == NVME_TEMP_THSEL_OVER
) {
2195 result
= NVME_TEMPERATURE_WARNING
;
2199 case NVME_NUMBER_OF_QUEUES
:
2200 result
= (n
->params
.max_ioqpairs
- 1) |
2201 ((n
->params
.max_ioqpairs
- 1) << 16);
2202 trace_pci_nvme_getfeat_numq(result
);
2204 case NVME_INTERRUPT_VECTOR_CONF
:
2206 if (iv
>= n
->params
.max_ioqpairs
+ 1) {
2207 return NVME_INVALID_FIELD
| NVME_DNR
;
2211 if (iv
== n
->admin_cq
.vector
) {
2212 result
|= NVME_INTVC_NOCOALESCING
;
2215 case NVME_COMMAND_SET_PROFILE
:
2219 result
= nvme_feature_default
[fid
];
2224 req
->cqe
.result
= cpu_to_le32(result
);
2225 return NVME_SUCCESS
;
2228 static uint16_t nvme_set_feature_timestamp(NvmeCtrl
*n
, NvmeRequest
*req
)
2233 ret
= nvme_dma(n
, (uint8_t *)×tamp
, sizeof(timestamp
),
2234 DMA_DIRECTION_TO_DEVICE
, req
);
2235 if (ret
!= NVME_SUCCESS
) {
2239 nvme_set_timestamp(n
, timestamp
);
2241 return NVME_SUCCESS
;
2244 static uint16_t nvme_set_feature(NvmeCtrl
*n
, NvmeRequest
*req
)
2246 NvmeNamespace
*ns
= NULL
;
2248 NvmeCmd
*cmd
= &req
->cmd
;
2249 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
2250 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
2251 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
2252 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
2253 uint8_t save
= NVME_SETFEAT_SAVE(dw10
);
2256 trace_pci_nvme_setfeat(nvme_cid(req
), nsid
, fid
, save
, dw11
);
2259 return NVME_FID_NOT_SAVEABLE
| NVME_DNR
;
2262 if (!nvme_feature_support
[fid
]) {
2263 return NVME_INVALID_FIELD
| NVME_DNR
;
2266 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
2267 if (nsid
!= NVME_NSID_BROADCAST
) {
2268 if (!nvme_nsid_valid(n
, nsid
)) {
2269 return NVME_INVALID_NSID
| NVME_DNR
;
2272 ns
= nvme_ns(n
, nsid
);
2273 if (unlikely(!ns
)) {
2274 return NVME_INVALID_FIELD
| NVME_DNR
;
2277 } else if (nsid
&& nsid
!= NVME_NSID_BROADCAST
) {
2278 if (!nvme_nsid_valid(n
, nsid
)) {
2279 return NVME_INVALID_NSID
| NVME_DNR
;
2282 return NVME_FEAT_NOT_NS_SPEC
| NVME_DNR
;
2285 if (!(nvme_feature_cap
[fid
] & NVME_FEAT_CAP_CHANGE
)) {
2286 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
2290 case NVME_TEMPERATURE_THRESHOLD
:
2291 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
2295 switch (NVME_TEMP_THSEL(dw11
)) {
2296 case NVME_TEMP_THSEL_OVER
:
2297 n
->features
.temp_thresh_hi
= NVME_TEMP_TMPTH(dw11
);
2299 case NVME_TEMP_THSEL_UNDER
:
2300 n
->features
.temp_thresh_low
= NVME_TEMP_TMPTH(dw11
);
2303 return NVME_INVALID_FIELD
| NVME_DNR
;
2306 if (((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
2307 (n
->temperature
<= n
->features
.temp_thresh_low
)) &&
2308 NVME_AEC_SMART(n
->features
.async_config
) & NVME_SMART_TEMPERATURE
) {
2309 nvme_enqueue_event(n
, NVME_AER_TYPE_SMART
,
2310 NVME_AER_INFO_SMART_TEMP_THRESH
,
2311 NVME_LOG_SMART_INFO
);
2315 case NVME_ERROR_RECOVERY
:
2316 if (nsid
== NVME_NSID_BROADCAST
) {
2317 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
2324 if (NVME_ID_NS_NSFEAT_DULBE(ns
->id_ns
.nsfeat
)) {
2325 ns
->features
.err_rec
= dw11
;
2333 ns
->features
.err_rec
= dw11
;
2335 case NVME_VOLATILE_WRITE_CACHE
:
2336 n
->features
.vwc
= dw11
& 0x1;
2338 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
2344 if (!(dw11
& 0x1) && blk_enable_write_cache(ns
->blkconf
.blk
)) {
2345 blk_flush(ns
->blkconf
.blk
);
2348 blk_set_enable_write_cache(ns
->blkconf
.blk
, dw11
& 1);
2353 case NVME_NUMBER_OF_QUEUES
:
2354 if (n
->qs_created
) {
2355 return NVME_CMD_SEQ_ERROR
| NVME_DNR
;
2359 * NVMe v1.3, Section 5.21.1.7: 0xffff is not an allowed value for NCQR
2362 if ((dw11
& 0xffff) == 0xffff || ((dw11
>> 16) & 0xffff) == 0xffff) {
2363 return NVME_INVALID_FIELD
| NVME_DNR
;
2366 trace_pci_nvme_setfeat_numq((dw11
& 0xFFFF) + 1,
2367 ((dw11
>> 16) & 0xFFFF) + 1,
2368 n
->params
.max_ioqpairs
,
2369 n
->params
.max_ioqpairs
);
2370 req
->cqe
.result
= cpu_to_le32((n
->params
.max_ioqpairs
- 1) |
2371 ((n
->params
.max_ioqpairs
- 1) << 16));
2373 case NVME_ASYNCHRONOUS_EVENT_CONF
:
2374 n
->features
.async_config
= dw11
;
2376 case NVME_TIMESTAMP
:
2377 return nvme_set_feature_timestamp(n
, req
);
2378 case NVME_COMMAND_SET_PROFILE
:
2380 trace_pci_nvme_err_invalid_iocsci(dw11
& 0x1ff);
2381 return NVME_CMD_SET_CMB_REJECTED
| NVME_DNR
;
2385 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
2387 return NVME_SUCCESS
;
2390 static uint16_t nvme_aer(NvmeCtrl
*n
, NvmeRequest
*req
)
2392 trace_pci_nvme_aer(nvme_cid(req
));
2394 if (n
->outstanding_aers
> n
->params
.aerl
) {
2395 trace_pci_nvme_aer_aerl_exceeded();
2396 return NVME_AER_LIMIT_EXCEEDED
;
2399 n
->aer_reqs
[n
->outstanding_aers
] = req
;
2400 n
->outstanding_aers
++;
2402 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
2403 nvme_process_aers(n
);
2406 return NVME_NO_COMPLETE
;
2409 static uint16_t nvme_admin_cmd(NvmeCtrl
*n
, NvmeRequest
*req
)
2411 trace_pci_nvme_admin_cmd(nvme_cid(req
), nvme_sqid(req
), req
->cmd
.opcode
,
2412 nvme_adm_opc_str(req
->cmd
.opcode
));
2414 if (!(nvme_cse_acs
[req
->cmd
.opcode
] & NVME_CMD_EFF_CSUPP
)) {
2415 trace_pci_nvme_err_invalid_admin_opc(req
->cmd
.opcode
);
2416 return NVME_INVALID_OPCODE
| NVME_DNR
;
2419 switch (req
->cmd
.opcode
) {
2420 case NVME_ADM_CMD_DELETE_SQ
:
2421 return nvme_del_sq(n
, req
);
2422 case NVME_ADM_CMD_CREATE_SQ
:
2423 return nvme_create_sq(n
, req
);
2424 case NVME_ADM_CMD_GET_LOG_PAGE
:
2425 return nvme_get_log(n
, req
);
2426 case NVME_ADM_CMD_DELETE_CQ
:
2427 return nvme_del_cq(n
, req
);
2428 case NVME_ADM_CMD_CREATE_CQ
:
2429 return nvme_create_cq(n
, req
);
2430 case NVME_ADM_CMD_IDENTIFY
:
2431 return nvme_identify(n
, req
);
2432 case NVME_ADM_CMD_ABORT
:
2433 return nvme_abort(n
, req
);
2434 case NVME_ADM_CMD_SET_FEATURES
:
2435 return nvme_set_feature(n
, req
);
2436 case NVME_ADM_CMD_GET_FEATURES
:
2437 return nvme_get_feature(n
, req
);
2438 case NVME_ADM_CMD_ASYNC_EV_REQ
:
2439 return nvme_aer(n
, req
);
2444 return NVME_INVALID_OPCODE
| NVME_DNR
;
2447 static void nvme_process_sq(void *opaque
)
2449 NvmeSQueue
*sq
= opaque
;
2450 NvmeCtrl
*n
= sq
->ctrl
;
2451 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
2458 while (!(nvme_sq_empty(sq
) || QTAILQ_EMPTY(&sq
->req_list
))) {
2459 addr
= sq
->dma_addr
+ sq
->head
* n
->sqe_size
;
2460 if (nvme_addr_read(n
, addr
, (void *)&cmd
, sizeof(cmd
))) {
2461 trace_pci_nvme_err_addr_read(addr
);
2462 trace_pci_nvme_err_cfs();
2463 n
->bar
.csts
= NVME_CSTS_FAILED
;
2466 nvme_inc_sq_head(sq
);
2468 req
= QTAILQ_FIRST(&sq
->req_list
);
2469 QTAILQ_REMOVE(&sq
->req_list
, req
, entry
);
2470 QTAILQ_INSERT_TAIL(&sq
->out_req_list
, req
, entry
);
2471 nvme_req_clear(req
);
2472 req
->cqe
.cid
= cmd
.cid
;
2473 memcpy(&req
->cmd
, &cmd
, sizeof(NvmeCmd
));
2475 status
= sq
->sqid
? nvme_io_cmd(n
, req
) :
2476 nvme_admin_cmd(n
, req
);
2477 if (status
!= NVME_NO_COMPLETE
) {
2478 req
->status
= status
;
2479 nvme_enqueue_req_completion(cq
, req
);
2484 static void nvme_clear_ctrl(NvmeCtrl
*n
)
2489 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
2498 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
2499 if (n
->sq
[i
] != NULL
) {
2500 nvme_free_sq(n
->sq
[i
], n
);
2503 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
2504 if (n
->cq
[i
] != NULL
) {
2505 nvme_free_cq(n
->cq
[i
], n
);
2509 while (!QTAILQ_EMPTY(&n
->aer_queue
)) {
2510 NvmeAsyncEvent
*event
= QTAILQ_FIRST(&n
->aer_queue
);
2511 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
2516 n
->outstanding_aers
= 0;
2517 n
->qs_created
= false;
2520 static void nvme_ctrl_reset(NvmeCtrl
*n
)
2526 static void nvme_ctrl_shutdown(NvmeCtrl
*n
)
2533 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
2539 nvme_ns_shutdown(ns
);
2543 static void nvme_select_ns_iocs(NvmeCtrl
*n
)
2548 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
2553 ns
->iocs
= nvme_cse_iocs_none
;
2556 if (NVME_CC_CSS(n
->bar
.cc
) != NVME_CC_CSS_ADMIN_ONLY
) {
2557 ns
->iocs
= nvme_cse_iocs_nvm
;
2564 static int nvme_start_ctrl(NvmeCtrl
*n
)
2566 uint32_t page_bits
= NVME_CC_MPS(n
->bar
.cc
) + 12;
2567 uint32_t page_size
= 1 << page_bits
;
2569 if (unlikely(n
->cq
[0])) {
2570 trace_pci_nvme_err_startfail_cq();
2573 if (unlikely(n
->sq
[0])) {
2574 trace_pci_nvme_err_startfail_sq();
2577 if (unlikely(!n
->bar
.asq
)) {
2578 trace_pci_nvme_err_startfail_nbarasq();
2581 if (unlikely(!n
->bar
.acq
)) {
2582 trace_pci_nvme_err_startfail_nbaracq();
2585 if (unlikely(n
->bar
.asq
& (page_size
- 1))) {
2586 trace_pci_nvme_err_startfail_asq_misaligned(n
->bar
.asq
);
2589 if (unlikely(n
->bar
.acq
& (page_size
- 1))) {
2590 trace_pci_nvme_err_startfail_acq_misaligned(n
->bar
.acq
);
2593 if (unlikely(!(NVME_CAP_CSS(n
->bar
.cap
) & (1 << NVME_CC_CSS(n
->bar
.cc
))))) {
2594 trace_pci_nvme_err_startfail_css(NVME_CC_CSS(n
->bar
.cc
));
2597 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) <
2598 NVME_CAP_MPSMIN(n
->bar
.cap
))) {
2599 trace_pci_nvme_err_startfail_page_too_small(
2600 NVME_CC_MPS(n
->bar
.cc
),
2601 NVME_CAP_MPSMIN(n
->bar
.cap
));
2604 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) >
2605 NVME_CAP_MPSMAX(n
->bar
.cap
))) {
2606 trace_pci_nvme_err_startfail_page_too_large(
2607 NVME_CC_MPS(n
->bar
.cc
),
2608 NVME_CAP_MPSMAX(n
->bar
.cap
));
2611 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) <
2612 NVME_CTRL_CQES_MIN(n
->id_ctrl
.cqes
))) {
2613 trace_pci_nvme_err_startfail_cqent_too_small(
2614 NVME_CC_IOCQES(n
->bar
.cc
),
2615 NVME_CTRL_CQES_MIN(n
->bar
.cap
));
2618 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) >
2619 NVME_CTRL_CQES_MAX(n
->id_ctrl
.cqes
))) {
2620 trace_pci_nvme_err_startfail_cqent_too_large(
2621 NVME_CC_IOCQES(n
->bar
.cc
),
2622 NVME_CTRL_CQES_MAX(n
->bar
.cap
));
2625 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) <
2626 NVME_CTRL_SQES_MIN(n
->id_ctrl
.sqes
))) {
2627 trace_pci_nvme_err_startfail_sqent_too_small(
2628 NVME_CC_IOSQES(n
->bar
.cc
),
2629 NVME_CTRL_SQES_MIN(n
->bar
.cap
));
2632 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) >
2633 NVME_CTRL_SQES_MAX(n
->id_ctrl
.sqes
))) {
2634 trace_pci_nvme_err_startfail_sqent_too_large(
2635 NVME_CC_IOSQES(n
->bar
.cc
),
2636 NVME_CTRL_SQES_MAX(n
->bar
.cap
));
2639 if (unlikely(!NVME_AQA_ASQS(n
->bar
.aqa
))) {
2640 trace_pci_nvme_err_startfail_asqent_sz_zero();
2643 if (unlikely(!NVME_AQA_ACQS(n
->bar
.aqa
))) {
2644 trace_pci_nvme_err_startfail_acqent_sz_zero();
2648 n
->page_bits
= page_bits
;
2649 n
->page_size
= page_size
;
2650 n
->max_prp_ents
= n
->page_size
/ sizeof(uint64_t);
2651 n
->cqe_size
= 1 << NVME_CC_IOCQES(n
->bar
.cc
);
2652 n
->sqe_size
= 1 << NVME_CC_IOSQES(n
->bar
.cc
);
2653 nvme_init_cq(&n
->admin_cq
, n
, n
->bar
.acq
, 0, 0,
2654 NVME_AQA_ACQS(n
->bar
.aqa
) + 1, 1);
2655 nvme_init_sq(&n
->admin_sq
, n
, n
->bar
.asq
, 0, 0,
2656 NVME_AQA_ASQS(n
->bar
.aqa
) + 1);
2658 nvme_set_timestamp(n
, 0ULL);
2660 QTAILQ_INIT(&n
->aer_queue
);
2662 nvme_select_ns_iocs(n
);
2667 static void nvme_write_bar(NvmeCtrl
*n
, hwaddr offset
, uint64_t data
,
2670 if (unlikely(offset
& (sizeof(uint32_t) - 1))) {
2671 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32
,
2672 "MMIO write not 32-bit aligned,"
2673 " offset=0x%"PRIx64
"", offset
);
2674 /* should be ignored, fall through for now */
2677 if (unlikely(size
< sizeof(uint32_t))) {
2678 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall
,
2679 "MMIO write smaller than 32-bits,"
2680 " offset=0x%"PRIx64
", size=%u",
2682 /* should be ignored, fall through for now */
2686 case 0xc: /* INTMS */
2687 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
2688 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
2689 "undefined access to interrupt mask set"
2690 " when MSI-X is enabled");
2691 /* should be ignored, fall through for now */
2693 n
->bar
.intms
|= data
& 0xffffffff;
2694 n
->bar
.intmc
= n
->bar
.intms
;
2695 trace_pci_nvme_mmio_intm_set(data
& 0xffffffff, n
->bar
.intmc
);
2698 case 0x10: /* INTMC */
2699 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
2700 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
2701 "undefined access to interrupt mask clr"
2702 " when MSI-X is enabled");
2703 /* should be ignored, fall through for now */
2705 n
->bar
.intms
&= ~(data
& 0xffffffff);
2706 n
->bar
.intmc
= n
->bar
.intms
;
2707 trace_pci_nvme_mmio_intm_clr(data
& 0xffffffff, n
->bar
.intmc
);
2711 trace_pci_nvme_mmio_cfg(data
& 0xffffffff);
2712 /* Windows first sends data, then sends enable bit */
2713 if (!NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
) &&
2714 !NVME_CC_SHN(data
) && !NVME_CC_SHN(n
->bar
.cc
))
2719 if (NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
)) {
2721 if (unlikely(nvme_start_ctrl(n
))) {
2722 trace_pci_nvme_err_startfail();
2723 n
->bar
.csts
= NVME_CSTS_FAILED
;
2725 trace_pci_nvme_mmio_start_success();
2726 n
->bar
.csts
= NVME_CSTS_READY
;
2728 } else if (!NVME_CC_EN(data
) && NVME_CC_EN(n
->bar
.cc
)) {
2729 trace_pci_nvme_mmio_stopped();
2731 n
->bar
.csts
&= ~NVME_CSTS_READY
;
2733 if (NVME_CC_SHN(data
) && !(NVME_CC_SHN(n
->bar
.cc
))) {
2734 trace_pci_nvme_mmio_shutdown_set();
2735 nvme_ctrl_shutdown(n
);
2737 n
->bar
.csts
|= NVME_CSTS_SHST_COMPLETE
;
2738 } else if (!NVME_CC_SHN(data
) && NVME_CC_SHN(n
->bar
.cc
)) {
2739 trace_pci_nvme_mmio_shutdown_cleared();
2740 n
->bar
.csts
&= ~NVME_CSTS_SHST_COMPLETE
;
2744 case 0x1C: /* CSTS */
2745 if (data
& (1 << 4)) {
2746 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported
,
2747 "attempted to W1C CSTS.NSSRO"
2748 " but CAP.NSSRS is zero (not supported)");
2749 } else if (data
!= 0) {
2750 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts
,
2751 "attempted to set a read only bit"
2752 " of controller status");
2755 case 0x20: /* NSSR */
2756 if (data
== 0x4E564D65) {
2757 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
2759 /* The spec says that writes of other values have no effect */
2763 case 0x24: /* AQA */
2764 n
->bar
.aqa
= data
& 0xffffffff;
2765 trace_pci_nvme_mmio_aqattr(data
& 0xffffffff);
2767 case 0x28: /* ASQ */
2769 trace_pci_nvme_mmio_asqaddr(data
);
2771 case 0x2c: /* ASQ hi */
2772 n
->bar
.asq
|= data
<< 32;
2773 trace_pci_nvme_mmio_asqaddr_hi(data
, n
->bar
.asq
);
2775 case 0x30: /* ACQ */
2776 trace_pci_nvme_mmio_acqaddr(data
);
2779 case 0x34: /* ACQ hi */
2780 n
->bar
.acq
|= data
<< 32;
2781 trace_pci_nvme_mmio_acqaddr_hi(data
, n
->bar
.acq
);
2783 case 0x38: /* CMBLOC */
2784 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved
,
2785 "invalid write to reserved CMBLOC"
2786 " when CMBSZ is zero, ignored");
2788 case 0x3C: /* CMBSZ */
2789 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly
,
2790 "invalid write to read only CMBSZ, ignored");
2792 case 0xE00: /* PMRCAP */
2793 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly
,
2794 "invalid write to PMRCAP register, ignored");
2796 case 0xE04: /* TODO PMRCTL */
2798 case 0xE08: /* PMRSTS */
2799 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly
,
2800 "invalid write to PMRSTS register, ignored");
2802 case 0xE0C: /* PMREBS */
2803 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly
,
2804 "invalid write to PMREBS register, ignored");
2806 case 0xE10: /* PMRSWTP */
2807 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly
,
2808 "invalid write to PMRSWTP register, ignored");
2810 case 0xE14: /* TODO PMRMSC */
2813 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid
,
2814 "invalid MMIO write,"
2815 " offset=0x%"PRIx64
", data=%"PRIx64
"",
2821 static uint64_t nvme_mmio_read(void *opaque
, hwaddr addr
, unsigned size
)
2823 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
2824 uint8_t *ptr
= (uint8_t *)&n
->bar
;
2827 trace_pci_nvme_mmio_read(addr
);
2829 if (unlikely(addr
& (sizeof(uint32_t) - 1))) {
2830 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32
,
2831 "MMIO read not 32-bit aligned,"
2832 " offset=0x%"PRIx64
"", addr
);
2833 /* should RAZ, fall through for now */
2834 } else if (unlikely(size
< sizeof(uint32_t))) {
2835 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall
,
2836 "MMIO read smaller than 32-bits,"
2837 " offset=0x%"PRIx64
"", addr
);
2838 /* should RAZ, fall through for now */
2841 if (addr
< sizeof(n
->bar
)) {
2843 * When PMRWBM bit 1 is set then read from
2844 * from PMRSTS should ensure prior writes
2845 * made it to persistent media
2847 if (addr
== 0xE08 &&
2848 (NVME_PMRCAP_PMRWBM(n
->bar
.pmrcap
) & 0x02)) {
2849 memory_region_msync(&n
->pmrdev
->mr
, 0, n
->pmrdev
->size
);
2851 memcpy(&val
, ptr
+ addr
, size
);
2853 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs
,
2854 "MMIO read beyond last register,"
2855 " offset=0x%"PRIx64
", returning 0", addr
);
2861 static void nvme_process_db(NvmeCtrl
*n
, hwaddr addr
, int val
)
2865 if (unlikely(addr
& ((1 << 2) - 1))) {
2866 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned
,
2867 "doorbell write not 32-bit aligned,"
2868 " offset=0x%"PRIx64
", ignoring", addr
);
2872 if (((addr
- 0x1000) >> 2) & 1) {
2873 /* Completion queue doorbell write */
2875 uint16_t new_head
= val
& 0xffff;
2879 qid
= (addr
- (0x1000 + (1 << 2))) >> 3;
2880 if (unlikely(nvme_check_cqid(n
, qid
))) {
2881 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq
,
2882 "completion queue doorbell write"
2883 " for nonexistent queue,"
2884 " sqid=%"PRIu32
", ignoring", qid
);
2887 * NVM Express v1.3d, Section 4.1 state: "If host software writes
2888 * an invalid value to the Submission Queue Tail Doorbell or
2889 * Completion Queue Head Doorbell regiter and an Asynchronous Event
2890 * Request command is outstanding, then an asynchronous event is
2891 * posted to the Admin Completion Queue with a status code of
2892 * Invalid Doorbell Write Value."
2894 * Also note that the spec includes the "Invalid Doorbell Register"
2895 * status code, but nowhere does it specify when to use it.
2896 * However, it seems reasonable to use it here in a similar
2899 if (n
->outstanding_aers
) {
2900 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
2901 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
2902 NVME_LOG_ERROR_INFO
);
2909 if (unlikely(new_head
>= cq
->size
)) {
2910 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead
,
2911 "completion queue doorbell write value"
2912 " beyond queue size, sqid=%"PRIu32
","
2913 " new_head=%"PRIu16
", ignoring",
2916 if (n
->outstanding_aers
) {
2917 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
2918 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
2919 NVME_LOG_ERROR_INFO
);
2925 trace_pci_nvme_mmio_doorbell_cq(cq
->cqid
, new_head
);
2927 start_sqs
= nvme_cq_full(cq
) ? 1 : 0;
2928 cq
->head
= new_head
;
2931 QTAILQ_FOREACH(sq
, &cq
->sq_list
, entry
) {
2932 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
2934 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
2937 if (cq
->tail
== cq
->head
) {
2938 nvme_irq_deassert(n
, cq
);
2941 /* Submission queue doorbell write */
2943 uint16_t new_tail
= val
& 0xffff;
2946 qid
= (addr
- 0x1000) >> 3;
2947 if (unlikely(nvme_check_sqid(n
, qid
))) {
2948 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq
,
2949 "submission queue doorbell write"
2950 " for nonexistent queue,"
2951 " sqid=%"PRIu32
", ignoring", qid
);
2953 if (n
->outstanding_aers
) {
2954 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
2955 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
2956 NVME_LOG_ERROR_INFO
);
2963 if (unlikely(new_tail
>= sq
->size
)) {
2964 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail
,
2965 "submission queue doorbell write value"
2966 " beyond queue size, sqid=%"PRIu32
","
2967 " new_tail=%"PRIu16
", ignoring",
2970 if (n
->outstanding_aers
) {
2971 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
2972 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
2973 NVME_LOG_ERROR_INFO
);
2979 trace_pci_nvme_mmio_doorbell_sq(sq
->sqid
, new_tail
);
2981 sq
->tail
= new_tail
;
2982 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
2986 static void nvme_mmio_write(void *opaque
, hwaddr addr
, uint64_t data
,
2989 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
2991 trace_pci_nvme_mmio_write(addr
, data
);
2993 if (addr
< sizeof(n
->bar
)) {
2994 nvme_write_bar(n
, addr
, data
, size
);
2996 nvme_process_db(n
, addr
, data
);
3000 static const MemoryRegionOps nvme_mmio_ops
= {
3001 .read
= nvme_mmio_read
,
3002 .write
= nvme_mmio_write
,
3003 .endianness
= DEVICE_LITTLE_ENDIAN
,
3005 .min_access_size
= 2,
3006 .max_access_size
= 8,
3010 static void nvme_cmb_write(void *opaque
, hwaddr addr
, uint64_t data
,
3013 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
3014 stn_le_p(&n
->cmbuf
[addr
], size
, data
);
3017 static uint64_t nvme_cmb_read(void *opaque
, hwaddr addr
, unsigned size
)
3019 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
3020 return ldn_le_p(&n
->cmbuf
[addr
], size
);
3023 static const MemoryRegionOps nvme_cmb_ops
= {
3024 .read
= nvme_cmb_read
,
3025 .write
= nvme_cmb_write
,
3026 .endianness
= DEVICE_LITTLE_ENDIAN
,
3028 .min_access_size
= 1,
3029 .max_access_size
= 8,
3033 static void nvme_check_constraints(NvmeCtrl
*n
, Error
**errp
)
3035 NvmeParams
*params
= &n
->params
;
3037 if (params
->num_queues
) {
3038 warn_report("num_queues is deprecated; please use max_ioqpairs "
3041 params
->max_ioqpairs
= params
->num_queues
- 1;
3045 warn_report("drive property is deprecated; "
3046 "please use an nvme-ns device instead");
3049 if (params
->max_ioqpairs
< 1 ||
3050 params
->max_ioqpairs
> NVME_MAX_IOQPAIRS
) {
3051 error_setg(errp
, "max_ioqpairs must be between 1 and %d",
3056 if (params
->msix_qsize
< 1 ||
3057 params
->msix_qsize
> PCI_MSIX_FLAGS_QSIZE
+ 1) {
3058 error_setg(errp
, "msix_qsize must be between 1 and %d",
3059 PCI_MSIX_FLAGS_QSIZE
+ 1);
3063 if (!params
->serial
) {
3064 error_setg(errp
, "serial property not set");
3068 if (!n
->params
.cmb_size_mb
&& n
->pmrdev
) {
3069 if (host_memory_backend_is_mapped(n
->pmrdev
)) {
3070 error_setg(errp
, "can't use already busy memdev: %s",
3071 object_get_canonical_path_component(OBJECT(n
->pmrdev
)));
3075 if (!is_power_of_2(n
->pmrdev
->size
)) {
3076 error_setg(errp
, "pmr backend size needs to be power of 2 in size");
3080 host_memory_backend_set_mapped(n
->pmrdev
, true);
3084 static void nvme_init_state(NvmeCtrl
*n
)
3086 n
->num_namespaces
= NVME_MAX_NAMESPACES
;
3087 /* add one to max_ioqpairs to account for the admin queue pair */
3088 n
->reg_size
= pow2ceil(sizeof(NvmeBar
) +
3089 2 * (n
->params
.max_ioqpairs
+ 1) * NVME_DB_SIZE
);
3090 n
->sq
= g_new0(NvmeSQueue
*, n
->params
.max_ioqpairs
+ 1);
3091 n
->cq
= g_new0(NvmeCQueue
*, n
->params
.max_ioqpairs
+ 1);
3092 n
->temperature
= NVME_TEMPERATURE
;
3093 n
->features
.temp_thresh_hi
= NVME_TEMPERATURE_WARNING
;
3094 n
->starttime_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
3095 n
->aer_reqs
= g_new0(NvmeRequest
*, n
->params
.aerl
+ 1);
3098 int nvme_register_namespace(NvmeCtrl
*n
, NvmeNamespace
*ns
, Error
**errp
)
3100 uint32_t nsid
= nvme_nsid(ns
);
3102 if (nsid
> NVME_MAX_NAMESPACES
) {
3103 error_setg(errp
, "invalid namespace id (must be between 0 and %d)",
3104 NVME_MAX_NAMESPACES
);
3109 for (int i
= 1; i
<= n
->num_namespaces
; i
++) {
3110 if (!nvme_ns(n
, i
)) {
3111 nsid
= ns
->params
.nsid
= i
;
3117 error_setg(errp
, "no free namespace id");
3121 if (n
->namespaces
[nsid
- 1]) {
3122 error_setg(errp
, "namespace id '%d' is already in use", nsid
);
3127 trace_pci_nvme_register_namespace(nsid
);
3129 n
->namespaces
[nsid
- 1] = ns
;
3134 static void nvme_init_cmb(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
3136 NVME_CMBLOC_SET_BIR(n
->bar
.cmbloc
, NVME_CMB_BIR
);
3137 NVME_CMBLOC_SET_OFST(n
->bar
.cmbloc
, 0);
3139 NVME_CMBSZ_SET_SQS(n
->bar
.cmbsz
, 1);
3140 NVME_CMBSZ_SET_CQS(n
->bar
.cmbsz
, 0);
3141 NVME_CMBSZ_SET_LISTS(n
->bar
.cmbsz
, 1);
3142 NVME_CMBSZ_SET_RDS(n
->bar
.cmbsz
, 1);
3143 NVME_CMBSZ_SET_WDS(n
->bar
.cmbsz
, 1);
3144 NVME_CMBSZ_SET_SZU(n
->bar
.cmbsz
, 2); /* MBs */
3145 NVME_CMBSZ_SET_SZ(n
->bar
.cmbsz
, n
->params
.cmb_size_mb
);
3147 n
->cmbuf
= g_malloc0(NVME_CMBSZ_GETSIZE(n
->bar
.cmbsz
));
3148 memory_region_init_io(&n
->ctrl_mem
, OBJECT(n
), &nvme_cmb_ops
, n
,
3149 "nvme-cmb", NVME_CMBSZ_GETSIZE(n
->bar
.cmbsz
));
3150 pci_register_bar(pci_dev
, NVME_CMBLOC_BIR(n
->bar
.cmbloc
),
3151 PCI_BASE_ADDRESS_SPACE_MEMORY
|
3152 PCI_BASE_ADDRESS_MEM_TYPE_64
|
3153 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->ctrl_mem
);
3156 static void nvme_init_pmr(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
3158 /* Controller Capabilities register */
3159 NVME_CAP_SET_PMRS(n
->bar
.cap
, 1);
3161 /* PMR Capabities register */
3163 NVME_PMRCAP_SET_RDS(n
->bar
.pmrcap
, 0);
3164 NVME_PMRCAP_SET_WDS(n
->bar
.pmrcap
, 0);
3165 NVME_PMRCAP_SET_BIR(n
->bar
.pmrcap
, NVME_PMR_BIR
);
3166 NVME_PMRCAP_SET_PMRTU(n
->bar
.pmrcap
, 0);
3167 /* Turn on bit 1 support */
3168 NVME_PMRCAP_SET_PMRWBM(n
->bar
.pmrcap
, 0x02);
3169 NVME_PMRCAP_SET_PMRTO(n
->bar
.pmrcap
, 0);
3170 NVME_PMRCAP_SET_CMSS(n
->bar
.pmrcap
, 0);
3172 /* PMR Control register */
3174 NVME_PMRCTL_SET_EN(n
->bar
.pmrctl
, 0);
3176 /* PMR Status register */
3178 NVME_PMRSTS_SET_ERR(n
->bar
.pmrsts
, 0);
3179 NVME_PMRSTS_SET_NRDY(n
->bar
.pmrsts
, 0);
3180 NVME_PMRSTS_SET_HSTS(n
->bar
.pmrsts
, 0);
3181 NVME_PMRSTS_SET_CBAI(n
->bar
.pmrsts
, 0);
3183 /* PMR Elasticity Buffer Size register */
3185 NVME_PMREBS_SET_PMRSZU(n
->bar
.pmrebs
, 0);
3186 NVME_PMREBS_SET_RBB(n
->bar
.pmrebs
, 0);
3187 NVME_PMREBS_SET_PMRWBZ(n
->bar
.pmrebs
, 0);
3189 /* PMR Sustained Write Throughput register */
3191 NVME_PMRSWTP_SET_PMRSWTU(n
->bar
.pmrswtp
, 0);
3192 NVME_PMRSWTP_SET_PMRSWTV(n
->bar
.pmrswtp
, 0);
3194 /* PMR Memory Space Control register */
3196 NVME_PMRMSC_SET_CMSE(n
->bar
.pmrmsc
, 0);
3197 NVME_PMRMSC_SET_CBA(n
->bar
.pmrmsc
, 0);
3199 pci_register_bar(pci_dev
, NVME_PMRCAP_BIR(n
->bar
.pmrcap
),
3200 PCI_BASE_ADDRESS_SPACE_MEMORY
|
3201 PCI_BASE_ADDRESS_MEM_TYPE_64
|
3202 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->pmrdev
->mr
);
3205 static void nvme_init_pci(NvmeCtrl
*n
, PCIDevice
*pci_dev
, Error
**errp
)
3207 uint8_t *pci_conf
= pci_dev
->config
;
3209 pci_conf
[PCI_INTERRUPT_PIN
] = 1;
3210 pci_config_set_prog_interface(pci_conf
, 0x2);
3212 if (n
->params
.use_intel_id
) {
3213 pci_config_set_vendor_id(pci_conf
, PCI_VENDOR_ID_INTEL
);
3214 pci_config_set_device_id(pci_conf
, 0x5845);
3216 pci_config_set_vendor_id(pci_conf
, PCI_VENDOR_ID_REDHAT
);
3217 pci_config_set_device_id(pci_conf
, PCI_DEVICE_ID_REDHAT_NVME
);
3220 pci_config_set_class(pci_conf
, PCI_CLASS_STORAGE_EXPRESS
);
3221 pcie_endpoint_cap_init(pci_dev
, 0x80);
3223 memory_region_init_io(&n
->iomem
, OBJECT(n
), &nvme_mmio_ops
, n
, "nvme",
3225 pci_register_bar(pci_dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
|
3226 PCI_BASE_ADDRESS_MEM_TYPE_64
, &n
->iomem
);
3227 if (msix_init_exclusive_bar(pci_dev
, n
->params
.msix_qsize
, 4, errp
)) {
3231 if (n
->params
.cmb_size_mb
) {
3232 nvme_init_cmb(n
, pci_dev
);
3233 } else if (n
->pmrdev
) {
3234 nvme_init_pmr(n
, pci_dev
);
3238 static void nvme_init_ctrl(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
3240 NvmeIdCtrl
*id
= &n
->id_ctrl
;
3241 uint8_t *pci_conf
= pci_dev
->config
;
3244 id
->vid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_VENDOR_ID
));
3245 id
->ssvid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_SUBSYSTEM_VENDOR_ID
));
3246 strpadcpy((char *)id
->mn
, sizeof(id
->mn
), "QEMU NVMe Ctrl", ' ');
3247 strpadcpy((char *)id
->fr
, sizeof(id
->fr
), "1.0", ' ');
3248 strpadcpy((char *)id
->sn
, sizeof(id
->sn
), n
->params
.serial
, ' ');
3253 id
->mdts
= n
->params
.mdts
;
3254 id
->ver
= cpu_to_le32(NVME_SPEC_VER
);
3255 id
->oacs
= cpu_to_le16(0);
3258 * Because the controller always completes the Abort command immediately,
3259 * there can never be more than one concurrently executing Abort command,
3260 * so this value is never used for anything. Note that there can easily be
3261 * many Abort commands in the queues, but they are not considered
3262 * "executing" until processed by nvme_abort.
3264 * The specification recommends a value of 3 for Abort Command Limit (four
3265 * concurrently outstanding Abort commands), so lets use that though it is
3269 id
->aerl
= n
->params
.aerl
;
3270 id
->frmw
= (NVME_NUM_FW_SLOTS
<< 1) | NVME_FRMW_SLOT1_RO
;
3271 id
->lpa
= NVME_LPA_NS_SMART
| NVME_LPA_CSE
| NVME_LPA_EXTENDED
;
3273 /* recommended default value (~70 C) */
3274 id
->wctemp
= cpu_to_le16(NVME_TEMPERATURE_WARNING
);
3275 id
->cctemp
= cpu_to_le16(NVME_TEMPERATURE_CRITICAL
);
3277 id
->sqes
= (0x6 << 4) | 0x6;
3278 id
->cqes
= (0x4 << 4) | 0x4;
3279 id
->nn
= cpu_to_le32(n
->num_namespaces
);
3280 id
->oncs
= cpu_to_le16(NVME_ONCS_WRITE_ZEROES
| NVME_ONCS_TIMESTAMP
|
3281 NVME_ONCS_FEATURES
| NVME_ONCS_DSM
|
3285 id
->sgls
= cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN
|
3286 NVME_CTRL_SGLS_BITBUCKET
);
3288 subnqn
= g_strdup_printf("nqn.2019-08.org.qemu:%s", n
->params
.serial
);
3289 strpadcpy((char *)id
->subnqn
, sizeof(id
->subnqn
), subnqn
, '\0');
3292 id
->psd
[0].mp
= cpu_to_le16(0x9c4);
3293 id
->psd
[0].enlat
= cpu_to_le32(0x10);
3294 id
->psd
[0].exlat
= cpu_to_le32(0x4);
3296 NVME_CAP_SET_MQES(n
->bar
.cap
, 0x7ff);
3297 NVME_CAP_SET_CQR(n
->bar
.cap
, 1);
3298 NVME_CAP_SET_TO(n
->bar
.cap
, 0xf);
3299 NVME_CAP_SET_CSS(n
->bar
.cap
, NVME_CAP_CSS_NVM
);
3300 NVME_CAP_SET_CSS(n
->bar
.cap
, NVME_CAP_CSS_CSI_SUPP
);
3301 NVME_CAP_SET_CSS(n
->bar
.cap
, NVME_CAP_CSS_ADMIN_ONLY
);
3302 NVME_CAP_SET_MPSMAX(n
->bar
.cap
, 4);
3304 n
->bar
.vs
= NVME_SPEC_VER
;
3305 n
->bar
.intmc
= n
->bar
.intms
= 0;
3308 static void nvme_realize(PCIDevice
*pci_dev
, Error
**errp
)
3310 NvmeCtrl
*n
= NVME(pci_dev
);
3312 Error
*local_err
= NULL
;
3314 nvme_check_constraints(n
, &local_err
);
3316 error_propagate(errp
, local_err
);
3320 qbus_create_inplace(&n
->bus
, sizeof(NvmeBus
), TYPE_NVME_BUS
,
3321 &pci_dev
->qdev
, n
->parent_obj
.qdev
.id
);
3324 nvme_init_pci(n
, pci_dev
, &local_err
);
3326 error_propagate(errp
, local_err
);
3330 nvme_init_ctrl(n
, pci_dev
);
3332 /* setup a namespace if the controller drive property was given */
3333 if (n
->namespace.blkconf
.blk
) {
3335 ns
->params
.nsid
= 1;
3337 if (nvme_ns_setup(n
, ns
, errp
)) {
3343 static void nvme_exit(PCIDevice
*pci_dev
)
3345 NvmeCtrl
*n
= NVME(pci_dev
);
3347 nvme_ctrl_shutdown(n
);
3350 g_free(n
->aer_reqs
);
3352 if (n
->params
.cmb_size_mb
) {
3357 host_memory_backend_set_mapped(n
->pmrdev
, false);
3359 msix_uninit_exclusive_bar(pci_dev
);
3362 static Property nvme_props
[] = {
3363 DEFINE_BLOCK_PROPERTIES(NvmeCtrl
, namespace.blkconf
),
3364 DEFINE_PROP_LINK("pmrdev", NvmeCtrl
, pmrdev
, TYPE_MEMORY_BACKEND
,
3365 HostMemoryBackend
*),
3366 DEFINE_PROP_STRING("serial", NvmeCtrl
, params
.serial
),
3367 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl
, params
.cmb_size_mb
, 0),
3368 DEFINE_PROP_UINT32("num_queues", NvmeCtrl
, params
.num_queues
, 0),
3369 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl
, params
.max_ioqpairs
, 64),
3370 DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl
, params
.msix_qsize
, 65),
3371 DEFINE_PROP_UINT8("aerl", NvmeCtrl
, params
.aerl
, 3),
3372 DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl
, params
.aer_max_queued
, 64),
3373 DEFINE_PROP_UINT8("mdts", NvmeCtrl
, params
.mdts
, 7),
3374 DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl
, params
.use_intel_id
, false),
3375 DEFINE_PROP_END_OF_LIST(),
3378 static const VMStateDescription nvme_vmstate
= {
3383 static void nvme_class_init(ObjectClass
*oc
, void *data
)
3385 DeviceClass
*dc
= DEVICE_CLASS(oc
);
3386 PCIDeviceClass
*pc
= PCI_DEVICE_CLASS(oc
);
3388 pc
->realize
= nvme_realize
;
3389 pc
->exit
= nvme_exit
;
3390 pc
->class_id
= PCI_CLASS_STORAGE_EXPRESS
;
3393 set_bit(DEVICE_CATEGORY_STORAGE
, dc
->categories
);
3394 dc
->desc
= "Non-Volatile Memory Express";
3395 device_class_set_props(dc
, nvme_props
);
3396 dc
->vmsd
= &nvme_vmstate
;
3399 static void nvme_instance_init(Object
*obj
)
3401 NvmeCtrl
*s
= NVME(obj
);
3403 if (s
->namespace.blkconf
.blk
) {
3404 device_add_bootindex_property(obj
, &s
->namespace.blkconf
.bootindex
,
3405 "bootindex", "/namespace@1,0",
3410 static const TypeInfo nvme_info
= {
3412 .parent
= TYPE_PCI_DEVICE
,
3413 .instance_size
= sizeof(NvmeCtrl
),
3414 .instance_init
= nvme_instance_init
,
3415 .class_init
= nvme_class_init
,
3416 .interfaces
= (InterfaceInfo
[]) {
3417 { INTERFACE_PCIE_DEVICE
},
3422 static const TypeInfo nvme_bus_info
= {
3423 .name
= TYPE_NVME_BUS
,
3425 .instance_size
= sizeof(NvmeBus
),
3428 static void nvme_register_types(void)
3430 type_register_static(&nvme_info
);
3431 type_register_static(&nvme_bus_info
);
3434 type_init(nvme_register_types
)