2 * QEMU NVM Express Controller
4 * Copyright (c) 2012, Intel Corporation
6 * Written by Keith Busch <keith.busch@intel.com>
8 * This code is licensed under the GNU GPL v2 or later.
12 * Reference Specs: http://www.nvmexpress.org, 1.2, 1.1, 1.0e
14 * https://nvmexpress.org/developers/nvme-specification/
19 * -drive file=<file>,if=none,id=<drive_id>
20 * -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>, \
21 * cmb_size_mb=<cmb_size_mb[optional]>, \
22 * [pmrdev=<mem_backend_file_id>,] \
23 * max_ioqpairs=<N[optional]>, \
24 * aerl=<N[optional]>, aer_max_queued=<N[optional]>, \
27 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
28 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
30 * cmb_size_mb= and pmrdev= options are mutually exclusive due to limitation
31 * in available BAR's. cmb_size_mb= will take precedence over pmrdev= when
33 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
35 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
36 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
39 * nvme device parameters
40 * ~~~~~~~~~~~~~~~~~~~~~~
42 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
43 * of concurrently outstanding Asynchronous Event Request commands suppoert
44 * by the controller. This is a 0's based value.
47 * This is the maximum number of events that the device will enqueue for
48 * completion when there are no oustanding AERs. When the maximum number of
49 * enqueued events are reached, subsequent events will be dropped.
53 #include "qemu/osdep.h"
54 #include "qemu/units.h"
55 #include "qemu/error-report.h"
56 #include "hw/block/block.h"
57 #include "hw/pci/msix.h"
58 #include "hw/pci/pci.h"
59 #include "hw/qdev-properties.h"
60 #include "migration/vmstate.h"
61 #include "sysemu/sysemu.h"
62 #include "qapi/error.h"
63 #include "qapi/visitor.h"
64 #include "sysemu/hostmem.h"
65 #include "sysemu/block-backend.h"
66 #include "exec/memory.h"
68 #include "qemu/module.h"
69 #include "qemu/cutils.h"
73 #define NVME_MAX_IOQPAIRS 0xffff
74 #define NVME_DB_SIZE 4
75 #define NVME_SPEC_VER 0x00010300
76 #define NVME_CMB_BIR 2
77 #define NVME_PMR_BIR 2
78 #define NVME_TEMPERATURE 0x143
79 #define NVME_TEMPERATURE_WARNING 0x157
80 #define NVME_TEMPERATURE_CRITICAL 0x175
81 #define NVME_NUM_FW_SLOTS 1
83 #define NVME_GUEST_ERR(trace, fmt, ...) \
85 (trace_##trace)(__VA_ARGS__); \
86 qemu_log_mask(LOG_GUEST_ERROR, #trace \
87 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
90 static const bool nvme_feature_support
[NVME_FID_MAX
] = {
91 [NVME_ARBITRATION
] = true,
92 [NVME_POWER_MANAGEMENT
] = true,
93 [NVME_TEMPERATURE_THRESHOLD
] = true,
94 [NVME_ERROR_RECOVERY
] = true,
95 [NVME_VOLATILE_WRITE_CACHE
] = true,
96 [NVME_NUMBER_OF_QUEUES
] = true,
97 [NVME_INTERRUPT_COALESCING
] = true,
98 [NVME_INTERRUPT_VECTOR_CONF
] = true,
99 [NVME_WRITE_ATOMICITY
] = true,
100 [NVME_ASYNCHRONOUS_EVENT_CONF
] = true,
101 [NVME_TIMESTAMP
] = true,
104 static const uint32_t nvme_feature_cap
[NVME_FID_MAX
] = {
105 [NVME_TEMPERATURE_THRESHOLD
] = NVME_FEAT_CAP_CHANGE
,
106 [NVME_VOLATILE_WRITE_CACHE
] = NVME_FEAT_CAP_CHANGE
,
107 [NVME_NUMBER_OF_QUEUES
] = NVME_FEAT_CAP_CHANGE
,
108 [NVME_ASYNCHRONOUS_EVENT_CONF
] = NVME_FEAT_CAP_CHANGE
,
109 [NVME_TIMESTAMP
] = NVME_FEAT_CAP_CHANGE
,
112 static void nvme_process_sq(void *opaque
);
114 static uint16_t nvme_cid(NvmeRequest
*req
)
120 return le16_to_cpu(req
->cqe
.cid
);
123 static uint16_t nvme_sqid(NvmeRequest
*req
)
125 return le16_to_cpu(req
->sq
->sqid
);
128 static bool nvme_addr_is_cmb(NvmeCtrl
*n
, hwaddr addr
)
130 hwaddr low
= n
->ctrl_mem
.addr
;
131 hwaddr hi
= n
->ctrl_mem
.addr
+ int128_get64(n
->ctrl_mem
.size
);
133 return addr
>= low
&& addr
< hi
;
136 static inline void *nvme_addr_to_cmb(NvmeCtrl
*n
, hwaddr addr
)
138 assert(nvme_addr_is_cmb(n
, addr
));
140 return &n
->cmbuf
[addr
- n
->ctrl_mem
.addr
];
143 static int nvme_addr_read(NvmeCtrl
*n
, hwaddr addr
, void *buf
, int size
)
145 if (n
->bar
.cmbsz
&& nvme_addr_is_cmb(n
, addr
)) {
146 memcpy(buf
, nvme_addr_to_cmb(n
, addr
), size
);
150 return pci_dma_read(&n
->parent_obj
, addr
, buf
, size
);
153 static int nvme_check_sqid(NvmeCtrl
*n
, uint16_t sqid
)
155 return sqid
< n
->params
.max_ioqpairs
+ 1 && n
->sq
[sqid
] != NULL
? 0 : -1;
158 static int nvme_check_cqid(NvmeCtrl
*n
, uint16_t cqid
)
160 return cqid
< n
->params
.max_ioqpairs
+ 1 && n
->cq
[cqid
] != NULL
? 0 : -1;
163 static void nvme_inc_cq_tail(NvmeCQueue
*cq
)
166 if (cq
->tail
>= cq
->size
) {
168 cq
->phase
= !cq
->phase
;
172 static void nvme_inc_sq_head(NvmeSQueue
*sq
)
174 sq
->head
= (sq
->head
+ 1) % sq
->size
;
177 static uint8_t nvme_cq_full(NvmeCQueue
*cq
)
179 return (cq
->tail
+ 1) % cq
->size
== cq
->head
;
182 static uint8_t nvme_sq_empty(NvmeSQueue
*sq
)
184 return sq
->head
== sq
->tail
;
187 static void nvme_irq_check(NvmeCtrl
*n
)
189 if (msix_enabled(&(n
->parent_obj
))) {
192 if (~n
->bar
.intms
& n
->irq_status
) {
193 pci_irq_assert(&n
->parent_obj
);
195 pci_irq_deassert(&n
->parent_obj
);
199 static void nvme_irq_assert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
201 if (cq
->irq_enabled
) {
202 if (msix_enabled(&(n
->parent_obj
))) {
203 trace_pci_nvme_irq_msix(cq
->vector
);
204 msix_notify(&(n
->parent_obj
), cq
->vector
);
206 trace_pci_nvme_irq_pin();
207 assert(cq
->vector
< 32);
208 n
->irq_status
|= 1 << cq
->vector
;
212 trace_pci_nvme_irq_masked();
216 static void nvme_irq_deassert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
218 if (cq
->irq_enabled
) {
219 if (msix_enabled(&(n
->parent_obj
))) {
222 assert(cq
->vector
< 32);
223 n
->irq_status
&= ~(1 << cq
->vector
);
229 static void nvme_req_clear(NvmeRequest
*req
)
232 memset(&req
->cqe
, 0x0, sizeof(req
->cqe
));
235 static void nvme_req_exit(NvmeRequest
*req
)
238 qemu_sglist_destroy(&req
->qsg
);
242 qemu_iovec_destroy(&req
->iov
);
246 static uint16_t nvme_map_addr_cmb(NvmeCtrl
*n
, QEMUIOVector
*iov
, hwaddr addr
,
253 trace_pci_nvme_map_addr_cmb(addr
, len
);
255 if (!nvme_addr_is_cmb(n
, addr
) || !nvme_addr_is_cmb(n
, addr
+ len
- 1)) {
256 return NVME_DATA_TRAS_ERROR
;
259 qemu_iovec_add(iov
, nvme_addr_to_cmb(n
, addr
), len
);
264 static uint16_t nvme_map_addr(NvmeCtrl
*n
, QEMUSGList
*qsg
, QEMUIOVector
*iov
,
265 hwaddr addr
, size_t len
)
271 trace_pci_nvme_map_addr(addr
, len
);
273 if (nvme_addr_is_cmb(n
, addr
)) {
274 if (qsg
&& qsg
->sg
) {
275 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
281 qemu_iovec_init(iov
, 1);
284 return nvme_map_addr_cmb(n
, iov
, addr
, len
);
287 if (iov
&& iov
->iov
) {
288 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
294 pci_dma_sglist_init(qsg
, &n
->parent_obj
, 1);
297 qemu_sglist_add(qsg
, addr
, len
);
302 static uint16_t nvme_map_prp(NvmeCtrl
*n
, uint64_t prp1
, uint64_t prp2
,
303 uint32_t len
, NvmeRequest
*req
)
305 hwaddr trans_len
= n
->page_size
- (prp1
% n
->page_size
);
306 trans_len
= MIN(len
, trans_len
);
307 int num_prps
= (len
>> n
->page_bits
) + 1;
309 bool prp_list_in_cmb
= false;
312 QEMUSGList
*qsg
= &req
->qsg
;
313 QEMUIOVector
*iov
= &req
->iov
;
315 trace_pci_nvme_map_prp(trans_len
, len
, prp1
, prp2
, num_prps
);
317 if (unlikely(!prp1
)) {
318 trace_pci_nvme_err_invalid_prp();
319 return NVME_INVALID_FIELD
| NVME_DNR
;
322 if (nvme_addr_is_cmb(n
, prp1
)) {
323 qemu_iovec_init(iov
, num_prps
);
325 pci_dma_sglist_init(qsg
, &n
->parent_obj
, num_prps
);
328 status
= nvme_map_addr(n
, qsg
, iov
, prp1
, trans_len
);
335 if (unlikely(!prp2
)) {
336 trace_pci_nvme_err_invalid_prp2_missing();
337 return NVME_INVALID_FIELD
| NVME_DNR
;
340 if (len
> n
->page_size
) {
341 uint64_t prp_list
[n
->max_prp_ents
];
342 uint32_t nents
, prp_trans
;
345 if (nvme_addr_is_cmb(n
, prp2
)) {
346 prp_list_in_cmb
= true;
349 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
350 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
351 ret
= nvme_addr_read(n
, prp2
, (void *)prp_list
, prp_trans
);
353 trace_pci_nvme_err_addr_read(prp2
);
354 return NVME_DATA_TRAS_ERROR
;
357 uint64_t prp_ent
= le64_to_cpu(prp_list
[i
]);
359 if (i
== n
->max_prp_ents
- 1 && len
> n
->page_size
) {
360 if (unlikely(!prp_ent
|| prp_ent
& (n
->page_size
- 1))) {
361 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
362 return NVME_INVALID_FIELD
| NVME_DNR
;
365 if (prp_list_in_cmb
!= nvme_addr_is_cmb(n
, prp_ent
)) {
366 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
370 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
371 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
372 ret
= nvme_addr_read(n
, prp_ent
, (void *)prp_list
,
375 trace_pci_nvme_err_addr_read(prp_ent
);
376 return NVME_DATA_TRAS_ERROR
;
378 prp_ent
= le64_to_cpu(prp_list
[i
]);
381 if (unlikely(!prp_ent
|| prp_ent
& (n
->page_size
- 1))) {
382 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
383 return NVME_INVALID_FIELD
| NVME_DNR
;
386 trans_len
= MIN(len
, n
->page_size
);
387 status
= nvme_map_addr(n
, qsg
, iov
, prp_ent
, trans_len
);
396 if (unlikely(prp2
& (n
->page_size
- 1))) {
397 trace_pci_nvme_err_invalid_prp2_align(prp2
);
398 return NVME_INVALID_FIELD
| NVME_DNR
;
400 status
= nvme_map_addr(n
, qsg
, iov
, prp2
, len
);
410 static uint16_t nvme_dma_prp(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
411 uint64_t prp1
, uint64_t prp2
, DMADirection dir
,
414 uint16_t status
= NVME_SUCCESS
;
416 status
= nvme_map_prp(n
, prp1
, prp2
, len
, req
);
421 /* assert that only one of qsg and iov carries data */
422 assert((req
->qsg
.nsg
> 0) != (req
->iov
.niov
> 0));
424 if (req
->qsg
.nsg
> 0) {
427 if (dir
== DMA_DIRECTION_TO_DEVICE
) {
428 residual
= dma_buf_write(ptr
, len
, &req
->qsg
);
430 residual
= dma_buf_read(ptr
, len
, &req
->qsg
);
433 if (unlikely(residual
)) {
434 trace_pci_nvme_err_invalid_dma();
435 status
= NVME_INVALID_FIELD
| NVME_DNR
;
440 if (dir
== DMA_DIRECTION_TO_DEVICE
) {
441 bytes
= qemu_iovec_to_buf(&req
->iov
, 0, ptr
, len
);
443 bytes
= qemu_iovec_from_buf(&req
->iov
, 0, ptr
, len
);
446 if (unlikely(bytes
!= len
)) {
447 trace_pci_nvme_err_invalid_dma();
448 status
= NVME_INVALID_FIELD
| NVME_DNR
;
455 static uint16_t nvme_map_dptr(NvmeCtrl
*n
, size_t len
, NvmeRequest
*req
)
457 NvmeCmd
*cmd
= &req
->cmd
;
458 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
459 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
461 return nvme_map_prp(n
, prp1
, prp2
, len
, req
);
464 static void nvme_post_cqes(void *opaque
)
466 NvmeCQueue
*cq
= opaque
;
467 NvmeCtrl
*n
= cq
->ctrl
;
468 NvmeRequest
*req
, *next
;
471 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
475 if (nvme_cq_full(cq
)) {
480 req
->cqe
.status
= cpu_to_le16((req
->status
<< 1) | cq
->phase
);
481 req
->cqe
.sq_id
= cpu_to_le16(sq
->sqid
);
482 req
->cqe
.sq_head
= cpu_to_le16(sq
->head
);
483 addr
= cq
->dma_addr
+ cq
->tail
* n
->cqe_size
;
484 ret
= pci_dma_write(&n
->parent_obj
, addr
, (void *)&req
->cqe
,
487 trace_pci_nvme_err_addr_write(addr
);
488 trace_pci_nvme_err_cfs();
489 n
->bar
.csts
= NVME_CSTS_FAILED
;
492 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
493 nvme_inc_cq_tail(cq
);
495 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
497 if (cq
->tail
!= cq
->head
) {
498 nvme_irq_assert(n
, cq
);
502 static void nvme_enqueue_req_completion(NvmeCQueue
*cq
, NvmeRequest
*req
)
504 assert(cq
->cqid
== req
->sq
->cqid
);
505 trace_pci_nvme_enqueue_req_completion(nvme_cid(req
), cq
->cqid
,
507 QTAILQ_REMOVE(&req
->sq
->out_req_list
, req
, entry
);
508 QTAILQ_INSERT_TAIL(&cq
->req_list
, req
, entry
);
509 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
512 static void nvme_process_aers(void *opaque
)
514 NvmeCtrl
*n
= opaque
;
515 NvmeAsyncEvent
*event
, *next
;
517 trace_pci_nvme_process_aers(n
->aer_queued
);
519 QTAILQ_FOREACH_SAFE(event
, &n
->aer_queue
, entry
, next
) {
521 NvmeAerResult
*result
;
523 /* can't post cqe if there is nothing to complete */
524 if (!n
->outstanding_aers
) {
525 trace_pci_nvme_no_outstanding_aers();
529 /* ignore if masked (cqe posted, but event not cleared) */
530 if (n
->aer_mask
& (1 << event
->result
.event_type
)) {
531 trace_pci_nvme_aer_masked(event
->result
.event_type
, n
->aer_mask
);
535 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
538 n
->aer_mask
|= 1 << event
->result
.event_type
;
539 n
->outstanding_aers
--;
541 req
= n
->aer_reqs
[n
->outstanding_aers
];
543 result
= (NvmeAerResult
*) &req
->cqe
.result
;
544 result
->event_type
= event
->result
.event_type
;
545 result
->event_info
= event
->result
.event_info
;
546 result
->log_page
= event
->result
.log_page
;
549 req
->status
= NVME_SUCCESS
;
551 trace_pci_nvme_aer_post_cqe(result
->event_type
, result
->event_info
,
554 nvme_enqueue_req_completion(&n
->admin_cq
, req
);
558 static void nvme_enqueue_event(NvmeCtrl
*n
, uint8_t event_type
,
559 uint8_t event_info
, uint8_t log_page
)
561 NvmeAsyncEvent
*event
;
563 trace_pci_nvme_enqueue_event(event_type
, event_info
, log_page
);
565 if (n
->aer_queued
== n
->params
.aer_max_queued
) {
566 trace_pci_nvme_enqueue_event_noqueue(n
->aer_queued
);
570 event
= g_new(NvmeAsyncEvent
, 1);
571 event
->result
= (NvmeAerResult
) {
572 .event_type
= event_type
,
573 .event_info
= event_info
,
574 .log_page
= log_page
,
577 QTAILQ_INSERT_TAIL(&n
->aer_queue
, event
, entry
);
580 nvme_process_aers(n
);
583 static void nvme_clear_events(NvmeCtrl
*n
, uint8_t event_type
)
585 n
->aer_mask
&= ~(1 << event_type
);
586 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
587 nvme_process_aers(n
);
591 static inline uint16_t nvme_check_mdts(NvmeCtrl
*n
, size_t len
)
593 uint8_t mdts
= n
->params
.mdts
;
595 if (mdts
&& len
> n
->page_size
<< mdts
) {
596 return NVME_INVALID_FIELD
| NVME_DNR
;
602 static inline uint16_t nvme_check_bounds(NvmeCtrl
*n
, NvmeNamespace
*ns
,
603 uint64_t slba
, uint32_t nlb
)
605 uint64_t nsze
= le64_to_cpu(ns
->id_ns
.nsze
);
607 if (unlikely(UINT64_MAX
- slba
< nlb
|| slba
+ nlb
> nsze
)) {
608 return NVME_LBA_RANGE
| NVME_DNR
;
614 static void nvme_rw_cb(void *opaque
, int ret
)
616 NvmeRequest
*req
= opaque
;
617 NvmeCtrl
*n
= nvme_ctrl(req
);
619 BlockBackend
*blk
= n
->conf
.blk
;
620 BlockAcctCookie
*acct
= &req
->acct
;
621 BlockAcctStats
*stats
= blk_get_stats(blk
);
623 Error
*local_err
= NULL
;
625 trace_pci_nvme_rw_cb(nvme_cid(req
), blk_name(blk
));
628 block_acct_done(stats
, acct
);
629 req
->status
= NVME_SUCCESS
;
633 block_acct_failed(stats
, acct
);
635 switch (req
->cmd
.opcode
) {
637 status
= NVME_UNRECOVERED_READ
;
641 case NVME_CMD_WRITE_ZEROES
:
642 status
= NVME_WRITE_FAULT
;
645 status
= NVME_INTERNAL_DEV_ERROR
;
649 trace_pci_nvme_err_aio(nvme_cid(req
), strerror(ret
), status
);
651 error_setg_errno(&local_err
, -ret
, "aio failed");
652 error_report_err(local_err
);
654 req
->status
= status
;
657 nvme_enqueue_req_completion(nvme_cq(req
), req
);
660 static uint16_t nvme_do_aio(BlockBackend
*blk
, int64_t offset
, size_t len
,
663 BlockAcctCookie
*acct
= &req
->acct
;
664 BlockAcctStats
*stats
= blk_get_stats(blk
);
666 bool is_write
= false;
668 trace_pci_nvme_do_aio(nvme_cid(req
), req
->cmd
.opcode
,
669 nvme_io_opc_str(req
->cmd
.opcode
), blk_name(blk
),
672 switch (req
->cmd
.opcode
) {
674 block_acct_start(stats
, acct
, 0, BLOCK_ACCT_FLUSH
);
675 req
->aiocb
= blk_aio_flush(blk
, nvme_rw_cb
, req
);
678 case NVME_CMD_WRITE_ZEROES
:
679 block_acct_start(stats
, acct
, len
, BLOCK_ACCT_WRITE
);
680 req
->aiocb
= blk_aio_pwrite_zeroes(blk
, offset
, len
,
681 BDRV_REQ_MAY_UNMAP
, nvme_rw_cb
,
691 block_acct_start(stats
, acct
, len
,
692 is_write
? BLOCK_ACCT_WRITE
: BLOCK_ACCT_READ
);
696 req
->aiocb
= dma_blk_write(blk
, &req
->qsg
, offset
,
697 BDRV_SECTOR_SIZE
, nvme_rw_cb
, req
);
699 req
->aiocb
= dma_blk_read(blk
, &req
->qsg
, offset
,
700 BDRV_SECTOR_SIZE
, nvme_rw_cb
, req
);
704 req
->aiocb
= blk_aio_pwritev(blk
, offset
, &req
->iov
, 0,
707 req
->aiocb
= blk_aio_preadv(blk
, offset
, &req
->iov
, 0,
715 return NVME_NO_COMPLETE
;
718 static uint16_t nvme_flush(NvmeCtrl
*n
, NvmeRequest
*req
)
720 return nvme_do_aio(n
->conf
.blk
, 0, 0, req
);
723 static uint16_t nvme_write_zeroes(NvmeCtrl
*n
, NvmeRequest
*req
)
725 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
726 NvmeNamespace
*ns
= req
->ns
;
727 uint64_t slba
= le64_to_cpu(rw
->slba
);
728 uint32_t nlb
= (uint32_t)le16_to_cpu(rw
->nlb
) + 1;
729 uint64_t offset
= nvme_l2b(ns
, slba
);
730 uint32_t count
= nvme_l2b(ns
, nlb
);
733 trace_pci_nvme_write_zeroes(nvme_cid(req
), slba
, nlb
);
735 status
= nvme_check_bounds(n
, ns
, slba
, nlb
);
737 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
741 return nvme_do_aio(n
->conf
.blk
, offset
, count
, req
);
744 static uint16_t nvme_rw(NvmeCtrl
*n
, NvmeRequest
*req
)
746 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
747 NvmeNamespace
*ns
= req
->ns
;
748 uint32_t nlb
= (uint32_t)le16_to_cpu(rw
->nlb
) + 1;
749 uint64_t slba
= le64_to_cpu(rw
->slba
);
751 uint64_t data_size
= nvme_l2b(ns
, nlb
);
752 uint64_t data_offset
= nvme_l2b(ns
, slba
);
753 enum BlockAcctType acct
= req
->cmd
.opcode
== NVME_CMD_WRITE
?
754 BLOCK_ACCT_WRITE
: BLOCK_ACCT_READ
;
757 trace_pci_nvme_rw(nvme_cid(req
), nvme_io_opc_str(rw
->opcode
), nlb
,
760 status
= nvme_check_mdts(n
, data_size
);
762 trace_pci_nvme_err_mdts(nvme_cid(req
), data_size
);
766 status
= nvme_check_bounds(n
, ns
, slba
, nlb
);
768 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
772 status
= nvme_map_dptr(n
, data_size
, req
);
777 return nvme_do_aio(n
->conf
.blk
, data_offset
, data_size
, req
);
780 block_acct_invalid(blk_get_stats(n
->conf
.blk
), acct
);
784 static uint16_t nvme_io_cmd(NvmeCtrl
*n
, NvmeRequest
*req
)
786 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
788 trace_pci_nvme_io_cmd(nvme_cid(req
), nsid
, nvme_sqid(req
),
789 req
->cmd
.opcode
, nvme_io_opc_str(req
->cmd
.opcode
));
791 if (unlikely(nsid
== 0 || nsid
> n
->num_namespaces
)) {
792 trace_pci_nvme_err_invalid_ns(nsid
, n
->num_namespaces
);
793 return NVME_INVALID_NSID
| NVME_DNR
;
796 req
->ns
= &n
->namespaces
[nsid
- 1];
797 switch (req
->cmd
.opcode
) {
799 return nvme_flush(n
, req
);
800 case NVME_CMD_WRITE_ZEROES
:
801 return nvme_write_zeroes(n
, req
);
804 return nvme_rw(n
, req
);
806 trace_pci_nvme_err_invalid_opc(req
->cmd
.opcode
);
807 return NVME_INVALID_OPCODE
| NVME_DNR
;
811 static void nvme_free_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
)
813 n
->sq
[sq
->sqid
] = NULL
;
814 timer_del(sq
->timer
);
815 timer_free(sq
->timer
);
822 static uint16_t nvme_del_sq(NvmeCtrl
*n
, NvmeRequest
*req
)
824 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)&req
->cmd
;
825 NvmeRequest
*r
, *next
;
828 uint16_t qid
= le16_to_cpu(c
->qid
);
830 if (unlikely(!qid
|| nvme_check_sqid(n
, qid
))) {
831 trace_pci_nvme_err_invalid_del_sq(qid
);
832 return NVME_INVALID_QID
| NVME_DNR
;
835 trace_pci_nvme_del_sq(qid
);
838 while (!QTAILQ_EMPTY(&sq
->out_req_list
)) {
839 r
= QTAILQ_FIRST(&sq
->out_req_list
);
841 blk_aio_cancel(r
->aiocb
);
843 if (!nvme_check_cqid(n
, sq
->cqid
)) {
844 cq
= n
->cq
[sq
->cqid
];
845 QTAILQ_REMOVE(&cq
->sq_list
, sq
, entry
);
848 QTAILQ_FOREACH_SAFE(r
, &cq
->req_list
, entry
, next
) {
850 QTAILQ_REMOVE(&cq
->req_list
, r
, entry
);
851 QTAILQ_INSERT_TAIL(&sq
->req_list
, r
, entry
);
860 static void nvme_init_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
, uint64_t dma_addr
,
861 uint16_t sqid
, uint16_t cqid
, uint16_t size
)
867 sq
->dma_addr
= dma_addr
;
871 sq
->head
= sq
->tail
= 0;
872 sq
->io_req
= g_new0(NvmeRequest
, sq
->size
);
874 QTAILQ_INIT(&sq
->req_list
);
875 QTAILQ_INIT(&sq
->out_req_list
);
876 for (i
= 0; i
< sq
->size
; i
++) {
877 sq
->io_req
[i
].sq
= sq
;
878 QTAILQ_INSERT_TAIL(&(sq
->req_list
), &sq
->io_req
[i
], entry
);
880 sq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_process_sq
, sq
);
884 QTAILQ_INSERT_TAIL(&(cq
->sq_list
), sq
, entry
);
888 static uint16_t nvme_create_sq(NvmeCtrl
*n
, NvmeRequest
*req
)
891 NvmeCreateSq
*c
= (NvmeCreateSq
*)&req
->cmd
;
893 uint16_t cqid
= le16_to_cpu(c
->cqid
);
894 uint16_t sqid
= le16_to_cpu(c
->sqid
);
895 uint16_t qsize
= le16_to_cpu(c
->qsize
);
896 uint16_t qflags
= le16_to_cpu(c
->sq_flags
);
897 uint64_t prp1
= le64_to_cpu(c
->prp1
);
899 trace_pci_nvme_create_sq(prp1
, sqid
, cqid
, qsize
, qflags
);
901 if (unlikely(!cqid
|| nvme_check_cqid(n
, cqid
))) {
902 trace_pci_nvme_err_invalid_create_sq_cqid(cqid
);
903 return NVME_INVALID_CQID
| NVME_DNR
;
905 if (unlikely(!sqid
|| !nvme_check_sqid(n
, sqid
))) {
906 trace_pci_nvme_err_invalid_create_sq_sqid(sqid
);
907 return NVME_INVALID_QID
| NVME_DNR
;
909 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
910 trace_pci_nvme_err_invalid_create_sq_size(qsize
);
911 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
913 if (unlikely(!prp1
|| prp1
& (n
->page_size
- 1))) {
914 trace_pci_nvme_err_invalid_create_sq_addr(prp1
);
915 return NVME_INVALID_FIELD
| NVME_DNR
;
917 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags
)))) {
918 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags
));
919 return NVME_INVALID_FIELD
| NVME_DNR
;
921 sq
= g_malloc0(sizeof(*sq
));
922 nvme_init_sq(sq
, n
, prp1
, sqid
, cqid
, qsize
+ 1);
926 static uint16_t nvme_smart_info(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
927 uint64_t off
, NvmeRequest
*req
)
929 NvmeCmd
*cmd
= &req
->cmd
;
930 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
931 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
932 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
936 uint64_t units_read
= 0, units_written
= 0;
937 uint64_t read_commands
= 0, write_commands
= 0;
941 if (nsid
&& nsid
!= 0xffffffff) {
942 return NVME_INVALID_FIELD
| NVME_DNR
;
945 s
= blk_get_stats(n
->conf
.blk
);
947 units_read
= s
->nr_bytes
[BLOCK_ACCT_READ
] >> BDRV_SECTOR_BITS
;
948 units_written
= s
->nr_bytes
[BLOCK_ACCT_WRITE
] >> BDRV_SECTOR_BITS
;
949 read_commands
= s
->nr_ops
[BLOCK_ACCT_READ
];
950 write_commands
= s
->nr_ops
[BLOCK_ACCT_WRITE
];
952 if (off
> sizeof(smart
)) {
953 return NVME_INVALID_FIELD
| NVME_DNR
;
956 trans_len
= MIN(sizeof(smart
) - off
, buf_len
);
958 memset(&smart
, 0x0, sizeof(smart
));
960 smart
.data_units_read
[0] = cpu_to_le64(DIV_ROUND_UP(units_read
, 1000));
961 smart
.data_units_written
[0] = cpu_to_le64(DIV_ROUND_UP(units_written
,
963 smart
.host_read_commands
[0] = cpu_to_le64(read_commands
);
964 smart
.host_write_commands
[0] = cpu_to_le64(write_commands
);
966 smart
.temperature
= cpu_to_le16(n
->temperature
);
968 if ((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
969 (n
->temperature
<= n
->features
.temp_thresh_low
)) {
970 smart
.critical_warning
|= NVME_SMART_TEMPERATURE
;
973 current_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
974 smart
.power_on_hours
[0] =
975 cpu_to_le64((((current_ms
- n
->starttime_ms
) / 1000) / 60) / 60);
978 nvme_clear_events(n
, NVME_AER_TYPE_SMART
);
981 return nvme_dma_prp(n
, (uint8_t *) &smart
+ off
, trans_len
, prp1
, prp2
,
982 DMA_DIRECTION_FROM_DEVICE
, req
);
985 static uint16_t nvme_fw_log_info(NvmeCtrl
*n
, uint32_t buf_len
, uint64_t off
,
989 NvmeCmd
*cmd
= &req
->cmd
;
990 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
991 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
992 NvmeFwSlotInfoLog fw_log
= {
996 strpadcpy((char *)&fw_log
.frs1
, sizeof(fw_log
.frs1
), "1.0", ' ');
998 if (off
> sizeof(fw_log
)) {
999 return NVME_INVALID_FIELD
| NVME_DNR
;
1002 trans_len
= MIN(sizeof(fw_log
) - off
, buf_len
);
1004 return nvme_dma_prp(n
, (uint8_t *) &fw_log
+ off
, trans_len
, prp1
, prp2
,
1005 DMA_DIRECTION_FROM_DEVICE
, req
);
1008 static uint16_t nvme_error_info(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
1009 uint64_t off
, NvmeRequest
*req
)
1012 NvmeCmd
*cmd
= &req
->cmd
;
1013 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
1014 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
1015 NvmeErrorLog errlog
;
1018 nvme_clear_events(n
, NVME_AER_TYPE_ERROR
);
1021 if (off
> sizeof(errlog
)) {
1022 return NVME_INVALID_FIELD
| NVME_DNR
;
1025 memset(&errlog
, 0x0, sizeof(errlog
));
1027 trans_len
= MIN(sizeof(errlog
) - off
, buf_len
);
1029 return nvme_dma_prp(n
, (uint8_t *)&errlog
, trans_len
, prp1
, prp2
,
1030 DMA_DIRECTION_FROM_DEVICE
, req
);
1033 static uint16_t nvme_get_log(NvmeCtrl
*n
, NvmeRequest
*req
)
1035 NvmeCmd
*cmd
= &req
->cmd
;
1037 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
1038 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
1039 uint32_t dw12
= le32_to_cpu(cmd
->cdw12
);
1040 uint32_t dw13
= le32_to_cpu(cmd
->cdw13
);
1041 uint8_t lid
= dw10
& 0xff;
1042 uint8_t lsp
= (dw10
>> 8) & 0xf;
1043 uint8_t rae
= (dw10
>> 15) & 0x1;
1044 uint32_t numdl
, numdu
;
1045 uint64_t off
, lpol
, lpou
;
1049 numdl
= (dw10
>> 16);
1050 numdu
= (dw11
& 0xffff);
1054 len
= (((numdu
<< 16) | numdl
) + 1) << 2;
1055 off
= (lpou
<< 32ULL) | lpol
;
1058 return NVME_INVALID_FIELD
| NVME_DNR
;
1061 trace_pci_nvme_get_log(nvme_cid(req
), lid
, lsp
, rae
, len
, off
);
1063 status
= nvme_check_mdts(n
, len
);
1065 trace_pci_nvme_err_mdts(nvme_cid(req
), len
);
1070 case NVME_LOG_ERROR_INFO
:
1071 return nvme_error_info(n
, rae
, len
, off
, req
);
1072 case NVME_LOG_SMART_INFO
:
1073 return nvme_smart_info(n
, rae
, len
, off
, req
);
1074 case NVME_LOG_FW_SLOT_INFO
:
1075 return nvme_fw_log_info(n
, len
, off
, req
);
1077 trace_pci_nvme_err_invalid_log_page(nvme_cid(req
), lid
);
1078 return NVME_INVALID_FIELD
| NVME_DNR
;
1082 static void nvme_free_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
)
1084 n
->cq
[cq
->cqid
] = NULL
;
1085 timer_del(cq
->timer
);
1086 timer_free(cq
->timer
);
1087 msix_vector_unuse(&n
->parent_obj
, cq
->vector
);
1093 static uint16_t nvme_del_cq(NvmeCtrl
*n
, NvmeRequest
*req
)
1095 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)&req
->cmd
;
1097 uint16_t qid
= le16_to_cpu(c
->qid
);
1099 if (unlikely(!qid
|| nvme_check_cqid(n
, qid
))) {
1100 trace_pci_nvme_err_invalid_del_cq_cqid(qid
);
1101 return NVME_INVALID_CQID
| NVME_DNR
;
1105 if (unlikely(!QTAILQ_EMPTY(&cq
->sq_list
))) {
1106 trace_pci_nvme_err_invalid_del_cq_notempty(qid
);
1107 return NVME_INVALID_QUEUE_DEL
;
1109 nvme_irq_deassert(n
, cq
);
1110 trace_pci_nvme_del_cq(qid
);
1111 nvme_free_cq(cq
, n
);
1112 return NVME_SUCCESS
;
1115 static void nvme_init_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
, uint64_t dma_addr
,
1116 uint16_t cqid
, uint16_t vector
, uint16_t size
,
1117 uint16_t irq_enabled
)
1121 ret
= msix_vector_use(&n
->parent_obj
, vector
);
1126 cq
->dma_addr
= dma_addr
;
1128 cq
->irq_enabled
= irq_enabled
;
1129 cq
->vector
= vector
;
1130 cq
->head
= cq
->tail
= 0;
1131 QTAILQ_INIT(&cq
->req_list
);
1132 QTAILQ_INIT(&cq
->sq_list
);
1134 cq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_post_cqes
, cq
);
1137 static uint16_t nvme_create_cq(NvmeCtrl
*n
, NvmeRequest
*req
)
1140 NvmeCreateCq
*c
= (NvmeCreateCq
*)&req
->cmd
;
1141 uint16_t cqid
= le16_to_cpu(c
->cqid
);
1142 uint16_t vector
= le16_to_cpu(c
->irq_vector
);
1143 uint16_t qsize
= le16_to_cpu(c
->qsize
);
1144 uint16_t qflags
= le16_to_cpu(c
->cq_flags
);
1145 uint64_t prp1
= le64_to_cpu(c
->prp1
);
1147 trace_pci_nvme_create_cq(prp1
, cqid
, vector
, qsize
, qflags
,
1148 NVME_CQ_FLAGS_IEN(qflags
) != 0);
1150 if (unlikely(!cqid
|| !nvme_check_cqid(n
, cqid
))) {
1151 trace_pci_nvme_err_invalid_create_cq_cqid(cqid
);
1152 return NVME_INVALID_CQID
| NVME_DNR
;
1154 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
1155 trace_pci_nvme_err_invalid_create_cq_size(qsize
);
1156 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
1158 if (unlikely(!prp1
)) {
1159 trace_pci_nvme_err_invalid_create_cq_addr(prp1
);
1160 return NVME_INVALID_FIELD
| NVME_DNR
;
1162 if (unlikely(!msix_enabled(&n
->parent_obj
) && vector
)) {
1163 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
1164 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
1166 if (unlikely(vector
>= n
->params
.msix_qsize
)) {
1167 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
1168 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
1170 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags
)))) {
1171 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags
));
1172 return NVME_INVALID_FIELD
| NVME_DNR
;
1175 cq
= g_malloc0(sizeof(*cq
));
1176 nvme_init_cq(cq
, n
, prp1
, cqid
, vector
, qsize
+ 1,
1177 NVME_CQ_FLAGS_IEN(qflags
));
1180 * It is only required to set qs_created when creating a completion queue;
1181 * creating a submission queue without a matching completion queue will
1184 n
->qs_created
= true;
1185 return NVME_SUCCESS
;
1188 static uint16_t nvme_identify_ctrl(NvmeCtrl
*n
, NvmeRequest
*req
)
1190 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1191 uint64_t prp1
= le64_to_cpu(c
->prp1
);
1192 uint64_t prp2
= le64_to_cpu(c
->prp2
);
1194 trace_pci_nvme_identify_ctrl();
1196 return nvme_dma_prp(n
, (uint8_t *)&n
->id_ctrl
, sizeof(n
->id_ctrl
), prp1
,
1197 prp2
, DMA_DIRECTION_FROM_DEVICE
, req
);
1200 static uint16_t nvme_identify_ns(NvmeCtrl
*n
, NvmeRequest
*req
)
1203 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1204 uint32_t nsid
= le32_to_cpu(c
->nsid
);
1205 uint64_t prp1
= le64_to_cpu(c
->prp1
);
1206 uint64_t prp2
= le64_to_cpu(c
->prp2
);
1208 trace_pci_nvme_identify_ns(nsid
);
1210 if (unlikely(nsid
== 0 || nsid
> n
->num_namespaces
)) {
1211 trace_pci_nvme_err_invalid_ns(nsid
, n
->num_namespaces
);
1212 return NVME_INVALID_NSID
| NVME_DNR
;
1215 ns
= &n
->namespaces
[nsid
- 1];
1217 return nvme_dma_prp(n
, (uint8_t *)&ns
->id_ns
, sizeof(ns
->id_ns
), prp1
,
1218 prp2
, DMA_DIRECTION_FROM_DEVICE
, req
);
1221 static uint16_t nvme_identify_nslist(NvmeCtrl
*n
, NvmeRequest
*req
)
1223 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1224 static const int data_len
= NVME_IDENTIFY_DATA_SIZE
;
1225 uint32_t min_nsid
= le32_to_cpu(c
->nsid
);
1226 uint64_t prp1
= le64_to_cpu(c
->prp1
);
1227 uint64_t prp2
= le64_to_cpu(c
->prp2
);
1232 trace_pci_nvme_identify_nslist(min_nsid
);
1235 * Both 0xffffffff (NVME_NSID_BROADCAST) and 0xfffffffe are invalid values
1236 * since the Active Namespace ID List should return namespaces with ids
1237 * *higher* than the NSID specified in the command. This is also specified
1238 * in the spec (NVM Express v1.3d, Section 5.15.4).
1240 if (min_nsid
>= NVME_NSID_BROADCAST
- 1) {
1241 return NVME_INVALID_NSID
| NVME_DNR
;
1244 list
= g_malloc0(data_len
);
1245 for (i
= 0; i
< n
->num_namespaces
; i
++) {
1249 list
[j
++] = cpu_to_le32(i
+ 1);
1250 if (j
== data_len
/ sizeof(uint32_t)) {
1254 ret
= nvme_dma_prp(n
, (uint8_t *)list
, data_len
, prp1
, prp2
,
1255 DMA_DIRECTION_FROM_DEVICE
, req
);
1260 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl
*n
, NvmeRequest
*req
)
1262 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1263 uint32_t nsid
= le32_to_cpu(c
->nsid
);
1264 uint64_t prp1
= le64_to_cpu(c
->prp1
);
1265 uint64_t prp2
= le64_to_cpu(c
->prp2
);
1267 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
];
1276 struct data
*ns_descrs
= (struct data
*)list
;
1278 trace_pci_nvme_identify_ns_descr_list(nsid
);
1280 if (unlikely(nsid
== 0 || nsid
> n
->num_namespaces
)) {
1281 trace_pci_nvme_err_invalid_ns(nsid
, n
->num_namespaces
);
1282 return NVME_INVALID_NSID
| NVME_DNR
;
1285 memset(list
, 0x0, sizeof(list
));
1288 * Because the NGUID and EUI64 fields are 0 in the Identify Namespace data
1289 * structure, a Namespace UUID (nidt = 0x3) must be reported in the
1290 * Namespace Identification Descriptor. Add a very basic Namespace UUID
1293 ns_descrs
->uuid
.hdr
.nidt
= NVME_NIDT_UUID
;
1294 ns_descrs
->uuid
.hdr
.nidl
= NVME_NIDT_UUID_LEN
;
1295 stl_be_p(&ns_descrs
->uuid
.v
, nsid
);
1297 return nvme_dma_prp(n
, list
, NVME_IDENTIFY_DATA_SIZE
, prp1
, prp2
,
1298 DMA_DIRECTION_FROM_DEVICE
, req
);
1301 static uint16_t nvme_identify(NvmeCtrl
*n
, NvmeRequest
*req
)
1303 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
1305 switch (le32_to_cpu(c
->cns
)) {
1306 case NVME_ID_CNS_NS
:
1307 return nvme_identify_ns(n
, req
);
1308 case NVME_ID_CNS_CTRL
:
1309 return nvme_identify_ctrl(n
, req
);
1310 case NVME_ID_CNS_NS_ACTIVE_LIST
:
1311 return nvme_identify_nslist(n
, req
);
1312 case NVME_ID_CNS_NS_DESCR_LIST
:
1313 return nvme_identify_ns_descr_list(n
, req
);
1315 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c
->cns
));
1316 return NVME_INVALID_FIELD
| NVME_DNR
;
1320 static uint16_t nvme_abort(NvmeCtrl
*n
, NvmeRequest
*req
)
1322 uint16_t sqid
= le32_to_cpu(req
->cmd
.cdw10
) & 0xffff;
1324 req
->cqe
.result
= 1;
1325 if (nvme_check_sqid(n
, sqid
)) {
1326 return NVME_INVALID_FIELD
| NVME_DNR
;
1329 return NVME_SUCCESS
;
1332 static inline void nvme_set_timestamp(NvmeCtrl
*n
, uint64_t ts
)
1334 trace_pci_nvme_setfeat_timestamp(ts
);
1336 n
->host_timestamp
= le64_to_cpu(ts
);
1337 n
->timestamp_set_qemu_clock_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
1340 static inline uint64_t nvme_get_timestamp(const NvmeCtrl
*n
)
1342 uint64_t current_time
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
1343 uint64_t elapsed_time
= current_time
- n
->timestamp_set_qemu_clock_ms
;
1345 union nvme_timestamp
{
1347 uint64_t timestamp
:48;
1355 union nvme_timestamp ts
;
1357 ts
.timestamp
= n
->host_timestamp
+ elapsed_time
;
1359 /* If the host timestamp is non-zero, set the timestamp origin */
1360 ts
.origin
= n
->host_timestamp
? 0x01 : 0x00;
1362 trace_pci_nvme_getfeat_timestamp(ts
.all
);
1364 return cpu_to_le64(ts
.all
);
1367 static uint16_t nvme_get_feature_timestamp(NvmeCtrl
*n
, NvmeRequest
*req
)
1369 NvmeCmd
*cmd
= &req
->cmd
;
1370 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
1371 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
1373 uint64_t timestamp
= nvme_get_timestamp(n
);
1375 return nvme_dma_prp(n
, (uint8_t *)×tamp
, sizeof(timestamp
), prp1
,
1376 prp2
, DMA_DIRECTION_FROM_DEVICE
, req
);
1379 static uint16_t nvme_get_feature(NvmeCtrl
*n
, NvmeRequest
*req
)
1381 NvmeCmd
*cmd
= &req
->cmd
;
1382 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
1383 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
1384 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
1386 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
1387 NvmeGetFeatureSelect sel
= NVME_GETFEAT_SELECT(dw10
);
1390 static const uint32_t nvme_feature_default
[NVME_FID_MAX
] = {
1391 [NVME_ARBITRATION
] = NVME_ARB_AB_NOLIMIT
,
1394 trace_pci_nvme_getfeat(nvme_cid(req
), fid
, sel
, dw11
);
1396 if (!nvme_feature_support
[fid
]) {
1397 return NVME_INVALID_FIELD
| NVME_DNR
;
1400 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
1401 if (!nsid
|| nsid
> n
->num_namespaces
) {
1403 * The Reservation Notification Mask and Reservation Persistence
1404 * features require a status code of Invalid Field in Command when
1405 * NSID is 0xFFFFFFFF. Since the device does not support those
1406 * features we can always return Invalid Namespace or Format as we
1407 * should do for all other features.
1409 return NVME_INVALID_NSID
| NVME_DNR
;
1414 case NVME_GETFEAT_SELECT_CURRENT
:
1416 case NVME_GETFEAT_SELECT_SAVED
:
1417 /* no features are saveable by the controller; fallthrough */
1418 case NVME_GETFEAT_SELECT_DEFAULT
:
1420 case NVME_GETFEAT_SELECT_CAP
:
1421 result
= nvme_feature_cap
[fid
];
1426 case NVME_TEMPERATURE_THRESHOLD
:
1430 * The controller only implements the Composite Temperature sensor, so
1431 * return 0 for all other sensors.
1433 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
1437 switch (NVME_TEMP_THSEL(dw11
)) {
1438 case NVME_TEMP_THSEL_OVER
:
1439 result
= n
->features
.temp_thresh_hi
;
1441 case NVME_TEMP_THSEL_UNDER
:
1442 result
= n
->features
.temp_thresh_low
;
1446 return NVME_INVALID_FIELD
| NVME_DNR
;
1447 case NVME_VOLATILE_WRITE_CACHE
:
1448 result
= blk_enable_write_cache(n
->conf
.blk
);
1449 trace_pci_nvme_getfeat_vwcache(result
? "enabled" : "disabled");
1451 case NVME_ASYNCHRONOUS_EVENT_CONF
:
1452 result
= n
->features
.async_config
;
1454 case NVME_TIMESTAMP
:
1455 return nvme_get_feature_timestamp(n
, req
);
1462 case NVME_TEMPERATURE_THRESHOLD
:
1465 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
1469 if (NVME_TEMP_THSEL(dw11
) == NVME_TEMP_THSEL_OVER
) {
1470 result
= NVME_TEMPERATURE_WARNING
;
1474 case NVME_NUMBER_OF_QUEUES
:
1475 result
= (n
->params
.max_ioqpairs
- 1) |
1476 ((n
->params
.max_ioqpairs
- 1) << 16);
1477 trace_pci_nvme_getfeat_numq(result
);
1479 case NVME_INTERRUPT_VECTOR_CONF
:
1481 if (iv
>= n
->params
.max_ioqpairs
+ 1) {
1482 return NVME_INVALID_FIELD
| NVME_DNR
;
1486 if (iv
== n
->admin_cq
.vector
) {
1487 result
|= NVME_INTVC_NOCOALESCING
;
1492 result
= nvme_feature_default
[fid
];
1497 req
->cqe
.result
= cpu_to_le32(result
);
1498 return NVME_SUCCESS
;
1501 static uint16_t nvme_set_feature_timestamp(NvmeCtrl
*n
, NvmeRequest
*req
)
1505 NvmeCmd
*cmd
= &req
->cmd
;
1506 uint64_t prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
1507 uint64_t prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
1509 ret
= nvme_dma_prp(n
, (uint8_t *)×tamp
, sizeof(timestamp
), prp1
,
1510 prp2
, DMA_DIRECTION_TO_DEVICE
, req
);
1511 if (ret
!= NVME_SUCCESS
) {
1515 nvme_set_timestamp(n
, timestamp
);
1517 return NVME_SUCCESS
;
1520 static uint16_t nvme_set_feature(NvmeCtrl
*n
, NvmeRequest
*req
)
1522 NvmeCmd
*cmd
= &req
->cmd
;
1523 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
1524 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
1525 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
1526 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
1527 uint8_t save
= NVME_SETFEAT_SAVE(dw10
);
1529 trace_pci_nvme_setfeat(nvme_cid(req
), fid
, save
, dw11
);
1532 return NVME_FID_NOT_SAVEABLE
| NVME_DNR
;
1535 if (!nvme_feature_support
[fid
]) {
1536 return NVME_INVALID_FIELD
| NVME_DNR
;
1539 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
1540 if (!nsid
|| (nsid
!= NVME_NSID_BROADCAST
&&
1541 nsid
> n
->num_namespaces
)) {
1542 return NVME_INVALID_NSID
| NVME_DNR
;
1544 } else if (nsid
&& nsid
!= NVME_NSID_BROADCAST
) {
1545 if (nsid
> n
->num_namespaces
) {
1546 return NVME_INVALID_NSID
| NVME_DNR
;
1549 return NVME_FEAT_NOT_NS_SPEC
| NVME_DNR
;
1552 if (!(nvme_feature_cap
[fid
] & NVME_FEAT_CAP_CHANGE
)) {
1553 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
1557 case NVME_TEMPERATURE_THRESHOLD
:
1558 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
1562 switch (NVME_TEMP_THSEL(dw11
)) {
1563 case NVME_TEMP_THSEL_OVER
:
1564 n
->features
.temp_thresh_hi
= NVME_TEMP_TMPTH(dw11
);
1566 case NVME_TEMP_THSEL_UNDER
:
1567 n
->features
.temp_thresh_low
= NVME_TEMP_TMPTH(dw11
);
1570 return NVME_INVALID_FIELD
| NVME_DNR
;
1573 if (((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
1574 (n
->temperature
<= n
->features
.temp_thresh_low
)) &&
1575 NVME_AEC_SMART(n
->features
.async_config
) & NVME_SMART_TEMPERATURE
) {
1576 nvme_enqueue_event(n
, NVME_AER_TYPE_SMART
,
1577 NVME_AER_INFO_SMART_TEMP_THRESH
,
1578 NVME_LOG_SMART_INFO
);
1582 case NVME_VOLATILE_WRITE_CACHE
:
1583 if (!(dw11
& 0x1) && blk_enable_write_cache(n
->conf
.blk
)) {
1584 blk_flush(n
->conf
.blk
);
1587 blk_set_enable_write_cache(n
->conf
.blk
, dw11
& 1);
1589 case NVME_NUMBER_OF_QUEUES
:
1590 if (n
->qs_created
) {
1591 return NVME_CMD_SEQ_ERROR
| NVME_DNR
;
1595 * NVMe v1.3, Section 5.21.1.7: 0xffff is not an allowed value for NCQR
1598 if ((dw11
& 0xffff) == 0xffff || ((dw11
>> 16) & 0xffff) == 0xffff) {
1599 return NVME_INVALID_FIELD
| NVME_DNR
;
1602 trace_pci_nvme_setfeat_numq((dw11
& 0xFFFF) + 1,
1603 ((dw11
>> 16) & 0xFFFF) + 1,
1604 n
->params
.max_ioqpairs
,
1605 n
->params
.max_ioqpairs
);
1606 req
->cqe
.result
= cpu_to_le32((n
->params
.max_ioqpairs
- 1) |
1607 ((n
->params
.max_ioqpairs
- 1) << 16));
1609 case NVME_ASYNCHRONOUS_EVENT_CONF
:
1610 n
->features
.async_config
= dw11
;
1612 case NVME_TIMESTAMP
:
1613 return nvme_set_feature_timestamp(n
, req
);
1615 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
1617 return NVME_SUCCESS
;
1620 static uint16_t nvme_aer(NvmeCtrl
*n
, NvmeRequest
*req
)
1622 trace_pci_nvme_aer(nvme_cid(req
));
1624 if (n
->outstanding_aers
> n
->params
.aerl
) {
1625 trace_pci_nvme_aer_aerl_exceeded();
1626 return NVME_AER_LIMIT_EXCEEDED
;
1629 n
->aer_reqs
[n
->outstanding_aers
] = req
;
1630 n
->outstanding_aers
++;
1632 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
1633 nvme_process_aers(n
);
1636 return NVME_NO_COMPLETE
;
1639 static uint16_t nvme_admin_cmd(NvmeCtrl
*n
, NvmeRequest
*req
)
1641 trace_pci_nvme_admin_cmd(nvme_cid(req
), nvme_sqid(req
), req
->cmd
.opcode
,
1642 nvme_adm_opc_str(req
->cmd
.opcode
));
1644 switch (req
->cmd
.opcode
) {
1645 case NVME_ADM_CMD_DELETE_SQ
:
1646 return nvme_del_sq(n
, req
);
1647 case NVME_ADM_CMD_CREATE_SQ
:
1648 return nvme_create_sq(n
, req
);
1649 case NVME_ADM_CMD_GET_LOG_PAGE
:
1650 return nvme_get_log(n
, req
);
1651 case NVME_ADM_CMD_DELETE_CQ
:
1652 return nvme_del_cq(n
, req
);
1653 case NVME_ADM_CMD_CREATE_CQ
:
1654 return nvme_create_cq(n
, req
);
1655 case NVME_ADM_CMD_IDENTIFY
:
1656 return nvme_identify(n
, req
);
1657 case NVME_ADM_CMD_ABORT
:
1658 return nvme_abort(n
, req
);
1659 case NVME_ADM_CMD_SET_FEATURES
:
1660 return nvme_set_feature(n
, req
);
1661 case NVME_ADM_CMD_GET_FEATURES
:
1662 return nvme_get_feature(n
, req
);
1663 case NVME_ADM_CMD_ASYNC_EV_REQ
:
1664 return nvme_aer(n
, req
);
1666 trace_pci_nvme_err_invalid_admin_opc(req
->cmd
.opcode
);
1667 return NVME_INVALID_OPCODE
| NVME_DNR
;
1671 static void nvme_process_sq(void *opaque
)
1673 NvmeSQueue
*sq
= opaque
;
1674 NvmeCtrl
*n
= sq
->ctrl
;
1675 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
1682 while (!(nvme_sq_empty(sq
) || QTAILQ_EMPTY(&sq
->req_list
))) {
1683 addr
= sq
->dma_addr
+ sq
->head
* n
->sqe_size
;
1684 if (nvme_addr_read(n
, addr
, (void *)&cmd
, sizeof(cmd
))) {
1685 trace_pci_nvme_err_addr_read(addr
);
1686 trace_pci_nvme_err_cfs();
1687 n
->bar
.csts
= NVME_CSTS_FAILED
;
1690 nvme_inc_sq_head(sq
);
1692 req
= QTAILQ_FIRST(&sq
->req_list
);
1693 QTAILQ_REMOVE(&sq
->req_list
, req
, entry
);
1694 QTAILQ_INSERT_TAIL(&sq
->out_req_list
, req
, entry
);
1695 nvme_req_clear(req
);
1696 req
->cqe
.cid
= cmd
.cid
;
1697 memcpy(&req
->cmd
, &cmd
, sizeof(NvmeCmd
));
1699 status
= sq
->sqid
? nvme_io_cmd(n
, req
) :
1700 nvme_admin_cmd(n
, req
);
1701 if (status
!= NVME_NO_COMPLETE
) {
1702 req
->status
= status
;
1703 nvme_enqueue_req_completion(cq
, req
);
1708 static void nvme_clear_ctrl(NvmeCtrl
*n
)
1712 blk_drain(n
->conf
.blk
);
1714 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
1715 if (n
->sq
[i
] != NULL
) {
1716 nvme_free_sq(n
->sq
[i
], n
);
1719 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
1720 if (n
->cq
[i
] != NULL
) {
1721 nvme_free_cq(n
->cq
[i
], n
);
1725 while (!QTAILQ_EMPTY(&n
->aer_queue
)) {
1726 NvmeAsyncEvent
*event
= QTAILQ_FIRST(&n
->aer_queue
);
1727 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
1732 n
->outstanding_aers
= 0;
1733 n
->qs_created
= false;
1735 blk_flush(n
->conf
.blk
);
1739 static int nvme_start_ctrl(NvmeCtrl
*n
)
1741 uint32_t page_bits
= NVME_CC_MPS(n
->bar
.cc
) + 12;
1742 uint32_t page_size
= 1 << page_bits
;
1744 if (unlikely(n
->cq
[0])) {
1745 trace_pci_nvme_err_startfail_cq();
1748 if (unlikely(n
->sq
[0])) {
1749 trace_pci_nvme_err_startfail_sq();
1752 if (unlikely(!n
->bar
.asq
)) {
1753 trace_pci_nvme_err_startfail_nbarasq();
1756 if (unlikely(!n
->bar
.acq
)) {
1757 trace_pci_nvme_err_startfail_nbaracq();
1760 if (unlikely(n
->bar
.asq
& (page_size
- 1))) {
1761 trace_pci_nvme_err_startfail_asq_misaligned(n
->bar
.asq
);
1764 if (unlikely(n
->bar
.acq
& (page_size
- 1))) {
1765 trace_pci_nvme_err_startfail_acq_misaligned(n
->bar
.acq
);
1768 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) <
1769 NVME_CAP_MPSMIN(n
->bar
.cap
))) {
1770 trace_pci_nvme_err_startfail_page_too_small(
1771 NVME_CC_MPS(n
->bar
.cc
),
1772 NVME_CAP_MPSMIN(n
->bar
.cap
));
1775 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) >
1776 NVME_CAP_MPSMAX(n
->bar
.cap
))) {
1777 trace_pci_nvme_err_startfail_page_too_large(
1778 NVME_CC_MPS(n
->bar
.cc
),
1779 NVME_CAP_MPSMAX(n
->bar
.cap
));
1782 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) <
1783 NVME_CTRL_CQES_MIN(n
->id_ctrl
.cqes
))) {
1784 trace_pci_nvme_err_startfail_cqent_too_small(
1785 NVME_CC_IOCQES(n
->bar
.cc
),
1786 NVME_CTRL_CQES_MIN(n
->bar
.cap
));
1789 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) >
1790 NVME_CTRL_CQES_MAX(n
->id_ctrl
.cqes
))) {
1791 trace_pci_nvme_err_startfail_cqent_too_large(
1792 NVME_CC_IOCQES(n
->bar
.cc
),
1793 NVME_CTRL_CQES_MAX(n
->bar
.cap
));
1796 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) <
1797 NVME_CTRL_SQES_MIN(n
->id_ctrl
.sqes
))) {
1798 trace_pci_nvme_err_startfail_sqent_too_small(
1799 NVME_CC_IOSQES(n
->bar
.cc
),
1800 NVME_CTRL_SQES_MIN(n
->bar
.cap
));
1803 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) >
1804 NVME_CTRL_SQES_MAX(n
->id_ctrl
.sqes
))) {
1805 trace_pci_nvme_err_startfail_sqent_too_large(
1806 NVME_CC_IOSQES(n
->bar
.cc
),
1807 NVME_CTRL_SQES_MAX(n
->bar
.cap
));
1810 if (unlikely(!NVME_AQA_ASQS(n
->bar
.aqa
))) {
1811 trace_pci_nvme_err_startfail_asqent_sz_zero();
1814 if (unlikely(!NVME_AQA_ACQS(n
->bar
.aqa
))) {
1815 trace_pci_nvme_err_startfail_acqent_sz_zero();
1819 n
->page_bits
= page_bits
;
1820 n
->page_size
= page_size
;
1821 n
->max_prp_ents
= n
->page_size
/ sizeof(uint64_t);
1822 n
->cqe_size
= 1 << NVME_CC_IOCQES(n
->bar
.cc
);
1823 n
->sqe_size
= 1 << NVME_CC_IOSQES(n
->bar
.cc
);
1824 nvme_init_cq(&n
->admin_cq
, n
, n
->bar
.acq
, 0, 0,
1825 NVME_AQA_ACQS(n
->bar
.aqa
) + 1, 1);
1826 nvme_init_sq(&n
->admin_sq
, n
, n
->bar
.asq
, 0, 0,
1827 NVME_AQA_ASQS(n
->bar
.aqa
) + 1);
1829 nvme_set_timestamp(n
, 0ULL);
1831 QTAILQ_INIT(&n
->aer_queue
);
1836 static void nvme_write_bar(NvmeCtrl
*n
, hwaddr offset
, uint64_t data
,
1839 if (unlikely(offset
& (sizeof(uint32_t) - 1))) {
1840 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32
,
1841 "MMIO write not 32-bit aligned,"
1842 " offset=0x%"PRIx64
"", offset
);
1843 /* should be ignored, fall through for now */
1846 if (unlikely(size
< sizeof(uint32_t))) {
1847 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall
,
1848 "MMIO write smaller than 32-bits,"
1849 " offset=0x%"PRIx64
", size=%u",
1851 /* should be ignored, fall through for now */
1855 case 0xc: /* INTMS */
1856 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
1857 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
1858 "undefined access to interrupt mask set"
1859 " when MSI-X is enabled");
1860 /* should be ignored, fall through for now */
1862 n
->bar
.intms
|= data
& 0xffffffff;
1863 n
->bar
.intmc
= n
->bar
.intms
;
1864 trace_pci_nvme_mmio_intm_set(data
& 0xffffffff, n
->bar
.intmc
);
1867 case 0x10: /* INTMC */
1868 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
1869 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
1870 "undefined access to interrupt mask clr"
1871 " when MSI-X is enabled");
1872 /* should be ignored, fall through for now */
1874 n
->bar
.intms
&= ~(data
& 0xffffffff);
1875 n
->bar
.intmc
= n
->bar
.intms
;
1876 trace_pci_nvme_mmio_intm_clr(data
& 0xffffffff, n
->bar
.intmc
);
1880 trace_pci_nvme_mmio_cfg(data
& 0xffffffff);
1881 /* Windows first sends data, then sends enable bit */
1882 if (!NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
) &&
1883 !NVME_CC_SHN(data
) && !NVME_CC_SHN(n
->bar
.cc
))
1888 if (NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
)) {
1890 if (unlikely(nvme_start_ctrl(n
))) {
1891 trace_pci_nvme_err_startfail();
1892 n
->bar
.csts
= NVME_CSTS_FAILED
;
1894 trace_pci_nvme_mmio_start_success();
1895 n
->bar
.csts
= NVME_CSTS_READY
;
1897 } else if (!NVME_CC_EN(data
) && NVME_CC_EN(n
->bar
.cc
)) {
1898 trace_pci_nvme_mmio_stopped();
1900 n
->bar
.csts
&= ~NVME_CSTS_READY
;
1902 if (NVME_CC_SHN(data
) && !(NVME_CC_SHN(n
->bar
.cc
))) {
1903 trace_pci_nvme_mmio_shutdown_set();
1906 n
->bar
.csts
|= NVME_CSTS_SHST_COMPLETE
;
1907 } else if (!NVME_CC_SHN(data
) && NVME_CC_SHN(n
->bar
.cc
)) {
1908 trace_pci_nvme_mmio_shutdown_cleared();
1909 n
->bar
.csts
&= ~NVME_CSTS_SHST_COMPLETE
;
1913 case 0x1C: /* CSTS */
1914 if (data
& (1 << 4)) {
1915 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported
,
1916 "attempted to W1C CSTS.NSSRO"
1917 " but CAP.NSSRS is zero (not supported)");
1918 } else if (data
!= 0) {
1919 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts
,
1920 "attempted to set a read only bit"
1921 " of controller status");
1924 case 0x20: /* NSSR */
1925 if (data
== 0x4E564D65) {
1926 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
1928 /* The spec says that writes of other values have no effect */
1932 case 0x24: /* AQA */
1933 n
->bar
.aqa
= data
& 0xffffffff;
1934 trace_pci_nvme_mmio_aqattr(data
& 0xffffffff);
1936 case 0x28: /* ASQ */
1938 trace_pci_nvme_mmio_asqaddr(data
);
1940 case 0x2c: /* ASQ hi */
1941 n
->bar
.asq
|= data
<< 32;
1942 trace_pci_nvme_mmio_asqaddr_hi(data
, n
->bar
.asq
);
1944 case 0x30: /* ACQ */
1945 trace_pci_nvme_mmio_acqaddr(data
);
1948 case 0x34: /* ACQ hi */
1949 n
->bar
.acq
|= data
<< 32;
1950 trace_pci_nvme_mmio_acqaddr_hi(data
, n
->bar
.acq
);
1952 case 0x38: /* CMBLOC */
1953 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved
,
1954 "invalid write to reserved CMBLOC"
1955 " when CMBSZ is zero, ignored");
1957 case 0x3C: /* CMBSZ */
1958 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly
,
1959 "invalid write to read only CMBSZ, ignored");
1961 case 0xE00: /* PMRCAP */
1962 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly
,
1963 "invalid write to PMRCAP register, ignored");
1965 case 0xE04: /* TODO PMRCTL */
1967 case 0xE08: /* PMRSTS */
1968 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly
,
1969 "invalid write to PMRSTS register, ignored");
1971 case 0xE0C: /* PMREBS */
1972 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly
,
1973 "invalid write to PMREBS register, ignored");
1975 case 0xE10: /* PMRSWTP */
1976 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly
,
1977 "invalid write to PMRSWTP register, ignored");
1979 case 0xE14: /* TODO PMRMSC */
1982 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid
,
1983 "invalid MMIO write,"
1984 " offset=0x%"PRIx64
", data=%"PRIx64
"",
1990 static uint64_t nvme_mmio_read(void *opaque
, hwaddr addr
, unsigned size
)
1992 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
1993 uint8_t *ptr
= (uint8_t *)&n
->bar
;
1996 trace_pci_nvme_mmio_read(addr
);
1998 if (unlikely(addr
& (sizeof(uint32_t) - 1))) {
1999 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32
,
2000 "MMIO read not 32-bit aligned,"
2001 " offset=0x%"PRIx64
"", addr
);
2002 /* should RAZ, fall through for now */
2003 } else if (unlikely(size
< sizeof(uint32_t))) {
2004 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall
,
2005 "MMIO read smaller than 32-bits,"
2006 " offset=0x%"PRIx64
"", addr
);
2007 /* should RAZ, fall through for now */
2010 if (addr
< sizeof(n
->bar
)) {
2012 * When PMRWBM bit 1 is set then read from
2013 * from PMRSTS should ensure prior writes
2014 * made it to persistent media
2016 if (addr
== 0xE08 &&
2017 (NVME_PMRCAP_PMRWBM(n
->bar
.pmrcap
) & 0x02)) {
2018 memory_region_msync(&n
->pmrdev
->mr
, 0, n
->pmrdev
->size
);
2020 memcpy(&val
, ptr
+ addr
, size
);
2022 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs
,
2023 "MMIO read beyond last register,"
2024 " offset=0x%"PRIx64
", returning 0", addr
);
2030 static void nvme_process_db(NvmeCtrl
*n
, hwaddr addr
, int val
)
2034 if (unlikely(addr
& ((1 << 2) - 1))) {
2035 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned
,
2036 "doorbell write not 32-bit aligned,"
2037 " offset=0x%"PRIx64
", ignoring", addr
);
2041 if (((addr
- 0x1000) >> 2) & 1) {
2042 /* Completion queue doorbell write */
2044 uint16_t new_head
= val
& 0xffff;
2048 qid
= (addr
- (0x1000 + (1 << 2))) >> 3;
2049 if (unlikely(nvme_check_cqid(n
, qid
))) {
2050 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq
,
2051 "completion queue doorbell write"
2052 " for nonexistent queue,"
2053 " sqid=%"PRIu32
", ignoring", qid
);
2056 * NVM Express v1.3d, Section 4.1 state: "If host software writes
2057 * an invalid value to the Submission Queue Tail Doorbell or
2058 * Completion Queue Head Doorbell regiter and an Asynchronous Event
2059 * Request command is outstanding, then an asynchronous event is
2060 * posted to the Admin Completion Queue with a status code of
2061 * Invalid Doorbell Write Value."
2063 * Also note that the spec includes the "Invalid Doorbell Register"
2064 * status code, but nowhere does it specify when to use it.
2065 * However, it seems reasonable to use it here in a similar
2068 if (n
->outstanding_aers
) {
2069 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
2070 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
2071 NVME_LOG_ERROR_INFO
);
2078 if (unlikely(new_head
>= cq
->size
)) {
2079 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead
,
2080 "completion queue doorbell write value"
2081 " beyond queue size, sqid=%"PRIu32
","
2082 " new_head=%"PRIu16
", ignoring",
2085 if (n
->outstanding_aers
) {
2086 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
2087 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
2088 NVME_LOG_ERROR_INFO
);
2094 trace_pci_nvme_mmio_doorbell_cq(cq
->cqid
, new_head
);
2096 start_sqs
= nvme_cq_full(cq
) ? 1 : 0;
2097 cq
->head
= new_head
;
2100 QTAILQ_FOREACH(sq
, &cq
->sq_list
, entry
) {
2101 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
2103 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
2106 if (cq
->tail
== cq
->head
) {
2107 nvme_irq_deassert(n
, cq
);
2110 /* Submission queue doorbell write */
2112 uint16_t new_tail
= val
& 0xffff;
2115 qid
= (addr
- 0x1000) >> 3;
2116 if (unlikely(nvme_check_sqid(n
, qid
))) {
2117 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq
,
2118 "submission queue doorbell write"
2119 " for nonexistent queue,"
2120 " sqid=%"PRIu32
", ignoring", qid
);
2122 if (n
->outstanding_aers
) {
2123 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
2124 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
2125 NVME_LOG_ERROR_INFO
);
2132 if (unlikely(new_tail
>= sq
->size
)) {
2133 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail
,
2134 "submission queue doorbell write value"
2135 " beyond queue size, sqid=%"PRIu32
","
2136 " new_tail=%"PRIu16
", ignoring",
2139 if (n
->outstanding_aers
) {
2140 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
2141 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
2142 NVME_LOG_ERROR_INFO
);
2148 trace_pci_nvme_mmio_doorbell_sq(sq
->sqid
, new_tail
);
2150 sq
->tail
= new_tail
;
2151 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
2155 static void nvme_mmio_write(void *opaque
, hwaddr addr
, uint64_t data
,
2158 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
2160 trace_pci_nvme_mmio_write(addr
, data
);
2162 if (addr
< sizeof(n
->bar
)) {
2163 nvme_write_bar(n
, addr
, data
, size
);
2165 nvme_process_db(n
, addr
, data
);
2169 static const MemoryRegionOps nvme_mmio_ops
= {
2170 .read
= nvme_mmio_read
,
2171 .write
= nvme_mmio_write
,
2172 .endianness
= DEVICE_LITTLE_ENDIAN
,
2174 .min_access_size
= 2,
2175 .max_access_size
= 8,
2179 static void nvme_cmb_write(void *opaque
, hwaddr addr
, uint64_t data
,
2182 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
2183 stn_le_p(&n
->cmbuf
[addr
], size
, data
);
2186 static uint64_t nvme_cmb_read(void *opaque
, hwaddr addr
, unsigned size
)
2188 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
2189 return ldn_le_p(&n
->cmbuf
[addr
], size
);
2192 static const MemoryRegionOps nvme_cmb_ops
= {
2193 .read
= nvme_cmb_read
,
2194 .write
= nvme_cmb_write
,
2195 .endianness
= DEVICE_LITTLE_ENDIAN
,
2197 .min_access_size
= 1,
2198 .max_access_size
= 8,
2202 static void nvme_check_constraints(NvmeCtrl
*n
, Error
**errp
)
2204 NvmeParams
*params
= &n
->params
;
2206 if (params
->num_queues
) {
2207 warn_report("num_queues is deprecated; please use max_ioqpairs "
2210 params
->max_ioqpairs
= params
->num_queues
- 1;
2213 if (params
->max_ioqpairs
< 1 ||
2214 params
->max_ioqpairs
> NVME_MAX_IOQPAIRS
) {
2215 error_setg(errp
, "max_ioqpairs must be between 1 and %d",
2220 if (params
->msix_qsize
< 1 ||
2221 params
->msix_qsize
> PCI_MSIX_FLAGS_QSIZE
+ 1) {
2222 error_setg(errp
, "msix_qsize must be between 1 and %d",
2223 PCI_MSIX_FLAGS_QSIZE
+ 1);
2228 error_setg(errp
, "drive property not set");
2232 if (!params
->serial
) {
2233 error_setg(errp
, "serial property not set");
2237 if (!n
->params
.cmb_size_mb
&& n
->pmrdev
) {
2238 if (host_memory_backend_is_mapped(n
->pmrdev
)) {
2239 error_setg(errp
, "can't use already busy memdev: %s",
2240 object_get_canonical_path_component(OBJECT(n
->pmrdev
)));
2244 if (!is_power_of_2(n
->pmrdev
->size
)) {
2245 error_setg(errp
, "pmr backend size needs to be power of 2 in size");
2249 host_memory_backend_set_mapped(n
->pmrdev
, true);
2253 static void nvme_init_state(NvmeCtrl
*n
)
2255 n
->num_namespaces
= 1;
2256 /* add one to max_ioqpairs to account for the admin queue pair */
2257 n
->reg_size
= pow2ceil(sizeof(NvmeBar
) +
2258 2 * (n
->params
.max_ioqpairs
+ 1) * NVME_DB_SIZE
);
2259 n
->namespaces
= g_new0(NvmeNamespace
, n
->num_namespaces
);
2260 n
->sq
= g_new0(NvmeSQueue
*, n
->params
.max_ioqpairs
+ 1);
2261 n
->cq
= g_new0(NvmeCQueue
*, n
->params
.max_ioqpairs
+ 1);
2262 n
->temperature
= NVME_TEMPERATURE
;
2263 n
->features
.temp_thresh_hi
= NVME_TEMPERATURE_WARNING
;
2264 n
->starttime_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
2265 n
->aer_reqs
= g_new0(NvmeRequest
*, n
->params
.aerl
+ 1);
2268 static void nvme_init_blk(NvmeCtrl
*n
, Error
**errp
)
2270 if (!blkconf_blocksizes(&n
->conf
, errp
)) {
2273 blkconf_apply_backend_options(&n
->conf
, blk_is_read_only(n
->conf
.blk
),
2277 static void nvme_init_namespace(NvmeCtrl
*n
, NvmeNamespace
*ns
, Error
**errp
)
2280 NvmeIdNs
*id_ns
= &ns
->id_ns
;
2282 bs_size
= blk_getlength(n
->conf
.blk
);
2284 error_setg_errno(errp
, -bs_size
, "could not get backing file size");
2288 n
->ns_size
= bs_size
;
2290 id_ns
->lbaf
[0].ds
= BDRV_SECTOR_BITS
;
2291 id_ns
->nsze
= cpu_to_le64(nvme_ns_nlbas(n
, ns
));
2293 /* no thin provisioning */
2294 id_ns
->ncap
= id_ns
->nsze
;
2295 id_ns
->nuse
= id_ns
->ncap
;
2298 static void nvme_init_cmb(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
2300 NVME_CMBLOC_SET_BIR(n
->bar
.cmbloc
, NVME_CMB_BIR
);
2301 NVME_CMBLOC_SET_OFST(n
->bar
.cmbloc
, 0);
2303 NVME_CMBSZ_SET_SQS(n
->bar
.cmbsz
, 1);
2304 NVME_CMBSZ_SET_CQS(n
->bar
.cmbsz
, 0);
2305 NVME_CMBSZ_SET_LISTS(n
->bar
.cmbsz
, 1);
2306 NVME_CMBSZ_SET_RDS(n
->bar
.cmbsz
, 1);
2307 NVME_CMBSZ_SET_WDS(n
->bar
.cmbsz
, 1);
2308 NVME_CMBSZ_SET_SZU(n
->bar
.cmbsz
, 2); /* MBs */
2309 NVME_CMBSZ_SET_SZ(n
->bar
.cmbsz
, n
->params
.cmb_size_mb
);
2311 n
->cmbuf
= g_malloc0(NVME_CMBSZ_GETSIZE(n
->bar
.cmbsz
));
2312 memory_region_init_io(&n
->ctrl_mem
, OBJECT(n
), &nvme_cmb_ops
, n
,
2313 "nvme-cmb", NVME_CMBSZ_GETSIZE(n
->bar
.cmbsz
));
2314 pci_register_bar(pci_dev
, NVME_CMBLOC_BIR(n
->bar
.cmbloc
),
2315 PCI_BASE_ADDRESS_SPACE_MEMORY
|
2316 PCI_BASE_ADDRESS_MEM_TYPE_64
|
2317 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->ctrl_mem
);
2320 static void nvme_init_pmr(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
2322 /* Controller Capabilities register */
2323 NVME_CAP_SET_PMRS(n
->bar
.cap
, 1);
2325 /* PMR Capabities register */
2327 NVME_PMRCAP_SET_RDS(n
->bar
.pmrcap
, 0);
2328 NVME_PMRCAP_SET_WDS(n
->bar
.pmrcap
, 0);
2329 NVME_PMRCAP_SET_BIR(n
->bar
.pmrcap
, NVME_PMR_BIR
);
2330 NVME_PMRCAP_SET_PMRTU(n
->bar
.pmrcap
, 0);
2331 /* Turn on bit 1 support */
2332 NVME_PMRCAP_SET_PMRWBM(n
->bar
.pmrcap
, 0x02);
2333 NVME_PMRCAP_SET_PMRTO(n
->bar
.pmrcap
, 0);
2334 NVME_PMRCAP_SET_CMSS(n
->bar
.pmrcap
, 0);
2336 /* PMR Control register */
2338 NVME_PMRCTL_SET_EN(n
->bar
.pmrctl
, 0);
2340 /* PMR Status register */
2342 NVME_PMRSTS_SET_ERR(n
->bar
.pmrsts
, 0);
2343 NVME_PMRSTS_SET_NRDY(n
->bar
.pmrsts
, 0);
2344 NVME_PMRSTS_SET_HSTS(n
->bar
.pmrsts
, 0);
2345 NVME_PMRSTS_SET_CBAI(n
->bar
.pmrsts
, 0);
2347 /* PMR Elasticity Buffer Size register */
2349 NVME_PMREBS_SET_PMRSZU(n
->bar
.pmrebs
, 0);
2350 NVME_PMREBS_SET_RBB(n
->bar
.pmrebs
, 0);
2351 NVME_PMREBS_SET_PMRWBZ(n
->bar
.pmrebs
, 0);
2353 /* PMR Sustained Write Throughput register */
2355 NVME_PMRSWTP_SET_PMRSWTU(n
->bar
.pmrswtp
, 0);
2356 NVME_PMRSWTP_SET_PMRSWTV(n
->bar
.pmrswtp
, 0);
2358 /* PMR Memory Space Control register */
2360 NVME_PMRMSC_SET_CMSE(n
->bar
.pmrmsc
, 0);
2361 NVME_PMRMSC_SET_CBA(n
->bar
.pmrmsc
, 0);
2363 pci_register_bar(pci_dev
, NVME_PMRCAP_BIR(n
->bar
.pmrcap
),
2364 PCI_BASE_ADDRESS_SPACE_MEMORY
|
2365 PCI_BASE_ADDRESS_MEM_TYPE_64
|
2366 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->pmrdev
->mr
);
2369 static void nvme_init_pci(NvmeCtrl
*n
, PCIDevice
*pci_dev
, Error
**errp
)
2371 uint8_t *pci_conf
= pci_dev
->config
;
2373 pci_conf
[PCI_INTERRUPT_PIN
] = 1;
2374 pci_config_set_prog_interface(pci_conf
, 0x2);
2375 pci_config_set_class(pci_conf
, PCI_CLASS_STORAGE_EXPRESS
);
2376 pcie_endpoint_cap_init(pci_dev
, 0x80);
2378 memory_region_init_io(&n
->iomem
, OBJECT(n
), &nvme_mmio_ops
, n
, "nvme",
2380 pci_register_bar(pci_dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
|
2381 PCI_BASE_ADDRESS_MEM_TYPE_64
, &n
->iomem
);
2382 if (msix_init_exclusive_bar(pci_dev
, n
->params
.msix_qsize
, 4, errp
)) {
2386 if (n
->params
.cmb_size_mb
) {
2387 nvme_init_cmb(n
, pci_dev
);
2388 } else if (n
->pmrdev
) {
2389 nvme_init_pmr(n
, pci_dev
);
2393 static void nvme_init_ctrl(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
2395 NvmeIdCtrl
*id
= &n
->id_ctrl
;
2396 uint8_t *pci_conf
= pci_dev
->config
;
2399 id
->vid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_VENDOR_ID
));
2400 id
->ssvid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_SUBSYSTEM_VENDOR_ID
));
2401 strpadcpy((char *)id
->mn
, sizeof(id
->mn
), "QEMU NVMe Ctrl", ' ');
2402 strpadcpy((char *)id
->fr
, sizeof(id
->fr
), "1.0", ' ');
2403 strpadcpy((char *)id
->sn
, sizeof(id
->sn
), n
->params
.serial
, ' ');
2408 id
->mdts
= n
->params
.mdts
;
2409 id
->ver
= cpu_to_le32(NVME_SPEC_VER
);
2410 id
->oacs
= cpu_to_le16(0);
2413 * Because the controller always completes the Abort command immediately,
2414 * there can never be more than one concurrently executing Abort command,
2415 * so this value is never used for anything. Note that there can easily be
2416 * many Abort commands in the queues, but they are not considered
2417 * "executing" until processed by nvme_abort.
2419 * The specification recommends a value of 3 for Abort Command Limit (four
2420 * concurrently outstanding Abort commands), so lets use that though it is
2424 id
->aerl
= n
->params
.aerl
;
2425 id
->frmw
= (NVME_NUM_FW_SLOTS
<< 1) | NVME_FRMW_SLOT1_RO
;
2426 id
->lpa
= NVME_LPA_EXTENDED
;
2428 /* recommended default value (~70 C) */
2429 id
->wctemp
= cpu_to_le16(NVME_TEMPERATURE_WARNING
);
2430 id
->cctemp
= cpu_to_le16(NVME_TEMPERATURE_CRITICAL
);
2432 id
->sqes
= (0x6 << 4) | 0x6;
2433 id
->cqes
= (0x4 << 4) | 0x4;
2434 id
->nn
= cpu_to_le32(n
->num_namespaces
);
2435 id
->oncs
= cpu_to_le16(NVME_ONCS_WRITE_ZEROES
| NVME_ONCS_TIMESTAMP
|
2436 NVME_ONCS_FEATURES
);
2438 subnqn
= g_strdup_printf("nqn.2019-08.org.qemu:%s", n
->params
.serial
);
2439 strpadcpy((char *)id
->subnqn
, sizeof(id
->subnqn
), subnqn
, '\0');
2442 id
->psd
[0].mp
= cpu_to_le16(0x9c4);
2443 id
->psd
[0].enlat
= cpu_to_le32(0x10);
2444 id
->psd
[0].exlat
= cpu_to_le32(0x4);
2445 if (blk_enable_write_cache(n
->conf
.blk
)) {
2450 NVME_CAP_SET_MQES(n
->bar
.cap
, 0x7ff);
2451 NVME_CAP_SET_CQR(n
->bar
.cap
, 1);
2452 NVME_CAP_SET_TO(n
->bar
.cap
, 0xf);
2453 NVME_CAP_SET_CSS(n
->bar
.cap
, 1);
2454 NVME_CAP_SET_MPSMAX(n
->bar
.cap
, 4);
2456 n
->bar
.vs
= NVME_SPEC_VER
;
2457 n
->bar
.intmc
= n
->bar
.intms
= 0;
2460 static void nvme_realize(PCIDevice
*pci_dev
, Error
**errp
)
2462 NvmeCtrl
*n
= NVME(pci_dev
);
2463 Error
*local_err
= NULL
;
2467 nvme_check_constraints(n
, &local_err
);
2469 error_propagate(errp
, local_err
);
2474 nvme_init_blk(n
, &local_err
);
2476 error_propagate(errp
, local_err
);
2480 nvme_init_pci(n
, pci_dev
, &local_err
);
2482 error_propagate(errp
, local_err
);
2486 nvme_init_ctrl(n
, pci_dev
);
2488 for (i
= 0; i
< n
->num_namespaces
; i
++) {
2489 nvme_init_namespace(n
, &n
->namespaces
[i
], &local_err
);
2491 error_propagate(errp
, local_err
);
2497 static void nvme_exit(PCIDevice
*pci_dev
)
2499 NvmeCtrl
*n
= NVME(pci_dev
);
2502 g_free(n
->namespaces
);
2505 g_free(n
->aer_reqs
);
2507 if (n
->params
.cmb_size_mb
) {
2512 host_memory_backend_set_mapped(n
->pmrdev
, false);
2514 msix_uninit_exclusive_bar(pci_dev
);
2517 static Property nvme_props
[] = {
2518 DEFINE_BLOCK_PROPERTIES(NvmeCtrl
, conf
),
2519 DEFINE_PROP_LINK("pmrdev", NvmeCtrl
, pmrdev
, TYPE_MEMORY_BACKEND
,
2520 HostMemoryBackend
*),
2521 DEFINE_PROP_STRING("serial", NvmeCtrl
, params
.serial
),
2522 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl
, params
.cmb_size_mb
, 0),
2523 DEFINE_PROP_UINT32("num_queues", NvmeCtrl
, params
.num_queues
, 0),
2524 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl
, params
.max_ioqpairs
, 64),
2525 DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl
, params
.msix_qsize
, 65),
2526 DEFINE_PROP_UINT8("aerl", NvmeCtrl
, params
.aerl
, 3),
2527 DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl
, params
.aer_max_queued
, 64),
2528 DEFINE_PROP_UINT8("mdts", NvmeCtrl
, params
.mdts
, 7),
2529 DEFINE_PROP_END_OF_LIST(),
2532 static const VMStateDescription nvme_vmstate
= {
2537 static void nvme_class_init(ObjectClass
*oc
, void *data
)
2539 DeviceClass
*dc
= DEVICE_CLASS(oc
);
2540 PCIDeviceClass
*pc
= PCI_DEVICE_CLASS(oc
);
2542 pc
->realize
= nvme_realize
;
2543 pc
->exit
= nvme_exit
;
2544 pc
->class_id
= PCI_CLASS_STORAGE_EXPRESS
;
2545 pc
->vendor_id
= PCI_VENDOR_ID_INTEL
;
2546 pc
->device_id
= 0x5845;
2549 set_bit(DEVICE_CATEGORY_STORAGE
, dc
->categories
);
2550 dc
->desc
= "Non-Volatile Memory Express";
2551 device_class_set_props(dc
, nvme_props
);
2552 dc
->vmsd
= &nvme_vmstate
;
2555 static void nvme_instance_init(Object
*obj
)
2557 NvmeCtrl
*s
= NVME(obj
);
2559 device_add_bootindex_property(obj
, &s
->conf
.bootindex
,
2560 "bootindex", "/namespace@1,0",
2564 static const TypeInfo nvme_info
= {
2566 .parent
= TYPE_PCI_DEVICE
,
2567 .instance_size
= sizeof(NvmeCtrl
),
2568 .class_init
= nvme_class_init
,
2569 .instance_init
= nvme_instance_init
,
2570 .interfaces
= (InterfaceInfo
[]) {
2571 { INTERFACE_PCIE_DEVICE
},
2576 static void nvme_register_types(void)
2578 type_register_static(&nvme_info
);
2581 type_init(nvme_register_types
)