2 * QEMU NVM Express Controller
4 * Copyright (c) 2012, Intel Corporation
6 * Written by Keith Busch <keith.busch@intel.com>
8 * This code is licensed under the GNU GPL v2 or later.
12 * Reference Specs: http://www.nvmexpress.org, 1.1, 1.0e
14 * http://www.nvmexpress.org/resources/
19 * -drive file=<file>,if=none,id=<drive_id>
20 * -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>
23 #include "qemu/osdep.h"
24 #include <hw/block/block.h>
26 #include <hw/pci/msix.h>
27 #include <hw/pci/pci.h>
28 #include "sysemu/sysemu.h"
29 #include "qapi/visitor.h"
30 #include "sysemu/block-backend.h"
34 static void nvme_process_sq(void *opaque
);
36 static int nvme_check_sqid(NvmeCtrl
*n
, uint16_t sqid
)
38 return sqid
< n
->num_queues
&& n
->sq
[sqid
] != NULL
? 0 : -1;
41 static int nvme_check_cqid(NvmeCtrl
*n
, uint16_t cqid
)
43 return cqid
< n
->num_queues
&& n
->cq
[cqid
] != NULL
? 0 : -1;
46 static void nvme_inc_cq_tail(NvmeCQueue
*cq
)
49 if (cq
->tail
>= cq
->size
) {
51 cq
->phase
= !cq
->phase
;
55 static void nvme_inc_sq_head(NvmeSQueue
*sq
)
57 sq
->head
= (sq
->head
+ 1) % sq
->size
;
60 static uint8_t nvme_cq_full(NvmeCQueue
*cq
)
62 return (cq
->tail
+ 1) % cq
->size
== cq
->head
;
65 static uint8_t nvme_sq_empty(NvmeSQueue
*sq
)
67 return sq
->head
== sq
->tail
;
70 static void nvme_isr_notify(NvmeCtrl
*n
, NvmeCQueue
*cq
)
72 if (cq
->irq_enabled
) {
73 if (msix_enabled(&(n
->parent_obj
))) {
74 msix_notify(&(n
->parent_obj
), cq
->vector
);
76 pci_irq_pulse(&n
->parent_obj
);
81 static uint16_t nvme_map_prp(QEMUSGList
*qsg
, uint64_t prp1
, uint64_t prp2
,
82 uint32_t len
, NvmeCtrl
*n
)
84 hwaddr trans_len
= n
->page_size
- (prp1
% n
->page_size
);
85 trans_len
= MIN(len
, trans_len
);
86 int num_prps
= (len
>> n
->page_bits
) + 1;
89 return NVME_INVALID_FIELD
| NVME_DNR
;
92 pci_dma_sglist_init(qsg
, &n
->parent_obj
, num_prps
);
93 qemu_sglist_add(qsg
, prp1
, trans_len
);
99 if (len
> n
->page_size
) {
100 uint64_t prp_list
[n
->max_prp_ents
];
101 uint32_t nents
, prp_trans
;
104 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
105 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
106 pci_dma_read(&n
->parent_obj
, prp2
, (void *)prp_list
, prp_trans
);
108 uint64_t prp_ent
= le64_to_cpu(prp_list
[i
]);
110 if (i
== n
->max_prp_ents
- 1 && len
> n
->page_size
) {
111 if (!prp_ent
|| prp_ent
& (n
->page_size
- 1)) {
116 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
117 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
118 pci_dma_read(&n
->parent_obj
, prp_ent
, (void *)prp_list
,
120 prp_ent
= le64_to_cpu(prp_list
[i
]);
123 if (!prp_ent
|| prp_ent
& (n
->page_size
- 1)) {
127 trans_len
= MIN(len
, n
->page_size
);
128 qemu_sglist_add(qsg
, prp_ent
, trans_len
);
133 if (prp2
& (n
->page_size
- 1)) {
136 qemu_sglist_add(qsg
, prp2
, len
);
142 qemu_sglist_destroy(qsg
);
143 return NVME_INVALID_FIELD
| NVME_DNR
;
146 static uint16_t nvme_dma_read_prp(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
147 uint64_t prp1
, uint64_t prp2
)
151 if (nvme_map_prp(&qsg
, prp1
, prp2
, len
, n
)) {
152 return NVME_INVALID_FIELD
| NVME_DNR
;
154 if (dma_buf_read(ptr
, len
, &qsg
)) {
155 qemu_sglist_destroy(&qsg
);
156 return NVME_INVALID_FIELD
| NVME_DNR
;
158 qemu_sglist_destroy(&qsg
);
162 static void nvme_post_cqes(void *opaque
)
164 NvmeCQueue
*cq
= opaque
;
165 NvmeCtrl
*n
= cq
->ctrl
;
166 NvmeRequest
*req
, *next
;
168 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
172 if (nvme_cq_full(cq
)) {
176 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
178 req
->cqe
.status
= cpu_to_le16((req
->status
<< 1) | cq
->phase
);
179 req
->cqe
.sq_id
= cpu_to_le16(sq
->sqid
);
180 req
->cqe
.sq_head
= cpu_to_le16(sq
->head
);
181 addr
= cq
->dma_addr
+ cq
->tail
* n
->cqe_size
;
182 nvme_inc_cq_tail(cq
);
183 pci_dma_write(&n
->parent_obj
, addr
, (void *)&req
->cqe
,
185 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
187 nvme_isr_notify(n
, cq
);
190 static void nvme_enqueue_req_completion(NvmeCQueue
*cq
, NvmeRequest
*req
)
192 assert(cq
->cqid
== req
->sq
->cqid
);
193 QTAILQ_REMOVE(&req
->sq
->out_req_list
, req
, entry
);
194 QTAILQ_INSERT_TAIL(&cq
->req_list
, req
, entry
);
195 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
198 static void nvme_rw_cb(void *opaque
, int ret
)
200 NvmeRequest
*req
= opaque
;
201 NvmeSQueue
*sq
= req
->sq
;
202 NvmeCtrl
*n
= sq
->ctrl
;
203 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
206 block_acct_done(blk_get_stats(n
->conf
.blk
), &req
->acct
);
207 req
->status
= NVME_SUCCESS
;
209 block_acct_failed(blk_get_stats(n
->conf
.blk
), &req
->acct
);
210 req
->status
= NVME_INTERNAL_DEV_ERROR
;
213 qemu_sglist_destroy(&req
->qsg
);
215 nvme_enqueue_req_completion(cq
, req
);
218 static uint16_t nvme_flush(NvmeCtrl
*n
, NvmeNamespace
*ns
, NvmeCmd
*cmd
,
222 block_acct_start(blk_get_stats(n
->conf
.blk
), &req
->acct
, 0,
224 req
->aiocb
= blk_aio_flush(n
->conf
.blk
, nvme_rw_cb
, req
);
226 return NVME_NO_COMPLETE
;
229 static uint16_t nvme_rw(NvmeCtrl
*n
, NvmeNamespace
*ns
, NvmeCmd
*cmd
,
232 NvmeRwCmd
*rw
= (NvmeRwCmd
*)cmd
;
233 uint32_t nlb
= le32_to_cpu(rw
->nlb
) + 1;
234 uint64_t slba
= le64_to_cpu(rw
->slba
);
235 uint64_t prp1
= le64_to_cpu(rw
->prp1
);
236 uint64_t prp2
= le64_to_cpu(rw
->prp2
);
238 uint8_t lba_index
= NVME_ID_NS_FLBAS_INDEX(ns
->id_ns
.flbas
);
239 uint8_t data_shift
= ns
->id_ns
.lbaf
[lba_index
].ds
;
240 uint64_t data_size
= (uint64_t)nlb
<< data_shift
;
241 uint64_t aio_slba
= slba
<< (data_shift
- BDRV_SECTOR_BITS
);
242 int is_write
= rw
->opcode
== NVME_CMD_WRITE
? 1 : 0;
243 enum BlockAcctType acct
= is_write
? BLOCK_ACCT_WRITE
: BLOCK_ACCT_READ
;
245 if ((slba
+ nlb
) > ns
->id_ns
.nsze
) {
246 block_acct_invalid(blk_get_stats(n
->conf
.blk
), acct
);
247 return NVME_LBA_RANGE
| NVME_DNR
;
250 if (nvme_map_prp(&req
->qsg
, prp1
, prp2
, data_size
, n
)) {
251 block_acct_invalid(blk_get_stats(n
->conf
.blk
), acct
);
252 return NVME_INVALID_FIELD
| NVME_DNR
;
255 assert((nlb
<< data_shift
) == req
->qsg
.size
);
258 dma_acct_start(n
->conf
.blk
, &req
->acct
, &req
->qsg
, acct
);
259 req
->aiocb
= is_write
?
260 dma_blk_write(n
->conf
.blk
, &req
->qsg
, aio_slba
, nvme_rw_cb
, req
) :
261 dma_blk_read(n
->conf
.blk
, &req
->qsg
, aio_slba
, nvme_rw_cb
, req
);
263 return NVME_NO_COMPLETE
;
266 static uint16_t nvme_io_cmd(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
269 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
271 if (nsid
== 0 || nsid
> n
->num_namespaces
) {
272 return NVME_INVALID_NSID
| NVME_DNR
;
275 ns
= &n
->namespaces
[nsid
- 1];
276 switch (cmd
->opcode
) {
278 return nvme_flush(n
, ns
, cmd
, req
);
281 return nvme_rw(n
, ns
, cmd
, req
);
283 return NVME_INVALID_OPCODE
| NVME_DNR
;
287 static void nvme_free_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
)
289 n
->sq
[sq
->sqid
] = NULL
;
290 timer_del(sq
->timer
);
291 timer_free(sq
->timer
);
298 static uint16_t nvme_del_sq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
300 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)cmd
;
301 NvmeRequest
*req
, *next
;
304 uint16_t qid
= le16_to_cpu(c
->qid
);
306 if (!qid
|| nvme_check_sqid(n
, qid
)) {
307 return NVME_INVALID_QID
| NVME_DNR
;
311 while (!QTAILQ_EMPTY(&sq
->out_req_list
)) {
312 req
= QTAILQ_FIRST(&sq
->out_req_list
);
314 blk_aio_cancel(req
->aiocb
);
316 if (!nvme_check_cqid(n
, sq
->cqid
)) {
317 cq
= n
->cq
[sq
->cqid
];
318 QTAILQ_REMOVE(&cq
->sq_list
, sq
, entry
);
321 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
323 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
324 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
333 static void nvme_init_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
, uint64_t dma_addr
,
334 uint16_t sqid
, uint16_t cqid
, uint16_t size
)
340 sq
->dma_addr
= dma_addr
;
344 sq
->head
= sq
->tail
= 0;
345 sq
->io_req
= g_new(NvmeRequest
, sq
->size
);
347 QTAILQ_INIT(&sq
->req_list
);
348 QTAILQ_INIT(&sq
->out_req_list
);
349 for (i
= 0; i
< sq
->size
; i
++) {
350 sq
->io_req
[i
].sq
= sq
;
351 QTAILQ_INSERT_TAIL(&(sq
->req_list
), &sq
->io_req
[i
], entry
);
353 sq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_process_sq
, sq
);
357 QTAILQ_INSERT_TAIL(&(cq
->sq_list
), sq
, entry
);
361 static uint16_t nvme_create_sq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
364 NvmeCreateSq
*c
= (NvmeCreateSq
*)cmd
;
366 uint16_t cqid
= le16_to_cpu(c
->cqid
);
367 uint16_t sqid
= le16_to_cpu(c
->sqid
);
368 uint16_t qsize
= le16_to_cpu(c
->qsize
);
369 uint16_t qflags
= le16_to_cpu(c
->sq_flags
);
370 uint64_t prp1
= le64_to_cpu(c
->prp1
);
372 if (!cqid
|| nvme_check_cqid(n
, cqid
)) {
373 return NVME_INVALID_CQID
| NVME_DNR
;
375 if (!sqid
|| (sqid
&& !nvme_check_sqid(n
, sqid
))) {
376 return NVME_INVALID_QID
| NVME_DNR
;
378 if (!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
)) {
379 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
381 if (!prp1
|| prp1
& (n
->page_size
- 1)) {
382 return NVME_INVALID_FIELD
| NVME_DNR
;
384 if (!(NVME_SQ_FLAGS_PC(qflags
))) {
385 return NVME_INVALID_FIELD
| NVME_DNR
;
387 sq
= g_malloc0(sizeof(*sq
));
388 nvme_init_sq(sq
, n
, prp1
, sqid
, cqid
, qsize
+ 1);
392 static void nvme_free_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
)
394 n
->cq
[cq
->cqid
] = NULL
;
395 timer_del(cq
->timer
);
396 timer_free(cq
->timer
);
397 msix_vector_unuse(&n
->parent_obj
, cq
->vector
);
403 static uint16_t nvme_del_cq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
405 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)cmd
;
407 uint16_t qid
= le16_to_cpu(c
->qid
);
409 if (!qid
|| nvme_check_cqid(n
, qid
)) {
410 return NVME_INVALID_CQID
| NVME_DNR
;
414 if (!QTAILQ_EMPTY(&cq
->sq_list
)) {
415 return NVME_INVALID_QUEUE_DEL
;
421 static void nvme_init_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
, uint64_t dma_addr
,
422 uint16_t cqid
, uint16_t vector
, uint16_t size
, uint16_t irq_enabled
)
427 cq
->dma_addr
= dma_addr
;
429 cq
->irq_enabled
= irq_enabled
;
431 cq
->head
= cq
->tail
= 0;
432 QTAILQ_INIT(&cq
->req_list
);
433 QTAILQ_INIT(&cq
->sq_list
);
434 msix_vector_use(&n
->parent_obj
, cq
->vector
);
436 cq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_post_cqes
, cq
);
439 static uint16_t nvme_create_cq(NvmeCtrl
*n
, NvmeCmd
*cmd
)
442 NvmeCreateCq
*c
= (NvmeCreateCq
*)cmd
;
443 uint16_t cqid
= le16_to_cpu(c
->cqid
);
444 uint16_t vector
= le16_to_cpu(c
->irq_vector
);
445 uint16_t qsize
= le16_to_cpu(c
->qsize
);
446 uint16_t qflags
= le16_to_cpu(c
->cq_flags
);
447 uint64_t prp1
= le64_to_cpu(c
->prp1
);
449 if (!cqid
|| (cqid
&& !nvme_check_cqid(n
, cqid
))) {
450 return NVME_INVALID_CQID
| NVME_DNR
;
452 if (!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
)) {
453 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
456 return NVME_INVALID_FIELD
| NVME_DNR
;
458 if (vector
> n
->num_queues
) {
459 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
461 if (!(NVME_CQ_FLAGS_PC(qflags
))) {
462 return NVME_INVALID_FIELD
| NVME_DNR
;
465 cq
= g_malloc0(sizeof(*cq
));
466 nvme_init_cq(cq
, n
, prp1
, cqid
, vector
, qsize
+ 1,
467 NVME_CQ_FLAGS_IEN(qflags
));
471 static uint16_t nvme_identify(NvmeCtrl
*n
, NvmeCmd
*cmd
)
474 NvmeIdentify
*c
= (NvmeIdentify
*)cmd
;
475 uint32_t cns
= le32_to_cpu(c
->cns
);
476 uint32_t nsid
= le32_to_cpu(c
->nsid
);
477 uint64_t prp1
= le64_to_cpu(c
->prp1
);
478 uint64_t prp2
= le64_to_cpu(c
->prp2
);
481 return nvme_dma_read_prp(n
, (uint8_t *)&n
->id_ctrl
, sizeof(n
->id_ctrl
),
484 if (nsid
== 0 || nsid
> n
->num_namespaces
) {
485 return NVME_INVALID_NSID
| NVME_DNR
;
488 ns
= &n
->namespaces
[nsid
- 1];
489 return nvme_dma_read_prp(n
, (uint8_t *)&ns
->id_ns
, sizeof(ns
->id_ns
),
493 static uint16_t nvme_get_feature(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
495 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
499 case NVME_VOLATILE_WRITE_CACHE
:
500 result
= blk_enable_write_cache(n
->conf
.blk
);
502 case NVME_NUMBER_OF_QUEUES
:
503 result
= cpu_to_le32((n
->num_queues
- 1) | ((n
->num_queues
- 1) << 16));
506 return NVME_INVALID_FIELD
| NVME_DNR
;
509 req
->cqe
.result
= result
;
513 static uint16_t nvme_set_feature(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
515 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
516 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
519 case NVME_VOLATILE_WRITE_CACHE
:
520 blk_set_enable_write_cache(n
->conf
.blk
, dw11
& 1);
522 case NVME_NUMBER_OF_QUEUES
:
524 cpu_to_le32((n
->num_queues
- 1) | ((n
->num_queues
- 1) << 16));
527 return NVME_INVALID_FIELD
| NVME_DNR
;
532 static uint16_t nvme_admin_cmd(NvmeCtrl
*n
, NvmeCmd
*cmd
, NvmeRequest
*req
)
534 switch (cmd
->opcode
) {
535 case NVME_ADM_CMD_DELETE_SQ
:
536 return nvme_del_sq(n
, cmd
);
537 case NVME_ADM_CMD_CREATE_SQ
:
538 return nvme_create_sq(n
, cmd
);
539 case NVME_ADM_CMD_DELETE_CQ
:
540 return nvme_del_cq(n
, cmd
);
541 case NVME_ADM_CMD_CREATE_CQ
:
542 return nvme_create_cq(n
, cmd
);
543 case NVME_ADM_CMD_IDENTIFY
:
544 return nvme_identify(n
, cmd
);
545 case NVME_ADM_CMD_SET_FEATURES
:
546 return nvme_set_feature(n
, cmd
, req
);
547 case NVME_ADM_CMD_GET_FEATURES
:
548 return nvme_get_feature(n
, cmd
, req
);
550 return NVME_INVALID_OPCODE
| NVME_DNR
;
554 static void nvme_process_sq(void *opaque
)
556 NvmeSQueue
*sq
= opaque
;
557 NvmeCtrl
*n
= sq
->ctrl
;
558 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
565 while (!(nvme_sq_empty(sq
) || QTAILQ_EMPTY(&sq
->req_list
))) {
566 addr
= sq
->dma_addr
+ sq
->head
* n
->sqe_size
;
567 pci_dma_read(&n
->parent_obj
, addr
, (void *)&cmd
, sizeof(cmd
));
568 nvme_inc_sq_head(sq
);
570 req
= QTAILQ_FIRST(&sq
->req_list
);
571 QTAILQ_REMOVE(&sq
->req_list
, req
, entry
);
572 QTAILQ_INSERT_TAIL(&sq
->out_req_list
, req
, entry
);
573 memset(&req
->cqe
, 0, sizeof(req
->cqe
));
574 req
->cqe
.cid
= cmd
.cid
;
576 status
= sq
->sqid
? nvme_io_cmd(n
, &cmd
, req
) :
577 nvme_admin_cmd(n
, &cmd
, req
);
578 if (status
!= NVME_NO_COMPLETE
) {
579 req
->status
= status
;
580 nvme_enqueue_req_completion(cq
, req
);
585 static void nvme_clear_ctrl(NvmeCtrl
*n
)
589 for (i
= 0; i
< n
->num_queues
; i
++) {
590 if (n
->sq
[i
] != NULL
) {
591 nvme_free_sq(n
->sq
[i
], n
);
594 for (i
= 0; i
< n
->num_queues
; i
++) {
595 if (n
->cq
[i
] != NULL
) {
596 nvme_free_cq(n
->cq
[i
], n
);
600 blk_flush(n
->conf
.blk
);
604 static int nvme_start_ctrl(NvmeCtrl
*n
)
606 uint32_t page_bits
= NVME_CC_MPS(n
->bar
.cc
) + 12;
607 uint32_t page_size
= 1 << page_bits
;
609 if (n
->cq
[0] || n
->sq
[0] || !n
->bar
.asq
|| !n
->bar
.acq
||
610 n
->bar
.asq
& (page_size
- 1) || n
->bar
.acq
& (page_size
- 1) ||
611 NVME_CC_MPS(n
->bar
.cc
) < NVME_CAP_MPSMIN(n
->bar
.cap
) ||
612 NVME_CC_MPS(n
->bar
.cc
) > NVME_CAP_MPSMAX(n
->bar
.cap
) ||
613 NVME_CC_IOCQES(n
->bar
.cc
) < NVME_CTRL_CQES_MIN(n
->id_ctrl
.cqes
) ||
614 NVME_CC_IOCQES(n
->bar
.cc
) > NVME_CTRL_CQES_MAX(n
->id_ctrl
.cqes
) ||
615 NVME_CC_IOSQES(n
->bar
.cc
) < NVME_CTRL_SQES_MIN(n
->id_ctrl
.sqes
) ||
616 NVME_CC_IOSQES(n
->bar
.cc
) > NVME_CTRL_SQES_MAX(n
->id_ctrl
.sqes
) ||
617 !NVME_AQA_ASQS(n
->bar
.aqa
) || !NVME_AQA_ACQS(n
->bar
.aqa
)) {
621 n
->page_bits
= page_bits
;
622 n
->page_size
= page_size
;
623 n
->max_prp_ents
= n
->page_size
/ sizeof(uint64_t);
624 n
->cqe_size
= 1 << NVME_CC_IOCQES(n
->bar
.cc
);
625 n
->sqe_size
= 1 << NVME_CC_IOSQES(n
->bar
.cc
);
626 nvme_init_cq(&n
->admin_cq
, n
, n
->bar
.acq
, 0, 0,
627 NVME_AQA_ACQS(n
->bar
.aqa
) + 1, 1);
628 nvme_init_sq(&n
->admin_sq
, n
, n
->bar
.asq
, 0, 0,
629 NVME_AQA_ASQS(n
->bar
.aqa
) + 1);
634 static void nvme_write_bar(NvmeCtrl
*n
, hwaddr offset
, uint64_t data
,
639 n
->bar
.intms
|= data
& 0xffffffff;
640 n
->bar
.intmc
= n
->bar
.intms
;
643 n
->bar
.intms
&= ~(data
& 0xffffffff);
644 n
->bar
.intmc
= n
->bar
.intms
;
647 /* Windows first sends data, then sends enable bit */
648 if (!NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
) &&
649 !NVME_CC_SHN(data
) && !NVME_CC_SHN(n
->bar
.cc
))
654 if (NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
)) {
656 if (nvme_start_ctrl(n
)) {
657 n
->bar
.csts
= NVME_CSTS_FAILED
;
659 n
->bar
.csts
= NVME_CSTS_READY
;
661 } else if (!NVME_CC_EN(data
) && NVME_CC_EN(n
->bar
.cc
)) {
663 n
->bar
.csts
&= ~NVME_CSTS_READY
;
665 if (NVME_CC_SHN(data
) && !(NVME_CC_SHN(n
->bar
.cc
))) {
668 n
->bar
.csts
|= NVME_CSTS_SHST_COMPLETE
;
669 } else if (!NVME_CC_SHN(data
) && NVME_CC_SHN(n
->bar
.cc
)) {
670 n
->bar
.csts
&= ~NVME_CSTS_SHST_COMPLETE
;
675 n
->bar
.aqa
= data
& 0xffffffff;
681 n
->bar
.asq
|= data
<< 32;
687 n
->bar
.acq
|= data
<< 32;
694 static uint64_t nvme_mmio_read(void *opaque
, hwaddr addr
, unsigned size
)
696 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
697 uint8_t *ptr
= (uint8_t *)&n
->bar
;
700 if (addr
< sizeof(n
->bar
)) {
701 memcpy(&val
, ptr
+ addr
, size
);
706 static void nvme_process_db(NvmeCtrl
*n
, hwaddr addr
, int val
)
710 if (addr
& ((1 << 2) - 1)) {
714 if (((addr
- 0x1000) >> 2) & 1) {
715 uint16_t new_head
= val
& 0xffff;
719 qid
= (addr
- (0x1000 + (1 << 2))) >> 3;
720 if (nvme_check_cqid(n
, qid
)) {
725 if (new_head
>= cq
->size
) {
729 start_sqs
= nvme_cq_full(cq
) ? 1 : 0;
733 QTAILQ_FOREACH(sq
, &cq
->sq_list
, entry
) {
734 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
736 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
739 if (cq
->tail
!= cq
->head
) {
740 nvme_isr_notify(n
, cq
);
743 uint16_t new_tail
= val
& 0xffff;
746 qid
= (addr
- 0x1000) >> 3;
747 if (nvme_check_sqid(n
, qid
)) {
752 if (new_tail
>= sq
->size
) {
757 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
761 static void nvme_mmio_write(void *opaque
, hwaddr addr
, uint64_t data
,
764 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
765 if (addr
< sizeof(n
->bar
)) {
766 nvme_write_bar(n
, addr
, data
, size
);
767 } else if (addr
>= 0x1000) {
768 nvme_process_db(n
, addr
, data
);
772 static const MemoryRegionOps nvme_mmio_ops
= {
773 .read
= nvme_mmio_read
,
774 .write
= nvme_mmio_write
,
775 .endianness
= DEVICE_LITTLE_ENDIAN
,
777 .min_access_size
= 2,
778 .max_access_size
= 8,
782 static int nvme_init(PCIDevice
*pci_dev
)
784 NvmeCtrl
*n
= NVME(pci_dev
);
785 NvmeIdCtrl
*id
= &n
->id_ctrl
;
795 bs_size
= blk_getlength(n
->conf
.blk
);
800 blkconf_serial(&n
->conf
, &n
->serial
);
804 blkconf_blocksizes(&n
->conf
);
806 pci_conf
= pci_dev
->config
;
807 pci_conf
[PCI_INTERRUPT_PIN
] = 1;
808 pci_config_set_prog_interface(pci_dev
->config
, 0x2);
809 pci_config_set_class(pci_dev
->config
, PCI_CLASS_STORAGE_EXPRESS
);
810 pcie_endpoint_cap_init(&n
->parent_obj
, 0x80);
812 n
->num_namespaces
= 1;
814 n
->reg_size
= pow2ceil(0x1004 + 2 * (n
->num_queues
+ 1) * 4);
815 n
->ns_size
= bs_size
/ (uint64_t)n
->num_namespaces
;
817 n
->namespaces
= g_new0(NvmeNamespace
, n
->num_namespaces
);
818 n
->sq
= g_new0(NvmeSQueue
*, n
->num_queues
);
819 n
->cq
= g_new0(NvmeCQueue
*, n
->num_queues
);
821 memory_region_init_io(&n
->iomem
, OBJECT(n
), &nvme_mmio_ops
, n
,
822 "nvme", n
->reg_size
);
823 pci_register_bar(&n
->parent_obj
, 0,
824 PCI_BASE_ADDRESS_SPACE_MEMORY
| PCI_BASE_ADDRESS_MEM_TYPE_64
,
826 msix_init_exclusive_bar(&n
->parent_obj
, n
->num_queues
, 4);
828 id
->vid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_VENDOR_ID
));
829 id
->ssvid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_SUBSYSTEM_VENDOR_ID
));
830 strpadcpy((char *)id
->mn
, sizeof(id
->mn
), "QEMU NVMe Ctrl", ' ');
831 strpadcpy((char *)id
->fr
, sizeof(id
->fr
), "1.0", ' ');
832 strpadcpy((char *)id
->sn
, sizeof(id
->sn
), n
->serial
, ' ');
837 id
->oacs
= cpu_to_le16(0);
840 id
->sqes
= (0x6 << 4) | 0x6;
841 id
->cqes
= (0x4 << 4) | 0x4;
842 id
->nn
= cpu_to_le32(n
->num_namespaces
);
843 id
->psd
[0].mp
= cpu_to_le16(0x9c4);
844 id
->psd
[0].enlat
= cpu_to_le32(0x10);
845 id
->psd
[0].exlat
= cpu_to_le32(0x4);
846 if (blk_enable_write_cache(n
->conf
.blk
)) {
851 NVME_CAP_SET_MQES(n
->bar
.cap
, 0x7ff);
852 NVME_CAP_SET_CQR(n
->bar
.cap
, 1);
853 NVME_CAP_SET_AMS(n
->bar
.cap
, 1);
854 NVME_CAP_SET_TO(n
->bar
.cap
, 0xf);
855 NVME_CAP_SET_CSS(n
->bar
.cap
, 1);
856 NVME_CAP_SET_MPSMAX(n
->bar
.cap
, 4);
858 n
->bar
.vs
= 0x00010100;
859 n
->bar
.intmc
= n
->bar
.intms
= 0;
861 for (i
= 0; i
< n
->num_namespaces
; i
++) {
862 NvmeNamespace
*ns
= &n
->namespaces
[i
];
863 NvmeIdNs
*id_ns
= &ns
->id_ns
;
870 id_ns
->lbaf
[0].ds
= BDRV_SECTOR_BITS
;
871 id_ns
->ncap
= id_ns
->nuse
= id_ns
->nsze
=
872 cpu_to_le64(n
->ns_size
>>
873 id_ns
->lbaf
[NVME_ID_NS_FLBAS_INDEX(ns
->id_ns
.flbas
)].ds
);
878 static void nvme_exit(PCIDevice
*pci_dev
)
880 NvmeCtrl
*n
= NVME(pci_dev
);
883 g_free(n
->namespaces
);
886 msix_uninit_exclusive_bar(pci_dev
);
889 static Property nvme_props
[] = {
890 DEFINE_BLOCK_PROPERTIES(NvmeCtrl
, conf
),
891 DEFINE_PROP_STRING("serial", NvmeCtrl
, serial
),
892 DEFINE_PROP_END_OF_LIST(),
895 static const VMStateDescription nvme_vmstate
= {
900 static void nvme_class_init(ObjectClass
*oc
, void *data
)
902 DeviceClass
*dc
= DEVICE_CLASS(oc
);
903 PCIDeviceClass
*pc
= PCI_DEVICE_CLASS(oc
);
905 pc
->init
= nvme_init
;
906 pc
->exit
= nvme_exit
;
907 pc
->class_id
= PCI_CLASS_STORAGE_EXPRESS
;
908 pc
->vendor_id
= PCI_VENDOR_ID_INTEL
;
909 pc
->device_id
= 0x5845;
913 set_bit(DEVICE_CATEGORY_STORAGE
, dc
->categories
);
914 dc
->desc
= "Non-Volatile Memory Express";
915 dc
->props
= nvme_props
;
916 dc
->vmsd
= &nvme_vmstate
;
919 static void nvme_get_bootindex(Object
*obj
, Visitor
*v
, void *opaque
,
920 const char *name
, Error
**errp
)
922 NvmeCtrl
*s
= NVME(obj
);
924 visit_type_int32(v
, &s
->conf
.bootindex
, name
, errp
);
927 static void nvme_set_bootindex(Object
*obj
, Visitor
*v
, void *opaque
,
928 const char *name
, Error
**errp
)
930 NvmeCtrl
*s
= NVME(obj
);
932 Error
*local_err
= NULL
;
934 visit_type_int32(v
, &boot_index
, name
, &local_err
);
938 /* check whether bootindex is present in fw_boot_order list */
939 check_boot_index(boot_index
, &local_err
);
943 /* change bootindex to a new one */
944 s
->conf
.bootindex
= boot_index
;
948 error_propagate(errp
, local_err
);
952 static void nvme_instance_init(Object
*obj
)
954 object_property_add(obj
, "bootindex", "int32",
956 nvme_set_bootindex
, NULL
, NULL
, NULL
);
957 object_property_set_int(obj
, -1, "bootindex", NULL
);
960 static const TypeInfo nvme_info
= {
962 .parent
= TYPE_PCI_DEVICE
,
963 .instance_size
= sizeof(NvmeCtrl
),
964 .class_init
= nvme_class_init
,
965 .instance_init
= nvme_instance_init
,
968 static void nvme_register_types(void)
970 type_register_static(&nvme_info
);
973 type_init(nvme_register_types
)