2 * QEMU NVM Express Controller
4 * Copyright (c) 2012, Intel Corporation
6 * Written by Keith Busch <keith.busch@intel.com>
8 * This code is licensed under the GNU GPL v2 or later.
12 * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e
14 * https://nvmexpress.org/developers/nvme-specification/
19 * -drive file=<file>,if=none,id=<drive_id>
20 * -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id>
21 * -device nvme,serial=<serial>,id=<bus_name>, \
22 * cmb_size_mb=<cmb_size_mb[optional]>, \
23 * [pmrdev=<mem_backend_file_id>,] \
24 * max_ioqpairs=<N[optional]>, \
25 * aerl=<N[optional]>,aer_max_queued=<N[optional]>, \
26 * mdts=<N[optional]>,zoned.zasl=<N[optional]>, \
28 * -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
29 * zoned=<true|false[optional]>, \
30 * subsys=<subsys_id>,detached=<true|false[optional]>
32 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
33 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
34 * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to
35 * always enable the CMBLOC and CMBSZ registers (v1.3 behavior).
37 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
39 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
40 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
42 * The PMR will use BAR 4/5 exclusively.
44 * To place controller(s) and namespace(s) to a subsystem, then provide
45 * nvme-subsys device as above.
47 * nvme subsystem device parameters
48 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
50 * This parameter provides the `<nqn_id>` part of the string
51 * `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field
52 * of subsystem controllers. Note that `<nqn_id>` should be unique per
53 * subsystem, but this is not enforced by QEMU. If not specified, it will
54 * default to the value of the `id` parameter (`<subsys_id>`).
56 * nvme device parameters
57 * ~~~~~~~~~~~~~~~~~~~~~~
59 * Specifying this parameter attaches the controller to the subsystem and
60 * the SUBNQN field in the controller will report the NQN of the subsystem
61 * device. This also enables multi controller capability represented in
62 * Identify Controller data structure in CMIC (Controller Multi-path I/O and
63 * Namesapce Sharing Capabilities).
66 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
67 * of concurrently outstanding Asynchronous Event Request commands support
68 * by the controller. This is a 0's based value.
71 * This is the maximum number of events that the device will enqueue for
72 * completion when there are no outstanding AERs. When the maximum number of
73 * enqueued events are reached, subsequent events will be dropped.
76 * Indicates the maximum data transfer size for a command that transfers data
77 * between host-accessible memory and the controller. The value is specified
78 * as a power of two (2^n) and is in units of the minimum memory page size
79 * (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB).
82 * Indicates the maximum data transfer size for the Zone Append command. Like
83 * `mdts`, the value is specified as a power of two (2^n) and is in units of
84 * the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e.
85 * defaulting to the value of `mdts`).
87 * nvme namespace device parameters
88 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
90 * If given, the namespace will be attached to all controllers in the
91 * subsystem. Otherwise, `bus` must be given to attach this namespace to a
92 * specific controller as a non-shared namespace.
95 * This parameter is only valid together with the `subsys` parameter. If left
96 * at the default value (`false/off`), the namespace will be attached to all
97 * controllers in the NVMe subsystem at boot-up. If set to `true/on`, the
98 * namespace will be be available in the subsystem not not attached to any
101 * Setting `zoned` to true selects Zoned Command Set at the namespace.
102 * In this case, the following namespace properties are available to configure
104 * zoned.zone_size=<zone size in bytes, default: 128MiB>
105 * The number may be followed by K, M, G as in kilo-, mega- or giga-.
107 * zoned.zone_capacity=<zone capacity in bytes, default: zone size>
108 * The value 0 (default) forces zone capacity to be the same as zone
109 * size. The value of this property may not exceed zone size.
111 * zoned.descr_ext_size=<zone descriptor extension size, default 0>
112 * This value needs to be specified in 64B units. If it is zero,
113 * namespace(s) will not support zone descriptor extensions.
115 * zoned.max_active=<Maximum Active Resources (zones), default: 0>
116 * The default value means there is no limit to the number of
117 * concurrently active zones.
119 * zoned.max_open=<Maximum Open Resources (zones), default: 0>
120 * The default value means there is no limit to the number of
121 * concurrently open zones.
123 * zoned.cross_read=<enable RAZB, default: false>
124 * Setting this property to true enables Read Across Zone Boundaries.
127 #include "qemu/osdep.h"
128 #include "qemu/units.h"
129 #include "qemu/error-report.h"
130 #include "hw/block/block.h"
131 #include "hw/pci/msix.h"
132 #include "hw/pci/pci.h"
133 #include "hw/qdev-properties.h"
134 #include "migration/vmstate.h"
135 #include "sysemu/sysemu.h"
136 #include "qapi/error.h"
137 #include "qapi/visitor.h"
138 #include "sysemu/hostmem.h"
139 #include "sysemu/block-backend.h"
140 #include "exec/memory.h"
141 #include "qemu/log.h"
142 #include "qemu/module.h"
143 #include "qemu/cutils.h"
148 #define NVME_MAX_IOQPAIRS 0xffff
149 #define NVME_DB_SIZE 4
150 #define NVME_SPEC_VER 0x00010400
151 #define NVME_CMB_BIR 2
152 #define NVME_PMR_BIR 4
153 #define NVME_TEMPERATURE 0x143
154 #define NVME_TEMPERATURE_WARNING 0x157
155 #define NVME_TEMPERATURE_CRITICAL 0x175
156 #define NVME_NUM_FW_SLOTS 1
158 #define NVME_GUEST_ERR(trace, fmt, ...) \
160 (trace_##trace)(__VA_ARGS__); \
161 qemu_log_mask(LOG_GUEST_ERROR, #trace \
162 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
165 static const bool nvme_feature_support
[NVME_FID_MAX
] = {
166 [NVME_ARBITRATION
] = true,
167 [NVME_POWER_MANAGEMENT
] = true,
168 [NVME_TEMPERATURE_THRESHOLD
] = true,
169 [NVME_ERROR_RECOVERY
] = true,
170 [NVME_VOLATILE_WRITE_CACHE
] = true,
171 [NVME_NUMBER_OF_QUEUES
] = true,
172 [NVME_INTERRUPT_COALESCING
] = true,
173 [NVME_INTERRUPT_VECTOR_CONF
] = true,
174 [NVME_WRITE_ATOMICITY
] = true,
175 [NVME_ASYNCHRONOUS_EVENT_CONF
] = true,
176 [NVME_TIMESTAMP
] = true,
179 static const uint32_t nvme_feature_cap
[NVME_FID_MAX
] = {
180 [NVME_TEMPERATURE_THRESHOLD
] = NVME_FEAT_CAP_CHANGE
,
181 [NVME_ERROR_RECOVERY
] = NVME_FEAT_CAP_CHANGE
| NVME_FEAT_CAP_NS
,
182 [NVME_VOLATILE_WRITE_CACHE
] = NVME_FEAT_CAP_CHANGE
,
183 [NVME_NUMBER_OF_QUEUES
] = NVME_FEAT_CAP_CHANGE
,
184 [NVME_ASYNCHRONOUS_EVENT_CONF
] = NVME_FEAT_CAP_CHANGE
,
185 [NVME_TIMESTAMP
] = NVME_FEAT_CAP_CHANGE
,
188 static const uint32_t nvme_cse_acs
[256] = {
189 [NVME_ADM_CMD_DELETE_SQ
] = NVME_CMD_EFF_CSUPP
,
190 [NVME_ADM_CMD_CREATE_SQ
] = NVME_CMD_EFF_CSUPP
,
191 [NVME_ADM_CMD_GET_LOG_PAGE
] = NVME_CMD_EFF_CSUPP
,
192 [NVME_ADM_CMD_DELETE_CQ
] = NVME_CMD_EFF_CSUPP
,
193 [NVME_ADM_CMD_CREATE_CQ
] = NVME_CMD_EFF_CSUPP
,
194 [NVME_ADM_CMD_IDENTIFY
] = NVME_CMD_EFF_CSUPP
,
195 [NVME_ADM_CMD_ABORT
] = NVME_CMD_EFF_CSUPP
,
196 [NVME_ADM_CMD_SET_FEATURES
] = NVME_CMD_EFF_CSUPP
,
197 [NVME_ADM_CMD_GET_FEATURES
] = NVME_CMD_EFF_CSUPP
,
198 [NVME_ADM_CMD_ASYNC_EV_REQ
] = NVME_CMD_EFF_CSUPP
,
201 static const uint32_t nvme_cse_iocs_none
[256];
203 static const uint32_t nvme_cse_iocs_nvm
[256] = {
204 [NVME_CMD_FLUSH
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
205 [NVME_CMD_WRITE_ZEROES
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
206 [NVME_CMD_WRITE
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
207 [NVME_CMD_READ
] = NVME_CMD_EFF_CSUPP
,
208 [NVME_CMD_DSM
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
209 [NVME_CMD_COPY
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
210 [NVME_CMD_COMPARE
] = NVME_CMD_EFF_CSUPP
,
213 static const uint32_t nvme_cse_iocs_zoned
[256] = {
214 [NVME_CMD_FLUSH
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
215 [NVME_CMD_WRITE_ZEROES
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
216 [NVME_CMD_WRITE
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
217 [NVME_CMD_READ
] = NVME_CMD_EFF_CSUPP
,
218 [NVME_CMD_DSM
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
219 [NVME_CMD_COPY
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
220 [NVME_CMD_COMPARE
] = NVME_CMD_EFF_CSUPP
,
221 [NVME_CMD_ZONE_APPEND
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
222 [NVME_CMD_ZONE_MGMT_SEND
] = NVME_CMD_EFF_CSUPP
| NVME_CMD_EFF_LBCC
,
223 [NVME_CMD_ZONE_MGMT_RECV
] = NVME_CMD_EFF_CSUPP
,
226 static void nvme_process_sq(void *opaque
);
228 static uint16_t nvme_cid(NvmeRequest
*req
)
234 return le16_to_cpu(req
->cqe
.cid
);
237 static uint16_t nvme_sqid(NvmeRequest
*req
)
239 return le16_to_cpu(req
->sq
->sqid
);
242 static void nvme_assign_zone_state(NvmeNamespace
*ns
, NvmeZone
*zone
,
245 if (QTAILQ_IN_USE(zone
, entry
)) {
246 switch (nvme_get_zone_state(zone
)) {
247 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
248 QTAILQ_REMOVE(&ns
->exp_open_zones
, zone
, entry
);
250 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
251 QTAILQ_REMOVE(&ns
->imp_open_zones
, zone
, entry
);
253 case NVME_ZONE_STATE_CLOSED
:
254 QTAILQ_REMOVE(&ns
->closed_zones
, zone
, entry
);
256 case NVME_ZONE_STATE_FULL
:
257 QTAILQ_REMOVE(&ns
->full_zones
, zone
, entry
);
263 nvme_set_zone_state(zone
, state
);
266 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
267 QTAILQ_INSERT_TAIL(&ns
->exp_open_zones
, zone
, entry
);
269 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
270 QTAILQ_INSERT_TAIL(&ns
->imp_open_zones
, zone
, entry
);
272 case NVME_ZONE_STATE_CLOSED
:
273 QTAILQ_INSERT_TAIL(&ns
->closed_zones
, zone
, entry
);
275 case NVME_ZONE_STATE_FULL
:
276 QTAILQ_INSERT_TAIL(&ns
->full_zones
, zone
, entry
);
277 case NVME_ZONE_STATE_READ_ONLY
:
285 * Check if we can open a zone without exceeding open/active limits.
286 * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5).
288 static int nvme_aor_check(NvmeNamespace
*ns
, uint32_t act
, uint32_t opn
)
290 if (ns
->params
.max_active_zones
!= 0 &&
291 ns
->nr_active_zones
+ act
> ns
->params
.max_active_zones
) {
292 trace_pci_nvme_err_insuff_active_res(ns
->params
.max_active_zones
);
293 return NVME_ZONE_TOO_MANY_ACTIVE
| NVME_DNR
;
295 if (ns
->params
.max_open_zones
!= 0 &&
296 ns
->nr_open_zones
+ opn
> ns
->params
.max_open_zones
) {
297 trace_pci_nvme_err_insuff_open_res(ns
->params
.max_open_zones
);
298 return NVME_ZONE_TOO_MANY_OPEN
| NVME_DNR
;
304 static bool nvme_addr_is_cmb(NvmeCtrl
*n
, hwaddr addr
)
312 lo
= n
->params
.legacy_cmb
? n
->cmb
.mem
.addr
: n
->cmb
.cba
;
313 hi
= lo
+ int128_get64(n
->cmb
.mem
.size
);
315 return addr
>= lo
&& addr
< hi
;
318 static inline void *nvme_addr_to_cmb(NvmeCtrl
*n
, hwaddr addr
)
320 hwaddr base
= n
->params
.legacy_cmb
? n
->cmb
.mem
.addr
: n
->cmb
.cba
;
321 return &n
->cmb
.buf
[addr
- base
];
324 static bool nvme_addr_is_pmr(NvmeCtrl
*n
, hwaddr addr
)
332 hi
= n
->pmr
.cba
+ int128_get64(n
->pmr
.dev
->mr
.size
);
334 return addr
>= n
->pmr
.cba
&& addr
< hi
;
337 static inline void *nvme_addr_to_pmr(NvmeCtrl
*n
, hwaddr addr
)
339 return memory_region_get_ram_ptr(&n
->pmr
.dev
->mr
) + (addr
- n
->pmr
.cba
);
342 static int nvme_addr_read(NvmeCtrl
*n
, hwaddr addr
, void *buf
, int size
)
344 hwaddr hi
= addr
+ size
- 1;
349 if (n
->bar
.cmbsz
&& nvme_addr_is_cmb(n
, addr
) && nvme_addr_is_cmb(n
, hi
)) {
350 memcpy(buf
, nvme_addr_to_cmb(n
, addr
), size
);
354 if (nvme_addr_is_pmr(n
, addr
) && nvme_addr_is_pmr(n
, hi
)) {
355 memcpy(buf
, nvme_addr_to_pmr(n
, addr
), size
);
359 return pci_dma_read(&n
->parent_obj
, addr
, buf
, size
);
362 static bool nvme_nsid_valid(NvmeCtrl
*n
, uint32_t nsid
)
364 return nsid
&& (nsid
== NVME_NSID_BROADCAST
|| nsid
<= n
->num_namespaces
);
367 static int nvme_check_sqid(NvmeCtrl
*n
, uint16_t sqid
)
369 return sqid
< n
->params
.max_ioqpairs
+ 1 && n
->sq
[sqid
] != NULL
? 0 : -1;
372 static int nvme_check_cqid(NvmeCtrl
*n
, uint16_t cqid
)
374 return cqid
< n
->params
.max_ioqpairs
+ 1 && n
->cq
[cqid
] != NULL
? 0 : -1;
377 static void nvme_inc_cq_tail(NvmeCQueue
*cq
)
380 if (cq
->tail
>= cq
->size
) {
382 cq
->phase
= !cq
->phase
;
386 static void nvme_inc_sq_head(NvmeSQueue
*sq
)
388 sq
->head
= (sq
->head
+ 1) % sq
->size
;
391 static uint8_t nvme_cq_full(NvmeCQueue
*cq
)
393 return (cq
->tail
+ 1) % cq
->size
== cq
->head
;
396 static uint8_t nvme_sq_empty(NvmeSQueue
*sq
)
398 return sq
->head
== sq
->tail
;
401 static void nvme_irq_check(NvmeCtrl
*n
)
403 if (msix_enabled(&(n
->parent_obj
))) {
406 if (~n
->bar
.intms
& n
->irq_status
) {
407 pci_irq_assert(&n
->parent_obj
);
409 pci_irq_deassert(&n
->parent_obj
);
413 static void nvme_irq_assert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
415 if (cq
->irq_enabled
) {
416 if (msix_enabled(&(n
->parent_obj
))) {
417 trace_pci_nvme_irq_msix(cq
->vector
);
418 msix_notify(&(n
->parent_obj
), cq
->vector
);
420 trace_pci_nvme_irq_pin();
421 assert(cq
->vector
< 32);
422 n
->irq_status
|= 1 << cq
->vector
;
426 trace_pci_nvme_irq_masked();
430 static void nvme_irq_deassert(NvmeCtrl
*n
, NvmeCQueue
*cq
)
432 if (cq
->irq_enabled
) {
433 if (msix_enabled(&(n
->parent_obj
))) {
436 assert(cq
->vector
< 32);
437 n
->irq_status
&= ~(1 << cq
->vector
);
443 static void nvme_req_clear(NvmeRequest
*req
)
447 memset(&req
->cqe
, 0x0, sizeof(req
->cqe
));
448 req
->status
= NVME_SUCCESS
;
451 static inline void nvme_sg_init(NvmeCtrl
*n
, NvmeSg
*sg
, bool dma
)
454 pci_dma_sglist_init(&sg
->qsg
, &n
->parent_obj
, 0);
455 sg
->flags
= NVME_SG_DMA
;
457 qemu_iovec_init(&sg
->iov
, 0);
460 sg
->flags
|= NVME_SG_ALLOC
;
463 static inline void nvme_sg_unmap(NvmeSg
*sg
)
465 if (!(sg
->flags
& NVME_SG_ALLOC
)) {
469 if (sg
->flags
& NVME_SG_DMA
) {
470 qemu_sglist_destroy(&sg
->qsg
);
472 qemu_iovec_destroy(&sg
->iov
);
475 memset(sg
, 0x0, sizeof(*sg
));
478 static uint16_t nvme_map_addr_cmb(NvmeCtrl
*n
, QEMUIOVector
*iov
, hwaddr addr
,
485 trace_pci_nvme_map_addr_cmb(addr
, len
);
487 if (!nvme_addr_is_cmb(n
, addr
) || !nvme_addr_is_cmb(n
, addr
+ len
- 1)) {
488 return NVME_DATA_TRAS_ERROR
;
491 qemu_iovec_add(iov
, nvme_addr_to_cmb(n
, addr
), len
);
496 static uint16_t nvme_map_addr_pmr(NvmeCtrl
*n
, QEMUIOVector
*iov
, hwaddr addr
,
503 if (!nvme_addr_is_pmr(n
, addr
) || !nvme_addr_is_pmr(n
, addr
+ len
- 1)) {
504 return NVME_DATA_TRAS_ERROR
;
507 qemu_iovec_add(iov
, nvme_addr_to_pmr(n
, addr
), len
);
512 static uint16_t nvme_map_addr(NvmeCtrl
*n
, NvmeSg
*sg
, hwaddr addr
, size_t len
)
514 bool cmb
= false, pmr
= false;
520 trace_pci_nvme_map_addr(addr
, len
);
522 if (nvme_addr_is_cmb(n
, addr
)) {
524 } else if (nvme_addr_is_pmr(n
, addr
)) {
529 if (sg
->flags
& NVME_SG_DMA
) {
530 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
534 return nvme_map_addr_cmb(n
, &sg
->iov
, addr
, len
);
536 return nvme_map_addr_pmr(n
, &sg
->iov
, addr
, len
);
540 if (!(sg
->flags
& NVME_SG_DMA
)) {
541 return NVME_INVALID_USE_OF_CMB
| NVME_DNR
;
544 qemu_sglist_add(&sg
->qsg
, addr
, len
);
549 static inline bool nvme_addr_is_dma(NvmeCtrl
*n
, hwaddr addr
)
551 return !(nvme_addr_is_cmb(n
, addr
) || nvme_addr_is_pmr(n
, addr
));
554 static uint16_t nvme_map_prp(NvmeCtrl
*n
, NvmeSg
*sg
, uint64_t prp1
,
555 uint64_t prp2
, uint32_t len
)
557 hwaddr trans_len
= n
->page_size
- (prp1
% n
->page_size
);
558 trans_len
= MIN(len
, trans_len
);
559 int num_prps
= (len
>> n
->page_bits
) + 1;
563 trace_pci_nvme_map_prp(trans_len
, len
, prp1
, prp2
, num_prps
);
565 nvme_sg_init(n
, sg
, nvme_addr_is_dma(n
, prp1
));
567 status
= nvme_map_addr(n
, sg
, prp1
, trans_len
);
574 if (len
> n
->page_size
) {
575 uint64_t prp_list
[n
->max_prp_ents
];
576 uint32_t nents
, prp_trans
;
579 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
580 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
581 ret
= nvme_addr_read(n
, prp2
, (void *)prp_list
, prp_trans
);
583 trace_pci_nvme_err_addr_read(prp2
);
584 status
= NVME_DATA_TRAS_ERROR
;
588 uint64_t prp_ent
= le64_to_cpu(prp_list
[i
]);
590 if (i
== n
->max_prp_ents
- 1 && len
> n
->page_size
) {
591 if (unlikely(prp_ent
& (n
->page_size
- 1))) {
592 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
593 status
= NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
598 nents
= (len
+ n
->page_size
- 1) >> n
->page_bits
;
599 prp_trans
= MIN(n
->max_prp_ents
, nents
) * sizeof(uint64_t);
600 ret
= nvme_addr_read(n
, prp_ent
, (void *)prp_list
,
603 trace_pci_nvme_err_addr_read(prp_ent
);
604 status
= NVME_DATA_TRAS_ERROR
;
607 prp_ent
= le64_to_cpu(prp_list
[i
]);
610 if (unlikely(prp_ent
& (n
->page_size
- 1))) {
611 trace_pci_nvme_err_invalid_prplist_ent(prp_ent
);
612 status
= NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
616 trans_len
= MIN(len
, n
->page_size
);
617 status
= nvme_map_addr(n
, sg
, prp_ent
, trans_len
);
626 if (unlikely(prp2
& (n
->page_size
- 1))) {
627 trace_pci_nvme_err_invalid_prp2_align(prp2
);
628 status
= NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
631 status
= nvme_map_addr(n
, sg
, prp2
, len
);
646 * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
647 * number of bytes mapped in len.
649 static uint16_t nvme_map_sgl_data(NvmeCtrl
*n
, NvmeSg
*sg
,
650 NvmeSglDescriptor
*segment
, uint64_t nsgld
,
651 size_t *len
, NvmeCmd
*cmd
)
653 dma_addr_t addr
, trans_len
;
657 for (int i
= 0; i
< nsgld
; i
++) {
658 uint8_t type
= NVME_SGL_TYPE(segment
[i
].type
);
661 case NVME_SGL_DESCR_TYPE_BIT_BUCKET
:
662 if (cmd
->opcode
== NVME_CMD_WRITE
) {
665 case NVME_SGL_DESCR_TYPE_DATA_BLOCK
:
667 case NVME_SGL_DESCR_TYPE_SEGMENT
:
668 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT
:
669 return NVME_INVALID_NUM_SGL_DESCRS
| NVME_DNR
;
671 return NVME_SGL_DESCR_TYPE_INVALID
| NVME_DNR
;
674 dlen
= le32_to_cpu(segment
[i
].len
);
682 * All data has been mapped, but the SGL contains additional
683 * segments and/or descriptors. The controller might accept
684 * ignoring the rest of the SGL.
686 uint32_t sgls
= le32_to_cpu(n
->id_ctrl
.sgls
);
687 if (sgls
& NVME_CTRL_SGLS_EXCESS_LENGTH
) {
691 trace_pci_nvme_err_invalid_sgl_excess_length(dlen
);
692 return NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
695 trans_len
= MIN(*len
, dlen
);
697 if (type
== NVME_SGL_DESCR_TYPE_BIT_BUCKET
) {
701 addr
= le64_to_cpu(segment
[i
].addr
);
703 if (UINT64_MAX
- addr
< dlen
) {
704 return NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
707 status
= nvme_map_addr(n
, sg
, addr
, trans_len
);
719 static uint16_t nvme_map_sgl(NvmeCtrl
*n
, NvmeSg
*sg
, NvmeSglDescriptor sgl
,
720 size_t len
, NvmeCmd
*cmd
)
723 * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
724 * dynamically allocating a potentially huge SGL. The spec allows the SGL
725 * to be larger (as in number of bytes required to describe the SGL
726 * descriptors and segment chain) than the command transfer size, so it is
727 * not bounded by MDTS.
729 const int SEG_CHUNK_SIZE
= 256;
731 NvmeSglDescriptor segment
[SEG_CHUNK_SIZE
], *sgld
, *last_sgld
;
739 addr
= le64_to_cpu(sgl
.addr
);
741 trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl
.type
), len
);
743 nvme_sg_init(n
, sg
, nvme_addr_is_dma(n
, addr
));
746 * If the entire transfer can be described with a single data block it can
747 * be mapped directly.
749 if (NVME_SGL_TYPE(sgl
.type
) == NVME_SGL_DESCR_TYPE_DATA_BLOCK
) {
750 status
= nvme_map_sgl_data(n
, sg
, sgld
, 1, &len
, cmd
);
759 switch (NVME_SGL_TYPE(sgld
->type
)) {
760 case NVME_SGL_DESCR_TYPE_SEGMENT
:
761 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT
:
764 return NVME_INVALID_SGL_SEG_DESCR
| NVME_DNR
;
767 seg_len
= le32_to_cpu(sgld
->len
);
769 /* check the length of the (Last) Segment descriptor */
770 if ((!seg_len
|| seg_len
& 0xf) &&
771 (NVME_SGL_TYPE(sgld
->type
) != NVME_SGL_DESCR_TYPE_BIT_BUCKET
)) {
772 return NVME_INVALID_SGL_SEG_DESCR
| NVME_DNR
;
775 if (UINT64_MAX
- addr
< seg_len
) {
776 return NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
779 nsgld
= seg_len
/ sizeof(NvmeSglDescriptor
);
781 while (nsgld
> SEG_CHUNK_SIZE
) {
782 if (nvme_addr_read(n
, addr
, segment
, sizeof(segment
))) {
783 trace_pci_nvme_err_addr_read(addr
);
784 status
= NVME_DATA_TRAS_ERROR
;
788 status
= nvme_map_sgl_data(n
, sg
, segment
, SEG_CHUNK_SIZE
,
794 nsgld
-= SEG_CHUNK_SIZE
;
795 addr
+= SEG_CHUNK_SIZE
* sizeof(NvmeSglDescriptor
);
798 ret
= nvme_addr_read(n
, addr
, segment
, nsgld
*
799 sizeof(NvmeSglDescriptor
));
801 trace_pci_nvme_err_addr_read(addr
);
802 status
= NVME_DATA_TRAS_ERROR
;
806 last_sgld
= &segment
[nsgld
- 1];
809 * If the segment ends with a Data Block or Bit Bucket Descriptor Type,
812 switch (NVME_SGL_TYPE(last_sgld
->type
)) {
813 case NVME_SGL_DESCR_TYPE_DATA_BLOCK
:
814 case NVME_SGL_DESCR_TYPE_BIT_BUCKET
:
815 status
= nvme_map_sgl_data(n
, sg
, segment
, nsgld
, &len
, cmd
);
827 * If the last descriptor was not a Data Block or Bit Bucket, then the
828 * current segment must not be a Last Segment.
830 if (NVME_SGL_TYPE(sgld
->type
) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT
) {
831 status
= NVME_INVALID_SGL_SEG_DESCR
| NVME_DNR
;
836 addr
= le64_to_cpu(sgld
->addr
);
839 * Do not map the last descriptor; it will be a Segment or Last Segment
840 * descriptor and is handled by the next iteration.
842 status
= nvme_map_sgl_data(n
, sg
, segment
, nsgld
- 1, &len
, cmd
);
849 /* if there is any residual left in len, the SGL was too short */
851 status
= NVME_DATA_SGL_LEN_INVALID
| NVME_DNR
;
862 static uint16_t nvme_map_dptr(NvmeCtrl
*n
, NvmeSg
*sg
, size_t len
,
867 switch (NVME_CMD_FLAGS_PSDT(cmd
->flags
)) {
869 prp1
= le64_to_cpu(cmd
->dptr
.prp1
);
870 prp2
= le64_to_cpu(cmd
->dptr
.prp2
);
872 return nvme_map_prp(n
, sg
, prp1
, prp2
, len
);
873 case NVME_PSDT_SGL_MPTR_CONTIGUOUS
:
874 case NVME_PSDT_SGL_MPTR_SGL
:
875 return nvme_map_sgl(n
, sg
, cmd
->dptr
.sgl
, len
, cmd
);
877 return NVME_INVALID_FIELD
;
881 typedef enum NvmeTxDirection
{
882 NVME_TX_DIRECTION_TO_DEVICE
= 0,
883 NVME_TX_DIRECTION_FROM_DEVICE
= 1,
886 static uint16_t nvme_tx(NvmeCtrl
*n
, NvmeSg
*sg
, uint8_t *ptr
, uint32_t len
,
889 assert(sg
->flags
& NVME_SG_ALLOC
);
891 if (sg
->flags
& NVME_SG_DMA
) {
894 if (dir
== NVME_TX_DIRECTION_TO_DEVICE
) {
895 residual
= dma_buf_write(ptr
, len
, &sg
->qsg
);
897 residual
= dma_buf_read(ptr
, len
, &sg
->qsg
);
900 if (unlikely(residual
)) {
901 trace_pci_nvme_err_invalid_dma();
902 return NVME_INVALID_FIELD
| NVME_DNR
;
907 if (dir
== NVME_TX_DIRECTION_TO_DEVICE
) {
908 bytes
= qemu_iovec_to_buf(&sg
->iov
, 0, ptr
, len
);
910 bytes
= qemu_iovec_from_buf(&sg
->iov
, 0, ptr
, len
);
913 if (unlikely(bytes
!= len
)) {
914 trace_pci_nvme_err_invalid_dma();
915 return NVME_INVALID_FIELD
| NVME_DNR
;
922 static inline uint16_t nvme_c2h(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
927 status
= nvme_map_dptr(n
, &req
->sg
, len
, &req
->cmd
);
932 return nvme_tx(n
, &req
->sg
, ptr
, len
, NVME_TX_DIRECTION_FROM_DEVICE
);
935 static inline uint16_t nvme_h2c(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
940 status
= nvme_map_dptr(n
, &req
->sg
, len
, &req
->cmd
);
945 return nvme_tx(n
, &req
->sg
, ptr
, len
, NVME_TX_DIRECTION_TO_DEVICE
);
948 static inline void nvme_blk_read(BlockBackend
*blk
, int64_t offset
,
949 BlockCompletionFunc
*cb
, NvmeRequest
*req
)
951 assert(req
->sg
.flags
& NVME_SG_ALLOC
);
953 if (req
->sg
.flags
& NVME_SG_DMA
) {
954 req
->aiocb
= dma_blk_read(blk
, &req
->sg
.qsg
, offset
, BDRV_SECTOR_SIZE
,
957 req
->aiocb
= blk_aio_preadv(blk
, offset
, &req
->sg
.iov
, 0, cb
, req
);
961 static inline void nvme_blk_write(BlockBackend
*blk
, int64_t offset
,
962 BlockCompletionFunc
*cb
, NvmeRequest
*req
)
964 assert(req
->sg
.flags
& NVME_SG_ALLOC
);
966 if (req
->sg
.flags
& NVME_SG_DMA
) {
967 req
->aiocb
= dma_blk_write(blk
, &req
->sg
.qsg
, offset
, BDRV_SECTOR_SIZE
,
970 req
->aiocb
= blk_aio_pwritev(blk
, offset
, &req
->sg
.iov
, 0, cb
, req
);
974 static void nvme_post_cqes(void *opaque
)
976 NvmeCQueue
*cq
= opaque
;
977 NvmeCtrl
*n
= cq
->ctrl
;
978 NvmeRequest
*req
, *next
;
981 QTAILQ_FOREACH_SAFE(req
, &cq
->req_list
, entry
, next
) {
985 if (nvme_cq_full(cq
)) {
990 req
->cqe
.status
= cpu_to_le16((req
->status
<< 1) | cq
->phase
);
991 req
->cqe
.sq_id
= cpu_to_le16(sq
->sqid
);
992 req
->cqe
.sq_head
= cpu_to_le16(sq
->head
);
993 addr
= cq
->dma_addr
+ cq
->tail
* n
->cqe_size
;
994 ret
= pci_dma_write(&n
->parent_obj
, addr
, (void *)&req
->cqe
,
997 trace_pci_nvme_err_addr_write(addr
);
998 trace_pci_nvme_err_cfs();
999 n
->bar
.csts
= NVME_CSTS_FAILED
;
1002 QTAILQ_REMOVE(&cq
->req_list
, req
, entry
);
1003 nvme_inc_cq_tail(cq
);
1004 nvme_sg_unmap(&req
->sg
);
1005 QTAILQ_INSERT_TAIL(&sq
->req_list
, req
, entry
);
1007 if (cq
->tail
!= cq
->head
) {
1008 nvme_irq_assert(n
, cq
);
1012 static void nvme_enqueue_req_completion(NvmeCQueue
*cq
, NvmeRequest
*req
)
1014 assert(cq
->cqid
== req
->sq
->cqid
);
1015 trace_pci_nvme_enqueue_req_completion(nvme_cid(req
), cq
->cqid
,
1019 trace_pci_nvme_err_req_status(nvme_cid(req
), nvme_nsid(req
->ns
),
1020 req
->status
, req
->cmd
.opcode
);
1023 QTAILQ_REMOVE(&req
->sq
->out_req_list
, req
, entry
);
1024 QTAILQ_INSERT_TAIL(&cq
->req_list
, req
, entry
);
1025 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
1028 static void nvme_process_aers(void *opaque
)
1030 NvmeCtrl
*n
= opaque
;
1031 NvmeAsyncEvent
*event
, *next
;
1033 trace_pci_nvme_process_aers(n
->aer_queued
);
1035 QTAILQ_FOREACH_SAFE(event
, &n
->aer_queue
, entry
, next
) {
1037 NvmeAerResult
*result
;
1039 /* can't post cqe if there is nothing to complete */
1040 if (!n
->outstanding_aers
) {
1041 trace_pci_nvme_no_outstanding_aers();
1045 /* ignore if masked (cqe posted, but event not cleared) */
1046 if (n
->aer_mask
& (1 << event
->result
.event_type
)) {
1047 trace_pci_nvme_aer_masked(event
->result
.event_type
, n
->aer_mask
);
1051 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
1054 n
->aer_mask
|= 1 << event
->result
.event_type
;
1055 n
->outstanding_aers
--;
1057 req
= n
->aer_reqs
[n
->outstanding_aers
];
1059 result
= (NvmeAerResult
*) &req
->cqe
.result
;
1060 result
->event_type
= event
->result
.event_type
;
1061 result
->event_info
= event
->result
.event_info
;
1062 result
->log_page
= event
->result
.log_page
;
1065 trace_pci_nvme_aer_post_cqe(result
->event_type
, result
->event_info
,
1068 nvme_enqueue_req_completion(&n
->admin_cq
, req
);
1072 static void nvme_enqueue_event(NvmeCtrl
*n
, uint8_t event_type
,
1073 uint8_t event_info
, uint8_t log_page
)
1075 NvmeAsyncEvent
*event
;
1077 trace_pci_nvme_enqueue_event(event_type
, event_info
, log_page
);
1079 if (n
->aer_queued
== n
->params
.aer_max_queued
) {
1080 trace_pci_nvme_enqueue_event_noqueue(n
->aer_queued
);
1084 event
= g_new(NvmeAsyncEvent
, 1);
1085 event
->result
= (NvmeAerResult
) {
1086 .event_type
= event_type
,
1087 .event_info
= event_info
,
1088 .log_page
= log_page
,
1091 QTAILQ_INSERT_TAIL(&n
->aer_queue
, event
, entry
);
1094 nvme_process_aers(n
);
1097 static void nvme_smart_event(NvmeCtrl
*n
, uint8_t event
)
1101 /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */
1102 if (!(NVME_AEC_SMART(n
->features
.async_config
) & event
)) {
1107 case NVME_SMART_SPARE
:
1108 aer_info
= NVME_AER_INFO_SMART_SPARE_THRESH
;
1110 case NVME_SMART_TEMPERATURE
:
1111 aer_info
= NVME_AER_INFO_SMART_TEMP_THRESH
;
1113 case NVME_SMART_RELIABILITY
:
1114 case NVME_SMART_MEDIA_READ_ONLY
:
1115 case NVME_SMART_FAILED_VOLATILE_MEDIA
:
1116 case NVME_SMART_PMR_UNRELIABLE
:
1117 aer_info
= NVME_AER_INFO_SMART_RELIABILITY
;
1123 nvme_enqueue_event(n
, NVME_AER_TYPE_SMART
, aer_info
, NVME_LOG_SMART_INFO
);
1126 static void nvme_clear_events(NvmeCtrl
*n
, uint8_t event_type
)
1128 n
->aer_mask
&= ~(1 << event_type
);
1129 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
1130 nvme_process_aers(n
);
1134 static inline uint16_t nvme_check_mdts(NvmeCtrl
*n
, size_t len
)
1136 uint8_t mdts
= n
->params
.mdts
;
1138 if (mdts
&& len
> n
->page_size
<< mdts
) {
1139 trace_pci_nvme_err_mdts(len
);
1140 return NVME_INVALID_FIELD
| NVME_DNR
;
1143 return NVME_SUCCESS
;
1146 static inline uint16_t nvme_check_bounds(NvmeNamespace
*ns
, uint64_t slba
,
1149 uint64_t nsze
= le64_to_cpu(ns
->id_ns
.nsze
);
1151 if (unlikely(UINT64_MAX
- slba
< nlb
|| slba
+ nlb
> nsze
)) {
1152 return NVME_LBA_RANGE
| NVME_DNR
;
1155 return NVME_SUCCESS
;
1158 static uint16_t nvme_check_dulbe(NvmeNamespace
*ns
, uint64_t slba
,
1161 BlockDriverState
*bs
= blk_bs(ns
->blkconf
.blk
);
1163 int64_t pnum
= 0, bytes
= nvme_l2b(ns
, nlb
);
1164 int64_t offset
= nvme_l2b(ns
, slba
);
1168 Error
*local_err
= NULL
;
1171 * `pnum` holds the number of bytes after offset that shares the same
1172 * allocation status as the byte at offset. If `pnum` is different from
1173 * `bytes`, we should check the allocation status of the next range and
1174 * continue this until all bytes have been checked.
1179 ret
= bdrv_block_status(bs
, offset
, bytes
, &pnum
, NULL
, NULL
);
1181 error_setg_errno(&local_err
, -ret
, "unable to get block status");
1182 error_report_err(local_err
);
1184 return NVME_INTERNAL_DEV_ERROR
;
1187 zeroed
= !!(ret
& BDRV_BLOCK_ZERO
);
1189 trace_pci_nvme_block_status(offset
, bytes
, pnum
, ret
, zeroed
);
1196 } while (pnum
!= bytes
);
1198 return NVME_SUCCESS
;
1201 static void nvme_aio_err(NvmeRequest
*req
, int ret
)
1203 uint16_t status
= NVME_SUCCESS
;
1204 Error
*local_err
= NULL
;
1206 switch (req
->cmd
.opcode
) {
1208 status
= NVME_UNRECOVERED_READ
;
1210 case NVME_CMD_FLUSH
:
1211 case NVME_CMD_WRITE
:
1212 case NVME_CMD_WRITE_ZEROES
:
1213 case NVME_CMD_ZONE_APPEND
:
1214 status
= NVME_WRITE_FAULT
;
1217 status
= NVME_INTERNAL_DEV_ERROR
;
1221 trace_pci_nvme_err_aio(nvme_cid(req
), strerror(-ret
), status
);
1223 error_setg_errno(&local_err
, -ret
, "aio failed");
1224 error_report_err(local_err
);
1227 * Set the command status code to the first encountered error but allow a
1228 * subsequent Internal Device Error to trump it.
1230 if (req
->status
&& status
!= NVME_INTERNAL_DEV_ERROR
) {
1234 req
->status
= status
;
1237 static inline uint32_t nvme_zone_idx(NvmeNamespace
*ns
, uint64_t slba
)
1239 return ns
->zone_size_log2
> 0 ? slba
>> ns
->zone_size_log2
:
1240 slba
/ ns
->zone_size
;
1243 static inline NvmeZone
*nvme_get_zone_by_slba(NvmeNamespace
*ns
, uint64_t slba
)
1245 uint32_t zone_idx
= nvme_zone_idx(ns
, slba
);
1247 assert(zone_idx
< ns
->num_zones
);
1248 return &ns
->zone_array
[zone_idx
];
1251 static uint16_t nvme_check_zone_state_for_write(NvmeZone
*zone
)
1253 uint64_t zslba
= zone
->d
.zslba
;
1255 switch (nvme_get_zone_state(zone
)) {
1256 case NVME_ZONE_STATE_EMPTY
:
1257 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1258 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1259 case NVME_ZONE_STATE_CLOSED
:
1260 return NVME_SUCCESS
;
1261 case NVME_ZONE_STATE_FULL
:
1262 trace_pci_nvme_err_zone_is_full(zslba
);
1263 return NVME_ZONE_FULL
;
1264 case NVME_ZONE_STATE_OFFLINE
:
1265 trace_pci_nvme_err_zone_is_offline(zslba
);
1266 return NVME_ZONE_OFFLINE
;
1267 case NVME_ZONE_STATE_READ_ONLY
:
1268 trace_pci_nvme_err_zone_is_read_only(zslba
);
1269 return NVME_ZONE_READ_ONLY
;
1274 return NVME_INTERNAL_DEV_ERROR
;
1277 static uint16_t nvme_check_zone_write(NvmeNamespace
*ns
, NvmeZone
*zone
,
1278 uint64_t slba
, uint32_t nlb
)
1280 uint64_t zcap
= nvme_zone_wr_boundary(zone
);
1283 status
= nvme_check_zone_state_for_write(zone
);
1288 if (unlikely(slba
!= zone
->w_ptr
)) {
1289 trace_pci_nvme_err_write_not_at_wp(slba
, zone
->d
.zslba
, zone
->w_ptr
);
1290 return NVME_ZONE_INVALID_WRITE
;
1293 if (unlikely((slba
+ nlb
) > zcap
)) {
1294 trace_pci_nvme_err_zone_boundary(slba
, nlb
, zcap
);
1295 return NVME_ZONE_BOUNDARY_ERROR
;
1298 return NVME_SUCCESS
;
1301 static uint16_t nvme_check_zone_state_for_read(NvmeZone
*zone
)
1303 switch (nvme_get_zone_state(zone
)) {
1304 case NVME_ZONE_STATE_EMPTY
:
1305 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1306 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1307 case NVME_ZONE_STATE_FULL
:
1308 case NVME_ZONE_STATE_CLOSED
:
1309 case NVME_ZONE_STATE_READ_ONLY
:
1310 return NVME_SUCCESS
;
1311 case NVME_ZONE_STATE_OFFLINE
:
1312 trace_pci_nvme_err_zone_is_offline(zone
->d
.zslba
);
1313 return NVME_ZONE_OFFLINE
;
1318 return NVME_INTERNAL_DEV_ERROR
;
1321 static uint16_t nvme_check_zone_read(NvmeNamespace
*ns
, uint64_t slba
,
1324 NvmeZone
*zone
= nvme_get_zone_by_slba(ns
, slba
);
1325 uint64_t bndry
= nvme_zone_rd_boundary(ns
, zone
);
1326 uint64_t end
= slba
+ nlb
;
1329 status
= nvme_check_zone_state_for_read(zone
);
1332 } else if (unlikely(end
> bndry
)) {
1333 if (!ns
->params
.cross_zone_read
) {
1334 status
= NVME_ZONE_BOUNDARY_ERROR
;
1337 * Read across zone boundary - check that all subsequent
1338 * zones that are being read have an appropriate state.
1342 status
= nvme_check_zone_state_for_read(zone
);
1346 } while (end
> nvme_zone_rd_boundary(ns
, zone
));
1353 static uint16_t nvme_zrm_finish(NvmeNamespace
*ns
, NvmeZone
*zone
)
1355 switch (nvme_get_zone_state(zone
)) {
1356 case NVME_ZONE_STATE_FULL
:
1357 return NVME_SUCCESS
;
1359 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1360 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1361 nvme_aor_dec_open(ns
);
1363 case NVME_ZONE_STATE_CLOSED
:
1364 nvme_aor_dec_active(ns
);
1366 case NVME_ZONE_STATE_EMPTY
:
1367 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_FULL
);
1368 return NVME_SUCCESS
;
1371 return NVME_ZONE_INVAL_TRANSITION
;
1375 static uint16_t nvme_zrm_close(NvmeNamespace
*ns
, NvmeZone
*zone
)
1377 switch (nvme_get_zone_state(zone
)) {
1378 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1379 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1380 nvme_aor_dec_open(ns
);
1381 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_CLOSED
);
1383 case NVME_ZONE_STATE_CLOSED
:
1384 return NVME_SUCCESS
;
1387 return NVME_ZONE_INVAL_TRANSITION
;
1391 static void nvme_zrm_auto_transition_zone(NvmeNamespace
*ns
)
1395 if (ns
->params
.max_open_zones
&&
1396 ns
->nr_open_zones
== ns
->params
.max_open_zones
) {
1397 zone
= QTAILQ_FIRST(&ns
->imp_open_zones
);
1400 * Automatically close this implicitly open zone.
1402 QTAILQ_REMOVE(&ns
->imp_open_zones
, zone
, entry
);
1403 nvme_zrm_close(ns
, zone
);
1408 static uint16_t __nvme_zrm_open(NvmeNamespace
*ns
, NvmeZone
*zone
,
1414 switch (nvme_get_zone_state(zone
)) {
1415 case NVME_ZONE_STATE_EMPTY
:
1420 case NVME_ZONE_STATE_CLOSED
:
1421 nvme_zrm_auto_transition_zone(ns
);
1422 status
= nvme_aor_check(ns
, act
, 1);
1428 nvme_aor_inc_active(ns
);
1431 nvme_aor_inc_open(ns
);
1434 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_IMPLICITLY_OPEN
);
1435 return NVME_SUCCESS
;
1440 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1442 return NVME_SUCCESS
;
1445 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_EXPLICITLY_OPEN
);
1449 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1450 return NVME_SUCCESS
;
1453 return NVME_ZONE_INVAL_TRANSITION
;
1457 static inline uint16_t nvme_zrm_auto(NvmeNamespace
*ns
, NvmeZone
*zone
)
1459 return __nvme_zrm_open(ns
, zone
, true);
1462 static inline uint16_t nvme_zrm_open(NvmeNamespace
*ns
, NvmeZone
*zone
)
1464 return __nvme_zrm_open(ns
, zone
, false);
1467 static void __nvme_advance_zone_wp(NvmeNamespace
*ns
, NvmeZone
*zone
,
1472 if (zone
->d
.wp
== nvme_zone_wr_boundary(zone
)) {
1473 nvme_zrm_finish(ns
, zone
);
1477 static void nvme_finalize_zoned_write(NvmeNamespace
*ns
, NvmeRequest
*req
)
1479 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1484 slba
= le64_to_cpu(rw
->slba
);
1485 nlb
= le16_to_cpu(rw
->nlb
) + 1;
1486 zone
= nvme_get_zone_by_slba(ns
, slba
);
1488 __nvme_advance_zone_wp(ns
, zone
, nlb
);
1491 static inline bool nvme_is_write(NvmeRequest
*req
)
1493 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1495 return rw
->opcode
== NVME_CMD_WRITE
||
1496 rw
->opcode
== NVME_CMD_ZONE_APPEND
||
1497 rw
->opcode
== NVME_CMD_WRITE_ZEROES
;
1500 static void nvme_rw_cb(void *opaque
, int ret
)
1502 NvmeRequest
*req
= opaque
;
1503 NvmeNamespace
*ns
= req
->ns
;
1505 BlockBackend
*blk
= ns
->blkconf
.blk
;
1506 BlockAcctCookie
*acct
= &req
->acct
;
1507 BlockAcctStats
*stats
= blk_get_stats(blk
);
1509 trace_pci_nvme_rw_cb(nvme_cid(req
), blk_name(blk
));
1511 if (ns
->params
.zoned
&& nvme_is_write(req
)) {
1512 nvme_finalize_zoned_write(ns
, req
);
1516 block_acct_done(stats
, acct
);
1518 block_acct_failed(stats
, acct
);
1519 nvme_aio_err(req
, ret
);
1522 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1525 struct nvme_aio_flush_ctx
{
1528 BlockAcctCookie acct
;
1531 static void nvme_aio_flush_cb(void *opaque
, int ret
)
1533 struct nvme_aio_flush_ctx
*ctx
= opaque
;
1534 NvmeRequest
*req
= ctx
->req
;
1535 uintptr_t *num_flushes
= (uintptr_t *)&req
->opaque
;
1537 BlockBackend
*blk
= ctx
->ns
->blkconf
.blk
;
1538 BlockAcctCookie
*acct
= &ctx
->acct
;
1539 BlockAcctStats
*stats
= blk_get_stats(blk
);
1541 trace_pci_nvme_aio_flush_cb(nvme_cid(req
), blk_name(blk
));
1544 block_acct_done(stats
, acct
);
1546 block_acct_failed(stats
, acct
);
1547 nvme_aio_err(req
, ret
);
1557 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1560 static void nvme_aio_discard_cb(void *opaque
, int ret
)
1562 NvmeRequest
*req
= opaque
;
1563 uintptr_t *discards
= (uintptr_t *)&req
->opaque
;
1565 trace_pci_nvme_aio_discard_cb(nvme_cid(req
));
1568 nvme_aio_err(req
, ret
);
1577 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1580 struct nvme_zone_reset_ctx
{
1585 static void nvme_aio_zone_reset_cb(void *opaque
, int ret
)
1587 struct nvme_zone_reset_ctx
*ctx
= opaque
;
1588 NvmeRequest
*req
= ctx
->req
;
1589 NvmeNamespace
*ns
= req
->ns
;
1590 NvmeZone
*zone
= ctx
->zone
;
1591 uintptr_t *resets
= (uintptr_t *)&req
->opaque
;
1595 trace_pci_nvme_aio_zone_reset_cb(nvme_cid(req
), zone
->d
.zslba
);
1598 switch (nvme_get_zone_state(zone
)) {
1599 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
1600 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
1601 nvme_aor_dec_open(ns
);
1603 case NVME_ZONE_STATE_CLOSED
:
1604 nvme_aor_dec_active(ns
);
1606 case NVME_ZONE_STATE_FULL
:
1607 zone
->w_ptr
= zone
->d
.zslba
;
1608 zone
->d
.wp
= zone
->w_ptr
;
1609 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_EMPTY
);
1615 nvme_aio_err(req
, ret
);
1624 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1627 struct nvme_copy_ctx
{
1633 struct nvme_copy_in_ctx
{
1638 static void nvme_copy_cb(void *opaque
, int ret
)
1640 NvmeRequest
*req
= opaque
;
1641 NvmeNamespace
*ns
= req
->ns
;
1642 struct nvme_copy_ctx
*ctx
= req
->opaque
;
1644 trace_pci_nvme_copy_cb(nvme_cid(req
));
1646 if (ns
->params
.zoned
) {
1647 NvmeCopyCmd
*copy
= (NvmeCopyCmd
*)&req
->cmd
;
1648 uint64_t sdlba
= le64_to_cpu(copy
->sdlba
);
1649 NvmeZone
*zone
= nvme_get_zone_by_slba(ns
, sdlba
);
1651 __nvme_advance_zone_wp(ns
, zone
, ctx
->nlb
);
1655 block_acct_done(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
);
1657 block_acct_failed(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
);
1658 nvme_aio_err(req
, ret
);
1661 g_free(ctx
->bounce
);
1664 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1667 static void nvme_copy_in_complete(NvmeRequest
*req
)
1669 NvmeNamespace
*ns
= req
->ns
;
1670 NvmeCopyCmd
*copy
= (NvmeCopyCmd
*)&req
->cmd
;
1671 struct nvme_copy_ctx
*ctx
= req
->opaque
;
1672 uint64_t sdlba
= le64_to_cpu(copy
->sdlba
);
1675 trace_pci_nvme_copy_in_complete(nvme_cid(req
));
1677 block_acct_done(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
);
1679 status
= nvme_check_bounds(ns
, sdlba
, ctx
->nlb
);
1681 trace_pci_nvme_err_invalid_lba_range(sdlba
, ctx
->nlb
, ns
->id_ns
.nsze
);
1685 if (ns
->params
.zoned
) {
1686 NvmeZone
*zone
= nvme_get_zone_by_slba(ns
, sdlba
);
1688 status
= nvme_check_zone_write(ns
, zone
, sdlba
, ctx
->nlb
);
1693 status
= nvme_zrm_auto(ns
, zone
);
1698 zone
->w_ptr
+= ctx
->nlb
;
1701 qemu_iovec_init(&req
->sg
.iov
, 1);
1702 qemu_iovec_add(&req
->sg
.iov
, ctx
->bounce
, nvme_l2b(ns
, ctx
->nlb
));
1704 block_acct_start(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
, 0,
1707 req
->aiocb
= blk_aio_pwritev(ns
->blkconf
.blk
, nvme_l2b(ns
, sdlba
),
1708 &req
->sg
.iov
, 0, nvme_copy_cb
, req
);
1713 req
->status
= status
;
1715 g_free(ctx
->bounce
);
1718 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1721 static void nvme_aio_copy_in_cb(void *opaque
, int ret
)
1723 struct nvme_copy_in_ctx
*in_ctx
= opaque
;
1724 NvmeRequest
*req
= in_ctx
->req
;
1725 NvmeNamespace
*ns
= req
->ns
;
1726 struct nvme_copy_ctx
*ctx
= req
->opaque
;
1728 qemu_iovec_destroy(&in_ctx
->iov
);
1731 trace_pci_nvme_aio_copy_in_cb(nvme_cid(req
));
1734 nvme_aio_err(req
, ret
);
1744 block_acct_failed(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
);
1746 g_free(ctx
->bounce
);
1749 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1754 nvme_copy_in_complete(req
);
1757 struct nvme_compare_ctx
{
1762 static void nvme_compare_cb(void *opaque
, int ret
)
1764 NvmeRequest
*req
= opaque
;
1765 NvmeNamespace
*ns
= req
->ns
;
1766 struct nvme_compare_ctx
*ctx
= req
->opaque
;
1767 g_autofree
uint8_t *buf
= NULL
;
1770 trace_pci_nvme_compare_cb(nvme_cid(req
));
1773 block_acct_done(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
);
1775 block_acct_failed(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
);
1776 nvme_aio_err(req
, ret
);
1780 buf
= g_malloc(ctx
->iov
.size
);
1782 status
= nvme_h2c(nvme_ctrl(req
), buf
, ctx
->iov
.size
, req
);
1784 req
->status
= status
;
1788 if (memcmp(buf
, ctx
->bounce
, ctx
->iov
.size
)) {
1789 req
->status
= NVME_CMP_FAILURE
;
1793 qemu_iovec_destroy(&ctx
->iov
);
1794 g_free(ctx
->bounce
);
1797 nvme_enqueue_req_completion(nvme_cq(req
), req
);
1800 static uint16_t nvme_dsm(NvmeCtrl
*n
, NvmeRequest
*req
)
1802 NvmeNamespace
*ns
= req
->ns
;
1803 NvmeDsmCmd
*dsm
= (NvmeDsmCmd
*) &req
->cmd
;
1805 uint32_t attr
= le32_to_cpu(dsm
->attributes
);
1806 uint32_t nr
= (le32_to_cpu(dsm
->nr
) & 0xff) + 1;
1808 uint16_t status
= NVME_SUCCESS
;
1810 trace_pci_nvme_dsm(nvme_cid(req
), nvme_nsid(ns
), nr
, attr
);
1812 if (attr
& NVME_DSMGMT_AD
) {
1815 NvmeDsmRange range
[nr
];
1816 uintptr_t *discards
= (uintptr_t *)&req
->opaque
;
1818 status
= nvme_h2c(n
, (uint8_t *)range
, sizeof(range
), req
);
1824 * AIO callbacks may be called immediately, so initialize discards to 1
1825 * to make sure the the callback does not complete the request before
1826 * all discards have been issued.
1830 for (int i
= 0; i
< nr
; i
++) {
1831 uint64_t slba
= le64_to_cpu(range
[i
].slba
);
1832 uint32_t nlb
= le32_to_cpu(range
[i
].nlb
);
1834 if (nvme_check_bounds(ns
, slba
, nlb
)) {
1835 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
,
1840 trace_pci_nvme_dsm_deallocate(nvme_cid(req
), nvme_nsid(ns
), slba
,
1843 if (nlb
> n
->dmrsl
) {
1844 trace_pci_nvme_dsm_single_range_limit_exceeded(nlb
, n
->dmrsl
);
1847 offset
= nvme_l2b(ns
, slba
);
1848 len
= nvme_l2b(ns
, nlb
);
1851 size_t bytes
= MIN(BDRV_REQUEST_MAX_BYTES
, len
);
1855 blk_aio_pdiscard(ns
->blkconf
.blk
, offset
, bytes
,
1856 nvme_aio_discard_cb
, req
);
1863 /* account for the 1-initialization */
1867 status
= NVME_NO_COMPLETE
;
1869 status
= req
->status
;
1876 static uint16_t nvme_copy(NvmeCtrl
*n
, NvmeRequest
*req
)
1878 NvmeNamespace
*ns
= req
->ns
;
1879 NvmeCopyCmd
*copy
= (NvmeCopyCmd
*)&req
->cmd
;
1880 g_autofree NvmeCopySourceRange
*range
= NULL
;
1882 uint16_t nr
= copy
->nr
+ 1;
1883 uint8_t format
= copy
->control
[0] & 0xf;
1886 uint8_t *bounce
= NULL
, *bouncep
= NULL
;
1887 struct nvme_copy_ctx
*ctx
;
1891 trace_pci_nvme_copy(nvme_cid(req
), nvme_nsid(ns
), nr
, format
);
1893 if (!(n
->id_ctrl
.ocfs
& (1 << format
))) {
1894 trace_pci_nvme_err_copy_invalid_format(format
);
1895 return NVME_INVALID_FIELD
| NVME_DNR
;
1898 if (nr
> ns
->id_ns
.msrc
+ 1) {
1899 return NVME_CMD_SIZE_LIMIT
| NVME_DNR
;
1902 range
= g_new(NvmeCopySourceRange
, nr
);
1904 status
= nvme_h2c(n
, (uint8_t *)range
, nr
* sizeof(NvmeCopySourceRange
),
1910 for (i
= 0; i
< nr
; i
++) {
1911 uint64_t slba
= le64_to_cpu(range
[i
].slba
);
1912 uint32_t _nlb
= le16_to_cpu(range
[i
].nlb
) + 1;
1914 if (_nlb
> le16_to_cpu(ns
->id_ns
.mssrl
)) {
1915 return NVME_CMD_SIZE_LIMIT
| NVME_DNR
;
1918 status
= nvme_check_bounds(ns
, slba
, _nlb
);
1920 trace_pci_nvme_err_invalid_lba_range(slba
, _nlb
, ns
->id_ns
.nsze
);
1924 if (NVME_ERR_REC_DULBE(ns
->features
.err_rec
)) {
1925 status
= nvme_check_dulbe(ns
, slba
, _nlb
);
1931 if (ns
->params
.zoned
) {
1932 status
= nvme_check_zone_read(ns
, slba
, _nlb
);
1941 if (nlb
> le32_to_cpu(ns
->id_ns
.mcl
)) {
1942 return NVME_CMD_SIZE_LIMIT
| NVME_DNR
;
1945 bounce
= bouncep
= g_malloc(nvme_l2b(ns
, nlb
));
1947 block_acct_start(blk_get_stats(ns
->blkconf
.blk
), &req
->acct
, 0,
1950 ctx
= g_new(struct nvme_copy_ctx
, 1);
1952 ctx
->bounce
= bounce
;
1958 for (i
= 0; i
< nr
; i
++) {
1959 uint64_t slba
= le64_to_cpu(range
[i
].slba
);
1960 uint32_t nlb
= le16_to_cpu(range
[i
].nlb
) + 1;
1962 size_t len
= nvme_l2b(ns
, nlb
);
1963 int64_t offset
= nvme_l2b(ns
, slba
);
1965 trace_pci_nvme_copy_source_range(slba
, nlb
);
1967 struct nvme_copy_in_ctx
*in_ctx
= g_new(struct nvme_copy_in_ctx
, 1);
1970 qemu_iovec_init(&in_ctx
->iov
, 1);
1971 qemu_iovec_add(&in_ctx
->iov
, bouncep
, len
);
1975 blk_aio_preadv(ns
->blkconf
.blk
, offset
, &in_ctx
->iov
, 0,
1976 nvme_aio_copy_in_cb
, in_ctx
);
1981 /* account for the 1-initialization */
1985 nvme_copy_in_complete(req
);
1988 return NVME_NO_COMPLETE
;
1991 static uint16_t nvme_compare(NvmeCtrl
*n
, NvmeRequest
*req
)
1993 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
1994 NvmeNamespace
*ns
= req
->ns
;
1995 BlockBackend
*blk
= ns
->blkconf
.blk
;
1996 uint64_t slba
= le64_to_cpu(rw
->slba
);
1997 uint32_t nlb
= le16_to_cpu(rw
->nlb
) + 1;
1998 size_t len
= nvme_l2b(ns
, nlb
);
1999 int64_t offset
= nvme_l2b(ns
, slba
);
2000 uint8_t *bounce
= NULL
;
2001 struct nvme_compare_ctx
*ctx
= NULL
;
2004 trace_pci_nvme_compare(nvme_cid(req
), nvme_nsid(ns
), slba
, nlb
);
2006 status
= nvme_check_mdts(n
, len
);
2011 status
= nvme_check_bounds(ns
, slba
, nlb
);
2013 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
2017 if (NVME_ERR_REC_DULBE(ns
->features
.err_rec
)) {
2018 status
= nvme_check_dulbe(ns
, slba
, nlb
);
2024 bounce
= g_malloc(len
);
2026 ctx
= g_new(struct nvme_compare_ctx
, 1);
2027 ctx
->bounce
= bounce
;
2031 qemu_iovec_init(&ctx
->iov
, 1);
2032 qemu_iovec_add(&ctx
->iov
, bounce
, len
);
2034 block_acct_start(blk_get_stats(blk
), &req
->acct
, len
, BLOCK_ACCT_READ
);
2035 blk_aio_preadv(blk
, offset
, &ctx
->iov
, 0, nvme_compare_cb
, req
);
2037 return NVME_NO_COMPLETE
;
2040 static uint16_t nvme_flush(NvmeCtrl
*n
, NvmeRequest
*req
)
2042 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
2043 uintptr_t *num_flushes
= (uintptr_t *)&req
->opaque
;
2045 struct nvme_aio_flush_ctx
*ctx
;
2048 trace_pci_nvme_flush(nvme_cid(req
), nsid
);
2050 if (nsid
!= NVME_NSID_BROADCAST
) {
2051 req
->ns
= nvme_ns(n
, nsid
);
2052 if (unlikely(!req
->ns
)) {
2053 return NVME_INVALID_FIELD
| NVME_DNR
;
2056 block_acct_start(blk_get_stats(req
->ns
->blkconf
.blk
), &req
->acct
, 0,
2058 req
->aiocb
= blk_aio_flush(req
->ns
->blkconf
.blk
, nvme_rw_cb
, req
);
2059 return NVME_NO_COMPLETE
;
2062 /* 1-initialize; see comment in nvme_dsm */
2065 for (int i
= 1; i
<= n
->num_namespaces
; i
++) {
2071 ctx
= g_new(struct nvme_aio_flush_ctx
, 1);
2077 block_acct_start(blk_get_stats(ns
->blkconf
.blk
), &ctx
->acct
, 0,
2079 blk_aio_flush(ns
->blkconf
.blk
, nvme_aio_flush_cb
, ctx
);
2082 /* account for the 1-initialization */
2086 status
= NVME_NO_COMPLETE
;
2088 status
= req
->status
;
2094 static uint16_t nvme_read(NvmeCtrl
*n
, NvmeRequest
*req
)
2096 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
2097 NvmeNamespace
*ns
= req
->ns
;
2098 uint64_t slba
= le64_to_cpu(rw
->slba
);
2099 uint32_t nlb
= (uint32_t)le16_to_cpu(rw
->nlb
) + 1;
2100 uint64_t data_size
= nvme_l2b(ns
, nlb
);
2101 uint64_t data_offset
;
2102 BlockBackend
*blk
= ns
->blkconf
.blk
;
2105 trace_pci_nvme_read(nvme_cid(req
), nvme_nsid(ns
), nlb
, data_size
, slba
);
2107 status
= nvme_check_mdts(n
, data_size
);
2112 status
= nvme_check_bounds(ns
, slba
, nlb
);
2114 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
2118 if (ns
->params
.zoned
) {
2119 status
= nvme_check_zone_read(ns
, slba
, nlb
);
2121 trace_pci_nvme_err_zone_read_not_ok(slba
, nlb
, status
);
2126 status
= nvme_map_dptr(n
, &req
->sg
, data_size
, &req
->cmd
);
2131 if (NVME_ERR_REC_DULBE(ns
->features
.err_rec
)) {
2132 status
= nvme_check_dulbe(ns
, slba
, nlb
);
2138 data_offset
= nvme_l2b(ns
, slba
);
2140 block_acct_start(blk_get_stats(blk
), &req
->acct
, data_size
,
2142 nvme_blk_read(blk
, data_offset
, nvme_rw_cb
, req
);
2143 return NVME_NO_COMPLETE
;
2146 block_acct_invalid(blk_get_stats(blk
), BLOCK_ACCT_READ
);
2147 return status
| NVME_DNR
;
2150 static uint16_t nvme_do_write(NvmeCtrl
*n
, NvmeRequest
*req
, bool append
,
2153 NvmeRwCmd
*rw
= (NvmeRwCmd
*)&req
->cmd
;
2154 NvmeNamespace
*ns
= req
->ns
;
2155 uint64_t slba
= le64_to_cpu(rw
->slba
);
2156 uint32_t nlb
= (uint32_t)le16_to_cpu(rw
->nlb
) + 1;
2157 uint64_t data_size
= nvme_l2b(ns
, nlb
);
2158 uint64_t data_offset
;
2160 NvmeZonedResult
*res
= (NvmeZonedResult
*)&req
->cqe
;
2161 BlockBackend
*blk
= ns
->blkconf
.blk
;
2164 trace_pci_nvme_write(nvme_cid(req
), nvme_io_opc_str(rw
->opcode
),
2165 nvme_nsid(ns
), nlb
, data_size
, slba
);
2168 status
= nvme_check_mdts(n
, data_size
);
2174 status
= nvme_check_bounds(ns
, slba
, nlb
);
2176 trace_pci_nvme_err_invalid_lba_range(slba
, nlb
, ns
->id_ns
.nsze
);
2180 if (ns
->params
.zoned
) {
2181 zone
= nvme_get_zone_by_slba(ns
, slba
);
2184 if (unlikely(slba
!= zone
->d
.zslba
)) {
2185 trace_pci_nvme_err_append_not_at_start(slba
, zone
->d
.zslba
);
2186 status
= NVME_INVALID_FIELD
;
2190 if (n
->params
.zasl
&& data_size
> n
->page_size
<< n
->params
.zasl
) {
2191 trace_pci_nvme_err_zasl(data_size
);
2192 return NVME_INVALID_FIELD
| NVME_DNR
;
2196 res
->slba
= cpu_to_le64(slba
);
2199 status
= nvme_check_zone_write(ns
, zone
, slba
, nlb
);
2204 status
= nvme_zrm_auto(ns
, zone
);
2212 data_offset
= nvme_l2b(ns
, slba
);
2215 status
= nvme_map_dptr(n
, &req
->sg
, data_size
, &req
->cmd
);
2220 block_acct_start(blk_get_stats(blk
), &req
->acct
, data_size
,
2222 nvme_blk_write(blk
, data_offset
, nvme_rw_cb
, req
);
2224 req
->aiocb
= blk_aio_pwrite_zeroes(blk
, data_offset
, data_size
,
2225 BDRV_REQ_MAY_UNMAP
, nvme_rw_cb
,
2228 return NVME_NO_COMPLETE
;
2231 block_acct_invalid(blk_get_stats(blk
), BLOCK_ACCT_WRITE
);
2232 return status
| NVME_DNR
;
2235 static inline uint16_t nvme_write(NvmeCtrl
*n
, NvmeRequest
*req
)
2237 return nvme_do_write(n
, req
, false, false);
2240 static inline uint16_t nvme_write_zeroes(NvmeCtrl
*n
, NvmeRequest
*req
)
2242 return nvme_do_write(n
, req
, false, true);
2245 static inline uint16_t nvme_zone_append(NvmeCtrl
*n
, NvmeRequest
*req
)
2247 return nvme_do_write(n
, req
, true, false);
2250 static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace
*ns
, NvmeCmd
*c
,
2251 uint64_t *slba
, uint32_t *zone_idx
)
2253 uint32_t dw10
= le32_to_cpu(c
->cdw10
);
2254 uint32_t dw11
= le32_to_cpu(c
->cdw11
);
2256 if (!ns
->params
.zoned
) {
2257 trace_pci_nvme_err_invalid_opc(c
->opcode
);
2258 return NVME_INVALID_OPCODE
| NVME_DNR
;
2261 *slba
= ((uint64_t)dw11
) << 32 | dw10
;
2262 if (unlikely(*slba
>= ns
->id_ns
.nsze
)) {
2263 trace_pci_nvme_err_invalid_lba_range(*slba
, 0, ns
->id_ns
.nsze
);
2265 return NVME_LBA_RANGE
| NVME_DNR
;
2268 *zone_idx
= nvme_zone_idx(ns
, *slba
);
2269 assert(*zone_idx
< ns
->num_zones
);
2271 return NVME_SUCCESS
;
2274 typedef uint16_t (*op_handler_t
)(NvmeNamespace
*, NvmeZone
*, NvmeZoneState
,
2277 enum NvmeZoneProcessingMask
{
2278 NVME_PROC_CURRENT_ZONE
= 0,
2279 NVME_PROC_OPENED_ZONES
= 1 << 0,
2280 NVME_PROC_CLOSED_ZONES
= 1 << 1,
2281 NVME_PROC_READ_ONLY_ZONES
= 1 << 2,
2282 NVME_PROC_FULL_ZONES
= 1 << 3,
2285 static uint16_t nvme_open_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
2286 NvmeZoneState state
, NvmeRequest
*req
)
2288 return nvme_zrm_open(ns
, zone
);
2291 static uint16_t nvme_close_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
2292 NvmeZoneState state
, NvmeRequest
*req
)
2294 return nvme_zrm_close(ns
, zone
);
2297 static uint16_t nvme_finish_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
2298 NvmeZoneState state
, NvmeRequest
*req
)
2300 return nvme_zrm_finish(ns
, zone
);
2303 static uint16_t nvme_reset_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
2304 NvmeZoneState state
, NvmeRequest
*req
)
2306 uintptr_t *resets
= (uintptr_t *)&req
->opaque
;
2307 struct nvme_zone_reset_ctx
*ctx
;
2310 case NVME_ZONE_STATE_EMPTY
:
2311 return NVME_SUCCESS
;
2312 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
2313 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
2314 case NVME_ZONE_STATE_CLOSED
:
2315 case NVME_ZONE_STATE_FULL
:
2318 return NVME_ZONE_INVAL_TRANSITION
;
2322 * The zone reset aio callback needs to know the zone that is being reset
2323 * in order to transition the zone on completion.
2325 ctx
= g_new(struct nvme_zone_reset_ctx
, 1);
2331 blk_aio_pwrite_zeroes(ns
->blkconf
.blk
, nvme_l2b(ns
, zone
->d
.zslba
),
2332 nvme_l2b(ns
, ns
->zone_size
), BDRV_REQ_MAY_UNMAP
,
2333 nvme_aio_zone_reset_cb
, ctx
);
2335 return NVME_NO_COMPLETE
;
2338 static uint16_t nvme_offline_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
2339 NvmeZoneState state
, NvmeRequest
*req
)
2342 case NVME_ZONE_STATE_READ_ONLY
:
2343 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_OFFLINE
);
2345 case NVME_ZONE_STATE_OFFLINE
:
2346 return NVME_SUCCESS
;
2348 return NVME_ZONE_INVAL_TRANSITION
;
2352 static uint16_t nvme_set_zd_ext(NvmeNamespace
*ns
, NvmeZone
*zone
)
2355 uint8_t state
= nvme_get_zone_state(zone
);
2357 if (state
== NVME_ZONE_STATE_EMPTY
) {
2358 status
= nvme_aor_check(ns
, 1, 0);
2362 nvme_aor_inc_active(ns
);
2363 zone
->d
.za
|= NVME_ZA_ZD_EXT_VALID
;
2364 nvme_assign_zone_state(ns
, zone
, NVME_ZONE_STATE_CLOSED
);
2365 return NVME_SUCCESS
;
2368 return NVME_ZONE_INVAL_TRANSITION
;
2371 static uint16_t nvme_bulk_proc_zone(NvmeNamespace
*ns
, NvmeZone
*zone
,
2372 enum NvmeZoneProcessingMask proc_mask
,
2373 op_handler_t op_hndlr
, NvmeRequest
*req
)
2375 uint16_t status
= NVME_SUCCESS
;
2376 NvmeZoneState zs
= nvme_get_zone_state(zone
);
2380 case NVME_ZONE_STATE_IMPLICITLY_OPEN
:
2381 case NVME_ZONE_STATE_EXPLICITLY_OPEN
:
2382 proc_zone
= proc_mask
& NVME_PROC_OPENED_ZONES
;
2384 case NVME_ZONE_STATE_CLOSED
:
2385 proc_zone
= proc_mask
& NVME_PROC_CLOSED_ZONES
;
2387 case NVME_ZONE_STATE_READ_ONLY
:
2388 proc_zone
= proc_mask
& NVME_PROC_READ_ONLY_ZONES
;
2390 case NVME_ZONE_STATE_FULL
:
2391 proc_zone
= proc_mask
& NVME_PROC_FULL_ZONES
;
2398 status
= op_hndlr(ns
, zone
, zs
, req
);
2404 static uint16_t nvme_do_zone_op(NvmeNamespace
*ns
, NvmeZone
*zone
,
2405 enum NvmeZoneProcessingMask proc_mask
,
2406 op_handler_t op_hndlr
, NvmeRequest
*req
)
2409 uint16_t status
= NVME_SUCCESS
;
2413 status
= op_hndlr(ns
, zone
, nvme_get_zone_state(zone
), req
);
2415 if (proc_mask
& NVME_PROC_CLOSED_ZONES
) {
2416 QTAILQ_FOREACH_SAFE(zone
, &ns
->closed_zones
, entry
, next
) {
2417 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
2419 if (status
&& status
!= NVME_NO_COMPLETE
) {
2424 if (proc_mask
& NVME_PROC_OPENED_ZONES
) {
2425 QTAILQ_FOREACH_SAFE(zone
, &ns
->imp_open_zones
, entry
, next
) {
2426 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
2428 if (status
&& status
!= NVME_NO_COMPLETE
) {
2433 QTAILQ_FOREACH_SAFE(zone
, &ns
->exp_open_zones
, entry
, next
) {
2434 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
2436 if (status
&& status
!= NVME_NO_COMPLETE
) {
2441 if (proc_mask
& NVME_PROC_FULL_ZONES
) {
2442 QTAILQ_FOREACH_SAFE(zone
, &ns
->full_zones
, entry
, next
) {
2443 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
2445 if (status
&& status
!= NVME_NO_COMPLETE
) {
2451 if (proc_mask
& NVME_PROC_READ_ONLY_ZONES
) {
2452 for (i
= 0; i
< ns
->num_zones
; i
++, zone
++) {
2453 status
= nvme_bulk_proc_zone(ns
, zone
, proc_mask
, op_hndlr
,
2455 if (status
&& status
!= NVME_NO_COMPLETE
) {
2466 static uint16_t nvme_zone_mgmt_send(NvmeCtrl
*n
, NvmeRequest
*req
)
2468 NvmeCmd
*cmd
= (NvmeCmd
*)&req
->cmd
;
2469 NvmeNamespace
*ns
= req
->ns
;
2473 uint32_t dw13
= le32_to_cpu(cmd
->cdw13
);
2475 uint32_t zone_idx
= 0;
2479 enum NvmeZoneProcessingMask proc_mask
= NVME_PROC_CURRENT_ZONE
;
2481 action
= dw13
& 0xff;
2484 req
->status
= NVME_SUCCESS
;
2487 status
= nvme_get_mgmt_zone_slba_idx(ns
, cmd
, &slba
, &zone_idx
);
2493 zone
= &ns
->zone_array
[zone_idx
];
2494 if (slba
!= zone
->d
.zslba
) {
2495 trace_pci_nvme_err_unaligned_zone_cmd(action
, slba
, zone
->d
.zslba
);
2496 return NVME_INVALID_FIELD
| NVME_DNR
;
2501 case NVME_ZONE_ACTION_OPEN
:
2503 proc_mask
= NVME_PROC_CLOSED_ZONES
;
2505 trace_pci_nvme_open_zone(slba
, zone_idx
, all
);
2506 status
= nvme_do_zone_op(ns
, zone
, proc_mask
, nvme_open_zone
, req
);
2509 case NVME_ZONE_ACTION_CLOSE
:
2511 proc_mask
= NVME_PROC_OPENED_ZONES
;
2513 trace_pci_nvme_close_zone(slba
, zone_idx
, all
);
2514 status
= nvme_do_zone_op(ns
, zone
, proc_mask
, nvme_close_zone
, req
);
2517 case NVME_ZONE_ACTION_FINISH
:
2519 proc_mask
= NVME_PROC_OPENED_ZONES
| NVME_PROC_CLOSED_ZONES
;
2521 trace_pci_nvme_finish_zone(slba
, zone_idx
, all
);
2522 status
= nvme_do_zone_op(ns
, zone
, proc_mask
, nvme_finish_zone
, req
);
2525 case NVME_ZONE_ACTION_RESET
:
2526 resets
= (uintptr_t *)&req
->opaque
;
2529 proc_mask
= NVME_PROC_OPENED_ZONES
| NVME_PROC_CLOSED_ZONES
|
2530 NVME_PROC_FULL_ZONES
;
2532 trace_pci_nvme_reset_zone(slba
, zone_idx
, all
);
2536 status
= nvme_do_zone_op(ns
, zone
, proc_mask
, nvme_reset_zone
, req
);
2540 return *resets
? NVME_NO_COMPLETE
: req
->status
;
2542 case NVME_ZONE_ACTION_OFFLINE
:
2544 proc_mask
= NVME_PROC_READ_ONLY_ZONES
;
2546 trace_pci_nvme_offline_zone(slba
, zone_idx
, all
);
2547 status
= nvme_do_zone_op(ns
, zone
, proc_mask
, nvme_offline_zone
, req
);
2550 case NVME_ZONE_ACTION_SET_ZD_EXT
:
2551 trace_pci_nvme_set_descriptor_extension(slba
, zone_idx
);
2552 if (all
|| !ns
->params
.zd_extension_size
) {
2553 return NVME_INVALID_FIELD
| NVME_DNR
;
2555 zd_ext
= nvme_get_zd_extension(ns
, zone_idx
);
2556 status
= nvme_h2c(n
, zd_ext
, ns
->params
.zd_extension_size
, req
);
2558 trace_pci_nvme_err_zd_extension_map_error(zone_idx
);
2562 status
= nvme_set_zd_ext(ns
, zone
);
2563 if (status
== NVME_SUCCESS
) {
2564 trace_pci_nvme_zd_extension_set(zone_idx
);
2570 trace_pci_nvme_err_invalid_mgmt_action(action
);
2571 status
= NVME_INVALID_FIELD
;
2574 if (status
== NVME_ZONE_INVAL_TRANSITION
) {
2575 trace_pci_nvme_err_invalid_zone_state_transition(action
, slba
,
2585 static bool nvme_zone_matches_filter(uint32_t zafs
, NvmeZone
*zl
)
2587 NvmeZoneState zs
= nvme_get_zone_state(zl
);
2590 case NVME_ZONE_REPORT_ALL
:
2592 case NVME_ZONE_REPORT_EMPTY
:
2593 return zs
== NVME_ZONE_STATE_EMPTY
;
2594 case NVME_ZONE_REPORT_IMPLICITLY_OPEN
:
2595 return zs
== NVME_ZONE_STATE_IMPLICITLY_OPEN
;
2596 case NVME_ZONE_REPORT_EXPLICITLY_OPEN
:
2597 return zs
== NVME_ZONE_STATE_EXPLICITLY_OPEN
;
2598 case NVME_ZONE_REPORT_CLOSED
:
2599 return zs
== NVME_ZONE_STATE_CLOSED
;
2600 case NVME_ZONE_REPORT_FULL
:
2601 return zs
== NVME_ZONE_STATE_FULL
;
2602 case NVME_ZONE_REPORT_READ_ONLY
:
2603 return zs
== NVME_ZONE_STATE_READ_ONLY
;
2604 case NVME_ZONE_REPORT_OFFLINE
:
2605 return zs
== NVME_ZONE_STATE_OFFLINE
;
2611 static uint16_t nvme_zone_mgmt_recv(NvmeCtrl
*n
, NvmeRequest
*req
)
2613 NvmeCmd
*cmd
= (NvmeCmd
*)&req
->cmd
;
2614 NvmeNamespace
*ns
= req
->ns
;
2615 /* cdw12 is zero-based number of dwords to return. Convert to bytes */
2616 uint32_t data_size
= (le32_to_cpu(cmd
->cdw12
) + 1) << 2;
2617 uint32_t dw13
= le32_to_cpu(cmd
->cdw13
);
2618 uint32_t zone_idx
, zra
, zrasf
, partial
;
2619 uint64_t max_zones
, nr_zones
= 0;
2621 uint64_t slba
, capacity
= nvme_ns_nlbas(ns
);
2624 NvmeZoneReportHeader
*header
;
2626 size_t zone_entry_sz
;
2628 req
->status
= NVME_SUCCESS
;
2630 status
= nvme_get_mgmt_zone_slba_idx(ns
, cmd
, &slba
, &zone_idx
);
2636 if (zra
!= NVME_ZONE_REPORT
&& zra
!= NVME_ZONE_REPORT_EXTENDED
) {
2637 return NVME_INVALID_FIELD
| NVME_DNR
;
2639 if (zra
== NVME_ZONE_REPORT_EXTENDED
&& !ns
->params
.zd_extension_size
) {
2640 return NVME_INVALID_FIELD
| NVME_DNR
;
2643 zrasf
= (dw13
>> 8) & 0xff;
2644 if (zrasf
> NVME_ZONE_REPORT_OFFLINE
) {
2645 return NVME_INVALID_FIELD
| NVME_DNR
;
2648 if (data_size
< sizeof(NvmeZoneReportHeader
)) {
2649 return NVME_INVALID_FIELD
| NVME_DNR
;
2652 status
= nvme_check_mdts(n
, data_size
);
2657 partial
= (dw13
>> 16) & 0x01;
2659 zone_entry_sz
= sizeof(NvmeZoneDescr
);
2660 if (zra
== NVME_ZONE_REPORT_EXTENDED
) {
2661 zone_entry_sz
+= ns
->params
.zd_extension_size
;
2664 max_zones
= (data_size
- sizeof(NvmeZoneReportHeader
)) / zone_entry_sz
;
2665 buf
= g_malloc0(data_size
);
2667 zone
= &ns
->zone_array
[zone_idx
];
2668 for (; slba
< capacity
; slba
+= ns
->zone_size
) {
2669 if (partial
&& nr_zones
>= max_zones
) {
2672 if (nvme_zone_matches_filter(zrasf
, zone
++)) {
2676 header
= (NvmeZoneReportHeader
*)buf
;
2677 header
->nr_zones
= cpu_to_le64(nr_zones
);
2679 buf_p
= buf
+ sizeof(NvmeZoneReportHeader
);
2680 for (; zone_idx
< ns
->num_zones
&& max_zones
> 0; zone_idx
++) {
2681 zone
= &ns
->zone_array
[zone_idx
];
2682 if (nvme_zone_matches_filter(zrasf
, zone
)) {
2683 z
= (NvmeZoneDescr
*)buf_p
;
2684 buf_p
+= sizeof(NvmeZoneDescr
);
2688 z
->zcap
= cpu_to_le64(zone
->d
.zcap
);
2689 z
->zslba
= cpu_to_le64(zone
->d
.zslba
);
2692 if (nvme_wp_is_valid(zone
)) {
2693 z
->wp
= cpu_to_le64(zone
->d
.wp
);
2695 z
->wp
= cpu_to_le64(~0ULL);
2698 if (zra
== NVME_ZONE_REPORT_EXTENDED
) {
2699 if (zone
->d
.za
& NVME_ZA_ZD_EXT_VALID
) {
2700 memcpy(buf_p
, nvme_get_zd_extension(ns
, zone_idx
),
2701 ns
->params
.zd_extension_size
);
2703 buf_p
+= ns
->params
.zd_extension_size
;
2710 status
= nvme_c2h(n
, (uint8_t *)buf
, data_size
, req
);
2717 static uint16_t nvme_io_cmd(NvmeCtrl
*n
, NvmeRequest
*req
)
2719 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
2721 trace_pci_nvme_io_cmd(nvme_cid(req
), nsid
, nvme_sqid(req
),
2722 req
->cmd
.opcode
, nvme_io_opc_str(req
->cmd
.opcode
));
2724 if (!nvme_nsid_valid(n
, nsid
)) {
2725 return NVME_INVALID_NSID
| NVME_DNR
;
2729 * In the base NVM command set, Flush may apply to all namespaces
2730 * (indicated by NSID being set to 0xFFFFFFFF). But if that feature is used
2731 * along with TP 4056 (Namespace Types), it may be pretty screwed up.
2733 * If NSID is indeed set to 0xFFFFFFFF, we simply cannot associate the
2734 * opcode with a specific command since we cannot determine a unique I/O
2735 * command set. Opcode 0x0 could have any other meaning than something
2736 * equivalent to flushing and say it DOES have completely different
2737 * semantics in some other command set - does an NSID of 0xFFFFFFFF then
2738 * mean "for all namespaces, apply whatever command set specific command
2739 * that uses the 0x0 opcode?" Or does it mean "for all namespaces, apply
2740 * whatever command that uses the 0x0 opcode if, and only if, it allows
2741 * NSID to be 0xFFFFFFFF"?
2743 * Anyway (and luckily), for now, we do not care about this since the
2744 * device only supports namespace types that includes the NVM Flush command
2745 * (NVM and Zoned), so always do an NVM Flush.
2747 if (req
->cmd
.opcode
== NVME_CMD_FLUSH
) {
2748 return nvme_flush(n
, req
);
2751 req
->ns
= nvme_ns(n
, nsid
);
2752 if (unlikely(!req
->ns
)) {
2753 return NVME_INVALID_FIELD
| NVME_DNR
;
2756 if (!(req
->ns
->iocs
[req
->cmd
.opcode
] & NVME_CMD_EFF_CSUPP
)) {
2757 trace_pci_nvme_err_invalid_opc(req
->cmd
.opcode
);
2758 return NVME_INVALID_OPCODE
| NVME_DNR
;
2761 switch (req
->cmd
.opcode
) {
2762 case NVME_CMD_WRITE_ZEROES
:
2763 return nvme_write_zeroes(n
, req
);
2764 case NVME_CMD_ZONE_APPEND
:
2765 return nvme_zone_append(n
, req
);
2766 case NVME_CMD_WRITE
:
2767 return nvme_write(n
, req
);
2769 return nvme_read(n
, req
);
2770 case NVME_CMD_COMPARE
:
2771 return nvme_compare(n
, req
);
2773 return nvme_dsm(n
, req
);
2775 return nvme_copy(n
, req
);
2776 case NVME_CMD_ZONE_MGMT_SEND
:
2777 return nvme_zone_mgmt_send(n
, req
);
2778 case NVME_CMD_ZONE_MGMT_RECV
:
2779 return nvme_zone_mgmt_recv(n
, req
);
2784 return NVME_INVALID_OPCODE
| NVME_DNR
;
2787 static void nvme_free_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
)
2789 n
->sq
[sq
->sqid
] = NULL
;
2790 timer_free(sq
->timer
);
2797 static uint16_t nvme_del_sq(NvmeCtrl
*n
, NvmeRequest
*req
)
2799 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)&req
->cmd
;
2800 NvmeRequest
*r
, *next
;
2803 uint16_t qid
= le16_to_cpu(c
->qid
);
2805 if (unlikely(!qid
|| nvme_check_sqid(n
, qid
))) {
2806 trace_pci_nvme_err_invalid_del_sq(qid
);
2807 return NVME_INVALID_QID
| NVME_DNR
;
2810 trace_pci_nvme_del_sq(qid
);
2813 while (!QTAILQ_EMPTY(&sq
->out_req_list
)) {
2814 r
= QTAILQ_FIRST(&sq
->out_req_list
);
2816 blk_aio_cancel(r
->aiocb
);
2818 if (!nvme_check_cqid(n
, sq
->cqid
)) {
2819 cq
= n
->cq
[sq
->cqid
];
2820 QTAILQ_REMOVE(&cq
->sq_list
, sq
, entry
);
2823 QTAILQ_FOREACH_SAFE(r
, &cq
->req_list
, entry
, next
) {
2825 QTAILQ_REMOVE(&cq
->req_list
, r
, entry
);
2826 QTAILQ_INSERT_TAIL(&sq
->req_list
, r
, entry
);
2831 nvme_free_sq(sq
, n
);
2832 return NVME_SUCCESS
;
2835 static void nvme_init_sq(NvmeSQueue
*sq
, NvmeCtrl
*n
, uint64_t dma_addr
,
2836 uint16_t sqid
, uint16_t cqid
, uint16_t size
)
2842 sq
->dma_addr
= dma_addr
;
2846 sq
->head
= sq
->tail
= 0;
2847 sq
->io_req
= g_new0(NvmeRequest
, sq
->size
);
2849 QTAILQ_INIT(&sq
->req_list
);
2850 QTAILQ_INIT(&sq
->out_req_list
);
2851 for (i
= 0; i
< sq
->size
; i
++) {
2852 sq
->io_req
[i
].sq
= sq
;
2853 QTAILQ_INSERT_TAIL(&(sq
->req_list
), &sq
->io_req
[i
], entry
);
2855 sq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_process_sq
, sq
);
2857 assert(n
->cq
[cqid
]);
2859 QTAILQ_INSERT_TAIL(&(cq
->sq_list
), sq
, entry
);
2863 static uint16_t nvme_create_sq(NvmeCtrl
*n
, NvmeRequest
*req
)
2866 NvmeCreateSq
*c
= (NvmeCreateSq
*)&req
->cmd
;
2868 uint16_t cqid
= le16_to_cpu(c
->cqid
);
2869 uint16_t sqid
= le16_to_cpu(c
->sqid
);
2870 uint16_t qsize
= le16_to_cpu(c
->qsize
);
2871 uint16_t qflags
= le16_to_cpu(c
->sq_flags
);
2872 uint64_t prp1
= le64_to_cpu(c
->prp1
);
2874 trace_pci_nvme_create_sq(prp1
, sqid
, cqid
, qsize
, qflags
);
2876 if (unlikely(!cqid
|| nvme_check_cqid(n
, cqid
))) {
2877 trace_pci_nvme_err_invalid_create_sq_cqid(cqid
);
2878 return NVME_INVALID_CQID
| NVME_DNR
;
2880 if (unlikely(!sqid
|| sqid
> n
->params
.max_ioqpairs
||
2881 n
->sq
[sqid
] != NULL
)) {
2882 trace_pci_nvme_err_invalid_create_sq_sqid(sqid
);
2883 return NVME_INVALID_QID
| NVME_DNR
;
2885 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
2886 trace_pci_nvme_err_invalid_create_sq_size(qsize
);
2887 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
2889 if (unlikely(prp1
& (n
->page_size
- 1))) {
2890 trace_pci_nvme_err_invalid_create_sq_addr(prp1
);
2891 return NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
2893 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags
)))) {
2894 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags
));
2895 return NVME_INVALID_FIELD
| NVME_DNR
;
2897 sq
= g_malloc0(sizeof(*sq
));
2898 nvme_init_sq(sq
, n
, prp1
, sqid
, cqid
, qsize
+ 1);
2899 return NVME_SUCCESS
;
2903 uint64_t units_read
;
2904 uint64_t units_written
;
2905 uint64_t read_commands
;
2906 uint64_t write_commands
;
2909 static void nvme_set_blk_stats(NvmeNamespace
*ns
, struct nvme_stats
*stats
)
2911 BlockAcctStats
*s
= blk_get_stats(ns
->blkconf
.blk
);
2913 stats
->units_read
+= s
->nr_bytes
[BLOCK_ACCT_READ
] >> BDRV_SECTOR_BITS
;
2914 stats
->units_written
+= s
->nr_bytes
[BLOCK_ACCT_WRITE
] >> BDRV_SECTOR_BITS
;
2915 stats
->read_commands
+= s
->nr_ops
[BLOCK_ACCT_READ
];
2916 stats
->write_commands
+= s
->nr_ops
[BLOCK_ACCT_WRITE
];
2919 static uint16_t nvme_smart_info(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
2920 uint64_t off
, NvmeRequest
*req
)
2922 uint32_t nsid
= le32_to_cpu(req
->cmd
.nsid
);
2923 struct nvme_stats stats
= { 0 };
2924 NvmeSmartLog smart
= { 0 };
2929 if (off
>= sizeof(smart
)) {
2930 return NVME_INVALID_FIELD
| NVME_DNR
;
2933 if (nsid
!= 0xffffffff) {
2934 ns
= nvme_ns(n
, nsid
);
2936 return NVME_INVALID_NSID
| NVME_DNR
;
2938 nvme_set_blk_stats(ns
, &stats
);
2942 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
2947 nvme_set_blk_stats(ns
, &stats
);
2951 trans_len
= MIN(sizeof(smart
) - off
, buf_len
);
2952 smart
.critical_warning
= n
->smart_critical_warning
;
2954 smart
.data_units_read
[0] = cpu_to_le64(DIV_ROUND_UP(stats
.units_read
,
2956 smart
.data_units_written
[0] = cpu_to_le64(DIV_ROUND_UP(stats
.units_written
,
2958 smart
.host_read_commands
[0] = cpu_to_le64(stats
.read_commands
);
2959 smart
.host_write_commands
[0] = cpu_to_le64(stats
.write_commands
);
2961 smart
.temperature
= cpu_to_le16(n
->temperature
);
2963 if ((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
2964 (n
->temperature
<= n
->features
.temp_thresh_low
)) {
2965 smart
.critical_warning
|= NVME_SMART_TEMPERATURE
;
2968 current_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
2969 smart
.power_on_hours
[0] =
2970 cpu_to_le64((((current_ms
- n
->starttime_ms
) / 1000) / 60) / 60);
2973 nvme_clear_events(n
, NVME_AER_TYPE_SMART
);
2976 return nvme_c2h(n
, (uint8_t *) &smart
+ off
, trans_len
, req
);
2979 static uint16_t nvme_fw_log_info(NvmeCtrl
*n
, uint32_t buf_len
, uint64_t off
,
2983 NvmeFwSlotInfoLog fw_log
= {
2987 if (off
>= sizeof(fw_log
)) {
2988 return NVME_INVALID_FIELD
| NVME_DNR
;
2991 strpadcpy((char *)&fw_log
.frs1
, sizeof(fw_log
.frs1
), "1.0", ' ');
2992 trans_len
= MIN(sizeof(fw_log
) - off
, buf_len
);
2994 return nvme_c2h(n
, (uint8_t *) &fw_log
+ off
, trans_len
, req
);
2997 static uint16_t nvme_error_info(NvmeCtrl
*n
, uint8_t rae
, uint32_t buf_len
,
2998 uint64_t off
, NvmeRequest
*req
)
3001 NvmeErrorLog errlog
;
3003 if (off
>= sizeof(errlog
)) {
3004 return NVME_INVALID_FIELD
| NVME_DNR
;
3008 nvme_clear_events(n
, NVME_AER_TYPE_ERROR
);
3011 memset(&errlog
, 0x0, sizeof(errlog
));
3012 trans_len
= MIN(sizeof(errlog
) - off
, buf_len
);
3014 return nvme_c2h(n
, (uint8_t *)&errlog
, trans_len
, req
);
3017 static uint16_t nvme_cmd_effects(NvmeCtrl
*n
, uint8_t csi
, uint32_t buf_len
,
3018 uint64_t off
, NvmeRequest
*req
)
3020 NvmeEffectsLog log
= {};
3021 const uint32_t *src_iocs
= NULL
;
3024 if (off
>= sizeof(log
)) {
3025 trace_pci_nvme_err_invalid_log_page_offset(off
, sizeof(log
));
3026 return NVME_INVALID_FIELD
| NVME_DNR
;
3029 switch (NVME_CC_CSS(n
->bar
.cc
)) {
3030 case NVME_CC_CSS_NVM
:
3031 src_iocs
= nvme_cse_iocs_nvm
;
3033 case NVME_CC_CSS_ADMIN_ONLY
:
3035 case NVME_CC_CSS_CSI
:
3038 src_iocs
= nvme_cse_iocs_nvm
;
3040 case NVME_CSI_ZONED
:
3041 src_iocs
= nvme_cse_iocs_zoned
;
3046 memcpy(log
.acs
, nvme_cse_acs
, sizeof(nvme_cse_acs
));
3049 memcpy(log
.iocs
, src_iocs
, sizeof(log
.iocs
));
3052 trans_len
= MIN(sizeof(log
) - off
, buf_len
);
3054 return nvme_c2h(n
, ((uint8_t *)&log
) + off
, trans_len
, req
);
3057 static uint16_t nvme_get_log(NvmeCtrl
*n
, NvmeRequest
*req
)
3059 NvmeCmd
*cmd
= &req
->cmd
;
3061 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
3062 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
3063 uint32_t dw12
= le32_to_cpu(cmd
->cdw12
);
3064 uint32_t dw13
= le32_to_cpu(cmd
->cdw13
);
3065 uint8_t lid
= dw10
& 0xff;
3066 uint8_t lsp
= (dw10
>> 8) & 0xf;
3067 uint8_t rae
= (dw10
>> 15) & 0x1;
3068 uint8_t csi
= le32_to_cpu(cmd
->cdw14
) >> 24;
3069 uint32_t numdl
, numdu
;
3070 uint64_t off
, lpol
, lpou
;
3074 numdl
= (dw10
>> 16);
3075 numdu
= (dw11
& 0xffff);
3079 len
= (((numdu
<< 16) | numdl
) + 1) << 2;
3080 off
= (lpou
<< 32ULL) | lpol
;
3083 return NVME_INVALID_FIELD
| NVME_DNR
;
3086 trace_pci_nvme_get_log(nvme_cid(req
), lid
, lsp
, rae
, len
, off
);
3088 status
= nvme_check_mdts(n
, len
);
3094 case NVME_LOG_ERROR_INFO
:
3095 return nvme_error_info(n
, rae
, len
, off
, req
);
3096 case NVME_LOG_SMART_INFO
:
3097 return nvme_smart_info(n
, rae
, len
, off
, req
);
3098 case NVME_LOG_FW_SLOT_INFO
:
3099 return nvme_fw_log_info(n
, len
, off
, req
);
3100 case NVME_LOG_CMD_EFFECTS
:
3101 return nvme_cmd_effects(n
, csi
, len
, off
, req
);
3103 trace_pci_nvme_err_invalid_log_page(nvme_cid(req
), lid
);
3104 return NVME_INVALID_FIELD
| NVME_DNR
;
3108 static void nvme_free_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
)
3110 n
->cq
[cq
->cqid
] = NULL
;
3111 timer_free(cq
->timer
);
3112 if (msix_enabled(&n
->parent_obj
)) {
3113 msix_vector_unuse(&n
->parent_obj
, cq
->vector
);
3120 static uint16_t nvme_del_cq(NvmeCtrl
*n
, NvmeRequest
*req
)
3122 NvmeDeleteQ
*c
= (NvmeDeleteQ
*)&req
->cmd
;
3124 uint16_t qid
= le16_to_cpu(c
->qid
);
3126 if (unlikely(!qid
|| nvme_check_cqid(n
, qid
))) {
3127 trace_pci_nvme_err_invalid_del_cq_cqid(qid
);
3128 return NVME_INVALID_CQID
| NVME_DNR
;
3132 if (unlikely(!QTAILQ_EMPTY(&cq
->sq_list
))) {
3133 trace_pci_nvme_err_invalid_del_cq_notempty(qid
);
3134 return NVME_INVALID_QUEUE_DEL
;
3136 nvme_irq_deassert(n
, cq
);
3137 trace_pci_nvme_del_cq(qid
);
3138 nvme_free_cq(cq
, n
);
3139 return NVME_SUCCESS
;
3142 static void nvme_init_cq(NvmeCQueue
*cq
, NvmeCtrl
*n
, uint64_t dma_addr
,
3143 uint16_t cqid
, uint16_t vector
, uint16_t size
,
3144 uint16_t irq_enabled
)
3148 if (msix_enabled(&n
->parent_obj
)) {
3149 ret
= msix_vector_use(&n
->parent_obj
, vector
);
3155 cq
->dma_addr
= dma_addr
;
3157 cq
->irq_enabled
= irq_enabled
;
3158 cq
->vector
= vector
;
3159 cq
->head
= cq
->tail
= 0;
3160 QTAILQ_INIT(&cq
->req_list
);
3161 QTAILQ_INIT(&cq
->sq_list
);
3163 cq
->timer
= timer_new_ns(QEMU_CLOCK_VIRTUAL
, nvme_post_cqes
, cq
);
3166 static uint16_t nvme_create_cq(NvmeCtrl
*n
, NvmeRequest
*req
)
3169 NvmeCreateCq
*c
= (NvmeCreateCq
*)&req
->cmd
;
3170 uint16_t cqid
= le16_to_cpu(c
->cqid
);
3171 uint16_t vector
= le16_to_cpu(c
->irq_vector
);
3172 uint16_t qsize
= le16_to_cpu(c
->qsize
);
3173 uint16_t qflags
= le16_to_cpu(c
->cq_flags
);
3174 uint64_t prp1
= le64_to_cpu(c
->prp1
);
3176 trace_pci_nvme_create_cq(prp1
, cqid
, vector
, qsize
, qflags
,
3177 NVME_CQ_FLAGS_IEN(qflags
) != 0);
3179 if (unlikely(!cqid
|| cqid
> n
->params
.max_ioqpairs
||
3180 n
->cq
[cqid
] != NULL
)) {
3181 trace_pci_nvme_err_invalid_create_cq_cqid(cqid
);
3182 return NVME_INVALID_QID
| NVME_DNR
;
3184 if (unlikely(!qsize
|| qsize
> NVME_CAP_MQES(n
->bar
.cap
))) {
3185 trace_pci_nvme_err_invalid_create_cq_size(qsize
);
3186 return NVME_MAX_QSIZE_EXCEEDED
| NVME_DNR
;
3188 if (unlikely(prp1
& (n
->page_size
- 1))) {
3189 trace_pci_nvme_err_invalid_create_cq_addr(prp1
);
3190 return NVME_INVALID_PRP_OFFSET
| NVME_DNR
;
3192 if (unlikely(!msix_enabled(&n
->parent_obj
) && vector
)) {
3193 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
3194 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
3196 if (unlikely(vector
>= n
->params
.msix_qsize
)) {
3197 trace_pci_nvme_err_invalid_create_cq_vector(vector
);
3198 return NVME_INVALID_IRQ_VECTOR
| NVME_DNR
;
3200 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags
)))) {
3201 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags
));
3202 return NVME_INVALID_FIELD
| NVME_DNR
;
3205 cq
= g_malloc0(sizeof(*cq
));
3206 nvme_init_cq(cq
, n
, prp1
, cqid
, vector
, qsize
+ 1,
3207 NVME_CQ_FLAGS_IEN(qflags
));
3210 * It is only required to set qs_created when creating a completion queue;
3211 * creating a submission queue without a matching completion queue will
3214 n
->qs_created
= true;
3215 return NVME_SUCCESS
;
3218 static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl
*n
, NvmeRequest
*req
)
3220 uint8_t id
[NVME_IDENTIFY_DATA_SIZE
] = {};
3222 return nvme_c2h(n
, id
, sizeof(id
), req
);
3225 static inline bool nvme_csi_has_nvm_support(NvmeNamespace
*ns
)
3229 case NVME_CSI_ZONED
:
3235 static uint16_t nvme_identify_ctrl(NvmeCtrl
*n
, NvmeRequest
*req
)
3237 trace_pci_nvme_identify_ctrl();
3239 return nvme_c2h(n
, (uint8_t *)&n
->id_ctrl
, sizeof(n
->id_ctrl
), req
);
3242 static uint16_t nvme_identify_ctrl_csi(NvmeCtrl
*n
, NvmeRequest
*req
)
3244 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
3245 uint8_t id
[NVME_IDENTIFY_DATA_SIZE
] = {};
3247 trace_pci_nvme_identify_ctrl_csi(c
->csi
);
3251 ((NvmeIdCtrlNvm
*)&id
)->dmrsl
= cpu_to_le32(n
->dmrsl
);
3254 case NVME_CSI_ZONED
:
3255 ((NvmeIdCtrlZoned
*)&id
)->zasl
= n
->params
.zasl
;
3259 return NVME_INVALID_FIELD
| NVME_DNR
;
3262 return nvme_c2h(n
, id
, sizeof(id
), req
);
3265 static uint16_t nvme_identify_ns(NvmeCtrl
*n
, NvmeRequest
*req
, bool active
)
3268 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
3269 uint32_t nsid
= le32_to_cpu(c
->nsid
);
3271 trace_pci_nvme_identify_ns(nsid
);
3273 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
3274 return NVME_INVALID_NSID
| NVME_DNR
;
3277 ns
= nvme_ns(n
, nsid
);
3278 if (unlikely(!ns
)) {
3280 ns
= nvme_subsys_ns(n
->subsys
, nsid
);
3282 return nvme_rpt_empty_id_struct(n
, req
);
3285 return nvme_rpt_empty_id_struct(n
, req
);
3289 if (c
->csi
== NVME_CSI_NVM
&& nvme_csi_has_nvm_support(ns
)) {
3290 return nvme_c2h(n
, (uint8_t *)&ns
->id_ns
, sizeof(NvmeIdNs
), req
);
3293 return NVME_INVALID_CMD_SET
| NVME_DNR
;
3296 static uint16_t nvme_identify_ns_csi(NvmeCtrl
*n
, NvmeRequest
*req
,
3300 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
3301 uint32_t nsid
= le32_to_cpu(c
->nsid
);
3303 trace_pci_nvme_identify_ns_csi(nsid
, c
->csi
);
3305 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
3306 return NVME_INVALID_NSID
| NVME_DNR
;
3309 ns
= nvme_ns(n
, nsid
);
3310 if (unlikely(!ns
)) {
3312 ns
= nvme_subsys_ns(n
->subsys
, nsid
);
3314 return nvme_rpt_empty_id_struct(n
, req
);
3317 return nvme_rpt_empty_id_struct(n
, req
);
3321 if (c
->csi
== NVME_CSI_NVM
&& nvme_csi_has_nvm_support(ns
)) {
3322 return nvme_rpt_empty_id_struct(n
, req
);
3323 } else if (c
->csi
== NVME_CSI_ZONED
&& ns
->csi
== NVME_CSI_ZONED
) {
3324 return nvme_c2h(n
, (uint8_t *)ns
->id_ns_zoned
, sizeof(NvmeIdNsZoned
),
3328 return NVME_INVALID_FIELD
| NVME_DNR
;
3331 static uint16_t nvme_identify_nslist(NvmeCtrl
*n
, NvmeRequest
*req
,
3335 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
3336 uint32_t min_nsid
= le32_to_cpu(c
->nsid
);
3337 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
3338 static const int data_len
= sizeof(list
);
3339 uint32_t *list_ptr
= (uint32_t *)list
;
3342 trace_pci_nvme_identify_nslist(min_nsid
);
3345 * Both 0xffffffff (NVME_NSID_BROADCAST) and 0xfffffffe are invalid values
3346 * since the Active Namespace ID List should return namespaces with ids
3347 * *higher* than the NSID specified in the command. This is also specified
3348 * in the spec (NVM Express v1.3d, Section 5.15.4).
3350 if (min_nsid
>= NVME_NSID_BROADCAST
- 1) {
3351 return NVME_INVALID_NSID
| NVME_DNR
;
3354 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
3358 ns
= nvme_subsys_ns(n
->subsys
, i
);
3366 if (ns
->params
.nsid
<= min_nsid
) {
3369 list_ptr
[j
++] = cpu_to_le32(ns
->params
.nsid
);
3370 if (j
== data_len
/ sizeof(uint32_t)) {
3375 return nvme_c2h(n
, list
, data_len
, req
);
3378 static uint16_t nvme_identify_nslist_csi(NvmeCtrl
*n
, NvmeRequest
*req
,
3382 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
3383 uint32_t min_nsid
= le32_to_cpu(c
->nsid
);
3384 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
3385 static const int data_len
= sizeof(list
);
3386 uint32_t *list_ptr
= (uint32_t *)list
;
3389 trace_pci_nvme_identify_nslist_csi(min_nsid
, c
->csi
);
3392 * Same as in nvme_identify_nslist(), 0xffffffff/0xfffffffe are invalid.
3394 if (min_nsid
>= NVME_NSID_BROADCAST
- 1) {
3395 return NVME_INVALID_NSID
| NVME_DNR
;
3398 if (c
->csi
!= NVME_CSI_NVM
&& c
->csi
!= NVME_CSI_ZONED
) {
3399 return NVME_INVALID_FIELD
| NVME_DNR
;
3402 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
3406 ns
= nvme_subsys_ns(n
->subsys
, i
);
3414 if (ns
->params
.nsid
<= min_nsid
|| c
->csi
!= ns
->csi
) {
3417 list_ptr
[j
++] = cpu_to_le32(ns
->params
.nsid
);
3418 if (j
== data_len
/ sizeof(uint32_t)) {
3423 return nvme_c2h(n
, list
, data_len
, req
);
3426 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl
*n
, NvmeRequest
*req
)
3429 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
3430 uint32_t nsid
= le32_to_cpu(c
->nsid
);
3431 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
3436 uint8_t v
[NVME_NIDL_UUID
];
3444 struct data
*ns_descrs
= (struct data
*)list
;
3446 trace_pci_nvme_identify_ns_descr_list(nsid
);
3448 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
3449 return NVME_INVALID_NSID
| NVME_DNR
;
3452 ns
= nvme_ns(n
, nsid
);
3453 if (unlikely(!ns
)) {
3454 return NVME_INVALID_FIELD
| NVME_DNR
;
3458 * Because the NGUID and EUI64 fields are 0 in the Identify Namespace data
3459 * structure, a Namespace UUID (nidt = 0x3) must be reported in the
3460 * Namespace Identification Descriptor. Add the namespace UUID here.
3462 ns_descrs
->uuid
.hdr
.nidt
= NVME_NIDT_UUID
;
3463 ns_descrs
->uuid
.hdr
.nidl
= NVME_NIDL_UUID
;
3464 memcpy(&ns_descrs
->uuid
.v
, ns
->params
.uuid
.data
, NVME_NIDL_UUID
);
3466 ns_descrs
->csi
.hdr
.nidt
= NVME_NIDT_CSI
;
3467 ns_descrs
->csi
.hdr
.nidl
= NVME_NIDL_CSI
;
3468 ns_descrs
->csi
.v
= ns
->csi
;
3470 return nvme_c2h(n
, list
, sizeof(list
), req
);
3473 static uint16_t nvme_identify_cmd_set(NvmeCtrl
*n
, NvmeRequest
*req
)
3475 uint8_t list
[NVME_IDENTIFY_DATA_SIZE
] = {};
3476 static const int data_len
= sizeof(list
);
3478 trace_pci_nvme_identify_cmd_set();
3480 NVME_SET_CSI(*list
, NVME_CSI_NVM
);
3481 NVME_SET_CSI(*list
, NVME_CSI_ZONED
);
3483 return nvme_c2h(n
, list
, data_len
, req
);
3486 static uint16_t nvme_identify(NvmeCtrl
*n
, NvmeRequest
*req
)
3488 NvmeIdentify
*c
= (NvmeIdentify
*)&req
->cmd
;
3490 trace_pci_nvme_identify(nvme_cid(req
), c
->cns
, le16_to_cpu(c
->ctrlid
),
3494 case NVME_ID_CNS_NS
:
3495 return nvme_identify_ns(n
, req
, true);
3496 case NVME_ID_CNS_NS_PRESENT
:
3497 return nvme_identify_ns(n
, req
, false);
3498 case NVME_ID_CNS_CS_NS
:
3499 return nvme_identify_ns_csi(n
, req
, true);
3500 case NVME_ID_CNS_CS_NS_PRESENT
:
3501 return nvme_identify_ns_csi(n
, req
, false);
3502 case NVME_ID_CNS_CTRL
:
3503 return nvme_identify_ctrl(n
, req
);
3504 case NVME_ID_CNS_CS_CTRL
:
3505 return nvme_identify_ctrl_csi(n
, req
);
3506 case NVME_ID_CNS_NS_ACTIVE_LIST
:
3507 return nvme_identify_nslist(n
, req
, true);
3508 case NVME_ID_CNS_NS_PRESENT_LIST
:
3509 return nvme_identify_nslist(n
, req
, false);
3510 case NVME_ID_CNS_CS_NS_ACTIVE_LIST
:
3511 return nvme_identify_nslist_csi(n
, req
, true);
3512 case NVME_ID_CNS_CS_NS_PRESENT_LIST
:
3513 return nvme_identify_nslist_csi(n
, req
, false);
3514 case NVME_ID_CNS_NS_DESCR_LIST
:
3515 return nvme_identify_ns_descr_list(n
, req
);
3516 case NVME_ID_CNS_IO_COMMAND_SET
:
3517 return nvme_identify_cmd_set(n
, req
);
3519 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c
->cns
));
3520 return NVME_INVALID_FIELD
| NVME_DNR
;
3524 static uint16_t nvme_abort(NvmeCtrl
*n
, NvmeRequest
*req
)
3526 uint16_t sqid
= le32_to_cpu(req
->cmd
.cdw10
) & 0xffff;
3528 req
->cqe
.result
= 1;
3529 if (nvme_check_sqid(n
, sqid
)) {
3530 return NVME_INVALID_FIELD
| NVME_DNR
;
3533 return NVME_SUCCESS
;
3536 static inline void nvme_set_timestamp(NvmeCtrl
*n
, uint64_t ts
)
3538 trace_pci_nvme_setfeat_timestamp(ts
);
3540 n
->host_timestamp
= le64_to_cpu(ts
);
3541 n
->timestamp_set_qemu_clock_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
3544 static inline uint64_t nvme_get_timestamp(const NvmeCtrl
*n
)
3546 uint64_t current_time
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
3547 uint64_t elapsed_time
= current_time
- n
->timestamp_set_qemu_clock_ms
;
3549 union nvme_timestamp
{
3551 uint64_t timestamp
:48;
3559 union nvme_timestamp ts
;
3561 ts
.timestamp
= n
->host_timestamp
+ elapsed_time
;
3563 /* If the host timestamp is non-zero, set the timestamp origin */
3564 ts
.origin
= n
->host_timestamp
? 0x01 : 0x00;
3566 trace_pci_nvme_getfeat_timestamp(ts
.all
);
3568 return cpu_to_le64(ts
.all
);
3571 static uint16_t nvme_get_feature_timestamp(NvmeCtrl
*n
, NvmeRequest
*req
)
3573 uint64_t timestamp
= nvme_get_timestamp(n
);
3575 return nvme_c2h(n
, (uint8_t *)×tamp
, sizeof(timestamp
), req
);
3578 static uint16_t nvme_get_feature(NvmeCtrl
*n
, NvmeRequest
*req
)
3580 NvmeCmd
*cmd
= &req
->cmd
;
3581 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
3582 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
3583 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
3585 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
3586 NvmeGetFeatureSelect sel
= NVME_GETFEAT_SELECT(dw10
);
3591 static const uint32_t nvme_feature_default
[NVME_FID_MAX
] = {
3592 [NVME_ARBITRATION
] = NVME_ARB_AB_NOLIMIT
,
3595 trace_pci_nvme_getfeat(nvme_cid(req
), nsid
, fid
, sel
, dw11
);
3597 if (!nvme_feature_support
[fid
]) {
3598 return NVME_INVALID_FIELD
| NVME_DNR
;
3601 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
3602 if (!nvme_nsid_valid(n
, nsid
) || nsid
== NVME_NSID_BROADCAST
) {
3604 * The Reservation Notification Mask and Reservation Persistence
3605 * features require a status code of Invalid Field in Command when
3606 * NSID is 0xFFFFFFFF. Since the device does not support those
3607 * features we can always return Invalid Namespace or Format as we
3608 * should do for all other features.
3610 return NVME_INVALID_NSID
| NVME_DNR
;
3613 if (!nvme_ns(n
, nsid
)) {
3614 return NVME_INVALID_FIELD
| NVME_DNR
;
3619 case NVME_GETFEAT_SELECT_CURRENT
:
3621 case NVME_GETFEAT_SELECT_SAVED
:
3622 /* no features are saveable by the controller; fallthrough */
3623 case NVME_GETFEAT_SELECT_DEFAULT
:
3625 case NVME_GETFEAT_SELECT_CAP
:
3626 result
= nvme_feature_cap
[fid
];
3631 case NVME_TEMPERATURE_THRESHOLD
:
3635 * The controller only implements the Composite Temperature sensor, so
3636 * return 0 for all other sensors.
3638 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
3642 switch (NVME_TEMP_THSEL(dw11
)) {
3643 case NVME_TEMP_THSEL_OVER
:
3644 result
= n
->features
.temp_thresh_hi
;
3646 case NVME_TEMP_THSEL_UNDER
:
3647 result
= n
->features
.temp_thresh_low
;
3651 return NVME_INVALID_FIELD
| NVME_DNR
;
3652 case NVME_ERROR_RECOVERY
:
3653 if (!nvme_nsid_valid(n
, nsid
)) {
3654 return NVME_INVALID_NSID
| NVME_DNR
;
3657 ns
= nvme_ns(n
, nsid
);
3658 if (unlikely(!ns
)) {
3659 return NVME_INVALID_FIELD
| NVME_DNR
;
3662 result
= ns
->features
.err_rec
;
3664 case NVME_VOLATILE_WRITE_CACHE
:
3666 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
3672 result
= blk_enable_write_cache(ns
->blkconf
.blk
);
3677 trace_pci_nvme_getfeat_vwcache(result
? "enabled" : "disabled");
3679 case NVME_ASYNCHRONOUS_EVENT_CONF
:
3680 result
= n
->features
.async_config
;
3682 case NVME_TIMESTAMP
:
3683 return nvme_get_feature_timestamp(n
, req
);
3690 case NVME_TEMPERATURE_THRESHOLD
:
3693 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
3697 if (NVME_TEMP_THSEL(dw11
) == NVME_TEMP_THSEL_OVER
) {
3698 result
= NVME_TEMPERATURE_WARNING
;
3702 case NVME_NUMBER_OF_QUEUES
:
3703 result
= (n
->params
.max_ioqpairs
- 1) |
3704 ((n
->params
.max_ioqpairs
- 1) << 16);
3705 trace_pci_nvme_getfeat_numq(result
);
3707 case NVME_INTERRUPT_VECTOR_CONF
:
3709 if (iv
>= n
->params
.max_ioqpairs
+ 1) {
3710 return NVME_INVALID_FIELD
| NVME_DNR
;
3714 if (iv
== n
->admin_cq
.vector
) {
3715 result
|= NVME_INTVC_NOCOALESCING
;
3718 case NVME_COMMAND_SET_PROFILE
:
3722 result
= nvme_feature_default
[fid
];
3727 req
->cqe
.result
= cpu_to_le32(result
);
3728 return NVME_SUCCESS
;
3731 static uint16_t nvme_set_feature_timestamp(NvmeCtrl
*n
, NvmeRequest
*req
)
3736 ret
= nvme_h2c(n
, (uint8_t *)×tamp
, sizeof(timestamp
), req
);
3741 nvme_set_timestamp(n
, timestamp
);
3743 return NVME_SUCCESS
;
3746 static uint16_t nvme_set_feature(NvmeCtrl
*n
, NvmeRequest
*req
)
3748 NvmeNamespace
*ns
= NULL
;
3750 NvmeCmd
*cmd
= &req
->cmd
;
3751 uint32_t dw10
= le32_to_cpu(cmd
->cdw10
);
3752 uint32_t dw11
= le32_to_cpu(cmd
->cdw11
);
3753 uint32_t nsid
= le32_to_cpu(cmd
->nsid
);
3754 uint8_t fid
= NVME_GETSETFEAT_FID(dw10
);
3755 uint8_t save
= NVME_SETFEAT_SAVE(dw10
);
3758 trace_pci_nvme_setfeat(nvme_cid(req
), nsid
, fid
, save
, dw11
);
3760 if (save
&& !(nvme_feature_cap
[fid
] & NVME_FEAT_CAP_SAVE
)) {
3761 return NVME_FID_NOT_SAVEABLE
| NVME_DNR
;
3764 if (!nvme_feature_support
[fid
]) {
3765 return NVME_INVALID_FIELD
| NVME_DNR
;
3768 if (nvme_feature_cap
[fid
] & NVME_FEAT_CAP_NS
) {
3769 if (nsid
!= NVME_NSID_BROADCAST
) {
3770 if (!nvme_nsid_valid(n
, nsid
)) {
3771 return NVME_INVALID_NSID
| NVME_DNR
;
3774 ns
= nvme_ns(n
, nsid
);
3775 if (unlikely(!ns
)) {
3776 return NVME_INVALID_FIELD
| NVME_DNR
;
3779 } else if (nsid
&& nsid
!= NVME_NSID_BROADCAST
) {
3780 if (!nvme_nsid_valid(n
, nsid
)) {
3781 return NVME_INVALID_NSID
| NVME_DNR
;
3784 return NVME_FEAT_NOT_NS_SPEC
| NVME_DNR
;
3787 if (!(nvme_feature_cap
[fid
] & NVME_FEAT_CAP_CHANGE
)) {
3788 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
3792 case NVME_TEMPERATURE_THRESHOLD
:
3793 if (NVME_TEMP_TMPSEL(dw11
) != NVME_TEMP_TMPSEL_COMPOSITE
) {
3797 switch (NVME_TEMP_THSEL(dw11
)) {
3798 case NVME_TEMP_THSEL_OVER
:
3799 n
->features
.temp_thresh_hi
= NVME_TEMP_TMPTH(dw11
);
3801 case NVME_TEMP_THSEL_UNDER
:
3802 n
->features
.temp_thresh_low
= NVME_TEMP_TMPTH(dw11
);
3805 return NVME_INVALID_FIELD
| NVME_DNR
;
3808 if ((n
->temperature
>= n
->features
.temp_thresh_hi
) ||
3809 (n
->temperature
<= n
->features
.temp_thresh_low
)) {
3810 nvme_smart_event(n
, NVME_AER_INFO_SMART_TEMP_THRESH
);
3814 case NVME_ERROR_RECOVERY
:
3815 if (nsid
== NVME_NSID_BROADCAST
) {
3816 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
3823 if (NVME_ID_NS_NSFEAT_DULBE(ns
->id_ns
.nsfeat
)) {
3824 ns
->features
.err_rec
= dw11
;
3832 if (NVME_ID_NS_NSFEAT_DULBE(ns
->id_ns
.nsfeat
)) {
3833 ns
->features
.err_rec
= dw11
;
3836 case NVME_VOLATILE_WRITE_CACHE
:
3837 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
3843 if (!(dw11
& 0x1) && blk_enable_write_cache(ns
->blkconf
.blk
)) {
3844 blk_flush(ns
->blkconf
.blk
);
3847 blk_set_enable_write_cache(ns
->blkconf
.blk
, dw11
& 1);
3852 case NVME_NUMBER_OF_QUEUES
:
3853 if (n
->qs_created
) {
3854 return NVME_CMD_SEQ_ERROR
| NVME_DNR
;
3858 * NVMe v1.3, Section 5.21.1.7: 0xffff is not an allowed value for NCQR
3861 if ((dw11
& 0xffff) == 0xffff || ((dw11
>> 16) & 0xffff) == 0xffff) {
3862 return NVME_INVALID_FIELD
| NVME_DNR
;
3865 trace_pci_nvme_setfeat_numq((dw11
& 0xFFFF) + 1,
3866 ((dw11
>> 16) & 0xFFFF) + 1,
3867 n
->params
.max_ioqpairs
,
3868 n
->params
.max_ioqpairs
);
3869 req
->cqe
.result
= cpu_to_le32((n
->params
.max_ioqpairs
- 1) |
3870 ((n
->params
.max_ioqpairs
- 1) << 16));
3872 case NVME_ASYNCHRONOUS_EVENT_CONF
:
3873 n
->features
.async_config
= dw11
;
3875 case NVME_TIMESTAMP
:
3876 return nvme_set_feature_timestamp(n
, req
);
3877 case NVME_COMMAND_SET_PROFILE
:
3879 trace_pci_nvme_err_invalid_iocsci(dw11
& 0x1ff);
3880 return NVME_CMD_SET_CMB_REJECTED
| NVME_DNR
;
3884 return NVME_FEAT_NOT_CHANGEABLE
| NVME_DNR
;
3886 return NVME_SUCCESS
;
3889 static uint16_t nvme_aer(NvmeCtrl
*n
, NvmeRequest
*req
)
3891 trace_pci_nvme_aer(nvme_cid(req
));
3893 if (n
->outstanding_aers
> n
->params
.aerl
) {
3894 trace_pci_nvme_aer_aerl_exceeded();
3895 return NVME_AER_LIMIT_EXCEEDED
;
3898 n
->aer_reqs
[n
->outstanding_aers
] = req
;
3899 n
->outstanding_aers
++;
3901 if (!QTAILQ_EMPTY(&n
->aer_queue
)) {
3902 nvme_process_aers(n
);
3905 return NVME_NO_COMPLETE
;
3908 static uint16_t nvme_admin_cmd(NvmeCtrl
*n
, NvmeRequest
*req
)
3910 trace_pci_nvme_admin_cmd(nvme_cid(req
), nvme_sqid(req
), req
->cmd
.opcode
,
3911 nvme_adm_opc_str(req
->cmd
.opcode
));
3913 if (!(nvme_cse_acs
[req
->cmd
.opcode
] & NVME_CMD_EFF_CSUPP
)) {
3914 trace_pci_nvme_err_invalid_admin_opc(req
->cmd
.opcode
);
3915 return NVME_INVALID_OPCODE
| NVME_DNR
;
3918 /* SGLs shall not be used for Admin commands in NVMe over PCIe */
3919 if (NVME_CMD_FLAGS_PSDT(req
->cmd
.flags
) != NVME_PSDT_PRP
) {
3920 return NVME_INVALID_FIELD
| NVME_DNR
;
3923 switch (req
->cmd
.opcode
) {
3924 case NVME_ADM_CMD_DELETE_SQ
:
3925 return nvme_del_sq(n
, req
);
3926 case NVME_ADM_CMD_CREATE_SQ
:
3927 return nvme_create_sq(n
, req
);
3928 case NVME_ADM_CMD_GET_LOG_PAGE
:
3929 return nvme_get_log(n
, req
);
3930 case NVME_ADM_CMD_DELETE_CQ
:
3931 return nvme_del_cq(n
, req
);
3932 case NVME_ADM_CMD_CREATE_CQ
:
3933 return nvme_create_cq(n
, req
);
3934 case NVME_ADM_CMD_IDENTIFY
:
3935 return nvme_identify(n
, req
);
3936 case NVME_ADM_CMD_ABORT
:
3937 return nvme_abort(n
, req
);
3938 case NVME_ADM_CMD_SET_FEATURES
:
3939 return nvme_set_feature(n
, req
);
3940 case NVME_ADM_CMD_GET_FEATURES
:
3941 return nvme_get_feature(n
, req
);
3942 case NVME_ADM_CMD_ASYNC_EV_REQ
:
3943 return nvme_aer(n
, req
);
3948 return NVME_INVALID_OPCODE
| NVME_DNR
;
3951 static void nvme_process_sq(void *opaque
)
3953 NvmeSQueue
*sq
= opaque
;
3954 NvmeCtrl
*n
= sq
->ctrl
;
3955 NvmeCQueue
*cq
= n
->cq
[sq
->cqid
];
3962 while (!(nvme_sq_empty(sq
) || QTAILQ_EMPTY(&sq
->req_list
))) {
3963 addr
= sq
->dma_addr
+ sq
->head
* n
->sqe_size
;
3964 if (nvme_addr_read(n
, addr
, (void *)&cmd
, sizeof(cmd
))) {
3965 trace_pci_nvme_err_addr_read(addr
);
3966 trace_pci_nvme_err_cfs();
3967 n
->bar
.csts
= NVME_CSTS_FAILED
;
3970 nvme_inc_sq_head(sq
);
3972 req
= QTAILQ_FIRST(&sq
->req_list
);
3973 QTAILQ_REMOVE(&sq
->req_list
, req
, entry
);
3974 QTAILQ_INSERT_TAIL(&sq
->out_req_list
, req
, entry
);
3975 nvme_req_clear(req
);
3976 req
->cqe
.cid
= cmd
.cid
;
3977 memcpy(&req
->cmd
, &cmd
, sizeof(NvmeCmd
));
3979 status
= sq
->sqid
? nvme_io_cmd(n
, req
) :
3980 nvme_admin_cmd(n
, req
);
3981 if (status
!= NVME_NO_COMPLETE
) {
3982 req
->status
= status
;
3983 nvme_enqueue_req_completion(cq
, req
);
3988 static void nvme_ctrl_reset(NvmeCtrl
*n
)
3993 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
4002 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
4003 if (n
->sq
[i
] != NULL
) {
4004 nvme_free_sq(n
->sq
[i
], n
);
4007 for (i
= 0; i
< n
->params
.max_ioqpairs
+ 1; i
++) {
4008 if (n
->cq
[i
] != NULL
) {
4009 nvme_free_cq(n
->cq
[i
], n
);
4013 while (!QTAILQ_EMPTY(&n
->aer_queue
)) {
4014 NvmeAsyncEvent
*event
= QTAILQ_FIRST(&n
->aer_queue
);
4015 QTAILQ_REMOVE(&n
->aer_queue
, event
, entry
);
4020 n
->outstanding_aers
= 0;
4021 n
->qs_created
= false;
4026 static void nvme_ctrl_shutdown(NvmeCtrl
*n
)
4032 memory_region_msync(&n
->pmr
.dev
->mr
, 0, n
->pmr
.dev
->size
);
4035 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
4041 nvme_ns_shutdown(ns
);
4045 static void __nvme_select_ns_iocs(NvmeCtrl
*n
, NvmeNamespace
*ns
)
4047 ns
->iocs
= nvme_cse_iocs_none
;
4050 if (NVME_CC_CSS(n
->bar
.cc
) != NVME_CC_CSS_ADMIN_ONLY
) {
4051 ns
->iocs
= nvme_cse_iocs_nvm
;
4054 case NVME_CSI_ZONED
:
4055 if (NVME_CC_CSS(n
->bar
.cc
) == NVME_CC_CSS_CSI
) {
4056 ns
->iocs
= nvme_cse_iocs_zoned
;
4057 } else if (NVME_CC_CSS(n
->bar
.cc
) == NVME_CC_CSS_NVM
) {
4058 ns
->iocs
= nvme_cse_iocs_nvm
;
4064 static void nvme_select_ns_iocs(NvmeCtrl
*n
)
4069 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
4075 __nvme_select_ns_iocs(n
, ns
);
4079 static int nvme_start_ctrl(NvmeCtrl
*n
)
4081 uint32_t page_bits
= NVME_CC_MPS(n
->bar
.cc
) + 12;
4082 uint32_t page_size
= 1 << page_bits
;
4084 if (unlikely(n
->cq
[0])) {
4085 trace_pci_nvme_err_startfail_cq();
4088 if (unlikely(n
->sq
[0])) {
4089 trace_pci_nvme_err_startfail_sq();
4092 if (unlikely(!n
->bar
.asq
)) {
4093 trace_pci_nvme_err_startfail_nbarasq();
4096 if (unlikely(!n
->bar
.acq
)) {
4097 trace_pci_nvme_err_startfail_nbaracq();
4100 if (unlikely(n
->bar
.asq
& (page_size
- 1))) {
4101 trace_pci_nvme_err_startfail_asq_misaligned(n
->bar
.asq
);
4104 if (unlikely(n
->bar
.acq
& (page_size
- 1))) {
4105 trace_pci_nvme_err_startfail_acq_misaligned(n
->bar
.acq
);
4108 if (unlikely(!(NVME_CAP_CSS(n
->bar
.cap
) & (1 << NVME_CC_CSS(n
->bar
.cc
))))) {
4109 trace_pci_nvme_err_startfail_css(NVME_CC_CSS(n
->bar
.cc
));
4112 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) <
4113 NVME_CAP_MPSMIN(n
->bar
.cap
))) {
4114 trace_pci_nvme_err_startfail_page_too_small(
4115 NVME_CC_MPS(n
->bar
.cc
),
4116 NVME_CAP_MPSMIN(n
->bar
.cap
));
4119 if (unlikely(NVME_CC_MPS(n
->bar
.cc
) >
4120 NVME_CAP_MPSMAX(n
->bar
.cap
))) {
4121 trace_pci_nvme_err_startfail_page_too_large(
4122 NVME_CC_MPS(n
->bar
.cc
),
4123 NVME_CAP_MPSMAX(n
->bar
.cap
));
4126 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) <
4127 NVME_CTRL_CQES_MIN(n
->id_ctrl
.cqes
))) {
4128 trace_pci_nvme_err_startfail_cqent_too_small(
4129 NVME_CC_IOCQES(n
->bar
.cc
),
4130 NVME_CTRL_CQES_MIN(n
->bar
.cap
));
4133 if (unlikely(NVME_CC_IOCQES(n
->bar
.cc
) >
4134 NVME_CTRL_CQES_MAX(n
->id_ctrl
.cqes
))) {
4135 trace_pci_nvme_err_startfail_cqent_too_large(
4136 NVME_CC_IOCQES(n
->bar
.cc
),
4137 NVME_CTRL_CQES_MAX(n
->bar
.cap
));
4140 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) <
4141 NVME_CTRL_SQES_MIN(n
->id_ctrl
.sqes
))) {
4142 trace_pci_nvme_err_startfail_sqent_too_small(
4143 NVME_CC_IOSQES(n
->bar
.cc
),
4144 NVME_CTRL_SQES_MIN(n
->bar
.cap
));
4147 if (unlikely(NVME_CC_IOSQES(n
->bar
.cc
) >
4148 NVME_CTRL_SQES_MAX(n
->id_ctrl
.sqes
))) {
4149 trace_pci_nvme_err_startfail_sqent_too_large(
4150 NVME_CC_IOSQES(n
->bar
.cc
),
4151 NVME_CTRL_SQES_MAX(n
->bar
.cap
));
4154 if (unlikely(!NVME_AQA_ASQS(n
->bar
.aqa
))) {
4155 trace_pci_nvme_err_startfail_asqent_sz_zero();
4158 if (unlikely(!NVME_AQA_ACQS(n
->bar
.aqa
))) {
4159 trace_pci_nvme_err_startfail_acqent_sz_zero();
4163 n
->page_bits
= page_bits
;
4164 n
->page_size
= page_size
;
4165 n
->max_prp_ents
= n
->page_size
/ sizeof(uint64_t);
4166 n
->cqe_size
= 1 << NVME_CC_IOCQES(n
->bar
.cc
);
4167 n
->sqe_size
= 1 << NVME_CC_IOSQES(n
->bar
.cc
);
4168 nvme_init_cq(&n
->admin_cq
, n
, n
->bar
.acq
, 0, 0,
4169 NVME_AQA_ACQS(n
->bar
.aqa
) + 1, 1);
4170 nvme_init_sq(&n
->admin_sq
, n
, n
->bar
.asq
, 0, 0,
4171 NVME_AQA_ASQS(n
->bar
.aqa
) + 1);
4173 nvme_set_timestamp(n
, 0ULL);
4175 QTAILQ_INIT(&n
->aer_queue
);
4177 nvme_select_ns_iocs(n
);
4182 static void nvme_cmb_enable_regs(NvmeCtrl
*n
)
4184 NVME_CMBLOC_SET_CDPCILS(n
->bar
.cmbloc
, 1);
4185 NVME_CMBLOC_SET_CDPMLS(n
->bar
.cmbloc
, 1);
4186 NVME_CMBLOC_SET_BIR(n
->bar
.cmbloc
, NVME_CMB_BIR
);
4188 NVME_CMBSZ_SET_SQS(n
->bar
.cmbsz
, 1);
4189 NVME_CMBSZ_SET_CQS(n
->bar
.cmbsz
, 0);
4190 NVME_CMBSZ_SET_LISTS(n
->bar
.cmbsz
, 1);
4191 NVME_CMBSZ_SET_RDS(n
->bar
.cmbsz
, 1);
4192 NVME_CMBSZ_SET_WDS(n
->bar
.cmbsz
, 1);
4193 NVME_CMBSZ_SET_SZU(n
->bar
.cmbsz
, 2); /* MBs */
4194 NVME_CMBSZ_SET_SZ(n
->bar
.cmbsz
, n
->params
.cmb_size_mb
);
4197 static void nvme_write_bar(NvmeCtrl
*n
, hwaddr offset
, uint64_t data
,
4200 if (unlikely(offset
& (sizeof(uint32_t) - 1))) {
4201 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32
,
4202 "MMIO write not 32-bit aligned,"
4203 " offset=0x%"PRIx64
"", offset
);
4204 /* should be ignored, fall through for now */
4207 if (unlikely(size
< sizeof(uint32_t))) {
4208 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall
,
4209 "MMIO write smaller than 32-bits,"
4210 " offset=0x%"PRIx64
", size=%u",
4212 /* should be ignored, fall through for now */
4216 case 0xc: /* INTMS */
4217 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
4218 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
4219 "undefined access to interrupt mask set"
4220 " when MSI-X is enabled");
4221 /* should be ignored, fall through for now */
4223 n
->bar
.intms
|= data
& 0xffffffff;
4224 n
->bar
.intmc
= n
->bar
.intms
;
4225 trace_pci_nvme_mmio_intm_set(data
& 0xffffffff, n
->bar
.intmc
);
4228 case 0x10: /* INTMC */
4229 if (unlikely(msix_enabled(&(n
->parent_obj
)))) {
4230 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix
,
4231 "undefined access to interrupt mask clr"
4232 " when MSI-X is enabled");
4233 /* should be ignored, fall through for now */
4235 n
->bar
.intms
&= ~(data
& 0xffffffff);
4236 n
->bar
.intmc
= n
->bar
.intms
;
4237 trace_pci_nvme_mmio_intm_clr(data
& 0xffffffff, n
->bar
.intmc
);
4241 trace_pci_nvme_mmio_cfg(data
& 0xffffffff);
4242 /* Windows first sends data, then sends enable bit */
4243 if (!NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
) &&
4244 !NVME_CC_SHN(data
) && !NVME_CC_SHN(n
->bar
.cc
))
4249 if (NVME_CC_EN(data
) && !NVME_CC_EN(n
->bar
.cc
)) {
4251 if (unlikely(nvme_start_ctrl(n
))) {
4252 trace_pci_nvme_err_startfail();
4253 n
->bar
.csts
= NVME_CSTS_FAILED
;
4255 trace_pci_nvme_mmio_start_success();
4256 n
->bar
.csts
= NVME_CSTS_READY
;
4258 } else if (!NVME_CC_EN(data
) && NVME_CC_EN(n
->bar
.cc
)) {
4259 trace_pci_nvme_mmio_stopped();
4261 n
->bar
.csts
&= ~NVME_CSTS_READY
;
4263 if (NVME_CC_SHN(data
) && !(NVME_CC_SHN(n
->bar
.cc
))) {
4264 trace_pci_nvme_mmio_shutdown_set();
4265 nvme_ctrl_shutdown(n
);
4267 n
->bar
.csts
|= NVME_CSTS_SHST_COMPLETE
;
4268 } else if (!NVME_CC_SHN(data
) && NVME_CC_SHN(n
->bar
.cc
)) {
4269 trace_pci_nvme_mmio_shutdown_cleared();
4270 n
->bar
.csts
&= ~NVME_CSTS_SHST_COMPLETE
;
4274 case 0x1C: /* CSTS */
4275 if (data
& (1 << 4)) {
4276 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported
,
4277 "attempted to W1C CSTS.NSSRO"
4278 " but CAP.NSSRS is zero (not supported)");
4279 } else if (data
!= 0) {
4280 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts
,
4281 "attempted to set a read only bit"
4282 " of controller status");
4285 case 0x20: /* NSSR */
4286 if (data
== 0x4E564D65) {
4287 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
4289 /* The spec says that writes of other values have no effect */
4293 case 0x24: /* AQA */
4294 n
->bar
.aqa
= data
& 0xffffffff;
4295 trace_pci_nvme_mmio_aqattr(data
& 0xffffffff);
4297 case 0x28: /* ASQ */
4298 n
->bar
.asq
= size
== 8 ? data
:
4299 (n
->bar
.asq
& ~0xffffffffULL
) | (data
& 0xffffffff);
4300 trace_pci_nvme_mmio_asqaddr(data
);
4302 case 0x2c: /* ASQ hi */
4303 n
->bar
.asq
= (n
->bar
.asq
& 0xffffffff) | (data
<< 32);
4304 trace_pci_nvme_mmio_asqaddr_hi(data
, n
->bar
.asq
);
4306 case 0x30: /* ACQ */
4307 trace_pci_nvme_mmio_acqaddr(data
);
4308 n
->bar
.acq
= size
== 8 ? data
:
4309 (n
->bar
.acq
& ~0xffffffffULL
) | (data
& 0xffffffff);
4311 case 0x34: /* ACQ hi */
4312 n
->bar
.acq
= (n
->bar
.acq
& 0xffffffff) | (data
<< 32);
4313 trace_pci_nvme_mmio_acqaddr_hi(data
, n
->bar
.acq
);
4315 case 0x38: /* CMBLOC */
4316 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved
,
4317 "invalid write to reserved CMBLOC"
4318 " when CMBSZ is zero, ignored");
4320 case 0x3C: /* CMBSZ */
4321 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly
,
4322 "invalid write to read only CMBSZ, ignored");
4324 case 0x50: /* CMBMSC */
4325 if (!NVME_CAP_CMBS(n
->bar
.cap
)) {
4329 n
->bar
.cmbmsc
= size
== 8 ? data
:
4330 (n
->bar
.cmbmsc
& ~0xffffffff) | (data
& 0xffffffff);
4331 n
->cmb
.cmse
= false;
4333 if (NVME_CMBMSC_CRE(data
)) {
4334 nvme_cmb_enable_regs(n
);
4336 if (NVME_CMBMSC_CMSE(data
)) {
4337 hwaddr cba
= NVME_CMBMSC_CBA(data
) << CMBMSC_CBA_SHIFT
;
4338 if (cba
+ int128_get64(n
->cmb
.mem
.size
) < cba
) {
4339 NVME_CMBSTS_SET_CBAI(n
->bar
.cmbsts
, 1);
4352 case 0x54: /* CMBMSC hi */
4353 n
->bar
.cmbmsc
= (n
->bar
.cmbmsc
& 0xffffffff) | (data
<< 32);
4356 case 0xE00: /* PMRCAP */
4357 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly
,
4358 "invalid write to PMRCAP register, ignored");
4360 case 0xE04: /* PMRCTL */
4361 n
->bar
.pmrctl
= data
;
4362 if (NVME_PMRCTL_EN(data
)) {
4363 memory_region_set_enabled(&n
->pmr
.dev
->mr
, true);
4366 memory_region_set_enabled(&n
->pmr
.dev
->mr
, false);
4367 NVME_PMRSTS_SET_NRDY(n
->bar
.pmrsts
, 1);
4368 n
->pmr
.cmse
= false;
4371 case 0xE08: /* PMRSTS */
4372 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly
,
4373 "invalid write to PMRSTS register, ignored");
4375 case 0xE0C: /* PMREBS */
4376 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly
,
4377 "invalid write to PMREBS register, ignored");
4379 case 0xE10: /* PMRSWTP */
4380 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly
,
4381 "invalid write to PMRSWTP register, ignored");
4383 case 0xE14: /* PMRMSCL */
4384 if (!NVME_CAP_PMRS(n
->bar
.cap
)) {
4388 n
->bar
.pmrmsc
= (n
->bar
.pmrmsc
& ~0xffffffff) | (data
& 0xffffffff);
4389 n
->pmr
.cmse
= false;
4391 if (NVME_PMRMSC_CMSE(n
->bar
.pmrmsc
)) {
4392 hwaddr cba
= NVME_PMRMSC_CBA(n
->bar
.pmrmsc
) << PMRMSC_CBA_SHIFT
;
4393 if (cba
+ int128_get64(n
->pmr
.dev
->mr
.size
) < cba
) {
4394 NVME_PMRSTS_SET_CBAI(n
->bar
.pmrsts
, 1);
4403 case 0xE18: /* PMRMSCU */
4404 if (!NVME_CAP_PMRS(n
->bar
.cap
)) {
4408 n
->bar
.pmrmsc
= (n
->bar
.pmrmsc
& 0xffffffff) | (data
<< 32);
4411 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid
,
4412 "invalid MMIO write,"
4413 " offset=0x%"PRIx64
", data=%"PRIx64
"",
4419 static uint64_t nvme_mmio_read(void *opaque
, hwaddr addr
, unsigned size
)
4421 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
4422 uint8_t *ptr
= (uint8_t *)&n
->bar
;
4425 trace_pci_nvme_mmio_read(addr
, size
);
4427 if (unlikely(addr
& (sizeof(uint32_t) - 1))) {
4428 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32
,
4429 "MMIO read not 32-bit aligned,"
4430 " offset=0x%"PRIx64
"", addr
);
4431 /* should RAZ, fall through for now */
4432 } else if (unlikely(size
< sizeof(uint32_t))) {
4433 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall
,
4434 "MMIO read smaller than 32-bits,"
4435 " offset=0x%"PRIx64
"", addr
);
4436 /* should RAZ, fall through for now */
4439 if (addr
< sizeof(n
->bar
)) {
4441 * When PMRWBM bit 1 is set then read from
4442 * from PMRSTS should ensure prior writes
4443 * made it to persistent media
4445 if (addr
== 0xE08 &&
4446 (NVME_PMRCAP_PMRWBM(n
->bar
.pmrcap
) & 0x02)) {
4447 memory_region_msync(&n
->pmr
.dev
->mr
, 0, n
->pmr
.dev
->size
);
4449 memcpy(&val
, ptr
+ addr
, size
);
4451 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs
,
4452 "MMIO read beyond last register,"
4453 " offset=0x%"PRIx64
", returning 0", addr
);
4459 static void nvme_process_db(NvmeCtrl
*n
, hwaddr addr
, int val
)
4463 if (unlikely(addr
& ((1 << 2) - 1))) {
4464 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned
,
4465 "doorbell write not 32-bit aligned,"
4466 " offset=0x%"PRIx64
", ignoring", addr
);
4470 if (((addr
- 0x1000) >> 2) & 1) {
4471 /* Completion queue doorbell write */
4473 uint16_t new_head
= val
& 0xffff;
4477 qid
= (addr
- (0x1000 + (1 << 2))) >> 3;
4478 if (unlikely(nvme_check_cqid(n
, qid
))) {
4479 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq
,
4480 "completion queue doorbell write"
4481 " for nonexistent queue,"
4482 " sqid=%"PRIu32
", ignoring", qid
);
4485 * NVM Express v1.3d, Section 4.1 state: "If host software writes
4486 * an invalid value to the Submission Queue Tail Doorbell or
4487 * Completion Queue Head Doorbell regiter and an Asynchronous Event
4488 * Request command is outstanding, then an asynchronous event is
4489 * posted to the Admin Completion Queue with a status code of
4490 * Invalid Doorbell Write Value."
4492 * Also note that the spec includes the "Invalid Doorbell Register"
4493 * status code, but nowhere does it specify when to use it.
4494 * However, it seems reasonable to use it here in a similar
4497 if (n
->outstanding_aers
) {
4498 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
4499 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
4500 NVME_LOG_ERROR_INFO
);
4507 if (unlikely(new_head
>= cq
->size
)) {
4508 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead
,
4509 "completion queue doorbell write value"
4510 " beyond queue size, sqid=%"PRIu32
","
4511 " new_head=%"PRIu16
", ignoring",
4514 if (n
->outstanding_aers
) {
4515 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
4516 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
4517 NVME_LOG_ERROR_INFO
);
4523 trace_pci_nvme_mmio_doorbell_cq(cq
->cqid
, new_head
);
4525 start_sqs
= nvme_cq_full(cq
) ? 1 : 0;
4526 cq
->head
= new_head
;
4529 QTAILQ_FOREACH(sq
, &cq
->sq_list
, entry
) {
4530 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
4532 timer_mod(cq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
4535 if (cq
->tail
== cq
->head
) {
4536 nvme_irq_deassert(n
, cq
);
4539 /* Submission queue doorbell write */
4541 uint16_t new_tail
= val
& 0xffff;
4544 qid
= (addr
- 0x1000) >> 3;
4545 if (unlikely(nvme_check_sqid(n
, qid
))) {
4546 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq
,
4547 "submission queue doorbell write"
4548 " for nonexistent queue,"
4549 " sqid=%"PRIu32
", ignoring", qid
);
4551 if (n
->outstanding_aers
) {
4552 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
4553 NVME_AER_INFO_ERR_INVALID_DB_REGISTER
,
4554 NVME_LOG_ERROR_INFO
);
4561 if (unlikely(new_tail
>= sq
->size
)) {
4562 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail
,
4563 "submission queue doorbell write value"
4564 " beyond queue size, sqid=%"PRIu32
","
4565 " new_tail=%"PRIu16
", ignoring",
4568 if (n
->outstanding_aers
) {
4569 nvme_enqueue_event(n
, NVME_AER_TYPE_ERROR
,
4570 NVME_AER_INFO_ERR_INVALID_DB_VALUE
,
4571 NVME_LOG_ERROR_INFO
);
4577 trace_pci_nvme_mmio_doorbell_sq(sq
->sqid
, new_tail
);
4579 sq
->tail
= new_tail
;
4580 timer_mod(sq
->timer
, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) + 500);
4584 static void nvme_mmio_write(void *opaque
, hwaddr addr
, uint64_t data
,
4587 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
4589 trace_pci_nvme_mmio_write(addr
, data
, size
);
4591 if (addr
< sizeof(n
->bar
)) {
4592 nvme_write_bar(n
, addr
, data
, size
);
4594 nvme_process_db(n
, addr
, data
);
4598 static const MemoryRegionOps nvme_mmio_ops
= {
4599 .read
= nvme_mmio_read
,
4600 .write
= nvme_mmio_write
,
4601 .endianness
= DEVICE_LITTLE_ENDIAN
,
4603 .min_access_size
= 2,
4604 .max_access_size
= 8,
4608 static void nvme_cmb_write(void *opaque
, hwaddr addr
, uint64_t data
,
4611 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
4612 stn_le_p(&n
->cmb
.buf
[addr
], size
, data
);
4615 static uint64_t nvme_cmb_read(void *opaque
, hwaddr addr
, unsigned size
)
4617 NvmeCtrl
*n
= (NvmeCtrl
*)opaque
;
4618 return ldn_le_p(&n
->cmb
.buf
[addr
], size
);
4621 static const MemoryRegionOps nvme_cmb_ops
= {
4622 .read
= nvme_cmb_read
,
4623 .write
= nvme_cmb_write
,
4624 .endianness
= DEVICE_LITTLE_ENDIAN
,
4626 .min_access_size
= 1,
4627 .max_access_size
= 8,
4631 static void nvme_check_constraints(NvmeCtrl
*n
, Error
**errp
)
4633 NvmeParams
*params
= &n
->params
;
4635 if (params
->num_queues
) {
4636 warn_report("num_queues is deprecated; please use max_ioqpairs "
4639 params
->max_ioqpairs
= params
->num_queues
- 1;
4643 warn_report("drive property is deprecated; "
4644 "please use an nvme-ns device instead");
4647 if (params
->max_ioqpairs
< 1 ||
4648 params
->max_ioqpairs
> NVME_MAX_IOQPAIRS
) {
4649 error_setg(errp
, "max_ioqpairs must be between 1 and %d",
4654 if (params
->msix_qsize
< 1 ||
4655 params
->msix_qsize
> PCI_MSIX_FLAGS_QSIZE
+ 1) {
4656 error_setg(errp
, "msix_qsize must be between 1 and %d",
4657 PCI_MSIX_FLAGS_QSIZE
+ 1);
4661 if (!params
->serial
) {
4662 error_setg(errp
, "serial property not set");
4667 if (host_memory_backend_is_mapped(n
->pmr
.dev
)) {
4668 error_setg(errp
, "can't use already busy memdev: %s",
4669 object_get_canonical_path_component(OBJECT(n
->pmr
.dev
)));
4673 if (!is_power_of_2(n
->pmr
.dev
->size
)) {
4674 error_setg(errp
, "pmr backend size needs to be power of 2 in size");
4678 host_memory_backend_set_mapped(n
->pmr
.dev
, true);
4681 if (n
->params
.zasl
> n
->params
.mdts
) {
4682 error_setg(errp
, "zoned.zasl (Zone Append Size Limit) must be less "
4683 "than or equal to mdts (Maximum Data Transfer Size)");
4688 static void nvme_init_state(NvmeCtrl
*n
)
4690 n
->num_namespaces
= NVME_MAX_NAMESPACES
;
4691 /* add one to max_ioqpairs to account for the admin queue pair */
4692 n
->reg_size
= pow2ceil(sizeof(NvmeBar
) +
4693 2 * (n
->params
.max_ioqpairs
+ 1) * NVME_DB_SIZE
);
4694 n
->sq
= g_new0(NvmeSQueue
*, n
->params
.max_ioqpairs
+ 1);
4695 n
->cq
= g_new0(NvmeCQueue
*, n
->params
.max_ioqpairs
+ 1);
4696 n
->temperature
= NVME_TEMPERATURE
;
4697 n
->features
.temp_thresh_hi
= NVME_TEMPERATURE_WARNING
;
4698 n
->starttime_ms
= qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
);
4699 n
->aer_reqs
= g_new0(NvmeRequest
*, n
->params
.aerl
+ 1);
4702 static int nvme_attach_namespace(NvmeCtrl
*n
, NvmeNamespace
*ns
, Error
**errp
)
4704 if (nvme_ns_is_attached(n
, ns
)) {
4706 "namespace %d is already attached to controller %d",
4707 nvme_nsid(ns
), n
->cntlid
);
4711 nvme_ns_attach(n
, ns
);
4716 int nvme_register_namespace(NvmeCtrl
*n
, NvmeNamespace
*ns
, Error
**errp
)
4718 uint32_t nsid
= nvme_nsid(ns
);
4720 if (nsid
> NVME_MAX_NAMESPACES
) {
4721 error_setg(errp
, "invalid namespace id (must be between 0 and %d)",
4722 NVME_MAX_NAMESPACES
);
4727 for (int i
= 1; i
<= n
->num_namespaces
; i
++) {
4728 if (!nvme_ns(n
, i
)) {
4729 nsid
= ns
->params
.nsid
= i
;
4735 error_setg(errp
, "no free namespace id");
4739 if (n
->namespaces
[nsid
- 1]) {
4740 error_setg(errp
, "namespace id '%d' is already in use", nsid
);
4745 trace_pci_nvme_register_namespace(nsid
);
4748 * If subsys is not given, namespae is always attached to the controller
4749 * because there's no subsystem to manage namespace allocation.
4752 if (ns
->params
.detached
) {
4754 "detached needs nvme-subsys specified nvme or nvme-ns");
4758 return nvme_attach_namespace(n
, ns
, errp
);
4760 if (!ns
->params
.detached
) {
4761 return nvme_attach_namespace(n
, ns
, errp
);
4765 n
->dmrsl
= MIN_NON_ZERO(n
->dmrsl
,
4766 BDRV_REQUEST_MAX_BYTES
/ nvme_l2b(ns
, 1));
4771 static void nvme_init_cmb(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
4773 uint64_t cmb_size
= n
->params
.cmb_size_mb
* MiB
;
4775 n
->cmb
.buf
= g_malloc0(cmb_size
);
4776 memory_region_init_io(&n
->cmb
.mem
, OBJECT(n
), &nvme_cmb_ops
, n
,
4777 "nvme-cmb", cmb_size
);
4778 pci_register_bar(pci_dev
, NVME_CMB_BIR
,
4779 PCI_BASE_ADDRESS_SPACE_MEMORY
|
4780 PCI_BASE_ADDRESS_MEM_TYPE_64
|
4781 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->cmb
.mem
);
4783 NVME_CAP_SET_CMBS(n
->bar
.cap
, 1);
4785 if (n
->params
.legacy_cmb
) {
4786 nvme_cmb_enable_regs(n
);
4791 static void nvme_init_pmr(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
4793 NVME_PMRCAP_SET_RDS(n
->bar
.pmrcap
, 1);
4794 NVME_PMRCAP_SET_WDS(n
->bar
.pmrcap
, 1);
4795 NVME_PMRCAP_SET_BIR(n
->bar
.pmrcap
, NVME_PMR_BIR
);
4796 /* Turn on bit 1 support */
4797 NVME_PMRCAP_SET_PMRWBM(n
->bar
.pmrcap
, 0x02);
4798 NVME_PMRCAP_SET_CMSS(n
->bar
.pmrcap
, 1);
4800 pci_register_bar(pci_dev
, NVME_PMRCAP_BIR(n
->bar
.pmrcap
),
4801 PCI_BASE_ADDRESS_SPACE_MEMORY
|
4802 PCI_BASE_ADDRESS_MEM_TYPE_64
|
4803 PCI_BASE_ADDRESS_MEM_PREFETCH
, &n
->pmr
.dev
->mr
);
4805 memory_region_set_enabled(&n
->pmr
.dev
->mr
, false);
4808 static int nvme_init_pci(NvmeCtrl
*n
, PCIDevice
*pci_dev
, Error
**errp
)
4810 uint8_t *pci_conf
= pci_dev
->config
;
4811 uint64_t bar_size
, msix_table_size
, msix_pba_size
;
4812 unsigned msix_table_offset
, msix_pba_offset
;
4817 pci_conf
[PCI_INTERRUPT_PIN
] = 1;
4818 pci_config_set_prog_interface(pci_conf
, 0x2);
4820 if (n
->params
.use_intel_id
) {
4821 pci_config_set_vendor_id(pci_conf
, PCI_VENDOR_ID_INTEL
);
4822 pci_config_set_device_id(pci_conf
, 0x5845);
4824 pci_config_set_vendor_id(pci_conf
, PCI_VENDOR_ID_REDHAT
);
4825 pci_config_set_device_id(pci_conf
, PCI_DEVICE_ID_REDHAT_NVME
);
4828 pci_config_set_class(pci_conf
, PCI_CLASS_STORAGE_EXPRESS
);
4829 pcie_endpoint_cap_init(pci_dev
, 0x80);
4831 bar_size
= QEMU_ALIGN_UP(n
->reg_size
, 4 * KiB
);
4832 msix_table_offset
= bar_size
;
4833 msix_table_size
= PCI_MSIX_ENTRY_SIZE
* n
->params
.msix_qsize
;
4835 bar_size
+= msix_table_size
;
4836 bar_size
= QEMU_ALIGN_UP(bar_size
, 4 * KiB
);
4837 msix_pba_offset
= bar_size
;
4838 msix_pba_size
= QEMU_ALIGN_UP(n
->params
.msix_qsize
, 64) / 8;
4840 bar_size
+= msix_pba_size
;
4841 bar_size
= pow2ceil(bar_size
);
4843 memory_region_init(&n
->bar0
, OBJECT(n
), "nvme-bar0", bar_size
);
4844 memory_region_init_io(&n
->iomem
, OBJECT(n
), &nvme_mmio_ops
, n
, "nvme",
4846 memory_region_add_subregion(&n
->bar0
, 0, &n
->iomem
);
4848 pci_register_bar(pci_dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
|
4849 PCI_BASE_ADDRESS_MEM_TYPE_64
, &n
->bar0
);
4850 ret
= msix_init(pci_dev
, n
->params
.msix_qsize
,
4851 &n
->bar0
, 0, msix_table_offset
,
4852 &n
->bar0
, 0, msix_pba_offset
, 0, &err
);
4854 if (ret
== -ENOTSUP
) {
4855 warn_report_err(err
);
4857 error_propagate(errp
, err
);
4862 if (n
->params
.cmb_size_mb
) {
4863 nvme_init_cmb(n
, pci_dev
);
4867 nvme_init_pmr(n
, pci_dev
);
4873 static void nvme_init_subnqn(NvmeCtrl
*n
)
4875 NvmeSubsystem
*subsys
= n
->subsys
;
4876 NvmeIdCtrl
*id
= &n
->id_ctrl
;
4879 snprintf((char *)id
->subnqn
, sizeof(id
->subnqn
),
4880 "nqn.2019-08.org.qemu:%s", n
->params
.serial
);
4882 pstrcpy((char *)id
->subnqn
, sizeof(id
->subnqn
), (char*)subsys
->subnqn
);
4886 static void nvme_init_ctrl(NvmeCtrl
*n
, PCIDevice
*pci_dev
)
4888 NvmeIdCtrl
*id
= &n
->id_ctrl
;
4889 uint8_t *pci_conf
= pci_dev
->config
;
4891 id
->vid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_VENDOR_ID
));
4892 id
->ssvid
= cpu_to_le16(pci_get_word(pci_conf
+ PCI_SUBSYSTEM_VENDOR_ID
));
4893 strpadcpy((char *)id
->mn
, sizeof(id
->mn
), "QEMU NVMe Ctrl", ' ');
4894 strpadcpy((char *)id
->fr
, sizeof(id
->fr
), "1.0", ' ');
4895 strpadcpy((char *)id
->sn
, sizeof(id
->sn
), n
->params
.serial
, ' ');
4897 id
->cntlid
= cpu_to_le16(n
->cntlid
);
4901 if (n
->params
.use_intel_id
) {
4911 id
->mdts
= n
->params
.mdts
;
4912 id
->ver
= cpu_to_le32(NVME_SPEC_VER
);
4913 id
->oacs
= cpu_to_le16(0);
4914 id
->cntrltype
= 0x1;
4917 * Because the controller always completes the Abort command immediately,
4918 * there can never be more than one concurrently executing Abort command,
4919 * so this value is never used for anything. Note that there can easily be
4920 * many Abort commands in the queues, but they are not considered
4921 * "executing" until processed by nvme_abort.
4923 * The specification recommends a value of 3 for Abort Command Limit (four
4924 * concurrently outstanding Abort commands), so lets use that though it is
4928 id
->aerl
= n
->params
.aerl
;
4929 id
->frmw
= (NVME_NUM_FW_SLOTS
<< 1) | NVME_FRMW_SLOT1_RO
;
4930 id
->lpa
= NVME_LPA_NS_SMART
| NVME_LPA_CSE
| NVME_LPA_EXTENDED
;
4932 /* recommended default value (~70 C) */
4933 id
->wctemp
= cpu_to_le16(NVME_TEMPERATURE_WARNING
);
4934 id
->cctemp
= cpu_to_le16(NVME_TEMPERATURE_CRITICAL
);
4936 id
->sqes
= (0x6 << 4) | 0x6;
4937 id
->cqes
= (0x4 << 4) | 0x4;
4938 id
->nn
= cpu_to_le32(n
->num_namespaces
);
4939 id
->oncs
= cpu_to_le16(NVME_ONCS_WRITE_ZEROES
| NVME_ONCS_TIMESTAMP
|
4940 NVME_ONCS_FEATURES
| NVME_ONCS_DSM
|
4941 NVME_ONCS_COMPARE
| NVME_ONCS_COPY
);
4944 * NOTE: If this device ever supports a command set that does NOT use 0x0
4945 * as a Flush-equivalent operation, support for the broadcast NSID in Flush
4946 * should probably be removed.
4948 * See comment in nvme_io_cmd.
4950 id
->vwc
= NVME_VWC_NSID_BROADCAST_SUPPORT
| NVME_VWC_PRESENT
;
4952 id
->ocfs
= cpu_to_le16(NVME_OCFS_COPY_FORMAT_0
);
4953 id
->sgls
= cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN
|
4954 NVME_CTRL_SGLS_BITBUCKET
);
4956 nvme_init_subnqn(n
);
4958 id
->psd
[0].mp
= cpu_to_le16(0x9c4);
4959 id
->psd
[0].enlat
= cpu_to_le32(0x10);
4960 id
->psd
[0].exlat
= cpu_to_le32(0x4);
4963 id
->cmic
|= NVME_CMIC_MULTI_CTRL
;
4966 NVME_CAP_SET_MQES(n
->bar
.cap
, 0x7ff);
4967 NVME_CAP_SET_CQR(n
->bar
.cap
, 1);
4968 NVME_CAP_SET_TO(n
->bar
.cap
, 0xf);
4969 NVME_CAP_SET_CSS(n
->bar
.cap
, NVME_CAP_CSS_NVM
);
4970 NVME_CAP_SET_CSS(n
->bar
.cap
, NVME_CAP_CSS_CSI_SUPP
);
4971 NVME_CAP_SET_CSS(n
->bar
.cap
, NVME_CAP_CSS_ADMIN_ONLY
);
4972 NVME_CAP_SET_MPSMAX(n
->bar
.cap
, 4);
4973 NVME_CAP_SET_CMBS(n
->bar
.cap
, n
->params
.cmb_size_mb
? 1 : 0);
4974 NVME_CAP_SET_PMRS(n
->bar
.cap
, n
->pmr
.dev
? 1 : 0);
4976 n
->bar
.vs
= NVME_SPEC_VER
;
4977 n
->bar
.intmc
= n
->bar
.intms
= 0;
4980 static int nvme_init_subsys(NvmeCtrl
*n
, Error
**errp
)
4988 cntlid
= nvme_subsys_register_ctrl(n
, errp
);
4998 static void nvme_realize(PCIDevice
*pci_dev
, Error
**errp
)
5000 NvmeCtrl
*n
= NVME(pci_dev
);
5002 Error
*local_err
= NULL
;
5004 nvme_check_constraints(n
, &local_err
);
5006 error_propagate(errp
, local_err
);
5010 qbus_create_inplace(&n
->bus
, sizeof(NvmeBus
), TYPE_NVME_BUS
,
5011 &pci_dev
->qdev
, n
->parent_obj
.qdev
.id
);
5014 if (nvme_init_pci(n
, pci_dev
, errp
)) {
5018 if (nvme_init_subsys(n
, errp
)) {
5019 error_propagate(errp
, local_err
);
5022 nvme_init_ctrl(n
, pci_dev
);
5024 /* setup a namespace if the controller drive property was given */
5025 if (n
->namespace.blkconf
.blk
) {
5027 ns
->params
.nsid
= 1;
5029 if (nvme_ns_setup(ns
, errp
)) {
5033 if (nvme_register_namespace(n
, ns
, errp
)) {
5039 static void nvme_exit(PCIDevice
*pci_dev
)
5041 NvmeCtrl
*n
= NVME(pci_dev
);
5047 for (i
= 1; i
<= n
->num_namespaces
; i
++) {
5053 nvme_ns_cleanup(ns
);
5058 g_free(n
->aer_reqs
);
5060 if (n
->params
.cmb_size_mb
) {
5065 host_memory_backend_set_mapped(n
->pmr
.dev
, false);
5067 msix_uninit_exclusive_bar(pci_dev
);
5070 static Property nvme_props
[] = {
5071 DEFINE_BLOCK_PROPERTIES(NvmeCtrl
, namespace.blkconf
),
5072 DEFINE_PROP_LINK("pmrdev", NvmeCtrl
, pmr
.dev
, TYPE_MEMORY_BACKEND
,
5073 HostMemoryBackend
*),
5074 DEFINE_PROP_LINK("subsys", NvmeCtrl
, subsys
, TYPE_NVME_SUBSYS
,
5076 DEFINE_PROP_STRING("serial", NvmeCtrl
, params
.serial
),
5077 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl
, params
.cmb_size_mb
, 0),
5078 DEFINE_PROP_UINT32("num_queues", NvmeCtrl
, params
.num_queues
, 0),
5079 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl
, params
.max_ioqpairs
, 64),
5080 DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl
, params
.msix_qsize
, 65),
5081 DEFINE_PROP_UINT8("aerl", NvmeCtrl
, params
.aerl
, 3),
5082 DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl
, params
.aer_max_queued
, 64),
5083 DEFINE_PROP_UINT8("mdts", NvmeCtrl
, params
.mdts
, 7),
5084 DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl
, params
.use_intel_id
, false),
5085 DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl
, params
.legacy_cmb
, false),
5086 DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl
, params
.zasl
, 0),
5087 DEFINE_PROP_END_OF_LIST(),
5090 static void nvme_get_smart_warning(Object
*obj
, Visitor
*v
, const char *name
,
5091 void *opaque
, Error
**errp
)
5093 NvmeCtrl
*n
= NVME(obj
);
5094 uint8_t value
= n
->smart_critical_warning
;
5096 visit_type_uint8(v
, name
, &value
, errp
);
5099 static void nvme_set_smart_warning(Object
*obj
, Visitor
*v
, const char *name
,
5100 void *opaque
, Error
**errp
)
5102 NvmeCtrl
*n
= NVME(obj
);
5103 uint8_t value
, old_value
, cap
= 0, index
, event
;
5105 if (!visit_type_uint8(v
, name
, &value
, errp
)) {
5109 cap
= NVME_SMART_SPARE
| NVME_SMART_TEMPERATURE
| NVME_SMART_RELIABILITY
5110 | NVME_SMART_MEDIA_READ_ONLY
| NVME_SMART_FAILED_VOLATILE_MEDIA
;
5111 if (NVME_CAP_PMRS(n
->bar
.cap
)) {
5112 cap
|= NVME_SMART_PMR_UNRELIABLE
;
5115 if ((value
& cap
) != value
) {
5116 error_setg(errp
, "unsupported smart critical warning bits: 0x%x",
5121 old_value
= n
->smart_critical_warning
;
5122 n
->smart_critical_warning
= value
;
5124 /* only inject new bits of smart critical warning */
5125 for (index
= 0; index
< NVME_SMART_WARN_MAX
; index
++) {
5127 if (value
& ~old_value
& event
)
5128 nvme_smart_event(n
, event
);
5132 static const VMStateDescription nvme_vmstate
= {
5137 static void nvme_class_init(ObjectClass
*oc
, void *data
)
5139 DeviceClass
*dc
= DEVICE_CLASS(oc
);
5140 PCIDeviceClass
*pc
= PCI_DEVICE_CLASS(oc
);
5142 pc
->realize
= nvme_realize
;
5143 pc
->exit
= nvme_exit
;
5144 pc
->class_id
= PCI_CLASS_STORAGE_EXPRESS
;
5147 set_bit(DEVICE_CATEGORY_STORAGE
, dc
->categories
);
5148 dc
->desc
= "Non-Volatile Memory Express";
5149 device_class_set_props(dc
, nvme_props
);
5150 dc
->vmsd
= &nvme_vmstate
;
5153 static void nvme_instance_init(Object
*obj
)
5155 NvmeCtrl
*n
= NVME(obj
);
5157 if (n
->namespace.blkconf
.blk
) {
5158 device_add_bootindex_property(obj
, &n
->namespace.blkconf
.bootindex
,
5159 "bootindex", "/namespace@1,0",
5163 object_property_add(obj
, "smart_critical_warning", "uint8",
5164 nvme_get_smart_warning
,
5165 nvme_set_smart_warning
, NULL
, NULL
);
5168 static const TypeInfo nvme_info
= {
5170 .parent
= TYPE_PCI_DEVICE
,
5171 .instance_size
= sizeof(NvmeCtrl
),
5172 .instance_init
= nvme_instance_init
,
5173 .class_init
= nvme_class_init
,
5174 .interfaces
= (InterfaceInfo
[]) {
5175 { INTERFACE_PCIE_DEVICE
},
5180 static const TypeInfo nvme_bus_info
= {
5181 .name
= TYPE_NVME_BUS
,
5183 .instance_size
= sizeof(NvmeBus
),
5186 static void nvme_register_types(void)
5188 type_register_static(&nvme_info
);
5189 type_register_static(&nvme_bus_info
);
5192 type_init(nvme_register_types
)