4 * Copyright (c) 2012 Intel Corporation
5 * Copyright (c) 2021 Minwoo Im
6 * Copyright (c) 2021 Samsung Electronics Co., Ltd.
9 * Keith Busch <kbusch@kernel.org>
10 * Klaus Jensen <k.jensen@samsung.com>
11 * Gollu Appalanaidu <anaidu.gollu@samsung.com>
12 * Dmitry Fomichev <dmitry.fomichev@wdc.com>
13 * Minwoo Im <minwoo.im.dev@gmail.com>
15 * This code is licensed under the GNU GPL v2 or later.
18 #ifndef HW_NVME_INTERNAL_H
19 #define HW_NVME_INTERNAL_H
21 #include "qemu/uuid.h"
22 #include "hw/pci/pci.h"
23 #include "hw/block/block.h"
25 #include "block/nvme.h"
27 #define NVME_MAX_CONTROLLERS 32
28 #define NVME_MAX_NAMESPACES 256
29 #define NVME_EUI64_DEFAULT ((uint64_t)0x5254000000000000)
31 QEMU_BUILD_BUG_ON(NVME_MAX_NAMESPACES
> NVME_NSID_BROADCAST
- 1);
33 typedef struct NvmeCtrl NvmeCtrl
;
34 typedef struct NvmeNamespace NvmeNamespace
;
36 #define TYPE_NVME_SUBSYS "nvme-subsys"
37 #define NVME_SUBSYS(obj) \
38 OBJECT_CHECK(NvmeSubsystem, (obj), TYPE_NVME_SUBSYS)
40 typedef struct NvmeSubsystem
{
41 DeviceState parent_obj
;
44 NvmeCtrl
*ctrls
[NVME_MAX_CONTROLLERS
];
45 NvmeNamespace
*namespaces
[NVME_MAX_NAMESPACES
+ 1];
52 int nvme_subsys_register_ctrl(NvmeCtrl
*n
, Error
**errp
);
54 static inline NvmeCtrl
*nvme_subsys_ctrl(NvmeSubsystem
*subsys
,
57 if (!subsys
|| cntlid
>= NVME_MAX_CONTROLLERS
) {
61 return subsys
->ctrls
[cntlid
];
64 static inline NvmeNamespace
*nvme_subsys_ns(NvmeSubsystem
*subsys
,
67 if (!subsys
|| !nsid
|| nsid
> NVME_MAX_NAMESPACES
) {
71 return subsys
->namespaces
[nsid
];
74 #define TYPE_NVME_NS "nvme-ns"
75 #define NVME_NS(obj) \
76 OBJECT_CHECK(NvmeNamespace, (obj), TYPE_NVME_NS)
78 typedef struct NvmeZone
{
81 QTAILQ_ENTRY(NvmeZone
) entry
;
84 typedef struct NvmeNamespaceParams
{
102 bool cross_zone_read
;
103 uint64_t zone_size_bs
;
104 uint64_t zone_cap_bs
;
105 uint32_t max_active_zones
;
106 uint32_t max_open_zones
;
107 uint32_t zd_extension_size
;
108 } NvmeNamespaceParams
;
110 typedef struct NvmeNamespace
{
111 DeviceState parent_obj
;
119 const uint32_t *iocs
;
124 QTAILQ_ENTRY(NvmeNamespace
) entry
;
126 NvmeIdNsZoned
*id_ns_zoned
;
127 NvmeZone
*zone_array
;
128 QTAILQ_HEAD(, NvmeZone
) exp_open_zones
;
129 QTAILQ_HEAD(, NvmeZone
) imp_open_zones
;
130 QTAILQ_HEAD(, NvmeZone
) closed_zones
;
131 QTAILQ_HEAD(, NvmeZone
) full_zones
;
134 uint64_t zone_capacity
;
135 uint32_t zone_size_log2
;
136 uint8_t *zd_extensions
;
137 int32_t nr_open_zones
;
138 int32_t nr_active_zones
;
140 NvmeNamespaceParams params
;
147 static inline uint32_t nvme_nsid(NvmeNamespace
*ns
)
150 return ns
->params
.nsid
;
156 static inline size_t nvme_l2b(NvmeNamespace
*ns
, uint64_t lba
)
158 return lba
<< ns
->lbaf
.ds
;
161 static inline size_t nvme_m2b(NvmeNamespace
*ns
, uint64_t lba
)
163 return ns
->lbaf
.ms
* lba
;
166 static inline int64_t nvme_moff(NvmeNamespace
*ns
, uint64_t lba
)
168 return ns
->moff
+ nvme_m2b(ns
, lba
);
171 static inline bool nvme_ns_ext(NvmeNamespace
*ns
)
173 return !!NVME_ID_NS_FLBAS_EXTENDED(ns
->id_ns
.flbas
);
176 static inline NvmeZoneState
nvme_get_zone_state(NvmeZone
*zone
)
178 return zone
->d
.zs
>> 4;
181 static inline void nvme_set_zone_state(NvmeZone
*zone
, NvmeZoneState state
)
183 zone
->d
.zs
= state
<< 4;
186 static inline uint64_t nvme_zone_rd_boundary(NvmeNamespace
*ns
, NvmeZone
*zone
)
188 return zone
->d
.zslba
+ ns
->zone_size
;
191 static inline uint64_t nvme_zone_wr_boundary(NvmeZone
*zone
)
193 return zone
->d
.zslba
+ zone
->d
.zcap
;
196 static inline bool nvme_wp_is_valid(NvmeZone
*zone
)
198 uint8_t st
= nvme_get_zone_state(zone
);
200 return st
!= NVME_ZONE_STATE_FULL
&&
201 st
!= NVME_ZONE_STATE_READ_ONLY
&&
202 st
!= NVME_ZONE_STATE_OFFLINE
;
205 static inline uint8_t *nvme_get_zd_extension(NvmeNamespace
*ns
,
208 return &ns
->zd_extensions
[zone_idx
* ns
->params
.zd_extension_size
];
211 static inline void nvme_aor_inc_open(NvmeNamespace
*ns
)
213 assert(ns
->nr_open_zones
>= 0);
214 if (ns
->params
.max_open_zones
) {
216 assert(ns
->nr_open_zones
<= ns
->params
.max_open_zones
);
220 static inline void nvme_aor_dec_open(NvmeNamespace
*ns
)
222 if (ns
->params
.max_open_zones
) {
223 assert(ns
->nr_open_zones
> 0);
226 assert(ns
->nr_open_zones
>= 0);
229 static inline void nvme_aor_inc_active(NvmeNamespace
*ns
)
231 assert(ns
->nr_active_zones
>= 0);
232 if (ns
->params
.max_active_zones
) {
233 ns
->nr_active_zones
++;
234 assert(ns
->nr_active_zones
<= ns
->params
.max_active_zones
);
238 static inline void nvme_aor_dec_active(NvmeNamespace
*ns
)
240 if (ns
->params
.max_active_zones
) {
241 assert(ns
->nr_active_zones
> 0);
242 ns
->nr_active_zones
--;
243 assert(ns
->nr_active_zones
>= ns
->nr_open_zones
);
245 assert(ns
->nr_active_zones
>= 0);
248 void nvme_ns_init_format(NvmeNamespace
*ns
);
249 int nvme_ns_setup(NvmeCtrl
*n
, NvmeNamespace
*ns
, Error
**errp
);
250 void nvme_ns_drain(NvmeNamespace
*ns
);
251 void nvme_ns_shutdown(NvmeNamespace
*ns
);
252 void nvme_ns_cleanup(NvmeNamespace
*ns
);
254 typedef struct NvmeAsyncEvent
{
255 QTAILQ_ENTRY(NvmeAsyncEvent
) entry
;
256 NvmeAerResult result
;
260 NVME_SG_ALLOC
= 1 << 0,
261 NVME_SG_DMA
= 1 << 1,
264 typedef struct NvmeSg
{
273 typedef enum NvmeTxDirection
{
274 NVME_TX_DIRECTION_TO_DEVICE
= 0,
275 NVME_TX_DIRECTION_FROM_DEVICE
= 1,
278 typedef struct NvmeRequest
{
279 struct NvmeSQueue
*sq
;
280 struct NvmeNamespace
*ns
;
286 BlockAcctCookie acct
;
288 QTAILQ_ENTRY(NvmeRequest
)entry
;
291 typedef struct NvmeBounceContext
{
300 static inline const char *nvme_adm_opc_str(uint8_t opc
)
303 case NVME_ADM_CMD_DELETE_SQ
: return "NVME_ADM_CMD_DELETE_SQ";
304 case NVME_ADM_CMD_CREATE_SQ
: return "NVME_ADM_CMD_CREATE_SQ";
305 case NVME_ADM_CMD_GET_LOG_PAGE
: return "NVME_ADM_CMD_GET_LOG_PAGE";
306 case NVME_ADM_CMD_DELETE_CQ
: return "NVME_ADM_CMD_DELETE_CQ";
307 case NVME_ADM_CMD_CREATE_CQ
: return "NVME_ADM_CMD_CREATE_CQ";
308 case NVME_ADM_CMD_IDENTIFY
: return "NVME_ADM_CMD_IDENTIFY";
309 case NVME_ADM_CMD_ABORT
: return "NVME_ADM_CMD_ABORT";
310 case NVME_ADM_CMD_SET_FEATURES
: return "NVME_ADM_CMD_SET_FEATURES";
311 case NVME_ADM_CMD_GET_FEATURES
: return "NVME_ADM_CMD_GET_FEATURES";
312 case NVME_ADM_CMD_ASYNC_EV_REQ
: return "NVME_ADM_CMD_ASYNC_EV_REQ";
313 case NVME_ADM_CMD_NS_ATTACHMENT
: return "NVME_ADM_CMD_NS_ATTACHMENT";
314 case NVME_ADM_CMD_FORMAT_NVM
: return "NVME_ADM_CMD_FORMAT_NVM";
315 default: return "NVME_ADM_CMD_UNKNOWN";
319 static inline const char *nvme_io_opc_str(uint8_t opc
)
322 case NVME_CMD_FLUSH
: return "NVME_NVM_CMD_FLUSH";
323 case NVME_CMD_WRITE
: return "NVME_NVM_CMD_WRITE";
324 case NVME_CMD_READ
: return "NVME_NVM_CMD_READ";
325 case NVME_CMD_COMPARE
: return "NVME_NVM_CMD_COMPARE";
326 case NVME_CMD_WRITE_ZEROES
: return "NVME_NVM_CMD_WRITE_ZEROES";
327 case NVME_CMD_DSM
: return "NVME_NVM_CMD_DSM";
328 case NVME_CMD_VERIFY
: return "NVME_NVM_CMD_VERIFY";
329 case NVME_CMD_COPY
: return "NVME_NVM_CMD_COPY";
330 case NVME_CMD_ZONE_MGMT_SEND
: return "NVME_ZONED_CMD_MGMT_SEND";
331 case NVME_CMD_ZONE_MGMT_RECV
: return "NVME_ZONED_CMD_MGMT_RECV";
332 case NVME_CMD_ZONE_APPEND
: return "NVME_ZONED_CMD_ZONE_APPEND";
333 default: return "NVME_NVM_CMD_UNKNOWN";
337 typedef struct NvmeSQueue
{
338 struct NvmeCtrl
*ctrl
;
347 QTAILQ_HEAD(, NvmeRequest
) req_list
;
348 QTAILQ_HEAD(, NvmeRequest
) out_req_list
;
349 QTAILQ_ENTRY(NvmeSQueue
) entry
;
352 typedef struct NvmeCQueue
{
353 struct NvmeCtrl
*ctrl
;
356 uint16_t irq_enabled
;
363 QTAILQ_HEAD(, NvmeSQueue
) sq_list
;
364 QTAILQ_HEAD(, NvmeRequest
) req_list
;
367 #define TYPE_NVME_BUS "nvme-bus"
368 #define NVME_BUS(obj) OBJECT_CHECK(NvmeBus, (obj), TYPE_NVME_BUS)
370 typedef struct NvmeBus
{
374 #define TYPE_NVME "nvme"
376 OBJECT_CHECK(NvmeCtrl, (obj), TYPE_NVME)
378 typedef struct NvmeParams
{
380 uint32_t num_queues
; /* deprecated since 5.1 */
381 uint32_t max_ioqpairs
;
383 uint32_t cmb_size_mb
;
385 uint32_t aer_max_queued
;
390 bool auto_transition_zones
;
394 typedef struct NvmeCtrl
{
395 PCIDevice parent_obj
;
406 uint16_t max_prp_ents
;
411 uint8_t outstanding_aers
;
413 uint64_t host_timestamp
; /* Timestamp sent by the host */
414 uint64_t timestamp_set_qemu_clock_ms
; /* QEMU clock time */
415 uint64_t starttime_ms
;
416 uint16_t temperature
;
417 uint8_t smart_critical_warning
;
427 HostMemoryBackend
*dev
;
433 NvmeRequest
**aer_reqs
;
434 QTAILQ_HEAD(, NvmeAsyncEvent
) aer_queue
;
439 /* Namespace ID is started with 1 so bitmap should be 1-based */
440 #define NVME_CHANGED_NSID_SIZE (NVME_MAX_NAMESPACES + 1)
441 DECLARE_BITMAP(changed_nsids
, NVME_CHANGED_NSID_SIZE
);
443 NvmeSubsystem
*subsys
;
445 NvmeNamespace
namespace;
446 NvmeNamespace
*namespaces
[NVME_MAX_NAMESPACES
+ 1];
455 uint16_t temp_thresh_hi
;
456 uint16_t temp_thresh_low
;
458 uint32_t async_config
;
462 static inline NvmeNamespace
*nvme_ns(NvmeCtrl
*n
, uint32_t nsid
)
464 if (!nsid
|| nsid
> NVME_MAX_NAMESPACES
) {
468 return n
->namespaces
[nsid
];
471 static inline NvmeCQueue
*nvme_cq(NvmeRequest
*req
)
473 NvmeSQueue
*sq
= req
->sq
;
474 NvmeCtrl
*n
= sq
->ctrl
;
476 return n
->cq
[sq
->cqid
];
479 static inline NvmeCtrl
*nvme_ctrl(NvmeRequest
*req
)
481 NvmeSQueue
*sq
= req
->sq
;
485 static inline uint16_t nvme_cid(NvmeRequest
*req
)
491 return le16_to_cpu(req
->cqe
.cid
);
494 void nvme_attach_ns(NvmeCtrl
*n
, NvmeNamespace
*ns
);
495 uint16_t nvme_bounce_data(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
496 NvmeTxDirection dir
, NvmeRequest
*req
);
497 uint16_t nvme_bounce_mdata(NvmeCtrl
*n
, uint8_t *ptr
, uint32_t len
,
498 NvmeTxDirection dir
, NvmeRequest
*req
);
499 void nvme_rw_complete_cb(void *opaque
, int ret
);
500 uint16_t nvme_map_dptr(NvmeCtrl
*n
, NvmeSg
*sg
, size_t len
,
503 /* from Linux kernel (crypto/crct10dif_common.c) */
504 static const uint16_t t10_dif_crc_table
[256] = {
505 0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
506 0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
507 0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
508 0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
509 0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
510 0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
511 0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
512 0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
513 0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
514 0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
515 0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
516 0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
517 0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
518 0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
519 0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
520 0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
521 0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
522 0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
523 0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
524 0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
525 0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
526 0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
527 0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
528 0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
529 0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
530 0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
531 0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
532 0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
533 0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
534 0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
535 0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
536 0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
539 uint16_t nvme_check_prinfo(NvmeNamespace
*ns
, uint8_t prinfo
, uint64_t slba
,
541 uint16_t nvme_dif_mangle_mdata(NvmeNamespace
*ns
, uint8_t *mbuf
, size_t mlen
,
543 void nvme_dif_pract_generate_dif(NvmeNamespace
*ns
, uint8_t *buf
, size_t len
,
544 uint8_t *mbuf
, size_t mlen
, uint16_t apptag
,
546 uint16_t nvme_dif_check(NvmeNamespace
*ns
, uint8_t *buf
, size_t len
,
547 uint8_t *mbuf
, size_t mlen
, uint8_t prinfo
,
548 uint64_t slba
, uint16_t apptag
,
549 uint16_t appmask
, uint32_t *reftag
);
550 uint16_t nvme_dif_rw(NvmeCtrl
*n
, NvmeRequest
*req
);
553 #endif /* HW_NVME_INTERNAL_H */