?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
15325 bhyve upstream sync 2023 January
[illumos-gate.git] / usr / src / cmd / bhyve / pci_nvme.c
blobb9d8873a872d388952cc6b105a25d68461de89de
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2017 Shunsuke Mie
5 * Copyright (c) 2018 Leon Dang
6 * Copyright (c) 2020 Chuck Tuffli
7 *
8 * Function crc16 Copyright (c) 2017, Fedor Uporov
9 * Obtained from function ext2_crc16() in sys/fs/ext2fs/ext2_csum.c
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 /*
34 * bhyve PCIe-NVMe device emulation.
35 *
36 * options:
37 * -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z,eui64=#,dsm=<opt>
38 *
39 * accepted devpath:
40 * /dev/blockdev
41 * /path/to/image
42 * ram=size_in_MiB
43 *
44 * maxq = max number of queues
45 * qsz = max elements in each queue
46 * ioslots = max number of concurrent io requests
47 * sectsz = sector size (defaults to blockif sector size)
48 * ser = serial number (20-chars max)
49 * eui64 = IEEE Extended Unique Identifier (8 byte value)
50 * dsm = DataSet Management support. Option is one of auto, enable,disable
51 *
52 */
53
54 /* TODO:
55 - create async event for smart and log
56 - intr coalesce
57 */
58
59 #include <sys/cdefs.h>
60 __FBSDID("$FreeBSD$");
61
62 #include <sys/errno.h>
63 #include <sys/types.h>
64 #include <net/ieee_oui.h>
65 #ifndef __FreeBSD__
66 #include <endian.h>
67 #endif
68
69 #include <assert.h>
70 #include <pthread.h>
71 #include <pthread_np.h>
72 #include <semaphore.h>
73 #include <stdbool.h>
74 #include <stddef.h>
75 #include <stdint.h>
76 #include <stdio.h>
77 #include <stdlib.h>
78 #include <string.h>
79
80 #include <machine/atomic.h>
81 #include <machine/vmm.h>
82 #include <vmmapi.h>
83
84 #include <dev/nvme/nvme.h>
85
86 #include "bhyverun.h"
87 #include "block_if.h"
88 #include "config.h"
89 #include "debug.h"
90 #include "pci_emul.h"
91
92
93 static int nvme_debug = 0;
94 #define DPRINTF(fmt, args...) if (nvme_debug) PRINTLN(fmt, ##args)
95 #define WPRINTF(fmt, args...) PRINTLN(fmt, ##args)
96
97 /* defaults; can be overridden */
98 #define NVME_MSIX_BAR 4
99
100 #define NVME_IOSLOTS 8
101
102 /* The NVMe spec defines bits 13:4 in BAR0 as reserved */
103 #define NVME_MMIO_SPACE_MIN (1 << 14)
104
105 #define NVME_QUEUES 16
106 #define NVME_MAX_QENTRIES 2048
107 /* Memory Page size Minimum reported in CAP register */
108 #define NVME_MPSMIN 0
109 /* MPSMIN converted to bytes */
110 #define NVME_MPSMIN_BYTES (1 << (12 + NVME_MPSMIN))
111
112 #define NVME_PRP2_ITEMS (PAGE_SIZE/sizeof(uint64_t))
113 #define NVME_MDTS 9
114 /* Note the + 1 allows for the initial descriptor to not be page aligned */
115 #define NVME_MAX_IOVEC ((1 << NVME_MDTS) + 1)
116 #define NVME_MAX_DATA_SIZE ((1 << NVME_MDTS) * NVME_MPSMIN_BYTES)
117
118 /* This is a synthetic status code to indicate there is no status */
119 #define NVME_NO_STATUS 0xffff
120 #define NVME_COMPLETION_VALID(c) ((c).status != NVME_NO_STATUS)
121
122 /* Reported temperature in Kelvin (i.e. room temperature) */
123 #define NVME_TEMPERATURE 296
124
125 /* helpers */
126
127 /* Convert a zero-based value into a one-based value */
128 #define ONE_BASED(zero) ((zero) + 1)
129 /* Convert a one-based value into a zero-based value */
130 #define ZERO_BASED(one) ((one) - 1)
131
132 /* Encode number of SQ's and CQ's for Set/Get Features */
133 #define NVME_FEATURE_NUM_QUEUES(sc) \
134 (ZERO_BASED((sc)->num_squeues) & 0xffff) | \
135 (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16;
136
137 #define NVME_DOORBELL_OFFSET offsetof(struct nvme_registers, doorbell)
138
139 enum nvme_controller_register_offsets {
140 NVME_CR_CAP_LOW = 0x00,
141 NVME_CR_CAP_HI = 0x04,
142 NVME_CR_VS = 0x08,
143 NVME_CR_INTMS = 0x0c,
144 NVME_CR_INTMC = 0x10,
145 NVME_CR_CC = 0x14,
146 NVME_CR_CSTS = 0x1c,
147 NVME_CR_NSSR = 0x20,
148 NVME_CR_AQA = 0x24,
149 NVME_CR_ASQ_LOW = 0x28,
150 NVME_CR_ASQ_HI = 0x2c,
151 NVME_CR_ACQ_LOW = 0x30,
152 NVME_CR_ACQ_HI = 0x34,
153 };
154
155 enum nvme_cmd_cdw11 {
156 NVME_CMD_CDW11_PC = 0x0001,
157 NVME_CMD_CDW11_IEN = 0x0002,
158 NVME_CMD_CDW11_IV = 0xFFFF0000,
159 };
160
161 enum nvme_copy_dir {
162 NVME_COPY_TO_PRP,
163 NVME_COPY_FROM_PRP,
164 };
165
166 #define NVME_CQ_INTEN 0x01
167 #define NVME_CQ_INTCOAL 0x02
168
169 struct nvme_completion_queue {
170 struct nvme_completion *qbase;
171 pthread_mutex_t mtx;
172 uint32_t size;
173 uint16_t tail; /* nvme progress */
174 uint16_t head; /* guest progress */
175 uint16_t intr_vec;
176 uint32_t intr_en;
177 };
178
179 struct nvme_submission_queue {
180 struct nvme_command *qbase;
181 pthread_mutex_t mtx;
182 uint32_t size;
183 uint16_t head; /* nvme progress */
184 uint16_t tail; /* guest progress */
185 uint16_t cqid; /* completion queue id */
186 int qpriority;
187 };
188
189 enum nvme_storage_type {
190 NVME_STOR_BLOCKIF = 0,
191 NVME_STOR_RAM = 1,
192 };
193
194 struct pci_nvme_blockstore {
195 enum nvme_storage_type type;
196 void *ctx;
197 uint64_t size;
198 uint32_t sectsz;
199 uint32_t sectsz_bits;
200 uint64_t eui64;
201 uint32_t deallocate:1;
202 };
203
204 /*
205 * Calculate the number of additional page descriptors for guest IO requests
206 * based on the advertised Max Data Transfer (MDTS) and given the number of
207 * default iovec's in a struct blockif_req.
208 */
209 #define MDTS_PAD_SIZE \
210 ( NVME_MAX_IOVEC > BLOCKIF_IOV_MAX ? \
211 NVME_MAX_IOVEC - BLOCKIF_IOV_MAX : \
212 0 )
213
214 struct pci_nvme_ioreq {
215 struct pci_nvme_softc *sc;
216 STAILQ_ENTRY(pci_nvme_ioreq) link;
217 struct nvme_submission_queue *nvme_sq;
218 uint16_t sqid;
219
220 /* command information */
221 uint16_t opc;
222 uint16_t cid;
223 uint32_t nsid;
224
225 uint64_t prev_gpaddr;
226 size_t prev_size;
227 size_t bytes;
228
229 struct blockif_req io_req;
230
231 struct iovec iovpadding[MDTS_PAD_SIZE];
232 };
233
234 enum nvme_dsm_type {
235 /* Dataset Management bit in ONCS reflects backing storage capability */
236 NVME_DATASET_MANAGEMENT_AUTO,
237 /* Unconditionally set Dataset Management bit in ONCS */
238 NVME_DATASET_MANAGEMENT_ENABLE,
239 /* Unconditionally clear Dataset Management bit in ONCS */
240 NVME_DATASET_MANAGEMENT_DISABLE,
241 };
242
243 struct pci_nvme_softc;
244 struct nvme_feature_obj;
245
246 typedef void (*nvme_feature_cb)(struct pci_nvme_softc *,
247 struct nvme_feature_obj *,
248 struct nvme_command *,
249 struct nvme_completion *);
250
251 struct nvme_feature_obj {
252 uint32_t cdw11;
253 nvme_feature_cb set;
254 nvme_feature_cb get;
255 bool namespace_specific;
256 };
257
258 #define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1)
259
260 typedef enum {
261 PCI_NVME_AE_TYPE_ERROR = 0,
262 PCI_NVME_AE_TYPE_SMART,
263 PCI_NVME_AE_TYPE_NOTICE,
264 PCI_NVME_AE_TYPE_IO_CMD = 6,
265 PCI_NVME_AE_TYPE_VENDOR = 7,
266 PCI_NVME_AE_TYPE_MAX /* Must be last */
267 } pci_nvme_async_type;
268
269 /* Asynchronous Event Requests */
270 struct pci_nvme_aer {
271 STAILQ_ENTRY(pci_nvme_aer) link;
272 uint16_t cid; /* Command ID of the submitted AER */
273 };
274
275 /** Asynchronous Event Information - Notice */
276 typedef enum {
277 PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED = 0,
278 PCI_NVME_AEI_NOTICE_FW_ACTIVATION,
279 PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE,
280 PCI_NVME_AEI_NOTICE_ANA_CHANGE,
281 PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE,
282 PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT,
283 PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE,
284 PCI_NVME_AEI_NOTICE_MAX,
285 } pci_nvme_async_event_info_notice;
286
287 #define PCI_NVME_AEI_NOTICE_SHIFT 8
288 #define PCI_NVME_AEI_NOTICE_MASK(event) (1 << (event + PCI_NVME_AEI_NOTICE_SHIFT))
289
290 /* Asynchronous Event Notifications */
291 struct pci_nvme_aen {
292 pci_nvme_async_type atype;
293 uint32_t event_data;
294 bool posted;
295 };
296
297 /*
298 * By default, enable all Asynchrnous Event Notifications:
299 * SMART / Health Critical Warnings
300 * Namespace Attribute Notices
301 */
302 #define PCI_NVME_AEN_DEFAULT_MASK 0x11f
303
304 typedef enum {
305 NVME_CNTRLTYPE_IO = 1,
306 NVME_CNTRLTYPE_DISCOVERY = 2,
307 NVME_CNTRLTYPE_ADMIN = 3,
308 } pci_nvme_cntrl_type;
309
310 struct pci_nvme_softc {
311 struct pci_devinst *nsc_pi;
312
313 pthread_mutex_t mtx;
314
315 struct nvme_registers regs;
316
317 struct nvme_namespace_data nsdata;
318 struct nvme_controller_data ctrldata;
319 struct nvme_error_information_entry err_log;
320 struct nvme_health_information_page health_log;
321 struct nvme_firmware_page fw_log;
322 struct nvme_ns_list ns_log;
323
324 struct pci_nvme_blockstore nvstore;
325
326 uint16_t max_qentries; /* max entries per queue */
327 uint32_t max_queues; /* max number of IO SQ's or CQ's */
328 uint32_t num_cqueues;
329 uint32_t num_squeues;
330 bool num_q_is_set; /* Has host set Number of Queues */
331
332 struct pci_nvme_ioreq *ioreqs;
333 STAILQ_HEAD(, pci_nvme_ioreq) ioreqs_free; /* free list of ioreqs */
334 uint32_t pending_ios;
335 uint32_t ioslots;
336 sem_t iosemlock;
337
338 /*
339 * Memory mapped Submission and Completion queues
340 * Each array includes both Admin and IO queues
341 */
342 struct nvme_completion_queue *compl_queues;
343 struct nvme_submission_queue *submit_queues;
344
345 struct nvme_feature_obj feat[NVME_FID_MAX];
346
347 enum nvme_dsm_type dataset_management;
348
349 /* Accounting for SMART data */
350 __uint128_t read_data_units;
351 __uint128_t write_data_units;
352 __uint128_t read_commands;
353 __uint128_t write_commands;
354 uint32_t read_dunits_remainder;
355 uint32_t write_dunits_remainder;
356
357 STAILQ_HEAD(, pci_nvme_aer) aer_list;
358 pthread_mutex_t aer_mtx;
359 uint32_t aer_count;
360 struct pci_nvme_aen aen[PCI_NVME_AE_TYPE_MAX];
361 pthread_t aen_tid;
362 pthread_mutex_t aen_mtx;
363 pthread_cond_t aen_cond;
364 };
365
366
367 static void pci_nvme_cq_update(struct pci_nvme_softc *sc,
368 struct nvme_completion_queue *cq,
369 uint32_t cdw0,
370 uint16_t cid,
371 uint16_t sqid,
372 uint16_t status);
373 static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *);
374 static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *);
375 static void pci_nvme_io_done(struct blockif_req *, int);
376
377 /* Controller Configuration utils */
378 #define NVME_CC_GET_EN(cc) \
379 ((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK)
380 #define NVME_CC_GET_CSS(cc) \
381 ((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK)
382 #define NVME_CC_GET_SHN(cc) \
383 ((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK)
384 #define NVME_CC_GET_IOSQES(cc) \
385 ((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK)
386 #define NVME_CC_GET_IOCQES(cc) \
387 ((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK)
388
389 #define NVME_CC_WRITE_MASK \
390 ((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \
391 (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \
392 (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT))
393
394 #define NVME_CC_NEN_WRITE_MASK \
395 ((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \
396 (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \
397 (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT))
398
399 /* Controller Status utils */
400 #define NVME_CSTS_GET_RDY(sts) \
401 ((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK)
402
403 #define NVME_CSTS_RDY (1 << NVME_CSTS_REG_RDY_SHIFT)
404 #define NVME_CSTS_CFS (1 << NVME_CSTS_REG_CFS_SHIFT)
405
406 /* Completion Queue status word utils */
407 #define NVME_STATUS_P (1 << NVME_STATUS_P_SHIFT)
408 #define NVME_STATUS_MASK \
409 ((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\
410 (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT))
411
412 #define NVME_ONCS_DSM (NVME_CTRLR_DATA_ONCS_DSM_MASK << \
413 NVME_CTRLR_DATA_ONCS_DSM_SHIFT)
414
415 static void nvme_feature_invalid_cb(struct pci_nvme_softc *,
416 struct nvme_feature_obj *,
417 struct nvme_command *,
418 struct nvme_completion *);
419 static void nvme_feature_temperature(struct pci_nvme_softc *,
420 struct nvme_feature_obj *,
421 struct nvme_command *,
422 struct nvme_completion *);
423 static void nvme_feature_num_queues(struct pci_nvme_softc *,
424 struct nvme_feature_obj *,
425 struct nvme_command *,
426 struct nvme_completion *);
427 static void nvme_feature_iv_config(struct pci_nvme_softc *,
428 struct nvme_feature_obj *,
429 struct nvme_command *,
430 struct nvme_completion *);
431 static void nvme_feature_async_event(struct pci_nvme_softc *,
432 struct nvme_feature_obj *,
433 struct nvme_command *,
434 struct nvme_completion *);
435
436 static void *aen_thr(void *arg);
437
438 static __inline void
439 cpywithpad(char *dst, size_t dst_size, const char *src, char pad)
440 {
441 size_t len;
442
443 len = strnlen(src, dst_size);
444 memset(dst, pad, dst_size);
445 memcpy(dst, src, len);
446 }
447
448 static __inline void
449 pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code)
450 {
451
452 *status &= ~NVME_STATUS_MASK;
453 *status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT |
454 (code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT;
455 }
456
457 static __inline void
458 pci_nvme_status_genc(uint16_t *status, uint16_t code)
459 {
460
461 pci_nvme_status_tc(status, NVME_SCT_GENERIC, code);
462 }
463
464 /*
465 * Initialize the requested number or IO Submission and Completion Queues.
466 * Admin queues are allocated implicitly.
467 */
468 static void
469 pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq)
470 {
471 uint32_t i;
472
473 /*
474 * Allocate and initialize the Submission Queues
475 */
476 if (nsq > NVME_QUEUES) {
477 WPRINTF("%s: clamping number of SQ from %u to %u",
478 __func__, nsq, NVME_QUEUES);
479 nsq = NVME_QUEUES;
480 }
481
482 sc->num_squeues = nsq;
483
484 sc->submit_queues = calloc(sc->num_squeues + 1,
485 sizeof(struct nvme_submission_queue));
486 if (sc->submit_queues == NULL) {
487 WPRINTF("%s: SQ allocation failed", __func__);
488 sc->num_squeues = 0;
489 } else {
490 struct nvme_submission_queue *sq = sc->submit_queues;
491
492 for (i = 0; i < sc->num_squeues + 1; i++)
493 pthread_mutex_init(&sq[i].mtx, NULL);
494 }
495
496 /*
497 * Allocate and initialize the Completion Queues
498 */
499 if (ncq > NVME_QUEUES) {
500 WPRINTF("%s: clamping number of CQ from %u to %u",
501 __func__, ncq, NVME_QUEUES);
502 ncq = NVME_QUEUES;
503 }
504
505 sc->num_cqueues = ncq;
506
507 sc->compl_queues = calloc(sc->num_cqueues + 1,
508 sizeof(struct nvme_completion_queue));
509 if (sc->compl_queues == NULL) {
510 WPRINTF("%s: CQ allocation failed", __func__);
511 sc->num_cqueues = 0;
512 } else {
513 struct nvme_completion_queue *cq = sc->compl_queues;
514
515 for (i = 0; i < sc->num_cqueues + 1; i++)
516 pthread_mutex_init(&cq[i].mtx, NULL);
517 }
518 }
519
520 static void
521 pci_nvme_init_ctrldata(struct pci_nvme_softc *sc)
522 {
523 struct nvme_controller_data *cd = &sc->ctrldata;
524
525 cd->vid = 0xFB5D;
526 cd->ssvid = 0x0000;
527
528 cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' ');
529 cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' ');
530
531 /* Num of submission commands that we can handle at a time (2^rab) */
532 cd->rab = 4;
533
534 /* FreeBSD OUI */
535 cd->ieee[0] = 0x58;
536 cd->ieee[1] = 0x9c;
537 cd->ieee[2] = 0xfc;
538
539 cd->mic = 0;
540
541 cd->mdts = NVME_MDTS; /* max data transfer size (2^mdts * CAP.MPSMIN) */
542
543 cd->ver = NVME_REV(1,4);
544
545 cd->cntrltype = NVME_CNTRLTYPE_IO;
546 cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT;
547 cd->oaes = NVMEB(NVME_CTRLR_DATA_OAES_NS_ATTR);
548 cd->acl = 2;
549 cd->aerl = 4;
550
551 /* Advertise 1, Read-only firmware slot */
552 cd->frmw = NVMEB(NVME_CTRLR_DATA_FRMW_SLOT1_RO) |
553 (1 << NVME_CTRLR_DATA_FRMW_NUM_SLOTS_SHIFT);
554 cd->lpa = 0; /* TODO: support some simple things like SMART */
555 cd->elpe = 0; /* max error log page entries */
556 /*
557 * Report a single power state (zero-based value)
558 * power_state[] values are left as zero to indicate "Not reported"
559 */
560 cd->npss = 0;
561
562 /* Warning Composite Temperature Threshold */
563 cd->wctemp = 0x0157;
564 cd->cctemp = 0x0157;
565
566 /* SANICAP must not be 0 for Revision 1.4 and later NVMe Controllers */
567 cd->sanicap = (NVME_CTRLR_DATA_SANICAP_NODMMAS_NO <<
568 NVME_CTRLR_DATA_SANICAP_NODMMAS_SHIFT);
569
570 cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) |
571 (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT);
572 cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) |
573 (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT);
574 cd->nn = 1; /* number of namespaces */
575
576 cd->oncs = 0;
577 switch (sc->dataset_management) {
578 case NVME_DATASET_MANAGEMENT_AUTO:
579 if (sc->nvstore.deallocate)
580 cd->oncs |= NVME_ONCS_DSM;
581 break;
582 case NVME_DATASET_MANAGEMENT_ENABLE:
583 cd->oncs |= NVME_ONCS_DSM;
584 break;
585 default:
586 break;
587 }
588
589 cd->fna = NVME_CTRLR_DATA_FNA_FORMAT_ALL_MASK <<
590 NVME_CTRLR_DATA_FNA_FORMAT_ALL_SHIFT;
591
592 cd->vwc = NVME_CTRLR_DATA_VWC_ALL_NO << NVME_CTRLR_DATA_VWC_ALL_SHIFT;
593 }
594
595 /*
596 * Calculate the CRC-16 of the given buffer
597 * See copyright attribution at top of file
598 */
599 static uint16_t
600 crc16(uint16_t crc, const void *buffer, unsigned int len)
601 {
602 const unsigned char *cp = buffer;
603 /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */
604 static uint16_t const crc16_table[256] = {
605 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241,
606 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440,
607 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40,
608 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841,
609 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40,
610 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41,
611 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641,
612 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040,
613 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240,
614 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441,
615 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41,
616 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840,
617 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41,
618 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40,
619 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640,
620 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041,
621 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240,
622 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441,
623 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41,
624 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840,
625 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41,
626 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40,
627 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640,
628 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041,
629 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241,
630 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440,
631 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40,
632 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841,
633 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40,
634 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41,
635 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641,
636 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040
637 };
638
639 while (len--)
640 crc = (((crc >> 8) & 0xffU) ^
641 crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU;
642 return crc;
643 }
644
645 static void
646 pci_nvme_init_nsdata_size(struct pci_nvme_blockstore *nvstore,
647 struct nvme_namespace_data *nd)
648 {
649
650 /* Get capacity and block size information from backing store */
651 nd->nsze = nvstore->size / nvstore->sectsz;
652 nd->ncap = nd->nsze;
653 nd->nuse = nd->nsze;
654 }
655
656 static void
657 pci_nvme_init_nsdata(struct pci_nvme_softc *sc,
658 struct nvme_namespace_data *nd, uint32_t nsid,
659 struct pci_nvme_blockstore *nvstore)
660 {
661
662 pci_nvme_init_nsdata_size(nvstore, nd);
663
664 if (nvstore->type == NVME_STOR_BLOCKIF)
665 nvstore->deallocate = blockif_candelete(nvstore->ctx);
666
667 nd->nlbaf = 0; /* NLBAF is a 0's based value (i.e. 1 LBA Format) */
668 nd->flbas = 0;
669
670 /* Create an EUI-64 if user did not provide one */
671 if (nvstore->eui64 == 0) {
672 char *data = NULL;
673 uint64_t eui64 = nvstore->eui64;
674
675 asprintf(&data, "%s%u%u%u", get_config_value("name"),
676 sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot,
677 sc->nsc_pi->pi_func);
678
679 if (data != NULL) {
680 eui64 = OUI_FREEBSD_NVME_LOW | crc16(0, data, strlen(data));
681 free(data);
682 }
683 nvstore->eui64 = (eui64 << 16) | (nsid & 0xffff);
684 }
685 be64enc(nd->eui64, nvstore->eui64);
686
687 /* LBA data-sz = 2^lbads */
688 nd->lbaf[0] = nvstore->sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT;
689 }
690
691 static void
692 pci_nvme_init_logpages(struct pci_nvme_softc *sc)
693 {
694 __uint128_t power_cycles = 1;
695
696 memset(&sc->err_log, 0, sizeof(sc->err_log));
697 memset(&sc->health_log, 0, sizeof(sc->health_log));
698 memset(&sc->fw_log, 0, sizeof(sc->fw_log));
699 memset(&sc->ns_log, 0, sizeof(sc->ns_log));
700
701 /* Set read/write remainder to round up according to spec */
702 sc->read_dunits_remainder = 999;
703 sc->write_dunits_remainder = 999;
704
705 /* Set nominal Health values checked by implementations */
706 sc->health_log.temperature = NVME_TEMPERATURE;
707 sc->health_log.available_spare = 100;
708 sc->health_log.available_spare_threshold = 10;
709
710 /* Set Active Firmware Info to slot 1 */
711 sc->fw_log.afi = (1 << NVME_FIRMWARE_PAGE_AFI_SLOT_SHIFT);
712 memcpy(&sc->fw_log.revision[0], sc->ctrldata.fr,
713 sizeof(sc->fw_log.revision[0]));
714
715 memcpy(&sc->health_log.power_cycles, &power_cycles,
716 sizeof(sc->health_log.power_cycles));
717 }
718
719 static void
720 pci_nvme_init_features(struct pci_nvme_softc *sc)
721 {
722 enum nvme_feature fid;
723
724 for (fid = 0; fid < NVME_FID_MAX; fid++) {
725 switch (fid) {
726 case NVME_FEAT_ARBITRATION:
727 case NVME_FEAT_POWER_MANAGEMENT:
728 case NVME_FEAT_INTERRUPT_COALESCING: //XXX
729 case NVME_FEAT_WRITE_ATOMICITY:
730 /* Mandatory but no special handling required */
731 //XXX hang - case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG:
732 //XXX hang - case NVME_FEAT_HOST_BEHAVIOR_SUPPORT:
733 // this returns a data buffer
734 break;
735 case NVME_FEAT_TEMPERATURE_THRESHOLD:
736 sc->feat[fid].set = nvme_feature_temperature;
737 break;
738 case NVME_FEAT_ERROR_RECOVERY:
739 sc->feat[fid].namespace_specific = true;
740 break;
741 case NVME_FEAT_NUMBER_OF_QUEUES:
742 sc->feat[fid].set = nvme_feature_num_queues;
743 break;
744 case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
745 sc->feat[fid].set = nvme_feature_iv_config;
746 break;
747 case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
748 sc->feat[fid].set = nvme_feature_async_event;
749 /* Enable all AENs by default */
750 sc->feat[fid].cdw11 = PCI_NVME_AEN_DEFAULT_MASK;
751 break;
752 default:
753 sc->feat[fid].set = nvme_feature_invalid_cb;
754 sc->feat[fid].get = nvme_feature_invalid_cb;
755 }
756 }
757 }
758
759 static void
760 pci_nvme_aer_reset(struct pci_nvme_softc *sc)
761 {
762
763 STAILQ_INIT(&sc->aer_list);
764 sc->aer_count = 0;
765 }
766
767 static void
768 pci_nvme_aer_init(struct pci_nvme_softc *sc)
769 {
770
771 pthread_mutex_init(&sc->aer_mtx, NULL);
772 pci_nvme_aer_reset(sc);
773 }
774
775 static void
776 pci_nvme_aer_destroy(struct pci_nvme_softc *sc)
777 {
778 struct pci_nvme_aer *aer = NULL;
779
780 pthread_mutex_lock(&sc->aer_mtx);
781 while (!STAILQ_EMPTY(&sc->aer_list)) {
782 aer = STAILQ_FIRST(&sc->aer_list);
783 STAILQ_REMOVE_HEAD(&sc->aer_list, link);
784 free(aer);
785 }
786 pthread_mutex_unlock(&sc->aer_mtx);
787
788 pci_nvme_aer_reset(sc);
789 }
790
791 static bool
792 pci_nvme_aer_available(struct pci_nvme_softc *sc)
793 {
794
795 return (sc->aer_count != 0);
796 }
797
798 static bool
799 pci_nvme_aer_limit_reached(struct pci_nvme_softc *sc)
800 {
801 struct nvme_controller_data *cd = &sc->ctrldata;
802
803 /* AERL is a zero based value while aer_count is one's based */
804 return (sc->aer_count == (cd->aerl + 1U));
805 }
806
807 /*
808 * Add an Async Event Request
809 *
810 * Stores an AER to be returned later if the Controller needs to notify the
811 * host of an event.
812 * Note that while the NVMe spec doesn't require Controllers to return AER's
813 * in order, this implementation does preserve the order.
814 */
815 static int
816 pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid)
817 {
818 struct pci_nvme_aer *aer = NULL;
819
820 aer = calloc(1, sizeof(struct pci_nvme_aer));
821 if (aer == NULL)
822 return (-1);
823
824 /* Save the Command ID for use in the completion message */
825 aer->cid = cid;
826
827 pthread_mutex_lock(&sc->aer_mtx);
828 sc->aer_count++;
829 STAILQ_INSERT_TAIL(&sc->aer_list, aer, link);
830 pthread_mutex_unlock(&sc->aer_mtx);
831
832 return (0);
833 }
834
835 /*
836 * Get an Async Event Request structure
837 *
838 * Returns a pointer to an AER previously submitted by the host or NULL if
839 * no AER's exist. Caller is responsible for freeing the returned struct.
840 */
841 static struct pci_nvme_aer *
842 pci_nvme_aer_get(struct pci_nvme_softc *sc)
843 {
844 struct pci_nvme_aer *aer = NULL;
845
846 pthread_mutex_lock(&sc->aer_mtx);
847 aer = STAILQ_FIRST(&sc->aer_list);
848 if (aer != NULL) {
849 STAILQ_REMOVE_HEAD(&sc->aer_list, link);
850 sc->aer_count--;
851 }
852 pthread_mutex_unlock(&sc->aer_mtx);
853
854 return (aer);
855 }
856
857 static void
858 pci_nvme_aen_reset(struct pci_nvme_softc *sc)
859 {
860 uint32_t atype;
861
862 memset(sc->aen, 0, PCI_NVME_AE_TYPE_MAX * sizeof(struct pci_nvme_aen));
863
864 for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) {
865 sc->aen[atype].atype = atype;
866 }
867 }
868
869 static void
870 pci_nvme_aen_init(struct pci_nvme_softc *sc)
871 {
872 char nstr[80];
873
874 pci_nvme_aen_reset(sc);
875
876 pthread_mutex_init(&sc->aen_mtx, NULL);
877 pthread_create(&sc->aen_tid, NULL, aen_thr, sc);
878 snprintf(nstr, sizeof(nstr), "nvme-aen-%d:%d", sc->nsc_pi->pi_slot,
879 sc->nsc_pi->pi_func);
880 pthread_set_name_np(sc->aen_tid, nstr);
881 }
882
883 static void
884 pci_nvme_aen_destroy(struct pci_nvme_softc *sc)
885 {
886
887 pci_nvme_aen_reset(sc);
888 }
889
890 /* Notify the AEN thread of pending work */
891 static void
892 pci_nvme_aen_notify(struct pci_nvme_softc *sc)
893 {
894
895 pthread_cond_signal(&sc->aen_cond);
896 }
897
898 /*
899 * Post an Asynchronous Event Notification
900 */
901 static int32_t
902 pci_nvme_aen_post(struct pci_nvme_softc *sc, pci_nvme_async_type atype,
903 uint32_t event_data)
904 {
905 struct pci_nvme_aen *aen;
906
907 if (atype >= PCI_NVME_AE_TYPE_MAX) {
908 return(EINVAL);
909 }
910
911 pthread_mutex_lock(&sc->aen_mtx);
912 aen = &sc->aen[atype];
913
914 /* Has the controller already posted an event of this type? */
915 if (aen->posted) {
916 pthread_mutex_unlock(&sc->aen_mtx);
917 return(EALREADY);
918 }
919
920 aen->event_data = event_data;
921 aen->posted = true;
922 pthread_mutex_unlock(&sc->aen_mtx);
923
924 pci_nvme_aen_notify(sc);
925
926 return(0);
927 }
928
929 static void
930 pci_nvme_aen_process(struct pci_nvme_softc *sc)
931 {
932 struct pci_nvme_aer *aer;
933 struct pci_nvme_aen *aen;
934 pci_nvme_async_type atype;
935 uint32_t mask;
936 uint16_t status;
937 uint8_t lid;
938
939 #ifndef __FreeBSD__
940 lid = 0;
941 #endif
942
943 assert(pthread_mutex_isowned_np(&sc->aen_mtx));
944 for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) {
945 aen = &sc->aen[atype];
946 /* Previous iterations may have depleted the available AER's */
947 if (!pci_nvme_aer_available(sc)) {
948 DPRINTF("%s: no AER", __func__);
949 break;
950 }
951
952 if (!aen->posted) {
953 DPRINTF("%s: no AEN posted for atype=%#x", __func__, atype);
954 continue;
955 }
956
957 status = NVME_SC_SUCCESS;
958
959 /* Is the event masked? */
960 mask =
961 sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11;
962
963 DPRINTF("%s: atype=%#x mask=%#x event_data=%#x", __func__, atype, mask, aen->event_data);
964 switch (atype) {
965 case PCI_NVME_AE_TYPE_ERROR:
966 lid = NVME_LOG_ERROR;
967 break;
968 case PCI_NVME_AE_TYPE_SMART:
969 mask &= 0xff;
970 if ((mask & aen->event_data) == 0)
971 continue;
972 lid = NVME_LOG_HEALTH_INFORMATION;
973 break;
974 case PCI_NVME_AE_TYPE_NOTICE:
975 if (aen->event_data >= PCI_NVME_AEI_NOTICE_MAX) {
976 EPRINTLN("%s unknown AEN notice type %u",
977 __func__, aen->event_data);
978 status = NVME_SC_INTERNAL_DEVICE_ERROR;
979 lid = 0;
980 break;
981 }
982 if ((PCI_NVME_AEI_NOTICE_MASK(aen->event_data) & mask) == 0)
983 continue;
984 switch (aen->event_data) {
985 case PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED:
986 lid = NVME_LOG_CHANGED_NAMESPACE;
987 break;
988 case PCI_NVME_AEI_NOTICE_FW_ACTIVATION:
989 lid = NVME_LOG_FIRMWARE_SLOT;
990 break;
991 case PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE:
992 lid = NVME_LOG_TELEMETRY_CONTROLLER_INITIATED;
993 break;
994 case PCI_NVME_AEI_NOTICE_ANA_CHANGE:
995 lid = NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS;
996 break;
997 case PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE:
998 lid = NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE;
999 break;
1000 case PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT:
1001 lid = NVME_LOG_LBA_STATUS_INFORMATION;
1002 break;
1003 case PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE:
1004 lid = NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE;
1005 break;
1006 default:
1007 lid = 0;
1008 }
1009 break;
1010 default:
1011 /* bad type?!? */
1012 EPRINTLN("%s unknown AEN type %u", __func__, atype);
1013 status = NVME_SC_INTERNAL_DEVICE_ERROR;
1014 lid = 0;
1015 break;
1016 }
1017
1018 aer = pci_nvme_aer_get(sc);
1019 assert(aer != NULL);
1020
1021 DPRINTF("%s: CID=%#x CDW0=%#x", __func__, aer->cid, (lid << 16) | (aen->event_data << 8) | atype);
1022 pci_nvme_cq_update(sc, &sc->compl_queues[0],
1023 (lid << 16) | (aen->event_data << 8) | atype, /* cdw0 */
1024 aer->cid,
1025 0, /* SQID */
1026 status);
1027
1028 aen->event_data = 0;
1029 aen->posted = false;
1030
1031 pci_generate_msix(sc->nsc_pi, 0);
1032 }
1033 }
1034
1035 static void *
1036 aen_thr(void *arg)
1037 {
1038 struct pci_nvme_softc *sc;
1039
1040 sc = arg;
1041
1042 pthread_mutex_lock(&sc->aen_mtx);
1043 for (;;) {
1044 pci_nvme_aen_process(sc);
1045 pthread_cond_wait(&sc->aen_cond, &sc->aen_mtx);
1046 }
1047 #ifdef __FreeBSD__ /* Smatch spots unreachable code */
1048 pthread_mutex_unlock(&sc->aen_mtx);
1049
1050 pthread_exit(NULL);
1051 #endif
1052 return (NULL);
1053 }
1054
1055 static void
1056 pci_nvme_reset_locked(struct pci_nvme_softc *sc)
1057 {
1058 uint32_t i;
1059
1060 DPRINTF("%s", __func__);
1061
1062 sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) |
1063 (1 << NVME_CAP_LO_REG_CQR_SHIFT) |
1064 (60 << NVME_CAP_LO_REG_TO_SHIFT);
1065
1066 sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT;
1067
1068 sc->regs.vs = NVME_REV(1,4); /* NVMe v1.4 */
1069
1070 sc->regs.cc = 0;
1071
1072 assert(sc->submit_queues != NULL);
1073
1074 for (i = 0; i < sc->num_squeues + 1; i++) {
1075 sc->submit_queues[i].qbase = NULL;
1076 sc->submit_queues[i].size = 0;
1077 sc->submit_queues[i].cqid = 0;
1078 sc->submit_queues[i].tail = 0;
1079 sc->submit_queues[i].head = 0;
1080 }
1081
1082 assert(sc->compl_queues != NULL);
1083
1084 for (i = 0; i < sc->num_cqueues + 1; i++) {
1085 sc->compl_queues[i].qbase = NULL;
1086 sc->compl_queues[i].size = 0;
1087 sc->compl_queues[i].tail = 0;
1088 sc->compl_queues[i].head = 0;
1089 }
1090
1091 sc->num_q_is_set = false;
1092
1093 pci_nvme_aer_destroy(sc);
1094 pci_nvme_aen_destroy(sc);
1095
1096 /*
1097 * Clear CSTS.RDY last to prevent the host from enabling Controller
1098 * before cleanup completes
1099 */
1100 sc->regs.csts = 0;
1101 }
1102
1103 static void
1104 pci_nvme_reset(struct pci_nvme_softc *sc)
1105 {
1106 pthread_mutex_lock(&sc->mtx);
1107 pci_nvme_reset_locked(sc);
1108 pthread_mutex_unlock(&sc->mtx);
1109 }
1110
1111 static int
1112 pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc)
1113 {
1114 uint16_t acqs, asqs;
1115
1116 DPRINTF("%s", __func__);
1117
1118 /*
1119 * NVMe 2.0 states that "enabling a controller while this field is
1120 * cleared to 0h produces undefined results" for both ACQS and
1121 * ASQS. If zero, set CFS and do not become ready.
1122 */
1123 asqs = ONE_BASED(sc->regs.aqa & NVME_AQA_REG_ASQS_MASK);
1124 if (asqs < 2) {
1125 EPRINTLN("%s: illegal ASQS value %#x (aqa=%#x)", __func__,
1126 asqs - 1, sc->regs.aqa);
1127 sc->regs.csts |= NVME_CSTS_CFS;
1128 return (-1);
1129 }
1130 sc->submit_queues[0].size = asqs;
1131 sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq,
1132 sizeof(struct nvme_command) * asqs);
1133 if (sc->submit_queues[0].qbase == NULL) {
1134 EPRINTLN("%s: ASQ vm_map_gpa(%lx) failed", __func__,
1135 sc->regs.asq);
1136 sc->regs.csts |= NVME_CSTS_CFS;
1137 return (-1);
1138 }
1139
1140 DPRINTF("%s mapping Admin-SQ guest 0x%lx, host: %p",
1141 __func__, sc->regs.asq, sc->submit_queues[0].qbase);
1142
1143 acqs = ONE_BASED((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &
1144 NVME_AQA_REG_ACQS_MASK);
1145 if (acqs < 2) {
1146 EPRINTLN("%s: illegal ACQS value %#x (aqa=%#x)", __func__,
1147 acqs - 1, sc->regs.aqa);
1148 sc->regs.csts |= NVME_CSTS_CFS;
1149 return (-1);
1150 }
1151 sc->compl_queues[0].size = acqs;
1152 sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq,
1153 sizeof(struct nvme_completion) * acqs);
1154 if (sc->compl_queues[0].qbase == NULL) {
1155 EPRINTLN("%s: ACQ vm_map_gpa(%lx) failed", __func__,
1156 sc->regs.acq);
1157 sc->regs.csts |= NVME_CSTS_CFS;
1158 return (-1);
1159 }
1160 sc->compl_queues[0].intr_en = NVME_CQ_INTEN;
1161
1162 DPRINTF("%s mapping Admin-CQ guest 0x%lx, host: %p",
1163 __func__, sc->regs.acq, sc->compl_queues[0].qbase);
1164
1165 return (0);
1166 }
1167
1168 static int
1169 nvme_prp_memcpy(struct vmctx *ctx, uint64_t prp1, uint64_t prp2, uint8_t *b,
1170 size_t len, enum nvme_copy_dir dir)
1171 {
1172 uint8_t *p;
1173 size_t bytes;
1174
1175 if (len > (8 * 1024)) {
1176 return (-1);
1177 }
1178
1179 /* Copy from the start of prp1 to the end of the physical page */
1180 bytes = PAGE_SIZE - (prp1 & PAGE_MASK);
1181 bytes = MIN(bytes, len);
1182
1183 p = vm_map_gpa(ctx, prp1, bytes);
1184 if (p == NULL) {
1185 return (-1);
1186 }
1187
1188 if (dir == NVME_COPY_TO_PRP)
1189 memcpy(p, b, bytes);
1190 else
1191 memcpy(b, p, bytes);
1192
1193 b += bytes;
1194
1195 len -= bytes;
1196 if (len == 0) {
1197 return (0);
1198 }
1199
1200 len = MIN(len, PAGE_SIZE);
1201
1202 p = vm_map_gpa(ctx, prp2, len);
1203 if (p == NULL) {
1204 return (-1);
1205 }
1206
1207 if (dir == NVME_COPY_TO_PRP)
1208 memcpy(p, b, len);
1209 else
1210 memcpy(b, p, len);
1211
1212 return (0);
1213 }
1214
1215 /*
1216 * Write a Completion Queue Entry update
1217 *
1218 * Write the completion and update the doorbell value
1219 */
1220 static void
1221 pci_nvme_cq_update(struct pci_nvme_softc *sc,
1222 struct nvme_completion_queue *cq,
1223 uint32_t cdw0,
1224 uint16_t cid,
1225 uint16_t sqid,
1226 uint16_t status)
1227 {
1228 struct nvme_submission_queue *sq = &sc->submit_queues[sqid];
1229 struct nvme_completion *cqe;
1230
1231 assert(cq->qbase != NULL);
1232
1233 pthread_mutex_lock(&cq->mtx);
1234
1235 cqe = &cq->qbase[cq->tail];
1236
1237 /* Flip the phase bit */
1238 status |= (cqe->status ^ NVME_STATUS_P) & NVME_STATUS_P_MASK;
1239
1240 cqe->cdw0 = cdw0;
1241 cqe->sqhd = sq->head;
1242 cqe->sqid = sqid;
1243 cqe->cid = cid;
1244 cqe->status = status;
1245
1246 cq->tail++;
1247 if (cq->tail >= cq->size) {
1248 cq->tail = 0;
1249 }
1250
1251 pthread_mutex_unlock(&cq->mtx);
1252 }
1253
1254 static int
1255 nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
1256 struct nvme_completion* compl)
1257 {
1258 uint16_t qid = command->cdw10 & 0xffff;
1259
1260 DPRINTF("%s DELETE_IO_SQ %u", __func__, qid);
1261 if (qid == 0 || qid > sc->num_squeues ||
1262 (sc->submit_queues[qid].qbase == NULL)) {
1263 WPRINTF("%s NOT PERMITTED queue id %u / num_squeues %u",
1264 __func__, qid, sc->num_squeues);
1265 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1266 NVME_SC_INVALID_QUEUE_IDENTIFIER);
1267 return (1);
1268 }
1269
1270 sc->submit_queues[qid].qbase = NULL;
1271 sc->submit_queues[qid].cqid = 0;
1272 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1273 return (1);
1274 }
1275
1276 static int
1277 nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
1278 struct nvme_completion* compl)
1279 {
1280 if (command->cdw11 & NVME_CMD_CDW11_PC) {
1281 uint16_t qid = command->cdw10 & 0xffff;
1282 struct nvme_submission_queue *nsq;
1283
1284 if ((qid == 0) || (qid > sc->num_squeues) ||
1285 (sc->submit_queues[qid].qbase != NULL)) {
1286 WPRINTF("%s queue index %u > num_squeues %u",
1287 __func__, qid, sc->num_squeues);
1288 pci_nvme_status_tc(&compl->status,
1289 NVME_SCT_COMMAND_SPECIFIC,
1290 NVME_SC_INVALID_QUEUE_IDENTIFIER);
1291 return (1);
1292 }
1293
1294 nsq = &sc->submit_queues[qid];
1295 nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff);
1296 DPRINTF("%s size=%u (max=%u)", __func__, nsq->size, sc->max_qentries);
1297 if ((nsq->size < 2) || (nsq->size > sc->max_qentries)) {
1298 /*
1299 * Queues must specify at least two entries
1300 * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to
1301 * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec
1302 */
1303 pci_nvme_status_tc(&compl->status,
1304 NVME_SCT_COMMAND_SPECIFIC,
1305 NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED);
1306 return (1);
1307 }
1308 nsq->head = nsq->tail = 0;
1309
1310 nsq->cqid = (command->cdw11 >> 16) & 0xffff;
1311 if ((nsq->cqid == 0) || (nsq->cqid > sc->num_cqueues)) {
1312 pci_nvme_status_tc(&compl->status,
1313 NVME_SCT_COMMAND_SPECIFIC,
1314 NVME_SC_INVALID_QUEUE_IDENTIFIER);
1315 return (1);
1316 }
1317
1318 if (sc->compl_queues[nsq->cqid].qbase == NULL) {
1319 pci_nvme_status_tc(&compl->status,
1320 NVME_SCT_COMMAND_SPECIFIC,
1321 NVME_SC_COMPLETION_QUEUE_INVALID);
1322 return (1);
1323 }
1324
1325 nsq->qpriority = (command->cdw11 >> 1) & 0x03;
1326
1327 nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
1328 sizeof(struct nvme_command) * (size_t)nsq->size);
1329
1330 DPRINTF("%s sq %u size %u gaddr %p cqid %u", __func__,
1331 qid, nsq->size, nsq->qbase, nsq->cqid);
1332
1333 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1334
1335 DPRINTF("%s completed creating IOSQ qid %u",
1336 __func__, qid);
1337 } else {
1338 /*
1339 * Guest sent non-cont submission queue request.
1340 * This setting is unsupported by this emulation.
1341 */
1342 WPRINTF("%s unsupported non-contig (list-based) "
1343 "create i/o submission queue", __func__);
1344
1345 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1346 }
1347 return (1);
1348 }
1349
1350 static int
1351 nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
1352 struct nvme_completion* compl)
1353 {
1354 uint16_t qid = command->cdw10 & 0xffff;
1355 uint16_t sqid;
1356
1357 DPRINTF("%s DELETE_IO_CQ %u", __func__, qid);
1358 if (qid == 0 || qid > sc->num_cqueues ||
1359 (sc->compl_queues[qid].qbase == NULL)) {
1360 WPRINTF("%s queue index %u / num_cqueues %u",
1361 __func__, qid, sc->num_cqueues);
1362 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1363 NVME_SC_INVALID_QUEUE_IDENTIFIER);
1364 return (1);
1365 }
1366
1367 /* Deleting an Active CQ is an error */
1368 for (sqid = 1; sqid < sc->num_squeues + 1; sqid++)
1369 if (sc->submit_queues[sqid].cqid == qid) {
1370 pci_nvme_status_tc(&compl->status,
1371 NVME_SCT_COMMAND_SPECIFIC,
1372 NVME_SC_INVALID_QUEUE_DELETION);
1373 return (1);
1374 }
1375
1376 sc->compl_queues[qid].qbase = NULL;
1377 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1378 return (1);
1379 }
1380
1381 static int
1382 nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
1383 struct nvme_completion* compl)
1384 {
1385 struct nvme_completion_queue *ncq;
1386 uint16_t qid = command->cdw10 & 0xffff;
1387
1388 /* Only support Physically Contiguous queues */
1389 if ((command->cdw11 & NVME_CMD_CDW11_PC) == 0) {
1390 WPRINTF("%s unsupported non-contig (list-based) "
1391 "create i/o completion queue",
1392 __func__);
1393
1394 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1395 return (1);
1396 }
1397
1398 if ((qid == 0) || (qid > sc->num_cqueues) ||
1399 (sc->compl_queues[qid].qbase != NULL)) {
1400 WPRINTF("%s queue index %u > num_cqueues %u",
1401 __func__, qid, sc->num_cqueues);
1402 pci_nvme_status_tc(&compl->status,
1403 NVME_SCT_COMMAND_SPECIFIC,
1404 NVME_SC_INVALID_QUEUE_IDENTIFIER);
1405 return (1);
1406 }
1407
1408 ncq = &sc->compl_queues[qid];
1409 ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1;
1410 ncq->intr_vec = (command->cdw11 >> 16) & 0xffff;
1411 if (ncq->intr_vec > (sc->max_queues + 1)) {
1412 pci_nvme_status_tc(&compl->status,
1413 NVME_SCT_COMMAND_SPECIFIC,
1414 NVME_SC_INVALID_INTERRUPT_VECTOR);
1415 return (1);
1416 }
1417
1418 ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff);
1419 if ((ncq->size < 2) || (ncq->size > sc->max_qentries)) {
1420 /*
1421 * Queues must specify at least two entries
1422 * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to
1423 * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec
1424 */
1425 pci_nvme_status_tc(&compl->status,
1426 NVME_SCT_COMMAND_SPECIFIC,
1427 NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED);
1428 return (1);
1429 }
1430 ncq->head = ncq->tail = 0;
1431 ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx,
1432 command->prp1,
1433 sizeof(struct nvme_command) * (size_t)ncq->size);
1434
1435 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1436
1437
1438 return (1);
1439 }
1440
1441 static int
1442 nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command,
1443 struct nvme_completion* compl)
1444 {
1445 uint64_t logoff;
1446 uint32_t logsize;
1447 uint8_t logpage;
1448
1449 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1450
1451 /*
1452 * Command specifies the number of dwords to return in fields NUMDU
1453 * and NUMDL. This is a zero-based value.
1454 */
1455 logpage = command->cdw10 & 0xFF;
1456 logsize = ((command->cdw11 << 16) | (command->cdw10 >> 16)) + 1;
1457 logsize *= sizeof(uint32_t);
1458 logoff = ((uint64_t)(command->cdw13) << 32) | command->cdw12;
1459
1460 DPRINTF("%s log page %u len %u", __func__, logpage, logsize);
1461
1462 switch (logpage) {
1463 case NVME_LOG_ERROR:
1464 if (logoff >= sizeof(sc->err_log)) {
1465 pci_nvme_status_genc(&compl->status,
1466 NVME_SC_INVALID_FIELD);
1467 break;
1468 }
1469
1470 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1471 command->prp2, (uint8_t *)&sc->err_log + logoff,
1472 MIN(logsize - logoff, sizeof(sc->err_log)),
1473 NVME_COPY_TO_PRP);
1474 break;
1475 case NVME_LOG_HEALTH_INFORMATION:
1476 if (logoff >= sizeof(sc->health_log)) {
1477 pci_nvme_status_genc(&compl->status,
1478 NVME_SC_INVALID_FIELD);
1479 break;
1480 }
1481
1482 pthread_mutex_lock(&sc->mtx);
1483 memcpy(&sc->health_log.data_units_read, &sc->read_data_units,
1484 sizeof(sc->health_log.data_units_read));
1485 memcpy(&sc->health_log.data_units_written, &sc->write_data_units,
1486 sizeof(sc->health_log.data_units_written));
1487 memcpy(&sc->health_log.host_read_commands, &sc->read_commands,
1488 sizeof(sc->health_log.host_read_commands));
1489 memcpy(&sc->health_log.host_write_commands, &sc->write_commands,
1490 sizeof(sc->health_log.host_write_commands));
1491 pthread_mutex_unlock(&sc->mtx);
1492
1493 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1494 command->prp2, (uint8_t *)&sc->health_log + logoff,
1495 MIN(logsize - logoff, sizeof(sc->health_log)),
1496 NVME_COPY_TO_PRP);
1497 break;
1498 case NVME_LOG_FIRMWARE_SLOT:
1499 if (logoff >= sizeof(sc->fw_log)) {
1500 pci_nvme_status_genc(&compl->status,
1501 NVME_SC_INVALID_FIELD);
1502 break;
1503 }
1504
1505 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1506 command->prp2, (uint8_t *)&sc->fw_log + logoff,
1507 MIN(logsize - logoff, sizeof(sc->fw_log)),
1508 NVME_COPY_TO_PRP);
1509 break;
1510 case NVME_LOG_CHANGED_NAMESPACE:
1511 if (logoff >= sizeof(sc->ns_log)) {
1512 pci_nvme_status_genc(&compl->status,
1513 NVME_SC_INVALID_FIELD);
1514 break;
1515 }
1516
1517 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1518 command->prp2, (uint8_t *)&sc->ns_log + logoff,
1519 MIN(logsize - logoff, sizeof(sc->ns_log)),
1520 NVME_COPY_TO_PRP);
1521 memset(&sc->ns_log, 0, sizeof(sc->ns_log));
1522 break;
1523 default:
1524 DPRINTF("%s get log page %x command not supported",
1525 __func__, logpage);
1526
1527 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1528 NVME_SC_INVALID_LOG_PAGE);
1529 }
1530
1531 return (1);
1532 }
1533
1534 static int
1535 nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command,
1536 struct nvme_completion* compl)
1537 {
1538 void *dest;
1539 uint16_t status;
1540
1541 DPRINTF("%s identify 0x%x nsid 0x%x", __func__,
1542 command->cdw10 & 0xFF, command->nsid);
1543
1544 status = 0;
1545 pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
1546
1547 switch (command->cdw10 & 0xFF) {
1548 case 0x00: /* return Identify Namespace data structure */
1549 /* Global NS only valid with NS Management */
1550 if (command->nsid == NVME_GLOBAL_NAMESPACE_TAG) {
1551 pci_nvme_status_genc(&status,
1552 NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
1553 break;
1554 }
1555 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1556 command->prp2, (uint8_t *)&sc->nsdata, sizeof(sc->nsdata),
1557 NVME_COPY_TO_PRP);
1558 break;
1559 case 0x01: /* return Identify Controller data structure */
1560 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1561 command->prp2, (uint8_t *)&sc->ctrldata,
1562 sizeof(sc->ctrldata),
1563 NVME_COPY_TO_PRP);
1564 break;
1565 case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */
1566 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
1567 sizeof(uint32_t) * 1024);
1568 /* All unused entries shall be zero */
1569 memset(dest, 0, sizeof(uint32_t) * 1024);
1570 ((uint32_t *)dest)[0] = 1;
1571 break;
1572 case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */
1573 if (command->nsid != 1) {
1574 pci_nvme_status_genc(&status,
1575 NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
1576 break;
1577 }
1578 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
1579 sizeof(uint32_t) * 1024);
1580 /* All bytes after the descriptor shall be zero */
1581 memset(dest, 0, sizeof(uint32_t) * 1024);
1582
1583 /* Return NIDT=1 (i.e. EUI64) descriptor */
1584 ((uint8_t *)dest)[0] = 1;
1585 ((uint8_t *)dest)[1] = sizeof(uint64_t);
1586 memcpy(((uint8_t *)dest) + 4, sc->nsdata.eui64, sizeof(uint64_t));
1587 break;
1588 case 0x13:
1589 /*
1590 * Controller list is optional but used by UNH tests. Return
1591 * a valid but empty list.
1592 */
1593 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
1594 sizeof(uint16_t) * 2048);
1595 memset(dest, 0, sizeof(uint16_t) * 2048);
1596 break;
1597 default:
1598 DPRINTF("%s unsupported identify command requested 0x%x",
1599 __func__, command->cdw10 & 0xFF);
1600 pci_nvme_status_genc(&status, NVME_SC_INVALID_FIELD);
1601 break;
1602 }
1603
1604 compl->status = status;
1605 return (1);
1606 }
1607
1608 static const char *
1609 nvme_fid_to_name(uint8_t fid)
1610 {
1611 const char *name;
1612
1613 switch (fid) {
1614 case NVME_FEAT_ARBITRATION:
1615 name = "Arbitration";
1616 break;
1617 case NVME_FEAT_POWER_MANAGEMENT:
1618 name = "Power Management";
1619 break;
1620 case NVME_FEAT_LBA_RANGE_TYPE:
1621 name = "LBA Range Type";
1622 break;
1623 case NVME_FEAT_TEMPERATURE_THRESHOLD:
1624 name = "Temperature Threshold";
1625 break;
1626 case NVME_FEAT_ERROR_RECOVERY:
1627 name = "Error Recovery";
1628 break;
1629 case NVME_FEAT_VOLATILE_WRITE_CACHE:
1630 name = "Volatile Write Cache";
1631 break;
1632 case NVME_FEAT_NUMBER_OF_QUEUES:
1633 name = "Number of Queues";
1634 break;
1635 case NVME_FEAT_INTERRUPT_COALESCING:
1636 name = "Interrupt Coalescing";
1637 break;
1638 case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
1639 name = "Interrupt Vector Configuration";
1640 break;
1641 case NVME_FEAT_WRITE_ATOMICITY:
1642 name = "Write Atomicity Normal";
1643 break;
1644 case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
1645 name = "Asynchronous Event Configuration";
1646 break;
1647 case NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION:
1648 name = "Autonomous Power State Transition";
1649 break;
1650 case NVME_FEAT_HOST_MEMORY_BUFFER:
1651 name = "Host Memory Buffer";
1652 break;
1653 case NVME_FEAT_TIMESTAMP:
1654 name = "Timestamp";
1655 break;
1656 case NVME_FEAT_KEEP_ALIVE_TIMER:
1657 name = "Keep Alive Timer";
1658 break;
1659 case NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT:
1660 name = "Host Controlled Thermal Management";
1661 break;
1662 case NVME_FEAT_NON_OP_POWER_STATE_CONFIG:
1663 name = "Non-Operation Power State Config";
1664 break;
1665 case NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG:
1666 name = "Read Recovery Level Config";
1667 break;
1668 case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG:
1669 name = "Predictable Latency Mode Config";
1670 break;
1671 case NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW:
1672 name = "Predictable Latency Mode Window";
1673 break;
1674 case NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES:
1675 name = "LBA Status Information Report Interval";
1676 break;
1677 case NVME_FEAT_HOST_BEHAVIOR_SUPPORT:
1678 name = "Host Behavior Support";
1679 break;
1680 case NVME_FEAT_SANITIZE_CONFIG:
1681 name = "Sanitize Config";
1682 break;
1683 case NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION:
1684 name = "Endurance Group Event Configuration";
1685 break;
1686 case NVME_FEAT_SOFTWARE_PROGRESS_MARKER:
1687 name = "Software Progress Marker";
1688 break;
1689 case NVME_FEAT_HOST_IDENTIFIER:
1690 name = "Host Identifier";
1691 break;
1692 case NVME_FEAT_RESERVATION_NOTIFICATION_MASK:
1693 name = "Reservation Notification Mask";
1694 break;
1695 case NVME_FEAT_RESERVATION_PERSISTENCE:
1696 name = "Reservation Persistence";
1697 break;
1698 case NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG:
1699 name = "Namespace Write Protection Config";
1700 break;
1701 default:
1702 name = "Unknown";
1703 break;
1704 }
1705
1706 return (name);
1707 }
1708
1709 static void
1710 nvme_feature_invalid_cb(struct pci_nvme_softc *sc __unused,
1711 struct nvme_feature_obj *feat __unused,
1712 struct nvme_command *command __unused,
1713 struct nvme_completion *compl)
1714 {
1715 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1716 }
1717
1718 static void
1719 nvme_feature_iv_config(struct pci_nvme_softc *sc,
1720 struct nvme_feature_obj *feat __unused,
1721 struct nvme_command *command,
1722 struct nvme_completion *compl)
1723 {
1724 uint32_t i;
1725 uint32_t cdw11 = command->cdw11;
1726 uint16_t iv;
1727 bool cd;
1728
1729 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1730
1731 iv = cdw11 & 0xffff;
1732 cd = cdw11 & (1 << 16);
1733
1734 if (iv > (sc->max_queues + 1)) {
1735 return;
1736 }
1737
1738 /* No Interrupt Coalescing (i.e. not Coalescing Disable) for Admin Q */
1739 if ((iv == 0) && !cd)
1740 return;
1741
1742 /* Requested Interrupt Vector must be used by a CQ */
1743 for (i = 0; i < sc->num_cqueues + 1; i++) {
1744 if (sc->compl_queues[i].intr_vec == iv) {
1745 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1746 }
1747 }
1748 }
1749
1750 #define NVME_ASYNC_EVENT_ENDURANCE_GROUP (0x4000)
1751 static void
1752 nvme_feature_async_event(struct pci_nvme_softc *sc __unused,
1753 struct nvme_feature_obj *feat __unused,
1754 struct nvme_command *command,
1755 struct nvme_completion *compl)
1756 {
1757 if (command->cdw11 & NVME_ASYNC_EVENT_ENDURANCE_GROUP)
1758 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1759 }
1760
1761 #define NVME_TEMP_THRESH_OVER 0
1762 #define NVME_TEMP_THRESH_UNDER 1
1763 static void
1764 nvme_feature_temperature(struct pci_nvme_softc *sc,
1765 struct nvme_feature_obj *feat __unused,
1766 struct nvme_command *command,
1767 struct nvme_completion *compl)
1768 {
1769 uint16_t tmpth; /* Temperature Threshold */
1770 uint8_t tmpsel; /* Threshold Temperature Select */
1771 uint8_t thsel; /* Threshold Type Select */
1772 bool set_crit = false;
1773 bool report_crit;
1774
1775 tmpth = command->cdw11 & 0xffff;
1776 tmpsel = (command->cdw11 >> 16) & 0xf;
1777 thsel = (command->cdw11 >> 20) & 0x3;
1778
1779 DPRINTF("%s: tmpth=%#x tmpsel=%#x thsel=%#x", __func__, tmpth, tmpsel, thsel);
1780
1781 /* Check for unsupported values */
1782 if (((tmpsel != 0) && (tmpsel != 0xf)) ||
1783 (thsel > NVME_TEMP_THRESH_UNDER)) {
1784 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1785 return;
1786 }
1787
1788 if (((thsel == NVME_TEMP_THRESH_OVER) && (NVME_TEMPERATURE >= tmpth)) ||
1789 ((thsel == NVME_TEMP_THRESH_UNDER) && (NVME_TEMPERATURE <= tmpth)))
1790 set_crit = true;
1791
1792 pthread_mutex_lock(&sc->mtx);
1793 if (set_crit)
1794 sc->health_log.critical_warning |=
1795 NVME_CRIT_WARN_ST_TEMPERATURE;
1796 else
1797 sc->health_log.critical_warning &=
1798 ~NVME_CRIT_WARN_ST_TEMPERATURE;
1799 pthread_mutex_unlock(&sc->mtx);
1800
1801 report_crit = sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11 &
1802 NVME_CRIT_WARN_ST_TEMPERATURE;
1803
1804 if (set_crit && report_crit)
1805 pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_SMART,
1806 sc->health_log.critical_warning);
1807
1808 DPRINTF("%s: set_crit=%c critical_warning=%#x status=%#x", __func__, set_crit ? 'T':'F', sc->health_log.critical_warning, compl->status);
1809 }
1810
1811 static void
1812 nvme_feature_num_queues(struct pci_nvme_softc *sc,
1813 struct nvme_feature_obj *feat __unused,
1814 struct nvme_command *command,
1815 struct nvme_completion *compl)
1816 {
1817 uint16_t nqr; /* Number of Queues Requested */
1818
1819 if (sc->num_q_is_set) {
1820 WPRINTF("%s: Number of Queues already set", __func__);
1821 pci_nvme_status_genc(&compl->status,
1822 NVME_SC_COMMAND_SEQUENCE_ERROR);
1823 return;
1824 }
1825
1826 nqr = command->cdw11 & 0xFFFF;
1827 if (nqr == 0xffff) {
1828 WPRINTF("%s: Illegal NSQR value %#x", __func__, nqr);
1829 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1830 return;
1831 }
1832
1833 sc->num_squeues = ONE_BASED(nqr);
1834 if (sc->num_squeues > sc->max_queues) {
1835 DPRINTF("NSQR=%u is greater than max %u", sc->num_squeues,
1836 sc->max_queues);
1837 sc->num_squeues = sc->max_queues;
1838 }
1839
1840 nqr = (command->cdw11 >> 16) & 0xFFFF;
1841 if (nqr == 0xffff) {
1842 WPRINTF("%s: Illegal NCQR value %#x", __func__, nqr);
1843 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1844 return;
1845 }
1846
1847 sc->num_cqueues = ONE_BASED(nqr);
1848 if (sc->num_cqueues > sc->max_queues) {
1849 DPRINTF("NCQR=%u is greater than max %u", sc->num_cqueues,
1850 sc->max_queues);
1851 sc->num_cqueues = sc->max_queues;
1852 }
1853
1854 /* Patch the command value which will be saved on callback's return */
1855 command->cdw11 = NVME_FEATURE_NUM_QUEUES(sc);
1856 compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc);
1857
1858 sc->num_q_is_set = true;
1859 }
1860
1861 static int
1862 nvme_opc_set_features(struct pci_nvme_softc *sc, struct nvme_command *command,
1863 struct nvme_completion *compl)
1864 {
1865 struct nvme_feature_obj *feat;
1866 uint32_t nsid = command->nsid;
1867 uint8_t fid = NVMEV(NVME_FEAT_SET_FID, command->cdw10);
1868 bool sv = NVMEV(NVME_FEAT_SET_SV, command->cdw10);
1869
1870 DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid));
1871
1872 if (fid >= NVME_FID_MAX) {
1873 DPRINTF("%s invalid feature 0x%x", __func__, fid);
1874 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1875 return (1);
1876 }
1877
1878 if (sv) {
1879 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1880 NVME_SC_FEATURE_NOT_SAVEABLE);
1881 return (1);
1882 }
1883
1884 feat = &sc->feat[fid];
1885
1886 if (feat->namespace_specific && (nsid == NVME_GLOBAL_NAMESPACE_TAG)) {
1887 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1888 return (1);
1889 }
1890
1891 if (!feat->namespace_specific &&
1892 !((nsid == 0) || (nsid == NVME_GLOBAL_NAMESPACE_TAG))) {
1893 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1894 NVME_SC_FEATURE_NOT_NS_SPECIFIC);
1895 return (1);
1896 }
1897
1898 compl->cdw0 = 0;
1899 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1900
1901 if (feat->set)
1902 feat->set(sc, feat, command, compl);
1903 else {
1904 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1905 NVME_SC_FEATURE_NOT_CHANGEABLE);
1906 return (1);
1907 }
1908
1909 DPRINTF("%s: status=%#x cdw11=%#x", __func__, compl->status, command->cdw11);
1910 if (compl->status == NVME_SC_SUCCESS) {
1911 feat->cdw11 = command->cdw11;
1912 if ((fid == NVME_FEAT_ASYNC_EVENT_CONFIGURATION) &&
1913 (command->cdw11 != 0))
1914 pci_nvme_aen_notify(sc);
1915 }
1916
1917 return (0);
1918 }
1919
1920 #define NVME_FEATURES_SEL_SUPPORTED 0x3
1921 #define NVME_FEATURES_NS_SPECIFIC (1 << 1)
1922
1923 static int
1924 nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command,
1925 struct nvme_completion* compl)
1926 {
1927 struct nvme_feature_obj *feat;
1928 uint8_t fid = command->cdw10 & 0xFF;
1929 uint8_t sel = (command->cdw10 >> 8) & 0x7;
1930
1931 DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid));
1932
1933 if (fid >= NVME_FID_MAX) {
1934 DPRINTF("%s invalid feature 0x%x", __func__, fid);
1935 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1936 return (1);
1937 }
1938
1939 compl->cdw0 = 0;
1940 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1941
1942 feat = &sc->feat[fid];
1943 if (feat->get) {
1944 feat->get(sc, feat, command, compl);
1945 }
1946
1947 if (compl->status == NVME_SC_SUCCESS) {
1948 if ((sel == NVME_FEATURES_SEL_SUPPORTED) && feat->namespace_specific)
1949 compl->cdw0 = NVME_FEATURES_NS_SPECIFIC;
1950 else
1951 compl->cdw0 = feat->cdw11;
1952 }
1953
1954 return (0);
1955 }
1956
1957 static int
1958 nvme_opc_format_nvm(struct pci_nvme_softc* sc, struct nvme_command* command,
1959 struct nvme_completion* compl)
1960 {
1961 uint8_t ses, lbaf, pi;
1962
1963 /* Only supports Secure Erase Setting - User Data Erase */
1964 ses = (command->cdw10 >> 9) & 0x7;
1965 if (ses > 0x1) {
1966 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1967 return (1);
1968 }
1969
1970 /* Only supports a single LBA Format */
1971 lbaf = command->cdw10 & 0xf;
1972 if (lbaf != 0) {
1973 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1974 NVME_SC_INVALID_FORMAT);
1975 return (1);
1976 }
1977
1978 /* Doesn't support Protection Infomation */
1979 pi = (command->cdw10 >> 5) & 0x7;
1980 if (pi != 0) {
1981 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1982 return (1);
1983 }
1984
1985 if (sc->nvstore.type == NVME_STOR_RAM) {
1986 if (sc->nvstore.ctx)
1987 free(sc->nvstore.ctx);
1988 sc->nvstore.ctx = calloc(1, sc->nvstore.size);
1989 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1990 } else {
1991 struct pci_nvme_ioreq *req;
1992 int err;
1993
1994 req = pci_nvme_get_ioreq(sc);
1995 if (req == NULL) {
1996 pci_nvme_status_genc(&compl->status,
1997 NVME_SC_INTERNAL_DEVICE_ERROR);
1998 WPRINTF("%s: unable to allocate IO req", __func__);
1999 return (1);
2000 }
2001 req->nvme_sq = &sc->submit_queues[0];
2002 req->sqid = 0;
2003 req->opc = command->opc;
2004 req->cid = command->cid;
2005 req->nsid = command->nsid;
2006
2007 req->io_req.br_offset = 0;
2008 req->io_req.br_resid = sc->nvstore.size;
2009 req->io_req.br_callback = pci_nvme_io_done;
2010
2011 err = blockif_delete(sc->nvstore.ctx, &req->io_req);
2012 if (err) {
2013 pci_nvme_status_genc(&compl->status,
2014 NVME_SC_INTERNAL_DEVICE_ERROR);
2015 pci_nvme_release_ioreq(sc, req);
2016 } else
2017 compl->status = NVME_NO_STATUS;
2018 }
2019
2020 return (1);
2021 }
2022
2023 static int
2024 nvme_opc_abort(struct pci_nvme_softc *sc __unused, struct nvme_command *command,
2025 struct nvme_completion *compl)
2026 {
2027 DPRINTF("%s submission queue %u, command ID 0x%x", __func__,
2028 command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF);
2029
2030 /* TODO: search for the command ID and abort it */
2031
2032 compl->cdw0 = 1;
2033 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
2034 return (1);
2035 }
2036
2037 static int
2038 nvme_opc_async_event_req(struct pci_nvme_softc* sc,
2039 struct nvme_command* command, struct nvme_completion* compl)
2040 {
2041 DPRINTF("%s async event request count=%u aerl=%u cid=%#x", __func__,
2042 sc->aer_count, sc->ctrldata.aerl, command->cid);
2043
2044 /* Don't exceed the Async Event Request Limit (AERL). */
2045 if (pci_nvme_aer_limit_reached(sc)) {
2046 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
2047 NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED);
2048 return (1);
2049 }
2050
2051 if (pci_nvme_aer_add(sc, command->cid)) {
2052 pci_nvme_status_tc(&compl->status, NVME_SCT_GENERIC,
2053 NVME_SC_INTERNAL_DEVICE_ERROR);
2054 return (1);
2055 }
2056
2057 /*
2058 * Raise events when they happen based on the Set Features cmd.
2059 * These events happen async, so only set completion successful if
2060 * there is an event reflective of the request to get event.
2061 */
2062 compl->status = NVME_NO_STATUS;
2063 pci_nvme_aen_notify(sc);
2064
2065 return (0);
2066 }
2067
2068 static void
2069 pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)
2070 {
2071 struct nvme_completion compl;
2072 struct nvme_command *cmd;
2073 struct nvme_submission_queue *sq;
2074 struct nvme_completion_queue *cq;
2075 uint16_t sqhead;
2076
2077 DPRINTF("%s index %u", __func__, (uint32_t)value);
2078
2079 sq = &sc->submit_queues[0];
2080 cq = &sc->compl_queues[0];
2081
2082 pthread_mutex_lock(&sq->mtx);
2083
2084 sqhead = sq->head;
2085 DPRINTF("sqhead %u, tail %u", sqhead, sq->tail);
2086
2087 while (sqhead != atomic_load_acq_short(&sq->tail)) {
2088 cmd = &(sq->qbase)[sqhead];
2089 compl.cdw0 = 0;
2090 compl.status = 0;
2091
2092 switch (cmd->opc) {
2093 case NVME_OPC_DELETE_IO_SQ:
2094 DPRINTF("%s command DELETE_IO_SQ", __func__);
2095 nvme_opc_delete_io_sq(sc, cmd, &compl);
2096 break;
2097 case NVME_OPC_CREATE_IO_SQ:
2098 DPRINTF("%s command CREATE_IO_SQ", __func__);
2099 nvme_opc_create_io_sq(sc, cmd, &compl);
2100 break;
2101 case NVME_OPC_DELETE_IO_CQ:
2102 DPRINTF("%s command DELETE_IO_CQ", __func__);
2103 nvme_opc_delete_io_cq(sc, cmd, &compl);
2104 break;
2105 case NVME_OPC_CREATE_IO_CQ:
2106 DPRINTF("%s command CREATE_IO_CQ", __func__);
2107 nvme_opc_create_io_cq(sc, cmd, &compl);
2108 break;
2109 case NVME_OPC_GET_LOG_PAGE:
2110 DPRINTF("%s command GET_LOG_PAGE", __func__);
2111 nvme_opc_get_log_page(sc, cmd, &compl);
2112 break;
2113 case NVME_OPC_IDENTIFY:
2114 DPRINTF("%s command IDENTIFY", __func__);
2115 nvme_opc_identify(sc, cmd, &compl);
2116 break;
2117 case NVME_OPC_ABORT:
2118 DPRINTF("%s command ABORT", __func__);
2119 nvme_opc_abort(sc, cmd, &compl);
2120 break;
2121 case NVME_OPC_SET_FEATURES:
2122 DPRINTF("%s command SET_FEATURES", __func__);
2123 nvme_opc_set_features(sc, cmd, &compl);
2124 break;
2125 case NVME_OPC_GET_FEATURES:
2126 DPRINTF("%s command GET_FEATURES", __func__);
2127 nvme_opc_get_features(sc, cmd, &compl);
2128 break;
2129 case NVME_OPC_FIRMWARE_ACTIVATE:
2130 DPRINTF("%s command FIRMWARE_ACTIVATE", __func__);
2131 pci_nvme_status_tc(&compl.status,
2132 NVME_SCT_COMMAND_SPECIFIC,
2133 NVME_SC_INVALID_FIRMWARE_SLOT);
2134 break;
2135 case NVME_OPC_ASYNC_EVENT_REQUEST:
2136 DPRINTF("%s command ASYNC_EVENT_REQ", __func__);
2137 nvme_opc_async_event_req(sc, cmd, &compl);
2138 break;
2139 case NVME_OPC_FORMAT_NVM:
2140 DPRINTF("%s command FORMAT_NVM", __func__);
2141 if ((sc->ctrldata.oacs &
2142 (1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT)) == 0) {
2143 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);
2144 break;
2145 }
2146 nvme_opc_format_nvm(sc, cmd, &compl);
2147 break;
2148 case NVME_OPC_SECURITY_SEND:
2149 case NVME_OPC_SECURITY_RECEIVE:
2150 case NVME_OPC_SANITIZE:
2151 case NVME_OPC_GET_LBA_STATUS:
2152 DPRINTF("%s command OPC=%#x (unsupported)", __func__,
2153 cmd->opc);
2154 /* Valid but unsupported opcodes */
2155 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_FIELD);
2156 break;
2157 default:
2158 DPRINTF("%s command OPC=%#X (not implemented)",
2159 __func__,
2160 cmd->opc);
2161 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);
2162 }
2163 sqhead = (sqhead + 1) % sq->size;
2164
2165 if (NVME_COMPLETION_VALID(compl)) {
2166 pci_nvme_cq_update(sc, &sc->compl_queues[0],
2167 compl.cdw0,
2168 cmd->cid,
2169 0, /* SQID */
2170 compl.status);
2171 }
2172 }
2173
2174 DPRINTF("setting sqhead %u", sqhead);
2175 sq->head = sqhead;
2176
2177 if (cq->head != cq->tail)
2178 pci_generate_msix(sc->nsc_pi, 0);
2179
2180 pthread_mutex_unlock(&sq->mtx);
2181 }
2182
2183 /*
2184 * Update the Write and Read statistics reported in SMART data
2185 *
2186 * NVMe defines "data unit" as thousand's of 512 byte blocks and is rounded up.
2187 * E.g. 1 data unit is 1 - 1,000 512 byte blocks. 3 data units are 2,001 - 3,000
2188 * 512 byte blocks. Rounding up is acheived by initializing the remainder to 999.
2189 */
2190 static void
2191 pci_nvme_stats_write_read_update(struct pci_nvme_softc *sc, uint8_t opc,
2192 size_t bytes, uint16_t status)
2193 {
2194
2195 pthread_mutex_lock(&sc->mtx);
2196 switch (opc) {
2197 case NVME_OPC_WRITE:
2198 sc->write_commands++;
2199 if (status != NVME_SC_SUCCESS)
2200 break;
2201 sc->write_dunits_remainder += (bytes / 512);
2202 while (sc->write_dunits_remainder >= 1000) {
2203 sc->write_data_units++;
2204 sc->write_dunits_remainder -= 1000;
2205 }
2206 break;
2207 case NVME_OPC_READ:
2208 sc->read_commands++;
2209 if (status != NVME_SC_SUCCESS)
2210 break;
2211 sc->read_dunits_remainder += (bytes / 512);
2212 while (sc->read_dunits_remainder >= 1000) {
2213 sc->read_data_units++;
2214 sc->read_dunits_remainder -= 1000;
2215 }
2216 break;
2217 default:
2218 DPRINTF("%s: Invalid OPC 0x%02x for stats", __func__, opc);
2219 break;
2220 }
2221 pthread_mutex_unlock(&sc->mtx);
2222 }
2223
2224 /*
2225 * Check if the combination of Starting LBA (slba) and number of blocks
2226 * exceeds the range of the underlying storage.
2227 *
2228 * Because NVMe specifies the SLBA in blocks as a uint64_t and blockif stores
2229 * the capacity in bytes as a uint64_t, care must be taken to avoid integer
2230 * overflow.
2231 */
2232 static bool
2233 pci_nvme_out_of_range(struct pci_nvme_blockstore *nvstore, uint64_t slba,
2234 uint32_t nblocks)
2235 {
2236 size_t offset, bytes;
2237
2238 /* Overflow check of multiplying Starting LBA by the sector size */
2239 if (slba >> (64 - nvstore->sectsz_bits))
2240 return (true);
2241
2242 offset = slba << nvstore->sectsz_bits;
2243 bytes = nblocks << nvstore->sectsz_bits;
2244
2245 /* Overflow check of Number of Logical Blocks */
2246 if ((nvstore->size <= offset) || ((nvstore->size - offset) < bytes))
2247 return (true);
2248
2249 return (false);
2250 }
2251
2252 static int
2253 pci_nvme_append_iov_req(struct pci_nvme_softc *sc __unused,
2254 struct pci_nvme_ioreq *req, uint64_t gpaddr, size_t size, uint64_t offset)
2255 {
2256 int iovidx;
2257 bool range_is_contiguous;
2258
2259 if (req == NULL)
2260 return (-1);
2261
2262 if (req->io_req.br_iovcnt == NVME_MAX_IOVEC) {
2263 return (-1);
2264 }
2265
2266 /*
2267 * Minimize the number of IOVs by concatenating contiguous address
2268 * ranges. If the IOV count is zero, there is no previous range to
2269 * concatenate.
2270 */
2271 if (req->io_req.br_iovcnt == 0)
2272 range_is_contiguous = false;
2273 else
2274 range_is_contiguous = (req->prev_gpaddr + req->prev_size) == gpaddr;
2275
2276 if (range_is_contiguous) {
2277 iovidx = req->io_req.br_iovcnt - 1;
2278
2279 req->io_req.br_iov[iovidx].iov_base =
2280 paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
2281 req->prev_gpaddr, size);
2282 if (req->io_req.br_iov[iovidx].iov_base == NULL)
2283 return (-1);
2284
2285 req->prev_size += size;
2286 req->io_req.br_resid += size;
2287
2288 req->io_req.br_iov[iovidx].iov_len = req->prev_size;
2289 } else {
2290 iovidx = req->io_req.br_iovcnt;
2291 if (iovidx == 0) {
2292 req->io_req.br_offset = offset;
2293 req->io_req.br_resid = 0;
2294 req->io_req.br_param = req;
2295 }
2296
2297 req->io_req.br_iov[iovidx].iov_base =
2298 paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
2299 gpaddr, size);
2300 if (req->io_req.br_iov[iovidx].iov_base == NULL)
2301 return (-1);
2302
2303 req->io_req.br_iov[iovidx].iov_len = size;
2304
2305 req->prev_gpaddr = gpaddr;
2306 req->prev_size = size;
2307 req->io_req.br_resid += size;
2308
2309 req->io_req.br_iovcnt++;
2310 }
2311
2312 return (0);
2313 }
2314
2315 static void
2316 pci_nvme_set_completion(struct pci_nvme_softc *sc,
2317 struct nvme_submission_queue *sq, int sqid, uint16_t cid, uint16_t status)
2318 {
2319 struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];
2320
2321 DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x",
2322 __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),
2323 NVME_STATUS_GET_SC(status));
2324
2325 pci_nvme_cq_update(sc, cq, 0, cid, sqid, status);
2326
2327 if (cq->head != cq->tail) {
2328 if (cq->intr_en & NVME_CQ_INTEN) {
2329 pci_generate_msix(sc->nsc_pi, cq->intr_vec);
2330 } else {
2331 DPRINTF("%s: CQ%u interrupt disabled",
2332 __func__, sq->cqid);
2333 }
2334 }
2335 }
2336
2337 static void
2338 pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req)
2339 {
2340 req->sc = NULL;
2341 req->nvme_sq = NULL;
2342 req->sqid = 0;
2343
2344 pthread_mutex_lock(&sc->mtx);
2345
2346 STAILQ_INSERT_TAIL(&sc->ioreqs_free, req, link);
2347 sc->pending_ios--;
2348
2349 /* when no more IO pending, can set to ready if device reset/enabled */
2350 if (sc->pending_ios == 0 &&
2351 NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts)))
2352 sc->regs.csts |= NVME_CSTS_RDY;
2353
2354 pthread_mutex_unlock(&sc->mtx);
2355
2356 sem_post(&sc->iosemlock);
2357 }
2358
2359 static struct pci_nvme_ioreq *
2360 pci_nvme_get_ioreq(struct pci_nvme_softc *sc)
2361 {
2362 struct pci_nvme_ioreq *req = NULL;
2363
2364 sem_wait(&sc->iosemlock);
2365 pthread_mutex_lock(&sc->mtx);
2366
2367 req = STAILQ_FIRST(&sc->ioreqs_free);
2368 assert(req != NULL);
2369 STAILQ_REMOVE_HEAD(&sc->ioreqs_free, link);
2370
2371 req->sc = sc;
2372
2373 sc->pending_ios++;
2374
2375 pthread_mutex_unlock(&sc->mtx);
2376
2377 req->io_req.br_iovcnt = 0;
2378 req->io_req.br_offset = 0;
2379 req->io_req.br_resid = 0;
2380 req->io_req.br_param = req;
2381 req->prev_gpaddr = 0;
2382 req->prev_size = 0;
2383
2384 return req;
2385 }
2386
2387 static void
2388 pci_nvme_io_done(struct blockif_req *br, int err)
2389 {
2390 struct pci_nvme_ioreq *req = br->br_param;
2391 struct nvme_submission_queue *sq = req->nvme_sq;
2392 uint16_t code, status;
2393
2394 DPRINTF("%s error %d %s", __func__, err, strerror(err));
2395
2396 /* TODO return correct error */
2397 code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS;
2398 status = 0;
2399 pci_nvme_status_genc(&status, code);
2400
2401 pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, status);
2402 pci_nvme_stats_write_read_update(req->sc, req->opc,
2403 req->bytes, status);
2404 pci_nvme_release_ioreq(req->sc, req);
2405 }
2406
2407 /*
2408 * Implements the Flush command. The specification states:
2409 * If a volatile write cache is not present, Flush commands complete
2410 * successfully and have no effect
2411 * in the description of the Volatile Write Cache (VWC) field of the Identify
2412 * Controller data. Therefore, set status to Success if the command is
2413 * not supported (i.e. RAM or as indicated by the blockif).
2414 */
2415 static bool
2416 nvme_opc_flush(struct pci_nvme_softc *sc __unused,
2417 struct nvme_command *cmd __unused,
2418 struct pci_nvme_blockstore *nvstore,
2419 struct pci_nvme_ioreq *req,
2420 uint16_t *status)
2421 {
2422 bool pending = false;
2423
2424 if (nvstore->type == NVME_STOR_RAM) {
2425 pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2426 } else {
2427 int err;
2428
2429 req->io_req.br_callback = pci_nvme_io_done;
2430
2431 err = blockif_flush(nvstore->ctx, &req->io_req);
2432 switch (err) {
2433 case 0:
2434 pending = true;
2435 break;
2436 case EOPNOTSUPP:
2437 pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2438 break;
2439 default:
2440 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
2441 }
2442 }
2443
2444 return (pending);
2445 }
2446
2447 static uint16_t
2448 nvme_write_read_ram(struct pci_nvme_softc *sc,
2449 struct pci_nvme_blockstore *nvstore,
2450 uint64_t prp1, uint64_t prp2,
2451 size_t offset, uint64_t bytes,
2452 bool is_write)
2453 {
2454 uint8_t *buf = nvstore->ctx;
2455 enum nvme_copy_dir dir;
2456 uint16_t status;
2457
2458 if (is_write)
2459 dir = NVME_COPY_TO_PRP;
2460 else
2461 dir = NVME_COPY_FROM_PRP;
2462
2463 status = 0;
2464 if (nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, prp1, prp2,
2465 buf + offset, bytes, dir))
2466 pci_nvme_status_genc(&status,
2467 NVME_SC_DATA_TRANSFER_ERROR);
2468 else
2469 pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
2470
2471 return (status);
2472 }
2473
2474 static uint16_t
2475 nvme_write_read_blockif(struct pci_nvme_softc *sc,
2476 struct pci_nvme_blockstore *nvstore,
2477 struct pci_nvme_ioreq *req,
2478 uint64_t prp1, uint64_t prp2,
2479 size_t offset, uint64_t bytes,
2480 bool is_write)
2481 {
2482 uint64_t size;
2483 int err;
2484 uint16_t status = NVME_NO_STATUS;
2485
2486 size = MIN(PAGE_SIZE - (prp1 % PAGE_SIZE), bytes);
2487 if (pci_nvme_append_iov_req(sc, req, prp1, size, offset)) {
2488 err = -1;
2489 goto out;
2490 }
2491
2492 offset += size;
2493 bytes -= size;
2494
2495 if (bytes == 0) {
2496 ;
2497 } else if (bytes <= PAGE_SIZE) {
2498 size = bytes;
2499 if (pci_nvme_append_iov_req(sc, req, prp2, size, offset)) {
2500 err = -1;
2501 goto out;
2502 }
2503 } else {
2504 void *vmctx = sc->nsc_pi->pi_vmctx;
2505 uint64_t *prp_list = &prp2;
2506 uint64_t *last = prp_list;
2507
2508 /* PRP2 is pointer to a physical region page list */
2509 while (bytes) {
2510 /* Last entry in list points to the next list */
2511 if ((prp_list == last) && (bytes > PAGE_SIZE)) {
2512 uint64_t prp = *prp_list;
2513
2514 prp_list = paddr_guest2host(vmctx, prp,
2515 PAGE_SIZE - (prp % PAGE_SIZE));
2516 if (prp_list == NULL) {
2517 err = -1;
2518 goto out;
2519 }
2520 last = prp_list + (NVME_PRP2_ITEMS - 1);
2521 }
2522
2523 size = MIN(bytes, PAGE_SIZE);
2524
2525 if (pci_nvme_append_iov_req(sc, req, *prp_list, size,
2526 offset)) {
2527 err = -1;
2528 goto out;
2529 }
2530
2531 offset += size;
2532 bytes -= size;
2533
2534 prp_list++;
2535 }
2536 }
2537 req->io_req.br_callback = pci_nvme_io_done;
2538 if (is_write)
2539 err = blockif_write(nvstore->ctx, &req->io_req);
2540 else
2541 err = blockif_read(nvstore->ctx, &req->io_req);
2542 out:
2543 if (err)
2544 pci_nvme_status_genc(&status, NVME_SC_DATA_TRANSFER_ERROR);
2545
2546 return (status);
2547 }
2548
2549 static bool
2550 nvme_opc_write_read(struct pci_nvme_softc *sc,
2551 struct nvme_command *cmd,
2552 struct pci_nvme_blockstore *nvstore,
2553 struct pci_nvme_ioreq *req,
2554 uint16_t *status)
2555 {
2556 uint64_t lba, nblocks, bytes;
2557 size_t offset;
2558 bool is_write = cmd->opc == NVME_OPC_WRITE;
2559 bool pending = false;
2560
2561 lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10;
2562 nblocks = (cmd->cdw12 & 0xFFFF) + 1;
2563 bytes = nblocks << nvstore->sectsz_bits;
2564 if (bytes > NVME_MAX_DATA_SIZE) {
2565 WPRINTF("%s command would exceed MDTS", __func__);
2566 pci_nvme_status_genc(status, NVME_SC_INVALID_FIELD);
2567 goto out;
2568 }
2569
2570 if (pci_nvme_out_of_range(nvstore, lba, nblocks)) {
2571 WPRINTF("%s command would exceed LBA range(slba=%#lx nblocks=%#lx)",
2572 __func__, lba, nblocks);
2573 pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE);
2574 goto out;
2575 }
2576
2577 offset = lba << nvstore->sectsz_bits;
2578
2579 req->bytes = bytes;
2580 req->io_req.br_offset = lba;
2581
2582 /* PRP bits 1:0 must be zero */
2583 cmd->prp1 &= ~0x3UL;
2584 cmd->prp2 &= ~0x3UL;
2585
2586 if (nvstore->type == NVME_STOR_RAM) {
2587 *status = nvme_write_read_ram(sc, nvstore, cmd->prp1,
2588 cmd->prp2, offset, bytes, is_write);
2589 } else {
2590 *status = nvme_write_read_blockif(sc, nvstore, req,
2591 cmd->prp1, cmd->prp2, offset, bytes, is_write);
2592
2593 if (*status == NVME_NO_STATUS)
2594 pending = true;
2595 }
2596 out:
2597 if (!pending)
2598 pci_nvme_stats_write_read_update(sc, cmd->opc, bytes, *status);
2599
2600 return (pending);
2601 }
2602
2603 static void
2604 pci_nvme_dealloc_sm(struct blockif_req *br, int err)
2605 {
2606 struct pci_nvme_ioreq *req = br->br_param;
2607 struct pci_nvme_softc *sc = req->sc;
2608 bool done = true;
2609 uint16_t status;
2610
2611 status = 0;
2612 if (err) {
2613 pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR);
2614 } else if ((req->prev_gpaddr + 1) == (req->prev_size)) {
2615 pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
2616 } else {
2617 struct iovec *iov = req->io_req.br_iov;
2618
2619 req->prev_gpaddr++;
2620 iov += req->prev_gpaddr;
2621
2622 /* The iov_* values already include the sector size */
2623 req->io_req.br_offset = (off_t)iov->iov_base;
2624 req->io_req.br_resid = iov->iov_len;
2625 if (blockif_delete(sc->nvstore.ctx, &req->io_req)) {
2626 pci_nvme_status_genc(&status,
2627 NVME_SC_INTERNAL_DEVICE_ERROR);
2628 } else
2629 done = false;
2630 }
2631
2632 if (done) {
2633 pci_nvme_set_completion(sc, req->nvme_sq, req->sqid, req->cid,
2634 status);
2635 pci_nvme_release_ioreq(sc, req);
2636 }
2637 }
2638
2639 static bool
2640 nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc,
2641 struct nvme_command *cmd,
2642 struct pci_nvme_blockstore *nvstore,
2643 struct pci_nvme_ioreq *req,
2644 uint16_t *status)
2645 {
2646 struct nvme_dsm_range *range = NULL;
2647 uint32_t nr, r, non_zero, dr;
2648 int err;
2649 bool pending = false;
2650
2651 if ((sc->ctrldata.oncs & NVME_ONCS_DSM) == 0) {
2652 pci_nvme_status_genc(status, NVME_SC_INVALID_OPCODE);
2653 goto out;
2654 }
2655
2656 nr = cmd->cdw10 & 0xff;
2657
2658 /* copy locally because a range entry could straddle PRPs */
2659 #ifdef __FreeBSD__
2660 range = calloc(1, NVME_MAX_DSM_TRIM);
2661 #else
2662 _Static_assert(NVME_MAX_DSM_TRIM % sizeof(struct nvme_dsm_range) == 0,
2663 "NVME_MAX_DSM_TRIM is not a multiple of struct size");
2664 range = calloc(NVME_MAX_DSM_TRIM / sizeof (*range), sizeof (*range));
2665 #endif
2666 if (range == NULL) {
2667 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
2668 goto out;
2669 }
2670 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, cmd->prp1, cmd->prp2,
2671 (uint8_t *)range, NVME_MAX_DSM_TRIM, NVME_COPY_FROM_PRP);
2672
2673 /* Check for invalid ranges and the number of non-zero lengths */
2674 non_zero = 0;
2675 for (r = 0; r <= nr; r++) {
2676 if (pci_nvme_out_of_range(nvstore,
2677 range[r].starting_lba, range[r].length)) {
2678 pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE);
2679 goto out;
2680 }
2681 if (range[r].length != 0)
2682 non_zero++;
2683 }
2684
2685 if (cmd->cdw11 & NVME_DSM_ATTR_DEALLOCATE) {
2686 size_t offset, bytes;
2687 int sectsz_bits = sc->nvstore.sectsz_bits;
2688
2689 /*
2690 * DSM calls are advisory only, and compliant controllers
2691 * may choose to take no actions (i.e. return Success).
2692 */
2693 if (!nvstore->deallocate) {
2694 pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2695 goto out;
2696 }
2697
2698 /* If all ranges have a zero length, return Success */
2699 if (non_zero == 0) {
2700 pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2701 goto out;
2702 }
2703
2704 if (req == NULL) {
2705 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
2706 goto out;
2707 }
2708
2709 offset = range[0].starting_lba << sectsz_bits;
2710 bytes = range[0].length << sectsz_bits;
2711
2712 /*
2713 * If the request is for more than a single range, store
2714 * the ranges in the br_iov. Optimize for the common case
2715 * of a single range.
2716 *
2717 * Note that NVMe Number of Ranges is a zero based value
2718 */
2719 req->io_req.br_iovcnt = 0;
2720 req->io_req.br_offset = offset;
2721 req->io_req.br_resid = bytes;
2722
2723 if (nr == 0) {
2724 req->io_req.br_callback = pci_nvme_io_done;
2725 } else {
2726 struct iovec *iov = req->io_req.br_iov;
2727
2728 for (r = 0, dr = 0; r <= nr; r++) {
2729 offset = range[r].starting_lba << sectsz_bits;
2730 bytes = range[r].length << sectsz_bits;
2731 if (bytes == 0)
2732 continue;
2733
2734 if ((nvstore->size - offset) < bytes) {
2735 pci_nvme_status_genc(status,
2736 NVME_SC_LBA_OUT_OF_RANGE);
2737 goto out;
2738 }
2739 iov[dr].iov_base = (void *)offset;
2740 iov[dr].iov_len = bytes;
2741 dr++;
2742 }
2743 req->io_req.br_callback = pci_nvme_dealloc_sm;
2744
2745 /*
2746 * Use prev_gpaddr to track the current entry and
2747 * prev_size to track the number of entries
2748 */
2749 req->prev_gpaddr = 0;
2750 req->prev_size = dr;
2751 }
2752
2753 err = blockif_delete(nvstore->ctx, &req->io_req);
2754 if (err)
2755 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
2756 else
2757 pending = true;
2758 }
2759 out:
2760 free(range);
2761 return (pending);
2762 }
2763
2764 static void
2765 pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx)
2766 {
2767 struct nvme_submission_queue *sq;
2768 uint16_t status;
2769 uint16_t sqhead;
2770
2771 /* handle all submissions up to sq->tail index */
2772 sq = &sc->submit_queues[idx];
2773
2774 pthread_mutex_lock(&sq->mtx);
2775
2776 sqhead = sq->head;
2777 DPRINTF("nvme_handle_io qid %u head %u tail %u cmdlist %p",
2778 idx, sqhead, sq->tail, sq->qbase);
2779
2780 while (sqhead != atomic_load_acq_short(&sq->tail)) {
2781 struct nvme_command *cmd;
2782 struct pci_nvme_ioreq *req;
2783 uint32_t nsid;
2784 bool pending;
2785
2786 pending = false;
2787 req = NULL;
2788 status = 0;
2789
2790 cmd = &sq->qbase[sqhead];
2791 sqhead = (sqhead + 1) % sq->size;
2792
2793 nsid = le32toh(cmd->nsid);
2794 if ((nsid == 0) || (nsid > sc->ctrldata.nn)) {
2795 pci_nvme_status_genc(&status,
2796 NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
2797 status |=
2798 NVME_STATUS_DNR_MASK << NVME_STATUS_DNR_SHIFT;
2799 goto complete;
2800 }
2801
2802 req = pci_nvme_get_ioreq(sc);
2803 if (req == NULL) {
2804 pci_nvme_status_genc(&status,
2805 NVME_SC_INTERNAL_DEVICE_ERROR);
2806 WPRINTF("%s: unable to allocate IO req", __func__);
2807 goto complete;
2808 }
2809 req->nvme_sq = sq;
2810 req->sqid = idx;
2811 req->opc = cmd->opc;
2812 req->cid = cmd->cid;
2813 req->nsid = cmd->nsid;
2814
2815 switch (cmd->opc) {
2816 case NVME_OPC_FLUSH:
2817 pending = nvme_opc_flush(sc, cmd, &sc->nvstore,
2818 req, &status);
2819 break;
2820 case NVME_OPC_WRITE:
2821 case NVME_OPC_READ:
2822 pending = nvme_opc_write_read(sc, cmd, &sc->nvstore,
2823 req, &status);
2824 break;
2825 case NVME_OPC_WRITE_ZEROES:
2826 /* TODO: write zeroes
2827 WPRINTF("%s write zeroes lba 0x%lx blocks %u",
2828 __func__, lba, cmd->cdw12 & 0xFFFF); */
2829 pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
2830 break;
2831 case NVME_OPC_DATASET_MANAGEMENT:
2832 pending = nvme_opc_dataset_mgmt(sc, cmd, &sc->nvstore,
2833 req, &status);
2834 break;
2835 default:
2836 WPRINTF("%s unhandled io command 0x%x",
2837 __func__, cmd->opc);
2838 pci_nvme_status_genc(&status, NVME_SC_INVALID_OPCODE);
2839 }
2840 complete:
2841 if (!pending) {
2842 pci_nvme_set_completion(sc, sq, idx, cmd->cid, status);
2843 if (req != NULL)
2844 pci_nvme_release_ioreq(sc, req);
2845 }
2846 }
2847
2848 sq->head = sqhead;
2849
2850 pthread_mutex_unlock(&sq->mtx);
2851 }
2852
2853 static void
2854 pci_nvme_handle_doorbell(struct pci_nvme_softc* sc,
2855 uint64_t idx, int is_sq, uint64_t value)
2856 {
2857 DPRINTF("nvme doorbell %lu, %s, val 0x%lx",
2858 idx, is_sq ? "SQ" : "CQ", value & 0xFFFF);
2859
2860 if (is_sq) {
2861 if (idx > sc->num_squeues) {
2862 WPRINTF("%s queue index %lu overflow from "
2863 "guest (max %u)",
2864 __func__, idx, sc->num_squeues);
2865 return;
2866 }
2867
2868 atomic_store_short(&sc->submit_queues[idx].tail,
2869 (uint16_t)value);
2870
2871 if (idx == 0) {
2872 pci_nvme_handle_admin_cmd(sc, value);
2873 } else {
2874 /* submission queue; handle new entries in SQ */
2875 if (idx > sc->num_squeues) {
2876 WPRINTF("%s SQ index %lu overflow from "
2877 "guest (max %u)",
2878 __func__, idx, sc->num_squeues);
2879 return;
2880 }
2881 pci_nvme_handle_io_cmd(sc, (uint16_t)idx);
2882 }
2883 } else {
2884 if (idx > sc->num_cqueues) {
2885 WPRINTF("%s queue index %lu overflow from "
2886 "guest (max %u)",
2887 __func__, idx, sc->num_cqueues);
2888 return;
2889 }
2890
2891 atomic_store_short(&sc->compl_queues[idx].head,
2892 (uint16_t)value);
2893 }
2894 }
2895
2896 static void
2897 pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite)
2898 {
2899 const char *s = iswrite ? "WRITE" : "READ";
2900
2901 switch (offset) {
2902 case NVME_CR_CAP_LOW:
2903 DPRINTF("%s %s NVME_CR_CAP_LOW", func, s);
2904 break;
2905 case NVME_CR_CAP_HI:
2906 DPRINTF("%s %s NVME_CR_CAP_HI", func, s);
2907 break;
2908 case NVME_CR_VS:
2909 DPRINTF("%s %s NVME_CR_VS", func, s);
2910 break;
2911 case NVME_CR_INTMS:
2912 DPRINTF("%s %s NVME_CR_INTMS", func, s);
2913 break;
2914 case NVME_CR_INTMC:
2915 DPRINTF("%s %s NVME_CR_INTMC", func, s);
2916 break;
2917 case NVME_CR_CC:
2918 DPRINTF("%s %s NVME_CR_CC", func, s);
2919 break;
2920 case NVME_CR_CSTS:
2921 DPRINTF("%s %s NVME_CR_CSTS", func, s);
2922 break;
2923 case NVME_CR_NSSR:
2924 DPRINTF("%s %s NVME_CR_NSSR", func, s);
2925 break;
2926 case NVME_CR_AQA:
2927 DPRINTF("%s %s NVME_CR_AQA", func, s);
2928 break;
2929 case NVME_CR_ASQ_LOW:
2930 DPRINTF("%s %s NVME_CR_ASQ_LOW", func, s);
2931 break;
2932 case NVME_CR_ASQ_HI:
2933 DPRINTF("%s %s NVME_CR_ASQ_HI", func, s);
2934 break;
2935 case NVME_CR_ACQ_LOW:
2936 DPRINTF("%s %s NVME_CR_ACQ_LOW", func, s);
2937 break;
2938 case NVME_CR_ACQ_HI:
2939 DPRINTF("%s %s NVME_CR_ACQ_HI", func, s);
2940 break;
2941 default:
2942 DPRINTF("unknown nvme bar-0 offset 0x%lx", offset);
2943 }
2944
2945 }
2946
2947 static void
2948 pci_nvme_write_bar_0(struct vmctx *ctx, struct pci_nvme_softc* sc,
2949 uint64_t offset, int size, uint64_t value)
2950 {
2951 uint32_t ccreg;
2952
2953 if (offset >= NVME_DOORBELL_OFFSET) {
2954 uint64_t belloffset = offset - NVME_DOORBELL_OFFSET;
2955 uint64_t idx = belloffset / 8; /* door bell size = 2*int */
2956 int is_sq = (belloffset % 8) < 4;
2957
2958 if ((sc->regs.csts & NVME_CSTS_RDY) == 0) {
2959 WPRINTF("doorbell write prior to RDY (offset=%#lx)\n",
2960 offset);
2961 return;
2962 }
2963
2964 if (belloffset > ((sc->max_queues+1) * 8 - 4)) {
2965 WPRINTF("guest attempted an overflow write offset "
2966 "0x%lx, val 0x%lx in %s",
2967 offset, value, __func__);
2968 return;
2969 }
2970
2971 if (is_sq) {
2972 if (sc->submit_queues[idx].qbase == NULL)
2973 return;
2974 } else if (sc->compl_queues[idx].qbase == NULL)
2975 return;
2976
2977 pci_nvme_handle_doorbell(sc, idx, is_sq, value);
2978 return;
2979 }
2980
2981 DPRINTF("nvme-write offset 0x%lx, size %d, value 0x%lx",
2982 offset, size, value);
2983
2984 if (size != 4) {
2985 WPRINTF("guest wrote invalid size %d (offset 0x%lx, "
2986 "val 0x%lx) to bar0 in %s",
2987 size, offset, value, __func__);
2988 /* TODO: shutdown device */
2989 return;
2990 }
2991
2992 pci_nvme_bar0_reg_dumps(__func__, offset, 1);
2993
2994 pthread_mutex_lock(&sc->mtx);
2995
2996 switch (offset) {
2997 case NVME_CR_CAP_LOW:
2998 case NVME_CR_CAP_HI:
2999 /* readonly */
3000 break;
3001 case NVME_CR_VS:
3002 /* readonly */
3003 break;
3004 case NVME_CR_INTMS:
3005 /* MSI-X, so ignore */
3006 break;
3007 case NVME_CR_INTMC:
3008 /* MSI-X, so ignore */
3009 break;
3010 case NVME_CR_CC:
3011 ccreg = (uint32_t)value;
3012
3013 DPRINTF("%s NVME_CR_CC en %x css %x shn %x iosqes %u "
3014 "iocqes %u",
3015 __func__,
3016 NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg),
3017 NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg),
3018 NVME_CC_GET_IOCQES(ccreg));
3019
3020 if (NVME_CC_GET_SHN(ccreg)) {
3021 /* perform shutdown - flush out data to backend */
3022 sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK <<
3023 NVME_CSTS_REG_SHST_SHIFT);
3024 sc->regs.csts |= NVME_SHST_COMPLETE <<
3025 NVME_CSTS_REG_SHST_SHIFT;
3026 }
3027 if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) {
3028 if (NVME_CC_GET_EN(ccreg) == 0)
3029 /* transition 1-> causes controller reset */
3030 pci_nvme_reset_locked(sc);
3031 else
3032 pci_nvme_init_controller(ctx, sc);
3033 }
3034
3035 /* Insert the iocqes, iosqes and en bits from the write */
3036 sc->regs.cc &= ~NVME_CC_WRITE_MASK;
3037 sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK;
3038 if (NVME_CC_GET_EN(ccreg) == 0) {
3039 /* Insert the ams, mps and css bit fields */
3040 sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK;
3041 sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK;
3042 sc->regs.csts &= ~NVME_CSTS_RDY;
3043 } else if ((sc->pending_ios == 0) &&
3044 !(sc->regs.csts & NVME_CSTS_CFS)) {
3045 sc->regs.csts |= NVME_CSTS_RDY;
3046 }
3047 break;
3048 case NVME_CR_CSTS:
3049 break;
3050 case NVME_CR_NSSR:
3051 /* ignore writes; don't support subsystem reset */
3052 break;
3053 case NVME_CR_AQA:
3054 sc->regs.aqa = (uint32_t)value;
3055 break;
3056 case NVME_CR_ASQ_LOW:
3057 sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) |
3058 (0xFFFFF000 & value);
3059 break;
3060 case NVME_CR_ASQ_HI:
3061 sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) |
3062 (value << 32);
3063 break;
3064 case NVME_CR_ACQ_LOW:
3065 sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) |
3066 (0xFFFFF000 & value);
3067 break;
3068 case NVME_CR_ACQ_HI:
3069 sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) |
3070 (value << 32);
3071 break;
3072 default:
3073 DPRINTF("%s unknown offset 0x%lx, value 0x%lx size %d",
3074 __func__, offset, value, size);
3075 }
3076 pthread_mutex_unlock(&sc->mtx);
3077 }
3078
3079 static void
3080 pci_nvme_write(struct vmctx *ctx, struct pci_devinst *pi,
3081 int baridx, uint64_t offset, int size, uint64_t value)
3082 {
3083 struct pci_nvme_softc* sc = pi->pi_arg;
3084
3085 if (baridx == pci_msix_table_bar(pi) ||
3086 baridx == pci_msix_pba_bar(pi)) {
3087 DPRINTF("nvme-write baridx %d, msix: off 0x%lx, size %d, "
3088 " value 0x%lx", baridx, offset, size, value);
3089
3090 pci_emul_msix_twrite(pi, offset, size, value);
3091 return;
3092 }
3093
3094 switch (baridx) {
3095 case 0:
3096 pci_nvme_write_bar_0(ctx, sc, offset, size, value);
3097 break;
3098
3099 default:
3100 DPRINTF("%s unknown baridx %d, val 0x%lx",
3101 __func__, baridx, value);
3102 }
3103 }
3104
3105 static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc,
3106 uint64_t offset, int size)
3107 {
3108 uint64_t value;
3109
3110 pci_nvme_bar0_reg_dumps(__func__, offset, 0);
3111
3112 if (offset < NVME_DOORBELL_OFFSET) {
3113 void *p = &(sc->regs);
3114 pthread_mutex_lock(&sc->mtx);
3115 memcpy(&value, (void *)((uintptr_t)p + offset), size);
3116 pthread_mutex_unlock(&sc->mtx);
3117 } else {
3118 value = 0;
3119 WPRINTF("pci_nvme: read invalid offset %ld", offset);
3120 }
3121
3122 switch (size) {
3123 case 1:
3124 value &= 0xFF;
3125 break;
3126 case 2:
3127 value &= 0xFFFF;
3128 break;
3129 case 4:
3130 value &= 0xFFFFFFFF;
3131 break;
3132 }
3133
3134 DPRINTF(" nvme-read offset 0x%lx, size %d -> value 0x%x",
3135 offset, size, (uint32_t)value);
3136
3137 return (value);
3138 }
3139
3140
3141
3142 static uint64_t
3143 pci_nvme_read(struct vmctx *ctx __unused,
3144 struct pci_devinst *pi, int baridx, uint64_t offset, int size)
3145 {
3146 struct pci_nvme_softc* sc = pi->pi_arg;
3147
3148 if (baridx == pci_msix_table_bar(pi) ||
3149 baridx == pci_msix_pba_bar(pi)) {
3150 DPRINTF("nvme-read bar: %d, msix: regoff 0x%lx, size %d",
3151 baridx, offset, size);
3152
3153 return pci_emul_msix_tread(pi, offset, size);
3154 }
3155
3156 switch (baridx) {
3157 case 0:
3158 return pci_nvme_read_bar_0(sc, offset, size);
3159
3160 default:
3161 DPRINTF("unknown bar %d, 0x%lx", baridx, offset);
3162 }
3163
3164 return (0);
3165 }
3166
3167 static int
3168 pci_nvme_parse_config(struct pci_nvme_softc *sc, nvlist_t *nvl)
3169 {
3170 char bident[sizeof("XXX:XXX")];
3171 const char *value;
3172 uint32_t sectsz;
3173
3174 sc->max_queues = NVME_QUEUES;
3175 sc->max_qentries = NVME_MAX_QENTRIES;
3176 sc->ioslots = NVME_IOSLOTS;
3177 sc->num_squeues = sc->max_queues;
3178 sc->num_cqueues = sc->max_queues;
3179 sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO;
3180 sectsz = 0;
3181 #ifdef __FreeBSD__
3182 snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn),
3183 "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
3184 #else
3185 snprintf((char *)sc->ctrldata.sn, sizeof(sc->ctrldata.sn),
3186 "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
3187 #endif
3188
3189 value = get_config_value_node(nvl, "maxq");
3190 if (value != NULL)
3191 sc->max_queues = atoi(value);
3192 value = get_config_value_node(nvl, "qsz");
3193 if (value != NULL) {
3194 sc->max_qentries = atoi(value);
3195 if (sc->max_qentries <= 0) {
3196 EPRINTLN("nvme: Invalid qsz option %d",
3197 sc->max_qentries);
3198 return (-1);
3199 }
3200 }
3201 value = get_config_value_node(nvl, "ioslots");
3202 if (value != NULL) {
3203 sc->ioslots = atoi(value);
3204 if (sc->ioslots <= 0) {
3205 EPRINTLN("Invalid ioslots option %d", sc->ioslots);
3206 return (-1);
3207 }
3208 }
3209 value = get_config_value_node(nvl, "sectsz");
3210 if (value != NULL)
3211 sectsz = atoi(value);
3212 value = get_config_value_node(nvl, "ser");
3213 if (value != NULL) {
3214 /*
3215 * This field indicates the Product Serial Number in
3216 * 7-bit ASCII, unused bytes should be space characters.
3217 * Ref: NVMe v1.3c.
3218 */
3219 cpywithpad((char *)sc->ctrldata.sn,
3220 sizeof(sc->ctrldata.sn), value, ' ');
3221 }
3222 value = get_config_value_node(nvl, "eui64");
3223 if (value != NULL)
3224 sc->nvstore.eui64 = htobe64(strtoull(value, NULL, 0));
3225 value = get_config_value_node(nvl, "dsm");
3226 if (value != NULL) {
3227 if (strcmp(value, "auto") == 0)
3228 sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO;
3229 else if (strcmp(value, "enable") == 0)
3230 sc->dataset_management = NVME_DATASET_MANAGEMENT_ENABLE;
3231 else if (strcmp(value, "disable") == 0)
3232 sc->dataset_management = NVME_DATASET_MANAGEMENT_DISABLE;
3233 }
3234
3235 value = get_config_value_node(nvl, "ram");
3236 if (value != NULL) {
3237 uint64_t sz = strtoull(value, NULL, 10);
3238
3239 sc->nvstore.type = NVME_STOR_RAM;
3240 sc->nvstore.size = sz * 1024 * 1024;
3241 sc->nvstore.ctx = calloc(1, sc->nvstore.size);
3242 sc->nvstore.sectsz = 4096;
3243 sc->nvstore.sectsz_bits = 12;
3244 if (sc->nvstore.ctx == NULL) {
3245 EPRINTLN("nvme: Unable to allocate RAM");
3246 return (-1);
3247 }
3248 } else {
3249 snprintf(bident, sizeof(bident), "%u:%u",
3250 sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
3251 sc->nvstore.ctx = blockif_open(nvl, bident);
3252 if (sc->nvstore.ctx == NULL) {
3253 EPRINTLN("nvme: Could not open backing file: %s",
3254 strerror(errno));
3255 return (-1);
3256 }
3257 sc->nvstore.type = NVME_STOR_BLOCKIF;
3258 sc->nvstore.size = blockif_size(sc->nvstore.ctx);
3259 }
3260
3261 if (sectsz == 512 || sectsz == 4096 || sectsz == 8192)
3262 sc->nvstore.sectsz = sectsz;
3263 else if (sc->nvstore.type != NVME_STOR_RAM)
3264 sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx);
3265 for (sc->nvstore.sectsz_bits = 9;
3266 (1U << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz;
3267 sc->nvstore.sectsz_bits++);
3268
3269 if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES)
3270 sc->max_queues = NVME_QUEUES;
3271
3272 return (0);
3273 }
3274
3275 static void
3276 pci_nvme_resized(struct blockif_ctxt *bctxt __unused, void *arg,
3277 size_t new_size)
3278 {
3279 struct pci_nvme_softc *sc;
3280 struct pci_nvme_blockstore *nvstore;
3281 struct nvme_namespace_data *nd;
3282
3283 sc = arg;
3284 nvstore = &sc->nvstore;
3285 nd = &sc->nsdata;
3286
3287 nvstore->size = new_size;
3288 pci_nvme_init_nsdata_size(nvstore, nd);
3289
3290 /* Add changed NSID to list */
3291 sc->ns_log.ns[0] = 1;
3292 sc->ns_log.ns[1] = 0;
3293
3294 pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_NOTICE,
3295 PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED);
3296 }
3297
3298 static int
3299 pci_nvme_init(struct vmctx *ctx __unused, struct pci_devinst *pi, nvlist_t *nvl)
3300 {
3301 struct pci_nvme_softc *sc;
3302 uint32_t pci_membar_sz;
3303 int error;
3304
3305 error = 0;
3306
3307 sc = calloc(1, sizeof(struct pci_nvme_softc));
3308 pi->pi_arg = sc;
3309 sc->nsc_pi = pi;
3310
3311 error = pci_nvme_parse_config(sc, nvl);
3312 if (error < 0)
3313 goto done;
3314 else
3315 error = 0;
3316
3317 STAILQ_INIT(&sc->ioreqs_free);
3318 sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq));
3319 for (uint32_t i = 0; i < sc->ioslots; i++) {
3320 STAILQ_INSERT_TAIL(&sc->ioreqs_free, &sc->ioreqs[i], link);
3321 }
3322
3323 pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A);
3324 pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D);
3325 pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
3326 pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM);
3327 pci_set_cfgdata8(pi, PCIR_PROGIF,
3328 PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0);
3329
3330 /*
3331 * Allocate size of NVMe registers + doorbell space for all queues.
3332 *
3333 * The specification requires a minimum memory I/O window size of 16K.
3334 * The Windows driver will refuse to start a device with a smaller
3335 * window.
3336 */
3337 pci_membar_sz = sizeof(struct nvme_registers) +
3338 2 * sizeof(uint32_t) * (sc->max_queues + 1);
3339 pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN);
3340
3341 DPRINTF("nvme membar size: %u", pci_membar_sz);
3342
3343 error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz);
3344 if (error) {
3345 WPRINTF("%s pci alloc mem bar failed", __func__);
3346 goto done;
3347 }
3348
3349 error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR);
3350 if (error) {
3351 WPRINTF("%s pci add msixcap failed", __func__);
3352 goto done;
3353 }
3354
3355 error = pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_INT_EP);
3356 if (error) {
3357 WPRINTF("%s pci add Express capability failed", __func__);
3358 goto done;
3359 }
3360
3361 pthread_mutex_init(&sc->mtx, NULL);
3362 sem_init(&sc->iosemlock, 0, sc->ioslots);
3363 blockif_register_resize_callback(sc->nvstore.ctx, pci_nvme_resized, sc);
3364
3365 pci_nvme_init_queues(sc, sc->max_queues, sc->max_queues);
3366 /*
3367 * Controller data depends on Namespace data so initialize Namespace
3368 * data first.
3369 */
3370 pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore);
3371 pci_nvme_init_ctrldata(sc);
3372 pci_nvme_init_logpages(sc);
3373 pci_nvme_init_features(sc);
3374
3375 pci_nvme_aer_init(sc);
3376 pci_nvme_aen_init(sc);
3377
3378 pci_nvme_reset(sc);
3379
3380 pci_lintr_request(pi);
3381
3382 done:
3383 return (error);
3384 }
3385
3386 static int
3387 pci_nvme_legacy_config(nvlist_t *nvl, const char *opts)
3388 {
3389 char *cp, *ram;
3390
3391 if (opts == NULL)
3392 return (0);
3393
3394 if (strncmp(opts, "ram=", 4) == 0) {
3395 cp = strchr(opts, ',');
3396 if (cp == NULL) {
3397 set_config_value_node(nvl, "ram", opts + 4);
3398 return (0);
3399 }
3400 ram = strndup(opts + 4, cp - opts - 4);
3401 set_config_value_node(nvl, "ram", ram);
3402 free(ram);
3403 return (pci_parse_legacy_config(nvl, cp + 1));
3404 } else
3405 return (blockif_legacy_config(nvl, opts));
3406 }
3407
3408 static const struct pci_devemu pci_de_nvme = {
3409 .pe_emu = "nvme",
3410 .pe_init = pci_nvme_init,
3411 .pe_legacy_config = pci_nvme_legacy_config,
3412 .pe_barwrite = pci_nvme_write,
3413 .pe_barread = pci_nvme_read
3414 };
3415 PCI_EMUL_SET(pci_de_nvme);
Community developed and maintained version of the OS/Net consolidation