2 * Copyright (c) 2016 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * NVME softc and structural definitions
39 #if defined(__DragonFly__)
40 #include "nvme_dragonfly.h"
42 #error "build for OS unknown"
46 #include "nvme_ident.h"
48 #include "nvme_chipset.h"
49 #include "nvme_ioctl.h"
51 MALLOC_DECLARE(M_NVME
);
54 * Choose some reasonable limit, even if the hardware supports more.
56 #define NVME_MAX_QUEUES 1024
57 #define NVME_MAX_NAMESPACES 1024
64 * Device matching array for special attach/detach codings.
67 pci_vendor_id_t vendor
;
68 pci_product_id_t product
;
69 int (*attach
)(device_t dev
);
70 int (*detach
)(device_t dev
);
75 * Kernel-level request structure. This structure allows the driver to
76 * construct, issue, wait for, and process responses to commands. Each
77 * queue manages its own request bank.
79 * In order to disconnect the request structure from the hardware queue
80 * mechanism itself, to reduce SMP conflicts and interactions, and allow
81 * command/response processing to block without interfering with queue
82 * operations, this structure embeds a copy of the HW command and response
83 * structures instead of referencing the ones in the actual hardware queues.
84 * These will be copied to/from the actual queue entries by lower-level
87 * Requests are associated with particular queues, completions can occur on
88 * any queue. Requests associated with the admin queue conveniently include
89 * an additional 4K 'info' block suitable for DMA.
91 typedef struct nvme_request
{
92 struct nvme_request
*next_avail
;
93 struct nvme_subqueue
*subq
; /* which queue is submission on */
94 struct nvme_comqueue
*comq
; /* which queue is completion on */
96 uint32_t cmd_id
; /* reqary[] index */
98 nvme_allcmd_t cmd
; /* hw submit structure for entry */
99 nvme_allres_t res
; /* hw completion structure for entry */
100 nvme_admin_data_t
*info
; /* DMA data (admin request only) */
101 bus_addr_t pinfo
; /* phys address for PRP */
104 * Aux fields to keep track of bio and other data, depending on
105 * the callback. If the callback is NULL the caller will poll for
108 void (*callback
)(struct nvme_request
*req
, struct lock
*lk
);
109 struct nvme_softns
*nsc
;
113 #define NVME_REQ_AVAIL 0
114 #define NVME_REQ_ALLOCATED 1
115 #define NVME_REQ_SUBMITTED 2
116 #define NVME_REQ_COMPLETED 3
118 typedef struct nvme_subqueue
{
122 struct lock lk
; /* queue lock controls access */
123 struct nvme_softc
*sc
;
124 nvme_request_t
*first_avail
;
125 nvme_request_t
*reqary
;
126 uint32_t nqe
; /* #of queue entries */
127 uint16_t qid
; /* which queue# are we on? */
128 uint16_t comqid
; /* we are tied to this completion qu */
129 uint32_t subq_doorbell_reg
;
130 uint32_t subq_head
; /* start of active requests */
131 uint32_t subq_tail
; /* new requests */
132 uint32_t unsubmitted
; /* #unsubmitted requests */
139 bus_dmamap_t sque_map
;
140 bus_dmamap_t prps_map
;
141 nvme_allcmd_t
*ksubq
; /* kernel-mapped addresses */
142 uint64_t *kprps
; /* enough PRPs per request for */
143 /* MAXPHYS bytes worth of mappings */
144 bus_addr_t psubq
; /* physical addresses */
148 * Additional DMA resources for admin queue (A NVME_MAX_ADMIN_BUFFER
149 * sized buffer for each queue entry).
151 bus_dmamap_t adm_map
;
153 nvme_admin_data_t
*kdatapgs
;
156 typedef struct nvme_comqueue
{
160 struct lock lk
; /* queue lock controls access */
161 struct nvme_softc
*sc
;
162 uint32_t nqe
; /* #of queue entries */
163 uint16_t phase
; /* phase to match (res->tail.status) */
164 uint16_t qid
; /* which queue# are we on? */
165 uint32_t comq_doorbell_reg
;
166 uint32_t comq_head
; /* consume responses */
172 bus_dmamap_t cque_map
;
173 nvme_allres_t
*kcomq
;
177 typedef struct nvme_softns
{
178 struct nvme_softc
*sc
;
179 nvme_ident_ns_data_t idns
;
180 struct bio_queue_head bioq
; /* excess BIOs */
181 struct lock lk
; /* mostly for bioq handling */
186 struct disk disk
; /* disk attachment */
187 struct devstat stats
;
188 cdev_t cdev
; /* disk device (cdev) */
191 #define NVME_NSC_STATE_UNATTACHED 0
192 #define NVME_NSC_STATE_ATTACHED 1
195 typedef struct nvme_softc
{
196 TAILQ_ENTRY(nvme_softc
) entry
; /* global list */
197 device_t dev
; /* core device */
198 const nvme_device_t
*ad
; /* quirk detection etc */
203 uint32_t dstrd4
; /* doorbell stride */
204 uint32_t entimo
; /* enable timeout in ticks */
205 uint16_t niosubqs
; /* #of I/O submission queues */
206 uint16_t niocomqs
; /* #of I/O completion queues */
209 uint16_t qmap
[SMP_MAXCPU
][2];
211 nvme_subqueue_t subqueues
[NVME_MAX_QUEUES
];
212 nvme_comqueue_t comqueues
[NVME_MAX_QUEUES
];
213 int cputovect
[SMP_MAXCPU
];
216 * bio/disk layer tracking
221 * admin queue irq resources
222 * register map resources
224 struct resource
*regs
;
225 struct resource
*bar4
;
227 bus_space_handle_t ioh
;
233 struct resource
*irq
[NVME_MAX_QUEUES
];
234 int rid_irq
[NVME_MAX_QUEUES
];
235 void *irq_handle
[NVME_MAX_QUEUES
];
240 bus_dma_tag_t prps_tag
; /* worst-case PRP table(s) per queue */
241 bus_dma_tag_t sque_tag
; /* submission queue */
242 bus_dma_tag_t cque_tag
; /* completion queue */
243 bus_dma_tag_t adm_tag
; /* DMA data buffers for admin cmds */
250 * Admin thread and controller identify data
252 uint32_t admin_signal
;
253 struct lock admin_lk
;
254 struct lock ioctl_lk
;
255 int (*admin_func
)(struct nvme_softc
*);
256 nvme_ident_ctlr_data_t idctlr
;
257 nvme_softns_t
*nscary
[NVME_MAX_NAMESPACES
];
261 #define NVME_SC_ATTACHED 0x00000001
262 #define NVME_SC_UNLOADING 0x00000002
264 #define ADMIN_SIG_STOP 0x00000001
265 #define ADMIN_SIG_RUNNING 0x00000002
266 #define ADMIN_SIG_PROBED 0x00000004
267 #define ADMIN_SIG_REQUEUE 0x00000008
268 #define ADMIN_SIG_RUN_MASK (ADMIN_SIG_STOP | ADMIN_SIG_REQUEUE)
270 #define NVME_QMAP_RD 0
271 #define NVME_QMAP_WR 1
276 const nvme_device_t
*nvme_lookup_device(device_t dev
);
277 void nvme_os_sleep(int ms
);
278 int nvme_os_softsleep(void);
279 void nvme_os_hardsleep(int us
);
280 u_int32_t
nvme_read(nvme_softc_t
*sc
, bus_size_t r
);
281 u_int64_t
nvme_read8(nvme_softc_t
*sc
, bus_size_t r
);
282 void nvme_write(nvme_softc_t
*sc
, bus_size_t r
, u_int32_t v
);
283 void nvme_write8(nvme_softc_t
*sc
, bus_size_t r
, u_int64_t v
);
284 int nvme_enable(nvme_softc_t
*sc
, int enable
);
285 int nvme_alloc_subqueue(nvme_softc_t
*sc
, uint16_t qid
);
286 int nvme_alloc_comqueue(nvme_softc_t
*sc
, uint16_t qid
);
287 void nvme_free_subqueue(nvme_softc_t
*sc
, uint16_t qid
);
288 void nvme_free_comqueue(nvme_softc_t
*sc
, uint16_t qid
);
290 nvme_request_t
*nvme_get_admin_request(nvme_softc_t
*sc
, uint8_t opcode
);
291 nvme_request_t
*nvme_get_request(nvme_subqueue_t
*queue
, uint8_t opcode
,
292 char *kva
, size_t bytes
);
293 void nvme_submit_request(nvme_request_t
*req
);
294 int nvme_wait_request(nvme_request_t
*req
, int ticks
);
295 void nvme_put_request(nvme_request_t
*req
);
296 void nvme_poll_completions(nvme_comqueue_t
*queue
, struct lock
*lk
);
298 int nvme_start_admin_thread(nvme_softc_t
*sc
);
299 void nvme_stop_admin_thread(nvme_softc_t
*sc
);
301 int nvme_create_subqueue(nvme_softc_t
*sc
, uint16_t qid
);
302 int nvme_create_comqueue(nvme_softc_t
*sc
, uint16_t qid
);
303 int nvme_delete_subqueue(nvme_softc_t
*sc
, uint16_t qid
);
304 int nvme_delete_comqueue(nvme_softc_t
*sc
, uint16_t qid
);
305 int nvme_issue_shutdown(nvme_softc_t
*sc
);
307 void nvme_disk_attach(nvme_softns_t
*nsc
);
308 void nvme_disk_detach(nvme_softns_t
*nsc
);
309 void nvme_disk_requeues(nvme_softc_t
*sc
);
310 int nvme_alloc_disk_unit(void);
312 int nvme_getlog_ioctl(nvme_softc_t
*sc
, nvme_getlog_ioctl_t
*ioc
);
314 void nvme_intr(void *arg
);
315 size_t string_cleanup(char *str
, int domiddle
);