2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2011 NetApp, Inc.
6 * Copyright 2020-2021 Joyent, Inc.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * This file and its contents are supplied under the terms of the
33 * Common Development and Distribution License ("CDDL"), version 1.0.
34 * You may only use this file in accordance with the terms of version
37 * A full copy of the text of the CDDL should have accompanied this
38 * source. A copy of the CDDL is also available via the Internet at
39 * http://www.illumos.org/license/CDDL.
41 * Copyright 2014 Pluribus Networks Inc.
44 #include <sys/cdefs.h>
45 __FBSDID("$FreeBSD$");
47 #include <sys/param.h>
48 #include <sys/linker_set.h>
51 #include <sys/ioctl.h>
73 #define VTBLK_BSIZE 512
74 #define VTBLK_RINGSZ 128
76 _Static_assert(VTBLK_RINGSZ
<= BLOCKIF_RING_MAX
, "Each ring entry must be able to queue a request");
79 #define VTBLK_S_IOERR 1
80 #define VTBLK_S_UNSUPP 2
82 #define VTBLK_BLK_ID_BYTES 20 + 1
85 #define VTBLK_F_BARRIER (1 << 0) /* Does host support barriers? */
86 #define VTBLK_F_SIZE_MAX (1 << 1) /* Indicates maximum segment size */
87 #define VTBLK_F_SEG_MAX (1 << 2) /* Indicates maximum # of segments */
88 #define VTBLK_F_GEOMETRY (1 << 4) /* Legacy geometry available */
89 #define VTBLK_F_RO (1 << 5) /* Disk is read-only */
90 #define VTBLK_F_BLK_SIZE (1 << 6) /* Block size of disk is available*/
91 #define VTBLK_F_SCSI (1 << 7) /* Supports scsi command passthru */
92 #define VTBLK_F_FLUSH (1 << 9) /* Writeback mode enabled after reset */
93 #define VTBLK_F_WCE (1 << 9) /* Legacy alias for FLUSH */
94 #define VTBLK_F_TOPOLOGY (1 << 10) /* Topology information is available */
95 #define VTBLK_F_CONFIG_WCE (1 << 11) /* Writeback mode available in config */
96 #define VTBLK_F_MQ (1 << 12) /* Multi-Queue */
97 #define VTBLK_F_DISCARD (1 << 13) /* Trim blocks */
98 #define VTBLK_F_WRITE_ZEROES (1 << 14) /* Write zeros */
103 #define VTBLK_S_HOSTCAPS \
104 ( VTBLK_F_SEG_MAX | \
108 VIRTIO_RING_F_INDIRECT_DESC ) /* indirect descriptors */
111 * The current blockif_delete() interface only allows a single delete
114 #define VTBLK_MAX_DISCARD_SEG 1
117 * An arbitrary limit to prevent excessive latency due to large
120 #define VTBLK_MAX_DISCARD_SECT ((16 << 20) / VTBLK_BSIZE) /* 16 MiB */
123 * Config space "registers"
125 struct vtblk_config
{
126 uint64_t vbc_capacity
;
127 uint32_t vbc_size_max
;
128 uint32_t vbc_seg_max
;
134 uint32_t vbc_blk_size
;
136 uint8_t physical_block_exp
;
137 uint8_t alignment_offset
;
138 uint16_t min_io_size
;
139 uint32_t opt_io_size
;
141 uint8_t vbc_writeback
;
144 uint32_t max_discard_sectors
;
145 uint32_t max_discard_seg
;
146 uint32_t discard_sector_alignment
;
147 uint32_t max_write_zeroes_sectors
;
148 uint32_t max_write_zeroes_seg
;
149 uint8_t write_zeroes_may_unmap
;
154 * Fixed-size block header
156 struct virtio_blk_hdr
{
157 #define VBH_OP_READ 0
158 #define VBH_OP_WRITE 1
159 #define VBH_OP_SCSI_CMD 2
160 #define VBH_OP_SCSI_CMD_OUT 3
161 #define VBH_OP_FLUSH 4
162 #define VBH_OP_FLUSH_OUT 5
163 #define VBH_OP_IDENT 8
164 #define VBH_OP_DISCARD 11
165 #define VBH_OP_WRITE_ZEROES 13
167 #define VBH_FLAG_BARRIER 0x80000000 /* OR'ed into vbh_type */
176 static int pci_vtblk_debug
;
177 #define DPRINTF(params) if (pci_vtblk_debug) PRINTLN params
178 #define WPRINTF(params) PRINTLN params
180 struct pci_vtblk_ioreq
{
181 struct blockif_req io_req
;
182 struct pci_vtblk_softc
*io_sc
;
187 struct virtio_blk_discard_write_zeroes
{
189 uint32_t num_sectors
;
192 uint32_t reserved
:31;
199 struct pci_vtblk_softc
{
200 struct virtio_softc vbsc_vs
;
201 pthread_mutex_t vsc_mtx
;
202 struct vqueue_info vbsc_vq
;
203 struct vtblk_config vbsc_cfg
;
204 struct virtio_consts vbsc_consts
;
205 struct blockif_ctxt
*bc
;
209 char vbsc_ident
[VTBLK_BLK_ID_BYTES
];
210 struct pci_vtblk_ioreq vbsc_ios
[VTBLK_RINGSZ
];
213 static void pci_vtblk_reset(void *);
214 static void pci_vtblk_notify(void *, struct vqueue_info
*);
215 static int pci_vtblk_cfgread(void *, int, int, uint32_t *);
216 static int pci_vtblk_cfgwrite(void *, int, int, uint32_t);
218 static void pci_vtblk_apply_feats(void *, uint64_t);
221 static struct virtio_consts vtblk_vi_consts
= {
224 .vc_cfgsize
= sizeof(struct vtblk_config
),
225 .vc_reset
= pci_vtblk_reset
,
226 .vc_qnotify
= pci_vtblk_notify
,
227 .vc_cfgread
= pci_vtblk_cfgread
,
228 .vc_cfgwrite
= pci_vtblk_cfgwrite
,
230 .vc_apply_features
= pci_vtblk_apply_feats
,
232 .vc_apply_features
= NULL
,
234 .vc_hv_caps
= VTBLK_S_HOSTCAPS
,
238 pci_vtblk_reset(void *vsc
)
240 struct pci_vtblk_softc
*sc
= vsc
;
242 DPRINTF(("vtblk: device reset requested !"));
243 vi_reset_dev(&sc
->vbsc_vs
);
245 /* Disable write cache until FLUSH feature is negotiated */
246 (void) blockif_set_wce(sc
->bc
, 0);
252 pci_vtblk_done_locked(struct pci_vtblk_ioreq
*io
, int err
)
254 struct pci_vtblk_softc
*sc
= io
->io_sc
;
256 /* convert errno into a virtio block error return */
257 if (err
== EOPNOTSUPP
|| err
== ENOSYS
)
258 *io
->io_status
= VTBLK_S_UNSUPP
;
260 *io
->io_status
= VTBLK_S_IOERR
;
262 *io
->io_status
= VTBLK_S_OK
;
265 * Return the descriptor back to the host.
266 * We wrote 1 byte (our status) to host.
268 vq_relchain(&sc
->vbsc_vq
, io
->io_idx
, 1);
269 vq_endchains(&sc
->vbsc_vq
, 0);
273 pci_vtblk_done(struct blockif_req
*br
, int err
)
275 struct pci_vtblk_ioreq
*io
= br
->br_param
;
276 struct pci_vtblk_softc
*sc
= io
->io_sc
;
278 pthread_mutex_lock(&sc
->vsc_mtx
);
279 pci_vtblk_done_locked(io
, err
);
280 pthread_mutex_unlock(&sc
->vsc_mtx
);
284 pci_vtblk_proc(struct pci_vtblk_softc
*sc
, struct vqueue_info
*vq
)
286 struct virtio_blk_hdr
*vbh
;
287 struct pci_vtblk_ioreq
*io
;
293 struct iovec iov
[BLOCKIF_IOV_MAX
+ 2];
294 struct virtio_blk_discard_write_zeroes
*discard
;
296 n
= vq_getchain(vq
, iov
, BLOCKIF_IOV_MAX
+ 2, &req
);
299 * The first descriptor will be the read-only fixed header,
300 * and the last is for status (hence +2 above and below).
301 * The remaining iov's are the actual data I/O vectors.
303 * XXX - note - this fails on crash dump, which does a
304 * VIRTIO_BLK_T_FLUSH with a zero transfer length
306 assert(n
>= 2 && n
<= BLOCKIF_IOV_MAX
+ 2);
308 io
= &sc
->vbsc_ios
[req
.idx
];
309 assert(req
.readable
!= 0);
310 assert(iov
[0].iov_len
== sizeof(struct virtio_blk_hdr
));
311 vbh
= (struct virtio_blk_hdr
*)iov
[0].iov_base
;
312 memcpy(&io
->io_req
.br_iov
, &iov
[1], sizeof(struct iovec
) * (n
- 2));
313 io
->io_req
.br_iovcnt
= n
- 2;
314 io
->io_req
.br_offset
= vbh
->vbh_sector
* VTBLK_BSIZE
;
315 io
->io_status
= (uint8_t *)iov
[--n
].iov_base
;
316 assert(req
.writable
!= 0);
317 assert(iov
[n
].iov_len
== 1);
321 * The guest should not be setting the BARRIER flag because
322 * we don't advertise the capability.
324 type
= vbh
->vbh_type
& ~VBH_FLAG_BARRIER
;
325 writeop
= (type
== VBH_OP_WRITE
|| type
== VBH_OP_DISCARD
);
327 * - Write op implies read-only descriptor
328 * - Read/ident op implies write-only descriptor
330 * By taking away either the read-only fixed header or the write-only
331 * status iovec, the following condition should hold true.
333 assert(n
== (writeop
? req
.readable
: req
.writable
));
336 for (i
= 1; i
< n
; i
++) {
337 iolen
+= iov
[i
].iov_len
;
339 io
->io_req
.br_resid
= iolen
;
341 DPRINTF(("virtio-block: %s op, %zd bytes, %d segs, offset %ld",
342 writeop
? "write/discard" : "read/ident", iolen
, i
- 1,
343 io
->io_req
.br_offset
));
347 err
= blockif_read(sc
->bc
, &io
->io_req
);
350 err
= blockif_write(sc
->bc
, &io
->io_req
);
354 * We currently only support a single request, if the guest
355 * has submitted a request that doesn't conform to the
356 * requirements, we return a error.
358 if (iov
[1].iov_len
!= sizeof (*discard
)) {
359 pci_vtblk_done_locked(io
, EINVAL
);
363 /* The segments to discard are provided rather than data */
364 discard
= (struct virtio_blk_discard_write_zeroes
*)
368 * virtio v1.1 5.2.6.2:
369 * The device MUST set the status byte to VIRTIO_BLK_S_UNSUPP
370 * for discard and write zeroes commands if any unknown flag is
371 * set. Furthermore, the device MUST set the status byte to
372 * VIRTIO_BLK_S_UNSUPP for discard commands if the unmap flag
375 * Currently there are no known flags for a DISCARD request.
377 if (discard
->flags
.unmap
!= 0 || discard
->flags
.reserved
!= 0) {
378 pci_vtblk_done_locked(io
, ENOTSUP
);
382 /* Make sure the request doesn't exceed our size limit */
383 if (discard
->num_sectors
> VTBLK_MAX_DISCARD_SECT
) {
384 pci_vtblk_done_locked(io
, EINVAL
);
388 io
->io_req
.br_offset
= discard
->sector
* VTBLK_BSIZE
;
389 io
->io_req
.br_resid
= discard
->num_sectors
* VTBLK_BSIZE
;
390 err
= blockif_delete(sc
->bc
, &io
->io_req
);
393 case VBH_OP_FLUSH_OUT
:
394 err
= blockif_flush(sc
->bc
, &io
->io_req
);
397 /* Assume a single buffer */
398 /* S/n equal to buffer is not zero-terminated. */
399 memset(iov
[1].iov_base
, 0, iov
[1].iov_len
);
400 strncpy(iov
[1].iov_base
, sc
->vbsc_ident
,
401 MIN(iov
[1].iov_len
, sizeof(sc
->vbsc_ident
)));
402 pci_vtblk_done_locked(io
, 0);
405 pci_vtblk_done_locked(io
, EOPNOTSUPP
);
412 pci_vtblk_notify(void *vsc
, struct vqueue_info
*vq
)
414 struct pci_vtblk_softc
*sc
= vsc
;
416 while (vq_has_descs(vq
))
417 pci_vtblk_proc(sc
, vq
);
421 pci_vtblk_resized(struct blockif_ctxt
*bctxt __unused
, void *arg
,
424 struct pci_vtblk_softc
*sc
;
428 sc
->vbsc_cfg
.vbc_capacity
= new_size
/ VTBLK_BSIZE
; /* 512-byte units */
429 vi_interrupt(&sc
->vbsc_vs
, VIRTIO_PCI_ISR_CONFIG
,
430 sc
->vbsc_vs
.vs_msix_cfg_idx
);
434 pci_vtblk_init(struct vmctx
*ctx __unused
, struct pci_devinst
*pi
,
437 char bident
[sizeof("XXX:XXX")];
438 struct blockif_ctxt
*bctxt
;
439 const char *path
, *serial
;
442 struct pci_vtblk_softc
*sc
;
444 int i
, sectsz
, sts
, sto
;
447 * The supplied backing file has to exist
449 snprintf(bident
, sizeof(bident
), "%u:%u", pi
->pi_slot
, pi
->pi_func
);
450 bctxt
= blockif_open(nvl
, bident
);
452 perror("Could not open backing file");
456 size
= blockif_size(bctxt
);
457 sectsz
= blockif_sectsz(bctxt
);
458 blockif_psectsz(bctxt
, &sts
, &sto
);
460 sc
= calloc(1, sizeof(struct pci_vtblk_softc
));
462 for (i
= 0; i
< VTBLK_RINGSZ
; i
++) {
463 struct pci_vtblk_ioreq
*io
= &sc
->vbsc_ios
[i
];
464 io
->io_req
.br_callback
= pci_vtblk_done
;
465 io
->io_req
.br_param
= io
;
470 bcopy(&vtblk_vi_consts
, &sc
->vbsc_consts
, sizeof (vtblk_vi_consts
));
471 if (blockif_candelete(sc
->bc
))
472 sc
->vbsc_consts
.vc_hv_caps
|= VTBLK_F_DISCARD
;
475 /* Disable write cache until FLUSH feature is negotiated */
476 (void) blockif_set_wce(sc
->bc
, 0);
480 pthread_mutex_init(&sc
->vsc_mtx
, NULL
);
482 /* init virtio softc and virtqueues */
483 vi_softc_linkup(&sc
->vbsc_vs
, &sc
->vbsc_consts
, sc
, pi
, &sc
->vbsc_vq
);
484 sc
->vbsc_vs
.vs_mtx
= &sc
->vsc_mtx
;
486 sc
->vbsc_vq
.vq_qsize
= VTBLK_RINGSZ
;
487 /* sc->vbsc_vq.vq_notify = we have no per-queue notify */
490 * If an explicit identifier is not given, create an
491 * identifier using parts of the md5 sum of the filename.
493 bzero(sc
->vbsc_ident
, VTBLK_BLK_ID_BYTES
);
494 if ((serial
= get_config_value_node(nvl
, "serial")) != NULL
||
495 (serial
= get_config_value_node(nvl
, "ser")) != NULL
) {
496 strlcpy(sc
->vbsc_ident
, serial
, VTBLK_BLK_ID_BYTES
);
498 path
= get_config_value_node(nvl
, "path");
500 MD5Update(&mdctx
, path
, strlen(path
));
501 MD5Final(digest
, &mdctx
);
502 snprintf(sc
->vbsc_ident
, VTBLK_BLK_ID_BYTES
,
503 "BHYVE-%02X%02X-%02X%02X-%02X%02X",
504 digest
[0], digest
[1], digest
[2], digest
[3], digest
[4],
508 /* setup virtio block config space */
509 sc
->vbsc_cfg
.vbc_capacity
= size
/ VTBLK_BSIZE
; /* 512-byte units */
510 sc
->vbsc_cfg
.vbc_size_max
= 0; /* not negotiated */
513 * If Linux is presented with a seg_max greater than the virtio queue
514 * size, it can stumble into situations where it violates its own
515 * invariants and panics. For safety, we keep seg_max clamped, paying
516 * heed to the two extra descriptors needed for the header and status
519 sc
->vbsc_cfg
.vbc_seg_max
= MIN(VTBLK_RINGSZ
- 2, BLOCKIF_IOV_MAX
);
520 sc
->vbsc_cfg
.vbc_geometry
.cylinders
= 0; /* no geometry */
521 sc
->vbsc_cfg
.vbc_geometry
.heads
= 0;
522 sc
->vbsc_cfg
.vbc_geometry
.sectors
= 0;
523 sc
->vbsc_cfg
.vbc_blk_size
= sectsz
;
524 sc
->vbsc_cfg
.vbc_topology
.physical_block_exp
=
525 (sts
> sectsz
) ? (ffsll(sts
/ sectsz
) - 1) : 0;
526 sc
->vbsc_cfg
.vbc_topology
.alignment_offset
=
527 (sto
!= 0) ? ((sts
- sto
) / sectsz
) : 0;
528 sc
->vbsc_cfg
.vbc_topology
.min_io_size
= 0;
529 sc
->vbsc_cfg
.vbc_topology
.opt_io_size
= 0;
530 sc
->vbsc_cfg
.vbc_writeback
= 0;
531 sc
->vbsc_cfg
.max_discard_sectors
= VTBLK_MAX_DISCARD_SECT
;
532 sc
->vbsc_cfg
.max_discard_seg
= VTBLK_MAX_DISCARD_SEG
;
533 sc
->vbsc_cfg
.discard_sector_alignment
= MAX(sectsz
, sts
) / VTBLK_BSIZE
;
536 * Should we move some of this into virtio.c? Could
537 * have the device, class, and subdev_0 as fields in
538 * the virtio constants structure.
540 pci_set_cfgdata16(pi
, PCIR_DEVICE
, VIRTIO_DEV_BLOCK
);
541 pci_set_cfgdata16(pi
, PCIR_VENDOR
, VIRTIO_VENDOR
);
542 pci_set_cfgdata8(pi
, PCIR_CLASS
, PCIC_STORAGE
);
543 pci_set_cfgdata16(pi
, PCIR_SUBDEV_0
, VIRTIO_ID_BLOCK
);
544 pci_set_cfgdata16(pi
, PCIR_SUBVEND_0
, VIRTIO_VENDOR
);
546 if (vi_intr_init(&sc
->vbsc_vs
, 1, fbsdrun_virtio_msix())) {
547 blockif_close(sc
->bc
);
551 vi_set_io_bar(&sc
->vbsc_vs
, 0);
552 blockif_register_resize_callback(sc
->bc
, pci_vtblk_resized
, sc
);
557 pci_vtblk_cfgwrite(void *vsc __unused
, int offset
, int size __unused
,
558 uint32_t value __unused
)
561 DPRINTF(("vtblk: write to readonly reg %d", offset
));
566 pci_vtblk_cfgread(void *vsc
, int offset
, int size
, uint32_t *retval
)
568 struct pci_vtblk_softc
*sc
= vsc
;
571 /* our caller has already verified offset and size */
572 ptr
= (uint8_t *)&sc
->vbsc_cfg
+ offset
;
573 memcpy(retval
, ptr
, size
);
579 pci_vtblk_apply_feats(void *vsc
, uint64_t caps
)
581 struct pci_vtblk_softc
*sc
= vsc
;
582 const int wce_next
= ((caps
& VTBLK_F_FLUSH
) != 0) ? 1 : 0;
584 if (sc
->vbsc_wce
!= wce_next
) {
585 (void) blockif_set_wce(sc
->bc
, wce_next
);
586 sc
->vbsc_wce
= wce_next
;
589 #endif /* __FreeBSD__ */
591 static const struct pci_devemu pci_de_vblk
= {
592 .pe_emu
= "virtio-blk",
593 .pe_init
= pci_vtblk_init
,
594 .pe_legacy_config
= blockif_legacy_config
,
595 .pe_barwrite
= vi_pci_write
,
596 .pe_barread
= vi_pci_read
,
598 PCI_EMUL_SET(pci_de_vblk
);