4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2012 Nexenta Systems, Inc.
24 * Copyright 2012 Alexey Zaytsev <alexey.zaytsev@gmail.com>
27 /* Based on the NetBSD virtio driver by Minoura Makoto. */
29 * Copyright (c) 2010 Minoura Makoto.
30 * All rights reserved.
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
42 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
43 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
44 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
45 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
46 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
47 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
48 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
49 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
50 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
56 #include <sys/debug.h>
57 #include <sys/modctl.h>
58 #include <sys/autoconf.h>
59 #include <sys/ddi_impldefs.h>
61 #include <sys/sunddi.h>
62 #include <sys/sunndi.h>
63 #include <sys/avintr.h>
65 #include <sys/promif.h>
67 #include <sys/bootconf.h>
68 #include <sys/bootsvcs.h>
69 #include <sys/sysmacros.h>
72 #include "virtiovar.h"
73 #include "virtioreg.h"
74 #define NDEVNAMES (sizeof (virtio_device_name) / sizeof (char *))
75 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */
76 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1)) & \
77 ~(VIRTIO_PAGE_SIZE-1))
80 virtio_set_status(struct virtio_softc
*sc
, unsigned int status
)
85 old
= ddi_get8(sc
->sc_ioh
,
86 (uint8_t *)(sc
->sc_io_addr
+
87 VIRTIO_CONFIG_DEVICE_STATUS
));
90 (uint8_t *)(sc
->sc_io_addr
+ VIRTIO_CONFIG_DEVICE_STATUS
),
95 * Negotiate features, save the result in sc->sc_features
98 virtio_negotiate_features(struct virtio_softc
*sc
, uint32_t guest_features
)
100 uint32_t host_features
;
103 host_features
= ddi_get32(sc
->sc_ioh
,
104 /* LINTED E_BAD_PTR_CAST_ALIGN */
105 (uint32_t *)(sc
->sc_io_addr
+ VIRTIO_CONFIG_DEVICE_FEATURES
));
107 dev_debug(sc
->sc_dev
, CE_NOTE
,
108 "host features: %x, guest features: %x",
109 host_features
, guest_features
);
111 features
= host_features
& guest_features
;
112 ddi_put32(sc
->sc_ioh
,
113 /* LINTED E_BAD_PTR_CAST_ALIGN */
114 (uint32_t *)(sc
->sc_io_addr
+ VIRTIO_CONFIG_GUEST_FEATURES
),
117 sc
->sc_features
= features
;
119 return (host_features
);
123 virtio_show_features(uint32_t features
,
124 char *buf
, size_t len
)
126 char *orig_buf
= buf
;
127 char *bufend
= buf
+ len
;
129 /* LINTED E_PTRDIFF_OVERFLOW */
130 buf
+= snprintf(buf
, bufend
- buf
, "Generic ( ");
131 if (features
& VIRTIO_F_RING_INDIRECT_DESC
)
132 /* LINTED E_PTRDIFF_OVERFLOW */
133 buf
+= snprintf(buf
, bufend
- buf
, "INDIRECT_DESC ");
135 /* LINTED E_PTRDIFF_OVERFLOW */
136 buf
+= snprintf(buf
, bufend
- buf
, ") ");
138 /* LINTED E_PTRDIFF_OVERFLOW */
139 return (buf
- orig_buf
);
143 virtio_has_feature(struct virtio_softc
*sc
, uint32_t feature
)
145 return (sc
->sc_features
& feature
);
149 * Device configuration registers.
152 virtio_read_device_config_1(struct virtio_softc
*sc
, unsigned int index
)
154 ASSERT(sc
->sc_config_offset
);
155 return ddi_get8(sc
->sc_ioh
,
156 (uint8_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+ index
));
160 virtio_read_device_config_2(struct virtio_softc
*sc
, unsigned int index
)
162 ASSERT(sc
->sc_config_offset
);
163 return ddi_get16(sc
->sc_ioh
,
164 /* LINTED E_BAD_PTR_CAST_ALIGN */
165 (uint16_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+ index
));
169 virtio_read_device_config_4(struct virtio_softc
*sc
, unsigned int index
)
171 ASSERT(sc
->sc_config_offset
);
172 return ddi_get32(sc
->sc_ioh
,
173 /* LINTED E_BAD_PTR_CAST_ALIGN */
174 (uint32_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+ index
));
178 virtio_read_device_config_8(struct virtio_softc
*sc
, unsigned int index
)
182 ASSERT(sc
->sc_config_offset
);
183 r
= ddi_get32(sc
->sc_ioh
,
184 /* LINTED E_BAD_PTR_CAST_ALIGN */
185 (uint32_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+
186 index
+ sizeof (uint32_t)));
190 r
+= ddi_get32(sc
->sc_ioh
,
191 /* LINTED E_BAD_PTR_CAST_ALIGN */
192 (uint32_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+ index
));
197 virtio_write_device_config_1(struct virtio_softc
*sc
,
198 unsigned int index
, uint8_t value
)
200 ASSERT(sc
->sc_config_offset
);
202 (uint8_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+ index
), value
);
206 virtio_write_device_config_2(struct virtio_softc
*sc
,
207 unsigned int index
, uint16_t value
)
209 ASSERT(sc
->sc_config_offset
);
210 ddi_put16(sc
->sc_ioh
,
211 /* LINTED E_BAD_PTR_CAST_ALIGN */
212 (uint16_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+ index
), value
);
216 virtio_write_device_config_4(struct virtio_softc
*sc
,
217 unsigned int index
, uint32_t value
)
219 ASSERT(sc
->sc_config_offset
);
220 ddi_put32(sc
->sc_ioh
,
221 /* LINTED E_BAD_PTR_CAST_ALIGN */
222 (uint32_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+ index
), value
);
226 virtio_write_device_config_8(struct virtio_softc
*sc
,
227 unsigned int index
, uint64_t value
)
229 ASSERT(sc
->sc_config_offset
);
230 ddi_put32(sc
->sc_ioh
,
231 /* LINTED E_BAD_PTR_CAST_ALIGN */
232 (uint32_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+ index
),
234 ddi_put32(sc
->sc_ioh
,
235 /* LINTED E_BAD_PTR_CAST_ALIGN */
236 (uint32_t *)(sc
->sc_io_addr
+ sc
->sc_config_offset
+
237 index
+ sizeof (uint32_t)), value
>> 32);
241 * Start/stop vq interrupt. No guarantee.
244 virtio_stop_vq_intr(struct virtqueue
*vq
)
246 vq
->vq_avail
->flags
|= VRING_AVAIL_F_NO_INTERRUPT
;
250 virtio_start_vq_intr(struct virtqueue
*vq
)
252 vq
->vq_avail
->flags
&= ~VRING_AVAIL_F_NO_INTERRUPT
;
255 static ddi_dma_attr_t virtio_vq_dma_attr
= {
256 DMA_ATTR_V0
, /* Version number */
259 * high address. Has to fit into 32 bits
260 * after page-shifting
263 0xFFFFFFFF, /* counter register max */
264 VIRTIO_PAGE_SIZE
, /* page alignment required */
265 0x3F, /* burst sizes: 1 - 32 */
266 0x1, /* minimum transfer size */
267 0xFFFFFFFF, /* max transfer size */
268 0xFFFFFFFF, /* address register max */
269 1, /* no scatter-gather */
270 1, /* device operates on bytes */
271 0, /* attr flag: set to 0 */
274 static ddi_dma_attr_t virtio_vq_indirect_dma_attr
= {
275 DMA_ATTR_V0
, /* Version number */
277 0xFFFFFFFFFFFFFFFF, /* high address */
278 0xFFFFFFFF, /* counter register max */
279 1, /* No specific alignment */
280 0x3F, /* burst sizes: 1 - 32 */
281 0x1, /* minimum transfer size */
282 0xFFFFFFFF, /* max transfer size */
283 0xFFFFFFFF, /* address register max */
284 1, /* no scatter-gather */
285 1, /* device operates on bytes */
286 0, /* attr flag: set to 0 */
289 /* Same for direct and indirect descriptors. */
290 static ddi_device_acc_attr_t virtio_vq_devattr
= {
293 DDI_STORECACHING_OK_ACC
,
298 virtio_free_indirect(struct vq_entry
*entry
)
301 (void) ddi_dma_unbind_handle(entry
->qe_indirect_dma_handle
);
302 ddi_dma_mem_free(&entry
->qe_indirect_dma_acch
);
303 ddi_dma_free_handle(&entry
->qe_indirect_dma_handle
);
305 entry
->qe_indirect_descs
= NULL
;
310 virtio_alloc_indirect(struct virtio_softc
*sc
, struct vq_entry
*entry
)
314 unsigned int ncookies
;
317 num
= entry
->qe_queue
->vq_indirect_num
;
320 allocsize
= sizeof (struct vring_desc
) * num
;
322 ret
= ddi_dma_alloc_handle(sc
->sc_dev
, &virtio_vq_indirect_dma_attr
,
323 DDI_DMA_SLEEP
, NULL
, &entry
->qe_indirect_dma_handle
);
324 if (ret
!= DDI_SUCCESS
) {
325 dev_err(sc
->sc_dev
, CE_WARN
,
326 "Failed to allocate dma handle for indirect descriptors,"
327 " entry %d, vq %d", entry
->qe_index
,
328 entry
->qe_queue
->vq_index
);
329 goto out_alloc_handle
;
332 ret
= ddi_dma_mem_alloc(entry
->qe_indirect_dma_handle
,
333 allocsize
, &virtio_vq_devattr
,
334 DDI_DMA_CONSISTENT
, DDI_DMA_SLEEP
, NULL
,
335 (caddr_t
*)&entry
->qe_indirect_descs
, &len
,
336 &entry
->qe_indirect_dma_acch
);
337 if (ret
!= DDI_SUCCESS
) {
338 dev_err(sc
->sc_dev
, CE_WARN
,
339 "Failed to alocate dma memory for indirect descriptors,"
340 " entry %d, vq %d,", entry
->qe_index
,
341 entry
->qe_queue
->vq_index
);
345 (void) memset(entry
->qe_indirect_descs
, 0xff, allocsize
);
347 ret
= ddi_dma_addr_bind_handle(entry
->qe_indirect_dma_handle
, NULL
,
348 (caddr_t
)entry
->qe_indirect_descs
, len
,
349 DDI_DMA_RDWR
| DDI_DMA_CONSISTENT
,
350 DDI_DMA_SLEEP
, NULL
, &entry
->qe_indirect_dma_cookie
, &ncookies
);
351 if (ret
!= DDI_DMA_MAPPED
) {
352 dev_err(sc
->sc_dev
, CE_WARN
,
353 "Failed to bind dma memory for indirect descriptors,"
354 "entry %d, vq %d", entry
->qe_index
,
355 entry
->qe_queue
->vq_index
);
359 /* We asked for a single segment */
360 ASSERT(ncookies
== 1);
365 ddi_dma_mem_free(&entry
->qe_indirect_dma_acch
);
367 ddi_dma_free_handle(&entry
->qe_indirect_dma_handle
);
374 * Initialize the vq structure.
377 virtio_init_vq(struct virtio_softc
*sc
, struct virtqueue
*vq
)
381 int vq_size
= vq
->vq_num
;
382 int indirect_num
= vq
->vq_indirect_num
;
384 /* free slot management */
385 list_create(&vq
->vq_freelist
, sizeof (struct vq_entry
),
386 offsetof(struct vq_entry
, qe_list
));
388 for (i
= 0; i
< vq_size
; i
++) {
389 struct vq_entry
*entry
= &vq
->vq_entries
[i
];
390 list_insert_tail(&vq
->vq_freelist
, entry
);
392 entry
->qe_desc
= &vq
->vq_descs
[i
];
393 entry
->qe_queue
= vq
;
396 ret
= virtio_alloc_indirect(sc
, entry
);
402 mutex_init(&vq
->vq_freelist_lock
, "virtio-freelist",
403 MUTEX_DRIVER
, DDI_INTR_PRI(sc
->sc_intr_prio
));
404 mutex_init(&vq
->vq_avail_lock
, "virtio-avail",
405 MUTEX_DRIVER
, DDI_INTR_PRI(sc
->sc_intr_prio
));
406 mutex_init(&vq
->vq_used_lock
, "virtio-used",
407 MUTEX_DRIVER
, DDI_INTR_PRI(sc
->sc_intr_prio
));
412 for (i
= 0; i
< vq_size
; i
++) {
413 struct vq_entry
*entry
= &vq
->vq_entries
[i
];
414 if (entry
->qe_indirect_descs
)
415 virtio_free_indirect(entry
);
424 * Allocate/free a vq.
427 virtio_alloc_vq(struct virtio_softc
*sc
,
430 unsigned int indirect_num
,
433 int vq_size
, allocsize1
, allocsize2
, allocsize
= 0;
435 unsigned int ncookies
;
437 struct virtqueue
*vq
;
440 ddi_put16(sc
->sc_ioh
,
441 /* LINTED E_BAD_PTR_CAST_ALIGN */
442 (uint16_t *)(sc
->sc_io_addr
+ VIRTIO_CONFIG_QUEUE_SELECT
), index
);
443 vq_size
= ddi_get16(sc
->sc_ioh
,
444 /* LINTED E_BAD_PTR_CAST_ALIGN */
445 (uint16_t *)(sc
->sc_io_addr
+ VIRTIO_CONFIG_QUEUE_SIZE
));
447 dev_err(sc
->sc_dev
, CE_WARN
,
448 "virtqueue dest not exist, index %d for %s\n", index
, name
);
452 vq
= kmem_zalloc(sizeof (struct virtqueue
), KM_SLEEP
);
454 /* size 0 => use native vq size, good for receive queues. */
456 vq_size
= MIN(vq_size
, size
);
458 /* allocsize1: descriptor table + avail ring + pad */
459 allocsize1
= VIRTQUEUE_ALIGN(sizeof (struct vring_desc
) * vq_size
+
460 sizeof (struct vring_avail
) +
461 sizeof (uint16_t) * vq_size
);
462 /* allocsize2: used ring + pad */
463 allocsize2
= VIRTQUEUE_ALIGN(sizeof (struct vring_used
)
464 + sizeof (struct vring_used_elem
) * vq_size
);
466 allocsize
= allocsize1
+ allocsize2
;
468 ret
= ddi_dma_alloc_handle(sc
->sc_dev
, &virtio_vq_dma_attr
,
469 DDI_DMA_SLEEP
, NULL
, &vq
->vq_dma_handle
);
470 if (ret
!= DDI_SUCCESS
) {
471 dev_err(sc
->sc_dev
, CE_WARN
,
472 "Failed to allocate dma handle for vq %d", index
);
473 goto out_alloc_handle
;
476 ret
= ddi_dma_mem_alloc(vq
->vq_dma_handle
, allocsize
,
477 &virtio_vq_devattr
, DDI_DMA_CONSISTENT
, DDI_DMA_SLEEP
, NULL
,
478 (caddr_t
*)&vq
->vq_vaddr
, &len
, &vq
->vq_dma_acch
);
479 if (ret
!= DDI_SUCCESS
) {
480 dev_err(sc
->sc_dev
, CE_WARN
,
481 "Failed to alocate dma memory for vq %d", index
);
486 ret
= ddi_dma_addr_bind_handle(vq
->vq_dma_handle
, NULL
,
487 (caddr_t
)vq
->vq_vaddr
, len
,
488 DDI_DMA_RDWR
| DDI_DMA_CONSISTENT
,
489 DDI_DMA_SLEEP
, NULL
, &vq
->vq_dma_cookie
, &ncookies
);
490 if (ret
!= DDI_DMA_MAPPED
) {
491 dev_err(sc
->sc_dev
, CE_WARN
,
492 "Failed to bind dma memory for vq %d", index
);
496 /* We asked for a single segment */
497 ASSERT(ncookies
== 1);
498 /* and page-ligned buffers. */
499 ASSERT(vq
->vq_dma_cookie
.dmac_laddress
% VIRTIO_PAGE_SIZE
== 0);
501 (void) memset(vq
->vq_vaddr
, 0, allocsize
);
503 /* Make sure all zeros hit the buffer before we point the host to it */
506 /* set the vq address */
507 ddi_put32(sc
->sc_ioh
,
508 /* LINTED E_BAD_PTR_CAST_ALIGN */
509 (uint32_t *)(sc
->sc_io_addr
+ VIRTIO_CONFIG_QUEUE_ADDRESS
),
510 (vq
->vq_dma_cookie
.dmac_laddress
/ VIRTIO_PAGE_SIZE
));
512 /* remember addresses and offsets for later use */
514 vq
->vq_num
= vq_size
;
515 vq
->vq_index
= index
;
516 vq
->vq_descs
= vq
->vq_vaddr
;
517 vq
->vq_availoffset
= sizeof (struct vring_desc
)*vq_size
;
518 vq
->vq_avail
= (void *)(((char *)vq
->vq_descs
) + vq
->vq_availoffset
);
519 vq
->vq_usedoffset
= allocsize1
;
520 vq
->vq_used
= (void *)(((char *)vq
->vq_descs
) + vq
->vq_usedoffset
);
522 ASSERT(indirect_num
== 0 ||
523 virtio_has_feature(sc
, VIRTIO_F_RING_INDIRECT_DESC
));
524 vq
->vq_indirect_num
= indirect_num
;
526 /* free slot management */
527 vq
->vq_entries
= kmem_zalloc(sizeof (struct vq_entry
) * vq_size
,
530 ret
= virtio_init_vq(sc
, vq
);
534 dev_debug(sc
->sc_dev
, CE_NOTE
,
535 "Allocated %d entries for vq %d:%s (%d incdirect descs)",
536 vq_size
, index
, name
, indirect_num
* vq_size
);
541 kmem_free(vq
->vq_entries
, sizeof (struct vq_entry
) * vq_size
);
542 (void) ddi_dma_unbind_handle(vq
->vq_dma_handle
);
544 ddi_dma_mem_free(&vq
->vq_dma_acch
);
546 ddi_dma_free_handle(&vq
->vq_dma_handle
);
548 kmem_free(vq
, sizeof (struct virtqueue
));
555 virtio_free_vq(struct virtqueue
*vq
)
557 struct virtio_softc
*sc
= vq
->vq_owner
;
560 /* tell device that there's no virtqueue any longer */
561 ddi_put16(sc
->sc_ioh
,
562 /* LINTED E_BAD_PTR_CAST_ALIGN */
563 (uint16_t *)(sc
->sc_io_addr
+ VIRTIO_CONFIG_QUEUE_SELECT
),
565 ddi_put32(sc
->sc_ioh
,
566 /* LINTED E_BAD_PTR_CAST_ALIGN */
567 (uint32_t *)(sc
->sc_io_addr
+ VIRTIO_CONFIG_QUEUE_ADDRESS
), 0);
569 /* Free the indirect descriptors, if any. */
570 for (i
= 0; i
< vq
->vq_num
; i
++) {
571 struct vq_entry
*entry
= &vq
->vq_entries
[i
];
572 if (entry
->qe_indirect_descs
)
573 virtio_free_indirect(entry
);
576 kmem_free(vq
->vq_entries
, sizeof (struct vq_entry
) * vq
->vq_num
);
578 (void) ddi_dma_unbind_handle(vq
->vq_dma_handle
);
579 ddi_dma_mem_free(&vq
->vq_dma_acch
);
580 ddi_dma_free_handle(&vq
->vq_dma_handle
);
582 mutex_destroy(&vq
->vq_used_lock
);
583 mutex_destroy(&vq
->vq_avail_lock
);
584 mutex_destroy(&vq
->vq_freelist_lock
);
586 kmem_free(vq
, sizeof (struct virtqueue
));
590 * Free descriptor management.
593 vq_alloc_entry(struct virtqueue
*vq
)
597 mutex_enter(&vq
->vq_freelist_lock
);
598 if (list_is_empty(&vq
->vq_freelist
)) {
599 mutex_exit(&vq
->vq_freelist_lock
);
602 qe
= list_remove_head(&vq
->vq_freelist
);
604 ASSERT(vq
->vq_used_entries
>= 0);
605 vq
->vq_used_entries
++;
607 mutex_exit(&vq
->vq_freelist_lock
);
610 qe
->qe_indirect_next
= 0;
611 (void) memset(qe
->qe_desc
, 0, sizeof (struct vring_desc
));
617 vq_free_entry(struct virtqueue
*vq
, struct vq_entry
*qe
)
619 mutex_enter(&vq
->vq_freelist_lock
);
621 list_insert_head(&vq
->vq_freelist
, qe
);
622 vq
->vq_used_entries
--;
623 ASSERT(vq
->vq_used_entries
>= 0);
624 mutex_exit(&vq
->vq_freelist_lock
);
628 * We (intentionally) don't have a global vq mutex, so you are
629 * responsible for external locking to avoid allocting/freeing any
630 * entries before using the returned value. Have fun.
633 vq_num_used(struct virtqueue
*vq
)
635 /* vq->vq_freelist_lock would not help here. */
636 return (vq
->vq_used_entries
);
640 virtio_ve_set_desc(struct vring_desc
*desc
, uint64_t paddr
, uint32_t len
,
648 /* 'write' - from the driver's point of view */
650 desc
->flags
= VRING_DESC_F_WRITE
;
656 virtio_ve_set(struct vq_entry
*qe
, uint64_t paddr
, uint32_t len
,
659 virtio_ve_set_desc(qe
->qe_desc
, paddr
, len
, write
);
663 virtio_ve_add_indirect_buf(struct vq_entry
*qe
, uint64_t paddr
, uint32_t len
,
666 struct vring_desc
*indirect_desc
;
668 ASSERT(qe
->qe_queue
->vq_indirect_num
);
669 ASSERT(qe
->qe_indirect_next
< qe
->qe_queue
->vq_indirect_num
);
671 indirect_desc
= &qe
->qe_indirect_descs
[qe
->qe_indirect_next
];
672 virtio_ve_set_desc(indirect_desc
, paddr
, len
, write
);
673 qe
->qe_indirect_next
++;
677 virtio_ve_add_cookie(struct vq_entry
*qe
, ddi_dma_handle_t dma_handle
,
678 ddi_dma_cookie_t dma_cookie
, unsigned int ncookies
, boolean_t write
)
682 for (i
= 0; i
< ncookies
; i
++) {
683 virtio_ve_add_indirect_buf(qe
, dma_cookie
.dmac_laddress
,
684 dma_cookie
.dmac_size
, write
);
685 ddi_dma_nextcookie(dma_handle
, &dma_cookie
);
690 virtio_sync_vq(struct virtqueue
*vq
)
692 struct virtio_softc
*vsc
= vq
->vq_owner
;
694 /* Make sure the avail ring update hit the buffer */
697 vq
->vq_avail
->idx
= vq
->vq_avail_idx
;
699 /* Make sure the avail idx update hits the buffer */
702 /* Make sure we see the flags update */
705 if (!(vq
->vq_used
->flags
& VRING_USED_F_NO_NOTIFY
))
706 ddi_put16(vsc
->sc_ioh
,
707 /* LINTED E_BAD_PTR_CAST_ALIGN */
708 (uint16_t *)(vsc
->sc_io_addr
+
709 VIRTIO_CONFIG_QUEUE_NOTIFY
),
714 virtio_push_chain(struct vq_entry
*qe
, boolean_t sync
)
716 struct virtqueue
*vq
= qe
->qe_queue
;
717 struct vq_entry
*head
= qe
;
718 struct vring_desc
*desc
;
724 * Bind the descs together, paddr and len should be already
725 * set with virtio_ve_set
728 /* Bind the indirect descriptors */
729 if (qe
->qe_indirect_next
> 1) {
733 * Set the pointer/flags to the
734 * first indirect descriptor
736 virtio_ve_set_desc(qe
->qe_desc
,
737 qe
->qe_indirect_dma_cookie
.dmac_laddress
,
738 sizeof (struct vring_desc
) * qe
->qe_indirect_next
,
740 qe
->qe_desc
->flags
|= VRING_DESC_F_INDIRECT
;
742 /* For all but the last one, add the next index/flag */
744 desc
= &qe
->qe_indirect_descs
[i
];
747 desc
->flags
|= VRING_DESC_F_NEXT
;
749 } while (i
< qe
->qe_indirect_next
- 1);
754 qe
->qe_desc
->flags
|= VRING_DESC_F_NEXT
;
755 qe
->qe_desc
->next
= qe
->qe_next
->qe_index
;
761 mutex_enter(&vq
->vq_avail_lock
);
762 idx
= vq
->vq_avail_idx
;
765 /* Make sure the bits hit the descriptor(s) */
767 vq
->vq_avail
->ring
[idx
% vq
->vq_num
] = head
->qe_index
;
769 /* Notify the device, if needed. */
773 mutex_exit(&vq
->vq_avail_lock
);
776 /* Get a chain of descriptors from the used ring, if one is available. */
778 virtio_pull_chain(struct virtqueue
*vq
, uint32_t *len
)
780 struct vq_entry
*head
;
784 mutex_enter(&vq
->vq_used_lock
);
786 /* No used entries? Bye. */
787 if (vq
->vq_used_idx
== vq
->vq_used
->idx
) {
788 mutex_exit(&vq
->vq_used_lock
);
792 usedidx
= vq
->vq_used_idx
;
794 mutex_exit(&vq
->vq_used_lock
);
796 usedidx
%= vq
->vq_num
;
798 /* Make sure we do the next step _after_ checking the idx. */
801 slot
= vq
->vq_used
->ring
[usedidx
].id
;
802 *len
= vq
->vq_used
->ring
[usedidx
].len
;
804 head
= &vq
->vq_entries
[slot
];
810 virtio_free_chain(struct vq_entry
*qe
)
812 struct vq_entry
*tmp
;
813 struct virtqueue
*vq
= qe
->qe_queue
;
818 ASSERT(qe
->qe_queue
== vq
);
820 vq_free_entry(vq
, qe
);
826 virtio_ventry_stick(struct vq_entry
*first
, struct vq_entry
*second
)
828 first
->qe_next
= second
;
832 virtio_register_msi(struct virtio_softc
*sc
,
833 struct virtio_int_handler
*config_handler
,
834 struct virtio_int_handler vq_handlers
[],
843 /* If both MSI and MSI-x are reported, prefer MSI-x. */
844 int_type
= DDI_INTR_TYPE_MSI
;
845 if (intr_types
& DDI_INTR_TYPE_MSIX
)
846 int_type
= DDI_INTR_TYPE_MSIX
;
848 /* Walk the handler table to get the number of handlers. */
849 for (handler_count
= 0;
850 vq_handlers
&& vq_handlers
[handler_count
].vh_func
;
854 /* +1 if there is a config change handler. */
858 /* Number of MSIs supported by the device. */
859 ret
= ddi_intr_get_nintrs(sc
->sc_dev
, int_type
, &count
);
860 if (ret
!= DDI_SUCCESS
) {
861 dev_err(sc
->sc_dev
, CE_WARN
, "ddi_intr_get_nintrs failed");
866 * Those who try to register more handlers then the device
867 * supports shall suffer.
869 ASSERT(handler_count
<= count
);
871 sc
->sc_intr_htable
= kmem_zalloc(
872 sizeof (ddi_intr_handle_t
) * handler_count
, KM_SLEEP
);
874 ret
= ddi_intr_alloc(sc
->sc_dev
, sc
->sc_intr_htable
, int_type
, 0,
875 handler_count
, &actual
, DDI_INTR_ALLOC_NORMAL
);
876 if (ret
!= DDI_SUCCESS
) {
877 dev_err(sc
->sc_dev
, CE_WARN
, "Failed to allocate MSI: %d", ret
);
881 if (actual
!= handler_count
) {
882 dev_err(sc
->sc_dev
, CE_WARN
,
883 "Not enough MSI available: need %d, available %d",
884 handler_count
, actual
);
885 goto out_msi_available
;
888 sc
->sc_intr_num
= handler_count
;
889 sc
->sc_intr_config
= B_FALSE
;
890 if (config_handler
) {
891 sc
->sc_intr_config
= B_TRUE
;
894 /* Assume they are all same priority */
895 ret
= ddi_intr_get_pri(sc
->sc_intr_htable
[0], &sc
->sc_intr_prio
);
896 if (ret
!= DDI_SUCCESS
) {
897 dev_err(sc
->sc_dev
, CE_WARN
, "ddi_intr_get_pri failed");
901 /* Add the vq handlers */
902 for (i
= 0; vq_handlers
[i
].vh_func
; i
++) {
903 ret
= ddi_intr_add_handler(sc
->sc_intr_htable
[i
],
904 vq_handlers
[i
].vh_func
,
905 sc
, vq_handlers
[i
].vh_priv
);
906 if (ret
!= DDI_SUCCESS
) {
907 dev_err(sc
->sc_dev
, CE_WARN
,
908 "ddi_intr_add_handler failed");
909 /* Remove the handlers that succeeded. */
911 (void) ddi_intr_remove_handler(
912 sc
->sc_intr_htable
[i
]);
914 goto out_add_handlers
;
918 /* Don't forget the config handler */
919 if (config_handler
) {
920 ret
= ddi_intr_add_handler(sc
->sc_intr_htable
[i
],
921 config_handler
->vh_func
,
922 sc
, config_handler
->vh_priv
);
923 if (ret
!= DDI_SUCCESS
) {
924 dev_err(sc
->sc_dev
, CE_WARN
,
925 "ddi_intr_add_handler failed");
926 /* Remove the handlers that succeeded. */
928 (void) ddi_intr_remove_handler(
929 sc
->sc_intr_htable
[i
]);
931 goto out_add_handlers
;
935 /* We know we are using MSI, so set the config offset. */
936 sc
->sc_config_offset
= VIRTIO_CONFIG_DEVICE_CONFIG_MSI
;
938 ret
= ddi_intr_get_cap(sc
->sc_intr_htable
[0],
941 if (ret
!= DDI_SUCCESS
)
947 for (i
= 0; i
< actual
; i
++)
948 (void) ddi_intr_free(sc
->sc_intr_htable
[i
]);
950 kmem_free(sc
->sc_intr_htable
, sizeof (ddi_intr_handle_t
) * count
);
955 struct virtio_handler_container
{
957 struct virtio_int_handler config_handler
;
958 struct virtio_int_handler vq_handlers
[];
962 virtio_intx_dispatch(caddr_t arg1
, caddr_t arg2
)
964 struct virtio_softc
*sc
= (void *)arg1
;
965 struct virtio_handler_container
*vhc
= (void *)arg2
;
969 isr_status
= ddi_get8(sc
->sc_ioh
, (uint8_t *)(sc
->sc_io_addr
+
970 VIRTIO_CONFIG_ISR_STATUS
));
973 return (DDI_INTR_UNCLAIMED
);
975 if ((isr_status
& VIRTIO_CONFIG_ISR_CONFIG_CHANGE
) &&
976 vhc
->config_handler
.vh_func
) {
977 vhc
->config_handler
.vh_func((void *)sc
,
978 vhc
->config_handler
.vh_priv
);
981 /* Notify all handlers */
982 for (i
= 0; i
< vhc
->nhandlers
; i
++) {
983 vhc
->vq_handlers
[i
].vh_func((void *)sc
,
984 vhc
->vq_handlers
[i
].vh_priv
);
987 return (DDI_INTR_CLAIMED
);
991 * config_handler and vq_handlers may be allocated on stack.
992 * Take precautions not to loose them.
995 virtio_register_intx(struct virtio_softc
*sc
,
996 struct virtio_int_handler
*config_handler
,
997 struct virtio_int_handler vq_handlers
[])
999 int vq_handler_count
;
1000 int config_handler_count
= 0;
1002 struct virtio_handler_container
*vhc
;
1003 int ret
= DDI_FAILURE
;
1005 /* Walk the handler table to get the number of handlers. */
1006 for (vq_handler_count
= 0;
1007 vq_handlers
&& vq_handlers
[vq_handler_count
].vh_func
;
1012 config_handler_count
= 1;
1014 vhc
= kmem_zalloc(sizeof (struct virtio_handler_container
) +
1015 sizeof (struct virtio_int_handler
) * vq_handler_count
,
1018 vhc
->nhandlers
= vq_handler_count
;
1019 (void) memcpy(vhc
->vq_handlers
, vq_handlers
,
1020 sizeof (struct virtio_int_handler
) * vq_handler_count
);
1022 if (config_handler
) {
1023 (void) memcpy(&vhc
->config_handler
, config_handler
,
1024 sizeof (struct virtio_int_handler
));
1027 /* Just a single entry for a single interrupt. */
1028 sc
->sc_intr_htable
= kmem_zalloc(sizeof (ddi_intr_handle_t
), KM_SLEEP
);
1030 ret
= ddi_intr_alloc(sc
->sc_dev
, sc
->sc_intr_htable
,
1031 DDI_INTR_TYPE_FIXED
, 0, 1, &actual
,
1032 DDI_INTR_ALLOC_NORMAL
);
1033 if (ret
!= DDI_SUCCESS
) {
1034 dev_err(sc
->sc_dev
, CE_WARN
,
1035 "Failed to allocate a fixed interrupt: %d", ret
);
1039 ASSERT(actual
== 1);
1040 sc
->sc_intr_num
= 1;
1042 ret
= ddi_intr_get_pri(sc
->sc_intr_htable
[0], &sc
->sc_intr_prio
);
1043 if (ret
!= DDI_SUCCESS
) {
1044 dev_err(sc
->sc_dev
, CE_WARN
, "ddi_intr_get_pri failed");
1048 ret
= ddi_intr_add_handler(sc
->sc_intr_htable
[0],
1049 virtio_intx_dispatch
, sc
, vhc
);
1050 if (ret
!= DDI_SUCCESS
) {
1051 dev_err(sc
->sc_dev
, CE_WARN
, "ddi_intr_add_handler failed");
1052 goto out_add_handlers
;
1055 /* We know we are not using MSI, so set the config offset. */
1056 sc
->sc_config_offset
= VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI
;
1058 return (DDI_SUCCESS
);
1062 (void) ddi_intr_free(sc
->sc_intr_htable
[0]);
1064 kmem_free(sc
->sc_intr_htable
, sizeof (ddi_intr_handle_t
));
1065 kmem_free(vhc
, sizeof (struct virtio_int_handler
) *
1066 (vq_handler_count
+ config_handler_count
));
1071 * We find out if we support MSI during this, and the register layout
1072 * depends on the MSI (doh). Don't acces the device specific bits in
1073 * BAR 0 before calling it!
1076 virtio_register_ints(struct virtio_softc
*sc
,
1077 struct virtio_int_handler
*config_handler
,
1078 struct virtio_int_handler vq_handlers
[])
1083 /* Determine which types of interrupts are supported */
1084 ret
= ddi_intr_get_supported_types(sc
->sc_dev
, &intr_types
);
1085 if (ret
!= DDI_SUCCESS
) {
1086 dev_err(sc
->sc_dev
, CE_WARN
, "Can't get supported int types");
1090 /* If we have msi, let's use them. */
1091 if (intr_types
& (DDI_INTR_TYPE_MSIX
| DDI_INTR_TYPE_MSI
)) {
1092 ret
= virtio_register_msi(sc
, config_handler
,
1093 vq_handlers
, intr_types
);
1098 /* Fall back to old-fashioned interrupts. */
1099 if (intr_types
& DDI_INTR_TYPE_FIXED
) {
1100 dev_debug(sc
->sc_dev
, CE_WARN
,
1101 "Using legacy interrupts");
1103 return (virtio_register_intx(sc
, config_handler
, vq_handlers
));
1106 dev_err(sc
->sc_dev
, CE_WARN
,
1107 "MSI failed and fixed interrupts not supported. Giving up.");
1116 virtio_enable_msi(struct virtio_softc
*sc
)
1119 int vq_handler_count
= sc
->sc_intr_num
;
1121 /* Number of handlers, not counting the counfig. */
1122 if (sc
->sc_intr_config
)
1125 /* Enable the iterrupts. Either the whole block, or one by one. */
1126 if (sc
->sc_intr_cap
& DDI_INTR_FLAG_BLOCK
) {
1127 ret
= ddi_intr_block_enable(sc
->sc_intr_htable
,
1129 if (ret
!= DDI_SUCCESS
) {
1130 dev_err(sc
->sc_dev
, CE_WARN
,
1131 "Failed to enable MSI, falling back to INTx");
1135 for (i
= 0; i
< sc
->sc_intr_num
; i
++) {
1136 ret
= ddi_intr_enable(sc
->sc_intr_htable
[i
]);
1137 if (ret
!= DDI_SUCCESS
) {
1138 dev_err(sc
->sc_dev
, CE_WARN
,
1139 "Failed to enable MSI %d, "
1140 "falling back to INTx", i
);
1143 (void) ddi_intr_disable(
1144 sc
->sc_intr_htable
[i
]);
1151 /* Bind the allocated MSI to the queues and config */
1152 for (i
= 0; i
< vq_handler_count
; i
++) {
1154 ddi_put16(sc
->sc_ioh
,
1155 /* LINTED E_BAD_PTR_CAST_ALIGN */
1156 (uint16_t *)(sc
->sc_io_addr
+
1157 VIRTIO_CONFIG_QUEUE_SELECT
), i
);
1159 ddi_put16(sc
->sc_ioh
,
1160 /* LINTED E_BAD_PTR_CAST_ALIGN */
1161 (uint16_t *)(sc
->sc_io_addr
+
1162 VIRTIO_CONFIG_QUEUE_VECTOR
), i
);
1164 check
= ddi_get16(sc
->sc_ioh
,
1165 /* LINTED E_BAD_PTR_CAST_ALIGN */
1166 (uint16_t *)(sc
->sc_io_addr
+
1167 VIRTIO_CONFIG_QUEUE_VECTOR
));
1169 dev_err(sc
->sc_dev
, CE_WARN
, "Failed to bind handler"
1170 "for VQ %d, MSI %d. Check = %x", i
, i
, check
);
1176 if (sc
->sc_intr_config
) {
1178 ddi_put16(sc
->sc_ioh
,
1179 /* LINTED E_BAD_PTR_CAST_ALIGN */
1180 (uint16_t *)(sc
->sc_io_addr
+
1181 VIRTIO_CONFIG_CONFIG_VECTOR
), i
);
1183 check
= ddi_get16(sc
->sc_ioh
,
1184 /* LINTED E_BAD_PTR_CAST_ALIGN */
1185 (uint16_t *)(sc
->sc_io_addr
+
1186 VIRTIO_CONFIG_CONFIG_VECTOR
));
1188 dev_err(sc
->sc_dev
, CE_WARN
, "Failed to bind handler "
1189 "for Config updates, MSI %d", i
);
1195 return (DDI_SUCCESS
);
1198 /* Unbind the vqs */
1199 for (i
= 0; i
< vq_handler_count
- 1; i
++) {
1200 ddi_put16(sc
->sc_ioh
,
1201 /* LINTED E_BAD_PTR_CAST_ALIGN */
1202 (uint16_t *)(sc
->sc_io_addr
+
1203 VIRTIO_CONFIG_QUEUE_SELECT
), i
);
1205 ddi_put16(sc
->sc_ioh
,
1206 /* LINTED E_BAD_PTR_CAST_ALIGN */
1207 (uint16_t *)(sc
->sc_io_addr
+
1208 VIRTIO_CONFIG_QUEUE_VECTOR
),
1209 VIRTIO_MSI_NO_VECTOR
);
1211 /* And the config */
1212 /* LINTED E_BAD_PTR_CAST_ALIGN */
1213 ddi_put16(sc
->sc_ioh
, (uint16_t *)(sc
->sc_io_addr
+
1214 VIRTIO_CONFIG_CONFIG_VECTOR
), VIRTIO_MSI_NO_VECTOR
);
1222 static int virtio_enable_intx(struct virtio_softc
*sc
)
1226 ret
= ddi_intr_enable(sc
->sc_intr_htable
[0]);
1227 if (ret
!= DDI_SUCCESS
)
1228 dev_err(sc
->sc_dev
, CE_WARN
,
1229 "Failed to enable interrupt: %d", ret
);
1234 * We can't enable/disable individual handlers in the INTx case so do
1235 * the whole bunch even in the msi case.
1238 virtio_enable_ints(struct virtio_softc
*sc
)
1241 /* See if we are using MSI. */
1242 if (sc
->sc_config_offset
== VIRTIO_CONFIG_DEVICE_CONFIG_MSI
)
1243 return (virtio_enable_msi(sc
));
1245 ASSERT(sc
->sc_config_offset
== VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI
);
1247 return (virtio_enable_intx(sc
));
1251 virtio_release_ints(struct virtio_softc
*sc
)
1256 /* We were running with MSI, unbind them. */
1257 if (sc
->sc_config_offset
== VIRTIO_CONFIG_DEVICE_CONFIG_MSI
) {
1258 /* Unbind all vqs */
1259 for (i
= 0; i
< sc
->sc_nvqs
; i
++) {
1260 ddi_put16(sc
->sc_ioh
,
1261 /* LINTED E_BAD_PTR_CAST_ALIGN */
1262 (uint16_t *)(sc
->sc_io_addr
+
1263 VIRTIO_CONFIG_QUEUE_SELECT
), i
);
1265 ddi_put16(sc
->sc_ioh
,
1266 /* LINTED E_BAD_PTR_CAST_ALIGN */
1267 (uint16_t *)(sc
->sc_io_addr
+
1268 VIRTIO_CONFIG_QUEUE_VECTOR
),
1269 VIRTIO_MSI_NO_VECTOR
);
1271 /* And the config */
1272 /* LINTED E_BAD_PTR_CAST_ALIGN */
1273 ddi_put16(sc
->sc_ioh
, (uint16_t *)(sc
->sc_io_addr
+
1274 VIRTIO_CONFIG_CONFIG_VECTOR
),
1275 VIRTIO_MSI_NO_VECTOR
);
1279 /* Disable the iterrupts. Either the whole block, or one by one. */
1280 if (sc
->sc_intr_cap
& DDI_INTR_FLAG_BLOCK
) {
1281 ret
= ddi_intr_block_disable(sc
->sc_intr_htable
,
1283 if (ret
!= DDI_SUCCESS
) {
1284 dev_err(sc
->sc_dev
, CE_WARN
,
1285 "Failed to disable MSIs, won't be able to"
1289 for (i
= 0; i
< sc
->sc_intr_num
; i
++) {
1290 ret
= ddi_intr_disable(sc
->sc_intr_htable
[i
]);
1291 if (ret
!= DDI_SUCCESS
) {
1292 dev_err(sc
->sc_dev
, CE_WARN
,
1293 "Failed to disable interrupt %d, "
1294 "won't be able to reuse", i
);
1301 for (i
= 0; i
< sc
->sc_intr_num
; i
++) {
1302 (void) ddi_intr_remove_handler(sc
->sc_intr_htable
[i
]);
1305 for (i
= 0; i
< sc
->sc_intr_num
; i
++)
1306 (void) ddi_intr_free(sc
->sc_intr_htable
[i
]);
1308 kmem_free(sc
->sc_intr_htable
,
1309 sizeof (ddi_intr_handle_t
) * sc
->sc_intr_num
);
1312 /* After disabling interrupts, the config offset is non-MSI. */
1313 sc
->sc_config_offset
= VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI
;
1317 * Module linkage information for the kernel.
1319 static struct modlmisc modlmisc
= {
1320 &mod_miscops
, /* Type of module */
1321 "VirtIO common library module",
1324 static struct modlinkage modlinkage
= {
1335 return (mod_install(&modlinkage
));
1341 return (mod_remove(&modlinkage
));
1345 _info(struct modinfo
*modinfop
)
1347 return (mod_info(&modlinkage
, modinfop
));