?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
virtio - Get rid of unused flags option in virtio_alloc_virtqueues().
[dragonfly.git] / sys / dev / virtual / virtio / pci / virtio_pci.c
blobfd50ccda7d2d3af6a22be7c0e0a93587afb68895
1 /*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * $FreeBSD: src/sys/dev/virtio/pci/virtio_pci.c,v 1.3 2012/04/14 05:48:04 grehan Exp $
27 */
28
29 /* Driver for the VirtIO PCI interface. */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/bus.h>
34 #include <sys/kernel.h>
35 #include <sys/module.h>
36 #include <sys/malloc.h>
37 #include <sys/serialize.h>
38
39 #include <bus/pci/pcivar.h>
40 #include <bus/pci/pcireg.h>
41
42 #include <sys/rman.h>
43
44 #include <dev/virtual/virtio/virtio/virtio.h>
45 #include <dev/virtual/virtio/virtio/virtqueue.h>
46 #include "virtio_pci.h"
47 #include "virtio_bus_if.h"
48
49 struct vqentry {
50 int what;
51 struct virtqueue *vq;
52 driver_intr_t *handler;
53 void *arg;
54 TAILQ_ENTRY(vqentry) entries;
55 };
56
57 TAILQ_HEAD(vqirq_list, vqentry);
58
59 struct vtpci_softc {
60 device_t vtpci_dev;
61 struct resource *vtpci_res;
62 struct resource *vtpci_msix_res;
63 uint64_t vtpci_features;
64 uint32_t vtpci_flags;
65 #define VIRTIO_PCI_FLAG_MSI 0x0001
66 #define VIRTIO_PCI_FLAG_MSIX 0x0010
67
68 device_t vtpci_child_dev;
69 struct virtio_feature_desc *vtpci_child_feat_desc;
70
71 /*
72 * Ideally, each virtqueue that the driver provides a callback for
73 * will receive its own MSIX vector. If there are not sufficient
74 * vectors available, we will then attempt to have all the VQs
75 * share one vector. Note that when using MSIX, the configuration
76 * changed notifications must be on their own vector.
77 *
78 * If MSIX is not available, we will attempt to have the whole
79 * device share one MSI vector, and then, finally, one legacy
80 * interrupt.
81 */
82 int vtpci_nvqs;
83 struct vtpci_virtqueue {
84 struct virtqueue *vq;
85
86 /* Index into vtpci_intr_res[] below. -1 if no IRQ assigned. */
87 int ires_idx;
88 } vtpci_vqx[VIRTIO_MAX_VIRTQUEUES];
89
90 /*
91 * When using MSIX interrupts, the first element of vtpci_intr_res[]
92 * is always the configuration changed notifications. The remaining
93 * element(s) are used for the virtqueues.
94 *
95 * With MSI and legacy interrupts, only the first element of
96 * vtpci_intr_res[] is used.
97 */
98 int vtpci_nintr_res;
99 int vtpci_irq_flags;
100 struct vtpci_intr_resource {
101 struct vtpci_softc *ires_sc;
102 struct resource *irq;
103 int rid;
104 void *intrhand;
105 struct vqirq_list ls;
106 } vtpci_intr_res[1 + VIRTIO_MAX_VIRTQUEUES];
107
108 int vtpci_config_irq;
109 };
110
111 static int vtpci_probe(device_t);
112 static int vtpci_attach(device_t);
113 static int vtpci_detach(device_t);
114 static int vtpci_suspend(device_t);
115 static int vtpci_resume(device_t);
116 static int vtpci_shutdown(device_t);
117 static void vtpci_driver_added(device_t, driver_t *);
118 static void vtpci_child_detached(device_t, device_t);
119 static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
120 static int vtpci_write_ivar(device_t, device_t, int, uintptr_t);
121
122 static uint64_t vtpci_negotiate_features(device_t, uint64_t);
123 static int vtpci_with_feature(device_t, uint64_t);
124 static int vtpci_intr_count(device_t dev);
125 static int vtpci_intr_alloc(device_t dev, int *cnt, int use_config,
126 int *cpus);
127 static int vtpci_intr_release(device_t dev);
128 static int vtpci_alloc_virtqueues(device_t, int, struct vq_alloc_info *);
129 static int vtpci_setup_intr(device_t, uint irq, lwkt_serialize_t);
130 static int vtpci_teardown_intr(device_t, uint irq);
131 static int vtpci_bind_intr(device_t, uint, int, driver_intr_t, void *);
132 static int vtpci_unbind_intr(device_t, int);
133 static void vtpci_stop(device_t);
134 static int vtpci_reinit(device_t, uint64_t);
135 static void vtpci_reinit_complete(device_t);
136 static void vtpci_notify_virtqueue(device_t, uint16_t);
137 static uint8_t vtpci_get_status(device_t);
138 static void vtpci_set_status(device_t, uint8_t);
139 static void vtpci_read_dev_config(device_t, bus_size_t, void *, int);
140 static void vtpci_write_dev_config(device_t, bus_size_t, void *, int);
141
142 static void vtpci_describe_features(struct vtpci_softc *, const char *,
143 uint64_t);
144 static void vtpci_probe_and_attach_child(struct vtpci_softc *);
145
146 static int vtpci_register_msix_vector(struct vtpci_softc *, int, int);
147
148 static void vtpci_free_interrupts(struct vtpci_softc *);
149 static void vtpci_free_virtqueues(struct vtpci_softc *);
150 static void vtpci_release_child_resources(struct vtpci_softc *);
151 static void vtpci_reset(struct vtpci_softc *);
152
153 static void vtpci_legacy_intr(void *);
154 static void vtpci_msix_intr(void *);
155
156 /*
157 * I/O port read/write wrappers.
158 */
159 #define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
160 #define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
161 #define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
162 #define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v))
163 #define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
164 #define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))
165
166 /* Tunables. */
167 static int vtpci_disable_msix = 0;
168 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
169
170 static device_method_t vtpci_methods[] = {
171 /* Device interface. */
172 DEVMETHOD(device_probe, vtpci_probe),
173 DEVMETHOD(device_attach, vtpci_attach),
174 DEVMETHOD(device_detach, vtpci_detach),
175 DEVMETHOD(device_suspend, vtpci_suspend),
176 DEVMETHOD(device_resume, vtpci_resume),
177 DEVMETHOD(device_shutdown, vtpci_shutdown),
178
179 /* Bus interface. */
180 DEVMETHOD(bus_driver_added, vtpci_driver_added),
181 DEVMETHOD(bus_child_detached, vtpci_child_detached),
182 DEVMETHOD(bus_read_ivar, vtpci_read_ivar),
183 DEVMETHOD(bus_write_ivar, vtpci_write_ivar),
184
185 /* VirtIO bus interface. */
186 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features),
187 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature),
188 DEVMETHOD(virtio_bus_intr_count, vtpci_intr_count),
189 DEVMETHOD(virtio_bus_intr_alloc, vtpci_intr_alloc),
190 DEVMETHOD(virtio_bus_intr_release, vtpci_intr_release),
191 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues),
192 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr),
193 DEVMETHOD(virtio_bus_teardown_intr, vtpci_teardown_intr),
194 DEVMETHOD(virtio_bus_bind_intr, vtpci_bind_intr),
195 DEVMETHOD(virtio_bus_unbind_intr, vtpci_unbind_intr),
196 DEVMETHOD(virtio_bus_stop, vtpci_stop),
197 DEVMETHOD(virtio_bus_reinit, vtpci_reinit),
198 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete),
199 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue),
200 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config),
201 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
202
203 DEVMETHOD_END
204 };
205
206 static driver_t vtpci_driver = {
207 "virtio_pci",
208 vtpci_methods,
209 sizeof(struct vtpci_softc)
210 };
211
212 devclass_t vtpci_devclass;
213
214 DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, NULL, NULL);
215 MODULE_VERSION(virtio_pci, 1);
216 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
217 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
218
219 static int
220 vtpci_probe(device_t dev)
221 {
222 char desc[36];
223 const char *name;
224
225 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
226 return (ENXIO);
227
228 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
229 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
230 return (ENXIO);
231
232 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
233 return (ENXIO);
234
235 name = virtio_device_name(pci_get_subdevice(dev));
236 if (name == NULL)
237 name = "Unknown";
238
239 ksnprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
240 device_set_desc_copy(dev, desc);
241
242 return (BUS_PROBE_DEFAULT);
243 }
244
245 static int
246 vtpci_attach(device_t dev)
247 {
248 struct vtpci_softc *sc;
249 device_t child;
250 int msix_cap, rid;
251
252 sc = device_get_softc(dev);
253 sc->vtpci_dev = dev;
254 sc->vtpci_config_irq = -1;
255
256 pci_enable_busmaster(dev);
257
258 rid = PCIR_BAR(0);
259 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
260 RF_ACTIVE);
261 if (sc->vtpci_res == NULL) {
262 device_printf(dev, "cannot map I/O space\n");
263 return (ENXIO);
264 }
265
266 if (pci_find_extcap(dev, PCIY_MSIX, &msix_cap) == 0) {
267 uint32_t val;
268 val = pci_read_config(dev, msix_cap + PCIR_MSIX_TABLE, 4);
269 rid = PCIR_BAR(val & PCIM_MSIX_BIR_MASK);
270 sc->vtpci_msix_res = bus_alloc_resource_any(dev,
271 SYS_RES_MEMORY, &rid, RF_ACTIVE);
272 }
273
274 vtpci_reset(sc);
275
276 /* Tell the host we've noticed this device. */
277 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
278
279 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
280 device_printf(dev, "cannot create child device\n");
281 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
282 vtpci_detach(dev);
283 return (ENOMEM);
284 }
285
286 sc->vtpci_child_dev = child;
287 vtpci_probe_and_attach_child(sc);
288
289 return (0);
290 }
291
292 static int
293 vtpci_detach(device_t dev)
294 {
295 struct vtpci_softc *sc;
296 device_t child;
297 int error;
298
299 sc = device_get_softc(dev);
300
301 if ((child = sc->vtpci_child_dev) != NULL) {
302 error = device_delete_child(dev, child);
303 if (error)
304 return (error);
305 sc->vtpci_child_dev = NULL;
306 }
307
308 vtpci_reset(sc);
309
310 if (sc->vtpci_msix_res != NULL) {
311 bus_release_resource(dev, SYS_RES_MEMORY,
312 rman_get_rid(sc->vtpci_msix_res), sc->vtpci_msix_res);
313 sc->vtpci_msix_res = NULL;
314 }
315
316 if (sc->vtpci_res != NULL) {
317 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
318 sc->vtpci_res);
319 sc->vtpci_res = NULL;
320 }
321
322 return (0);
323 }
324
325 static int
326 vtpci_suspend(device_t dev)
327 {
328
329 return (bus_generic_suspend(dev));
330 }
331
332 static int
333 vtpci_resume(device_t dev)
334 {
335
336 return (bus_generic_resume(dev));
337 }
338
339 static int
340 vtpci_shutdown(device_t dev)
341 {
342
343 (void) bus_generic_shutdown(dev);
344 /* Forcibly stop the host device. */
345 vtpci_stop(dev);
346
347 return (0);
348 }
349
350 static void
351 vtpci_driver_added(device_t dev, driver_t *driver)
352 {
353 struct vtpci_softc *sc;
354
355 sc = device_get_softc(dev);
356
357 vtpci_probe_and_attach_child(sc);
358 }
359
360 static void
361 vtpci_child_detached(device_t dev, device_t child)
362 {
363 struct vtpci_softc *sc;
364
365 sc = device_get_softc(dev);
366
367 vtpci_reset(sc);
368 vtpci_release_child_resources(sc);
369 }
370
371 static int
372 vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
373 {
374 struct vtpci_softc *sc;
375
376 sc = device_get_softc(dev);
377
378 if (sc->vtpci_child_dev != child)
379 return (ENOENT);
380
381 switch (index) {
382 case VIRTIO_IVAR_DEVTYPE:
383 *result = pci_get_subdevice(dev);
384 break;
385 default:
386 return (ENOENT);
387 }
388
389 return (0);
390 }
391
392 static int
393 vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
394 {
395 struct vtpci_softc *sc;
396
397 sc = device_get_softc(dev);
398
399 if (sc->vtpci_child_dev != child)
400 return (ENOENT);
401
402 switch (index) {
403 case VIRTIO_IVAR_FEATURE_DESC:
404 sc->vtpci_child_feat_desc = (void *) value;
405 break;
406 default:
407 return (ENOENT);
408 }
409
410 return (0);
411 }
412
413 static uint64_t
414 vtpci_negotiate_features(device_t dev, uint64_t child_features)
415 {
416 struct vtpci_softc *sc;
417 uint64_t host_features, features;
418
419 sc = device_get_softc(dev);
420
421 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
422 vtpci_describe_features(sc, "host", host_features);
423
424 /*
425 * Limit negotiated features to what the driver, virtqueue, and
426 * host all support.
427 */
428 features = host_features & child_features;
429 features = virtqueue_filter_features(features);
430 sc->vtpci_features = features;
431
432 vtpci_describe_features(sc, "negotiated", features);
433 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
434
435 return (features);
436 }
437
438 static int
439 vtpci_with_feature(device_t dev, uint64_t feature)
440 {
441 struct vtpci_softc *sc;
442
443 sc = device_get_softc(dev);
444
445 return ((sc->vtpci_features & feature) != 0);
446 }
447
448 static int
449 vtpci_intr_count(device_t dev)
450 {
451 struct vtpci_softc *sc = device_get_softc(dev);
452
453 if (vtpci_disable_msix != 0 || sc->vtpci_msix_res == NULL)
454 return 1;
455 else
456 return pci_msix_count(dev);
457 }
458
459 /* Will never return 0, with *cnt <= 0. */
460 static int
461 vtpci_intr_alloc(device_t dev, int *cnt, int use_config, int *cpus)
462 {
463 struct vtpci_softc *sc = device_get_softc(dev);
464 int i;
465
466 if (sc->vtpci_nintr_res > 0)
467 return (EINVAL);
468
469 if (*cnt <= 0)
470 return (EINVAL);
471
472 if (vtpci_disable_msix == 0 && sc->vtpci_msix_res != NULL) {
473 int nmsix = pci_msix_count(dev);
474 if (nmsix < *cnt)
475 *cnt = nmsix;
476 }
477
478 if ((*cnt > 1 || use_config == 0) &&
479 vtpci_disable_msix == 0 && sc->vtpci_msix_res != NULL) {
480 if (pci_setup_msix(dev) != 0) {
481 device_printf(dev, "pci_setup_msix failed\n");
482 /* Just fallthrough to legacy IRQ code instead. */
483 } else {
484 for (i = 0; i < *cnt; i++) {
485 int cpu, rid;
486
487 if (cpus != NULL && cpus[i] >= 0 &&
488 cpus[i] < ncpus) {
489 cpu = cpus[i];
490 } else {
491 cpu = device_get_unit(dev) + i;
492 cpu %= ncpus;
493 }
494 if (pci_alloc_msix_vector(dev, i, &rid, cpu)
495 != 0) {
496 if (i > 1 || (i == 1 && !use_config)) {
497 *cnt = i;
498 /* Got some MSI-X vectors. */
499 sc->vtpci_irq_flags = RF_ACTIVE;
500 sc->vtpci_flags |=
501 VIRTIO_PCI_FLAG_MSIX;
502 goto finish;
503 }
504 /*
505 * Allocate the legacy IRQ instead.
506 */
507 if (i == 1) {
508 pci_release_msix_vector(dev, 0);
509 }
510 pci_teardown_msix(dev);
511 break;
512 }
513 sc->vtpci_intr_res[i].rid = rid;
514 }
515 /* Got all the MSI-X vectors we wanted. */
516 sc->vtpci_irq_flags = RF_ACTIVE;
517 sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSIX;
518 /* Successfully allocated all MSI-X vectors */
519 goto finish;
520 }
521 }
522
523 /* Legacy IRQ code: */
524 *cnt = 1;
525 /*
526 * Use MSI interrupts if available. Otherwise, we fallback
527 * to legacy interrupts.
528 */
529 sc->vtpci_intr_res[0].rid = 0;
530 if (pci_alloc_1intr(sc->vtpci_dev, 1,
531 &sc->vtpci_intr_res[0].rid,
532 &sc->vtpci_irq_flags) == PCI_INTR_TYPE_MSI) {
533 sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSI;
534 }
535
536 finish:
537 KKASSERT(!((sc->vtpci_flags & VIRTIO_PCI_FLAG_MSI) != 0 &&
538 (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) != 0));
539
540 sc->vtpci_nintr_res = *cnt;
541 for (i = 0; i < sc->vtpci_nintr_res; i++) {
542 struct resource *irq;
543
544 TAILQ_INIT(&sc->vtpci_intr_res[i].ls);
545 sc->vtpci_intr_res[i].ires_sc = sc;
546 irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
547 &sc->vtpci_intr_res[i].rid, sc->vtpci_irq_flags);
548 if (irq == NULL)
549 return (ENXIO);
550 if (cpus != NULL)
551 cpus[i] = rman_get_cpuid(irq);
552
553 sc->vtpci_intr_res[i].irq = irq;
554 }
555
556 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
557 device_printf(dev, "using %d MSI-X vectors\n", *cnt);
558 pci_enable_msix(dev);
559 }
560
561 return (0);
562 }
563
564 static int
565 vtpci_intr_release(device_t dev)
566 {
567 struct vtpci_softc *sc = device_get_softc(dev);
568 struct vtpci_intr_resource *ires;
569 int i;
570
571 if (sc->vtpci_nintr_res == 0)
572 return (EINVAL);
573
574 /* XXX Make sure none of the interrupts is used at the moment. */
575
576 for (i = 0; i < sc->vtpci_nintr_res; i++) {
577 ires = &sc->vtpci_intr_res[i];
578
579 KKASSERT(TAILQ_EMPTY(&ires->ls));
580 if (ires->irq != NULL) {
581 bus_release_resource(dev, SYS_RES_IRQ, ires->rid,
582 ires->irq);
583 ires->irq = NULL;
584 }
585 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX)
586 pci_release_msix_vector(dev, ires->rid);
587 ires->rid = 0;
588 }
589 sc->vtpci_nintr_res = 0;
590 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSI) {
591 pci_release_msi(dev);
592 sc->vtpci_flags &= ~VIRTIO_PCI_FLAG_MSI;
593 }
594 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
595 pci_teardown_msix(dev);
596 sc->vtpci_flags &= ~VIRTIO_PCI_FLAG_MSIX;
597 }
598 return (0);
599 }
600
601 static int
602 vtpci_alloc_virtqueues(device_t dev, int nvqs, struct vq_alloc_info *vq_info)
603 {
604 struct vtpci_softc *sc;
605 struct vtpci_virtqueue *vqx;
606 struct vq_alloc_info *info;
607 int queue, error;
608 uint16_t vq_size;
609
610 sc = device_get_softc(dev);
611
612 if (sc->vtpci_nvqs != 0 || nvqs <= 0 ||
613 nvqs > VIRTIO_MAX_VIRTQUEUES)
614 return (EINVAL);
615
616 for (queue = 0; queue < nvqs; queue++) {
617 vqx = &sc->vtpci_vqx[queue];
618 info = &vq_info[queue];
619
620 vqx->ires_idx = -1;
621 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue);
622
623 vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
624 error = virtqueue_alloc(dev, queue, vq_size,
625 VIRTIO_PCI_VRING_ALIGN, 0xFFFFFFFFUL, info, &vqx->vq);
626 if (error)
627 return (error);
628
629 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
630 virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
631
632 *info->vqai_vq = vqx->vq;
633 sc->vtpci_nvqs++;
634 }
635
636 return (0);
637 }
638
639 /* XXX Add argument to specify the callback function here. */
640 static int
641 vtpci_setup_intr(device_t dev, uint irq, lwkt_serialize_t slz)
642 {
643 struct vtpci_softc *sc;
644 struct vtpci_intr_resource *ires;
645 int flags, error;
646
647 sc = device_get_softc(dev);
648 flags = INTR_MPSAFE;
649
650 if ((int)irq >= sc->vtpci_nintr_res)
651 return (EINVAL);
652 ires = &sc->vtpci_intr_res[irq];
653
654 if ((sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) == 0) {
655 error = bus_setup_intr(dev, ires->irq, flags,
656 vtpci_legacy_intr,
657 ires, &ires->intrhand, slz);
658 } else {
659 error = bus_setup_intr(dev, ires->irq, flags,
660 vtpci_msix_intr,
661 ires, &ires->intrhand, slz);
662 }
663 return (error);
664 }
665
666 static int
667 vtpci_teardown_intr(device_t dev, uint irq)
668 {
669 struct vtpci_softc *sc = device_get_softc(dev);
670 struct vtpci_intr_resource *ires;
671
672 if ((int)irq >= sc->vtpci_nintr_res)
673 return (EINVAL);
674
675 ires = &sc->vtpci_intr_res[irq];
676
677 if (ires->intrhand == NULL)
678 return (ENXIO);
679
680 bus_teardown_intr(dev, ires->irq, ires->intrhand);
681 ires->intrhand = NULL;
682 return (0);
683 }
684
685 static void
686 vtpci_add_irqentry(struct vtpci_intr_resource *intr_res, int what,
687 driver_intr_t handler, void *arg)
688 {
689 struct vqentry *e;
690
691 TAILQ_FOREACH(e, &intr_res->ls, entries) {
692 if (e->what == what)
693 return;
694 }
695 e = kmalloc(sizeof(*e), M_DEVBUF, M_WAITOK | M_ZERO);
696 e->what = what;
697 if (e->what == -1) {
698 e->vq = NULL;
699 } else {
700 e->vq = intr_res->ires_sc->vtpci_vqx[e->what].vq;
701 }
702 e->handler = handler;
703 e->arg = arg;
704 TAILQ_INSERT_TAIL(&intr_res->ls, e, entries);
705 }
706
707 static void
708 vtpci_del_irqentry(struct vtpci_intr_resource *intr_res, int what)
709 {
710 struct vqentry *e;
711
712 TAILQ_FOREACH(e, &intr_res->ls, entries) {
713 if (e->what == what)
714 break;
715 }
716 if (e != NULL) {
717 TAILQ_REMOVE(&intr_res->ls, e, entries);
718 kfree(e, M_DEVBUF);
719 }
720 }
721
722 /*
723 * Config intr can be bound after intr_alloc, virtqueue intrs can be bound
724 * after intr_alloc and alloc_virtqueues.
725 */
726 static int
727 vtpci_bind_intr(device_t dev, uint irq, int what,
728 driver_intr_t handler, void *arg)
729 {
730 struct vtpci_softc *sc = device_get_softc(dev);
731 struct vtpci_virtqueue *vqx;
732 int error;
733
734 if (irq >= sc->vtpci_nintr_res)
735 return (EINVAL);
736
737 if (what == -1) {
738 if (sc->vtpci_config_irq != -1)
739 return (EINVAL);
740
741 sc->vtpci_config_irq = irq;
742 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
743 error = vtpci_register_msix_vector(sc,
744 VIRTIO_MSI_CONFIG_VECTOR, irq);
745 if (error)
746 return (error);
747 }
748 goto done;
749 }
750
751 if (sc->vtpci_nvqs <= what || what < 0)
752 return (EINVAL);
753
754 vqx = &sc->vtpci_vqx[what];
755 if (vqx->ires_idx != -1)
756 return (EINVAL);
757
758 vqx->ires_idx = irq;
759 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
760 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, what);
761 error = vtpci_register_msix_vector(sc, VIRTIO_MSI_QUEUE_VECTOR,
762 irq);
763 if (error)
764 return (error);
765 }
766 done:
767 vtpci_add_irqentry(&sc->vtpci_intr_res[irq], what, handler, arg);
768 return (0);
769 }
770
771 static int
772 vtpci_unbind_intr(device_t dev, int what)
773 {
774 struct vtpci_softc *sc = device_get_softc(dev);
775 struct vtpci_virtqueue *vqx;
776 uint irq;
777
778 if (what == -1) {
779 if (sc->vtpci_config_irq == -1)
780 return (EINVAL);
781
782 irq = sc->vtpci_config_irq;
783 sc->vtpci_config_irq = -1;
784 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
785 vtpci_register_msix_vector(sc,
786 VIRTIO_MSI_CONFIG_VECTOR, -1);
787 }
788 goto done;
789 }
790
791 if (sc->vtpci_nvqs <= what || what < 0)
792 return (EINVAL);
793
794 vqx = &sc->vtpci_vqx[what];
795 if (vqx->ires_idx == -1)
796 return (EINVAL);
797
798 irq = vqx->ires_idx;
799 vqx->ires_idx = -1;
800 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
801 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, what);
802 vtpci_register_msix_vector(sc, VIRTIO_MSI_QUEUE_VECTOR, -1);
803 }
804 done:
805 KKASSERT(irq >= 0 && irq < sc->vtpci_nintr_res);
806 vtpci_del_irqentry(&sc->vtpci_intr_res[irq], what);
807 return (0);
808 }
809
810 static void
811 vtpci_stop(device_t dev)
812 {
813 vtpci_reset(device_get_softc(dev));
814 }
815
816 static int
817 vtpci_reinit(device_t dev, uint64_t features)
818 {
819 struct vtpci_softc *sc;
820 struct vtpci_virtqueue *vqx;
821 struct virtqueue *vq;
822 int queue, error;
823 uint16_t vq_size;
824
825 sc = device_get_softc(dev);
826
827 /*
828 * Redrive the device initialization. This is a bit of an abuse
829 * of the specification, but both VirtualBox and QEMU/KVM seem
830 * to play nice. We do not allow the host device to change from
831 * what was originally negotiated beyond what the guest driver
832 * changed (MSIX state should not change, number of virtqueues
833 * and their size remain the same, etc).
834 */
835
836 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
837 vtpci_stop(dev);
838
839 /*
840 * Quickly drive the status through ACK and DRIVER. The device
841 * does not become usable again until vtpci_reinit_complete().
842 */
843 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
844 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
845
846 vtpci_negotiate_features(dev, features);
847
848 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
849 pci_enable_msix(dev);
850 error = vtpci_register_msix_vector(sc,
851 VIRTIO_MSI_CONFIG_VECTOR, 0);
852 if (error)
853 return (error);
854 }
855
856 for (queue = 0; queue < sc->vtpci_nvqs; queue++) {
857 vqx = &sc->vtpci_vqx[queue];
858 vq = vqx->vq;
859
860 KASSERT(vq != NULL, ("vq %d not allocated", queue));
861 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue);
862
863 vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
864 error = virtqueue_reinit(vq, vq_size);
865 if (error)
866 return (error);
867
868 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
869 error = vtpci_register_msix_vector(sc,
870 VIRTIO_MSI_QUEUE_VECTOR, vqx->ires_idx);
871 if (error)
872 return (error);
873 }
874
875 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
876 virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
877 }
878
879 return (0);
880 }
881
882 static void
883 vtpci_reinit_complete(device_t dev)
884 {
885
886 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
887 }
888
889 static void
890 vtpci_notify_virtqueue(device_t dev, uint16_t queue)
891 {
892 struct vtpci_softc *sc;
893
894 sc = device_get_softc(dev);
895
896 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
897 }
898
899 static uint8_t
900 vtpci_get_status(device_t dev)
901 {
902 struct vtpci_softc *sc;
903
904 sc = device_get_softc(dev);
905
906 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
907 }
908
909 static void
910 vtpci_set_status(device_t dev, uint8_t status)
911 {
912 struct vtpci_softc *sc;
913
914 sc = device_get_softc(dev);
915
916 if (status != VIRTIO_CONFIG_STATUS_RESET)
917 status |= vtpci_get_status(dev);
918
919 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
920 }
921
922 static void
923 vtpci_read_dev_config(device_t dev, bus_size_t offset,
924 void *dst, int length)
925 {
926 struct vtpci_softc *sc;
927 bus_size_t off;
928 uint8_t *d;
929 int size;
930
931 sc = device_get_softc(dev);
932 off = VIRTIO_PCI_CONFIG(sc) + offset;
933
934 for (d = dst; length > 0; d += size, off += size, length -= size) {
935 if (length >= 4) {
936 size = 4;
937 *(uint32_t *)d = vtpci_read_config_4(sc, off);
938 } else if (length >= 2) {
939 size = 2;
940 *(uint16_t *)d = vtpci_read_config_2(sc, off);
941 } else {
942 size = 1;
943 *d = vtpci_read_config_1(sc, off);
944 }
945 }
946 }
947
948 static void
949 vtpci_write_dev_config(device_t dev, bus_size_t offset,
950 void *src, int length)
951 {
952 struct vtpci_softc *sc;
953 bus_size_t off;
954 uint8_t *s;
955 int size;
956
957 sc = device_get_softc(dev);
958 off = VIRTIO_PCI_CONFIG(sc) + offset;
959
960 for (s = src; length > 0; s += size, off += size, length -= size) {
961 if (length >= 4) {
962 size = 4;
963 vtpci_write_config_4(sc, off, *(uint32_t *)s);
964 } else if (length >= 2) {
965 size = 2;
966 vtpci_write_config_2(sc, off, *(uint16_t *)s);
967 } else {
968 size = 1;
969 vtpci_write_config_1(sc, off, *s);
970 }
971 }
972 }
973
974 static void
975 vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
976 uint64_t features)
977 {
978 device_t dev, child;
979
980 dev = sc->vtpci_dev;
981 child = sc->vtpci_child_dev;
982
983 if (device_is_attached(child) && bootverbose == 0)
984 return;
985
986 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
987 }
988
989 static void
990 vtpci_probe_and_attach_child(struct vtpci_softc *sc)
991 {
992 device_t dev, child;
993 int error;
994
995 dev = sc->vtpci_dev;
996 child = sc->vtpci_child_dev;
997
998 if (child == NULL)
999 return;
1000
1001 if (device_get_state(child) != DS_NOTPRESENT)
1002 return;
1003
1004 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
1005 error = device_probe_and_attach(child);
1006 if (error != 0 || device_get_state(child) == DS_NOTPRESENT) {
1007 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
1008 vtpci_reset(sc);
1009 vtpci_release_child_resources(sc);
1010
1011 /* Reset status for future attempt. */
1012 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
1013 } else
1014 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
1015 }
1016
1017 static int
1018 vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, int res_idx)
1019 {
1020 device_t dev;
1021 uint16_t vector;
1022
1023 dev = sc->vtpci_dev;
1024
1025 if (offset != VIRTIO_MSI_CONFIG_VECTOR &&
1026 offset != VIRTIO_MSI_QUEUE_VECTOR)
1027 return (EINVAL);
1028
1029 if (res_idx != -1) {
1030 /* Map from rid to host vector. */
1031 vector = res_idx;
1032 } else {
1033 vector = VIRTIO_MSI_NO_VECTOR;
1034 }
1035
1036 vtpci_write_config_2(sc, offset, vector);
1037
1038 if (vtpci_read_config_2(sc, offset) != vector) {
1039 device_printf(dev, "insufficient host resources for "
1040 "MSIX interrupts\n");
1041 return (ENODEV);
1042 }
1043
1044 return (0);
1045 }
1046
1047 static void
1048 vtpci_free_interrupts(struct vtpci_softc *sc)
1049 {
1050 device_t dev = sc->vtpci_dev;
1051 struct vtpci_intr_resource *ires;
1052 int i;
1053
1054 for (i = 0; i < sc->vtpci_nintr_res; i++) {
1055 ires = &sc->vtpci_intr_res[i];
1056
1057 if (ires->intrhand != NULL) {
1058 bus_teardown_intr(dev, ires->irq, ires->intrhand);
1059 ires->intrhand = NULL;
1060 }
1061 if (ires->irq != NULL) {
1062 bus_release_resource(dev, SYS_RES_IRQ, ires->rid,
1063 ires->irq);
1064 ires->irq = NULL;
1065 }
1066 }
1067
1068 vtpci_unbind_intr(sc->vtpci_dev, -1);
1069 for (i = 0; i < sc->vtpci_nvqs; i++)
1070 vtpci_unbind_intr(sc->vtpci_dev, i);
1071
1072 for (i = 0; i < sc->vtpci_nintr_res; i++) {
1073 ires = &sc->vtpci_intr_res[i];
1074
1075 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX)
1076 pci_release_msix_vector(dev, ires->rid);
1077 ires->rid = 0;
1078 }
1079 sc->vtpci_nintr_res = 0;
1080 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSI) {
1081 pci_release_msi(dev);
1082 sc->vtpci_flags &= ~VIRTIO_PCI_FLAG_MSI;
1083 }
1084 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
1085 pci_disable_msix(dev);
1086 pci_teardown_msix(dev);
1087 sc->vtpci_flags &= ~VIRTIO_PCI_FLAG_MSIX;
1088 }
1089
1090 }
1091
1092 static void
1093 vtpci_free_virtqueues(struct vtpci_softc *sc)
1094 {
1095 struct vtpci_virtqueue *vqx;
1096 int i;
1097
1098 sc->vtpci_nvqs = 0;
1099
1100 for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; i++) {
1101 vqx = &sc->vtpci_vqx[i];
1102
1103 if (vqx->vq != NULL) {
1104 virtqueue_free(vqx->vq);
1105 vqx->vq = NULL;
1106 }
1107 }
1108 }
1109
1110 static void
1111 vtpci_release_child_resources(struct vtpci_softc *sc)
1112 {
1113 vtpci_free_interrupts(sc);
1114 vtpci_free_virtqueues(sc);
1115 }
1116
1117 static void
1118 vtpci_reset(struct vtpci_softc *sc)
1119 {
1120
1121 /*
1122 * Setting the status to RESET sets the host device to
1123 * the original, uninitialized state.
1124 */
1125 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
1126 }
1127
1128 static void
1129 vtpci_legacy_intr(void *arg)
1130 {
1131 struct vtpci_intr_resource *ires;
1132 struct vtpci_softc *sc;
1133 struct vqentry *e;
1134 uint8_t isr;
1135
1136 ires = arg;
1137 sc = ires->ires_sc;
1138
1139 /* Reading the ISR also clears it. */
1140 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
1141
1142 TAILQ_FOREACH(e, &ires->ls, entries) {
1143 /*
1144 * The lwkt_serialize_handler_call API doesn't seem to fit
1145 * properly here. Instead move the virtqueue pending check
1146 * into the driver, who can then properly implement masking
1147 * of the handler itself.
1148 */
1149 if (e->what == -1) {
1150 if (isr & VIRTIO_PCI_ISR_CONFIG)
1151 e->handler(e->arg);
1152 } else if (isr & VIRTIO_PCI_ISR_INTR) {
1153 e->handler(e->arg);
1154 }
1155 }
1156 }
1157
1158 static void
1159 vtpci_msix_intr(void *arg)
1160 {
1161 struct vtpci_intr_resource *ires;
1162 struct vtpci_softc *sc;
1163 struct vqentry *e;
1164
1165 ires = arg;
1166 sc = ires->ires_sc;
1167 TAILQ_FOREACH(e, &ires->ls, entries) {
1168 /*
1169 * The lwkt_serialize_handler_call API doesn't seem to fit
1170 * properly here. Instead move the virtqueue pending check
1171 * into the driver, who can then properly implement masking
1172 * of the handler itself.
1173 */
1174 e->handler(e->arg);
1175 }
1176 }
DragonFly BSD