4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
5 * Author: Alex Williamson <alex.williamson@redhat.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * Derived from original vfio:
12 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
13 * Author: Tom Lyon, pugs@cisco.com
16 #include <linux/cdev.h>
17 #include <linux/compat.h>
18 #include <linux/device.h>
19 #include <linux/file.h>
20 #include <linux/anon_inodes.h>
22 #include <linux/idr.h>
23 #include <linux/iommu.h>
24 #include <linux/list.h>
25 #include <linux/module.h>
26 #include <linux/mutex.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
29 #include <linux/string.h>
30 #include <linux/uaccess.h>
31 #include <linux/vfio.h>
32 #include <linux/wait.h>
34 #define DRIVER_VERSION "0.3"
35 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
36 #define DRIVER_DESC "VFIO - User Level meta-driver"
40 struct list_head iommu_drivers_list
;
41 struct mutex iommu_drivers_lock
;
42 struct list_head group_list
;
44 struct mutex group_lock
;
45 struct cdev group_cdev
;
49 wait_queue_head_t release_q
;
52 struct vfio_iommu_driver
{
53 const struct vfio_iommu_driver_ops
*ops
;
54 struct list_head vfio_next
;
57 struct vfio_container
{
59 struct list_head group_list
;
60 struct mutex group_lock
;
61 struct vfio_iommu_driver
*iommu_driver
;
68 atomic_t container_users
;
69 struct iommu_group
*iommu_group
;
70 struct vfio_container
*container
;
71 struct list_head device_list
;
72 struct mutex device_lock
;
74 struct notifier_block nb
;
75 struct list_head vfio_next
;
76 struct list_head container_next
;
82 const struct vfio_device_ops
*ops
;
83 struct vfio_group
*group
;
84 struct list_head group_next
;
89 * IOMMU driver registration
91 int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops
*ops
)
93 struct vfio_iommu_driver
*driver
, *tmp
;
95 driver
= kzalloc(sizeof(*driver
), GFP_KERNEL
);
101 mutex_lock(&vfio
.iommu_drivers_lock
);
103 /* Check for duplicates */
104 list_for_each_entry(tmp
, &vfio
.iommu_drivers_list
, vfio_next
) {
105 if (tmp
->ops
== ops
) {
106 mutex_unlock(&vfio
.iommu_drivers_lock
);
112 list_add(&driver
->vfio_next
, &vfio
.iommu_drivers_list
);
114 mutex_unlock(&vfio
.iommu_drivers_lock
);
118 EXPORT_SYMBOL_GPL(vfio_register_iommu_driver
);
120 void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops
*ops
)
122 struct vfio_iommu_driver
*driver
;
124 mutex_lock(&vfio
.iommu_drivers_lock
);
125 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
, vfio_next
) {
126 if (driver
->ops
== ops
) {
127 list_del(&driver
->vfio_next
);
128 mutex_unlock(&vfio
.iommu_drivers_lock
);
133 mutex_unlock(&vfio
.iommu_drivers_lock
);
135 EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver
);
138 * Group minor allocation/free - both called with vfio.group_lock held
140 static int vfio_alloc_group_minor(struct vfio_group
*group
)
145 if (unlikely(idr_pre_get(&vfio
.group_idr
, GFP_KERNEL
) == 0))
148 /* index 0 is used by /dev/vfio/vfio */
149 ret
= idr_get_new_above(&vfio
.group_idr
, group
, 1, &minor
);
152 if (ret
|| minor
> MINORMASK
) {
153 if (minor
> MINORMASK
)
154 idr_remove(&vfio
.group_idr
, minor
);
161 static void vfio_free_group_minor(int minor
)
163 idr_remove(&vfio
.group_idr
, minor
);
166 static int vfio_iommu_group_notifier(struct notifier_block
*nb
,
167 unsigned long action
, void *data
);
168 static void vfio_group_get(struct vfio_group
*group
);
171 * Container objects - containers are created when /dev/vfio/vfio is
172 * opened, but their lifecycle extends until the last user is done, so
173 * it's freed via kref. Must support container/group/device being
174 * closed in any order.
176 static void vfio_container_get(struct vfio_container
*container
)
178 kref_get(&container
->kref
);
181 static void vfio_container_release(struct kref
*kref
)
183 struct vfio_container
*container
;
184 container
= container_of(kref
, struct vfio_container
, kref
);
189 static void vfio_container_put(struct vfio_container
*container
)
191 kref_put(&container
->kref
, vfio_container_release
);
195 * Group objects - create, release, get, put, search
197 static struct vfio_group
*vfio_create_group(struct iommu_group
*iommu_group
)
199 struct vfio_group
*group
, *tmp
;
203 group
= kzalloc(sizeof(*group
), GFP_KERNEL
);
205 return ERR_PTR(-ENOMEM
);
207 kref_init(&group
->kref
);
208 INIT_LIST_HEAD(&group
->device_list
);
209 mutex_init(&group
->device_lock
);
210 atomic_set(&group
->container_users
, 0);
211 group
->iommu_group
= iommu_group
;
213 group
->nb
.notifier_call
= vfio_iommu_group_notifier
;
216 * blocking notifiers acquire a rwsem around registering and hold
217 * it around callback. Therefore, need to register outside of
218 * vfio.group_lock to avoid A-B/B-A contention. Our callback won't
219 * do anything unless it can find the group in vfio.group_list, so
220 * no harm in registering early.
222 ret
= iommu_group_register_notifier(iommu_group
, &group
->nb
);
228 mutex_lock(&vfio
.group_lock
);
230 minor
= vfio_alloc_group_minor(group
);
232 mutex_unlock(&vfio
.group_lock
);
234 return ERR_PTR(minor
);
237 /* Did we race creating this group? */
238 list_for_each_entry(tmp
, &vfio
.group_list
, vfio_next
) {
239 if (tmp
->iommu_group
== iommu_group
) {
241 vfio_free_group_minor(minor
);
242 mutex_unlock(&vfio
.group_lock
);
248 dev
= device_create(vfio
.class, NULL
, MKDEV(MAJOR(vfio
.devt
), minor
),
249 group
, "%d", iommu_group_id(iommu_group
));
251 vfio_free_group_minor(minor
);
252 mutex_unlock(&vfio
.group_lock
);
254 return (struct vfio_group
*)dev
; /* ERR_PTR */
257 group
->minor
= minor
;
260 list_add(&group
->vfio_next
, &vfio
.group_list
);
262 mutex_unlock(&vfio
.group_lock
);
267 /* called with vfio.group_lock held */
268 static void vfio_group_release(struct kref
*kref
)
270 struct vfio_group
*group
= container_of(kref
, struct vfio_group
, kref
);
272 WARN_ON(!list_empty(&group
->device_list
));
274 device_destroy(vfio
.class, MKDEV(MAJOR(vfio
.devt
), group
->minor
));
275 list_del(&group
->vfio_next
);
276 vfio_free_group_minor(group
->minor
);
278 mutex_unlock(&vfio
.group_lock
);
281 * Unregister outside of lock. A spurious callback is harmless now
282 * that the group is no longer in vfio.group_list.
284 iommu_group_unregister_notifier(group
->iommu_group
, &group
->nb
);
289 static void vfio_group_put(struct vfio_group
*group
)
291 kref_put_mutex(&group
->kref
, vfio_group_release
, &vfio
.group_lock
);
294 /* Assume group_lock or group reference is held */
295 static void vfio_group_get(struct vfio_group
*group
)
297 kref_get(&group
->kref
);
301 * Not really a try as we will sleep for mutex, but we need to make
302 * sure the group pointer is valid under lock and get a reference.
304 static struct vfio_group
*vfio_group_try_get(struct vfio_group
*group
)
306 struct vfio_group
*target
= group
;
308 mutex_lock(&vfio
.group_lock
);
309 list_for_each_entry(group
, &vfio
.group_list
, vfio_next
) {
310 if (group
== target
) {
311 vfio_group_get(group
);
312 mutex_unlock(&vfio
.group_lock
);
316 mutex_unlock(&vfio
.group_lock
);
322 struct vfio_group
*vfio_group_get_from_iommu(struct iommu_group
*iommu_group
)
324 struct vfio_group
*group
;
326 mutex_lock(&vfio
.group_lock
);
327 list_for_each_entry(group
, &vfio
.group_list
, vfio_next
) {
328 if (group
->iommu_group
== iommu_group
) {
329 vfio_group_get(group
);
330 mutex_unlock(&vfio
.group_lock
);
334 mutex_unlock(&vfio
.group_lock
);
339 static struct vfio_group
*vfio_group_get_from_minor(int minor
)
341 struct vfio_group
*group
;
343 mutex_lock(&vfio
.group_lock
);
344 group
= idr_find(&vfio
.group_idr
, minor
);
346 mutex_unlock(&vfio
.group_lock
);
349 vfio_group_get(group
);
350 mutex_unlock(&vfio
.group_lock
);
356 * Device objects - create, release, get, put, search
359 struct vfio_device
*vfio_group_create_device(struct vfio_group
*group
,
361 const struct vfio_device_ops
*ops
,
364 struct vfio_device
*device
;
367 device
= kzalloc(sizeof(*device
), GFP_KERNEL
);
369 return ERR_PTR(-ENOMEM
);
371 kref_init(&device
->kref
);
373 device
->group
= group
;
375 device
->device_data
= device_data
;
377 ret
= dev_set_drvdata(dev
, device
);
383 /* No need to get group_lock, caller has group reference */
384 vfio_group_get(group
);
386 mutex_lock(&group
->device_lock
);
387 list_add(&device
->group_next
, &group
->device_list
);
388 mutex_unlock(&group
->device_lock
);
393 static void vfio_device_release(struct kref
*kref
)
395 struct vfio_device
*device
= container_of(kref
,
396 struct vfio_device
, kref
);
397 struct vfio_group
*group
= device
->group
;
399 list_del(&device
->group_next
);
400 mutex_unlock(&group
->device_lock
);
402 dev_set_drvdata(device
->dev
, NULL
);
406 /* vfio_del_group_dev may be waiting for this device */
407 wake_up(&vfio
.release_q
);
410 /* Device reference always implies a group reference */
411 static void vfio_device_put(struct vfio_device
*device
)
413 struct vfio_group
*group
= device
->group
;
414 kref_put_mutex(&device
->kref
, vfio_device_release
, &group
->device_lock
);
415 vfio_group_put(group
);
418 static void vfio_device_get(struct vfio_device
*device
)
420 vfio_group_get(device
->group
);
421 kref_get(&device
->kref
);
424 static struct vfio_device
*vfio_group_get_device(struct vfio_group
*group
,
427 struct vfio_device
*device
;
429 mutex_lock(&group
->device_lock
);
430 list_for_each_entry(device
, &group
->device_list
, group_next
) {
431 if (device
->dev
== dev
) {
432 vfio_device_get(device
);
433 mutex_unlock(&group
->device_lock
);
437 mutex_unlock(&group
->device_lock
);
442 * Whitelist some drivers that we know are safe (no dma) or just sit on
443 * a device. It's not always practical to leave a device within a group
444 * driverless as it could get re-bound to something unsafe.
446 static const char * const vfio_driver_whitelist
[] = { "pci-stub" };
448 static bool vfio_whitelisted_driver(struct device_driver
*drv
)
452 for (i
= 0; i
< ARRAY_SIZE(vfio_driver_whitelist
); i
++) {
453 if (!strcmp(drv
->name
, vfio_driver_whitelist
[i
]))
461 * A vfio group is viable for use by userspace if all devices are either
462 * driver-less or bound to a vfio or whitelisted driver. We test the
463 * latter by the existence of a struct vfio_device matching the dev.
465 static int vfio_dev_viable(struct device
*dev
, void *data
)
467 struct vfio_group
*group
= data
;
468 struct vfio_device
*device
;
470 if (!dev
->driver
|| vfio_whitelisted_driver(dev
->driver
))
473 device
= vfio_group_get_device(group
, dev
);
475 vfio_device_put(device
);
483 * Async device support
485 static int vfio_group_nb_add_dev(struct vfio_group
*group
, struct device
*dev
)
487 struct vfio_device
*device
;
489 /* Do we already know about it? We shouldn't */
490 device
= vfio_group_get_device(group
, dev
);
491 if (WARN_ON_ONCE(device
)) {
492 vfio_device_put(device
);
496 /* Nothing to do for idle groups */
497 if (!atomic_read(&group
->container_users
))
500 /* TODO Prevent device auto probing */
501 WARN("Device %s added to live group %d!\n", dev_name(dev
),
502 iommu_group_id(group
->iommu_group
));
507 static int vfio_group_nb_del_dev(struct vfio_group
*group
, struct device
*dev
)
509 struct vfio_device
*device
;
512 * Expect to fall out here. If a device was in use, it would
513 * have been bound to a vfio sub-driver, which would have blocked
514 * in .remove at vfio_del_group_dev. Sanity check that we no
515 * longer track the device, so it's safe to remove.
517 device
= vfio_group_get_device(group
, dev
);
521 WARN("Device %s removed from live group %d!\n", dev_name(dev
),
522 iommu_group_id(group
->iommu_group
));
524 vfio_device_put(device
);
528 static int vfio_group_nb_verify(struct vfio_group
*group
, struct device
*dev
)
530 /* We don't care what happens when the group isn't in use */
531 if (!atomic_read(&group
->container_users
))
534 return vfio_dev_viable(dev
, group
);
537 static int vfio_iommu_group_notifier(struct notifier_block
*nb
,
538 unsigned long action
, void *data
)
540 struct vfio_group
*group
= container_of(nb
, struct vfio_group
, nb
);
541 struct device
*dev
= data
;
544 * Need to go through a group_lock lookup to get a reference or
545 * we risk racing a group being removed. Leave a WARN_ON for
546 * debuging, but if the group no longer exists, a spurious notify
549 group
= vfio_group_try_get(group
);
554 case IOMMU_GROUP_NOTIFY_ADD_DEVICE
:
555 vfio_group_nb_add_dev(group
, dev
);
557 case IOMMU_GROUP_NOTIFY_DEL_DEVICE
:
558 vfio_group_nb_del_dev(group
, dev
);
560 case IOMMU_GROUP_NOTIFY_BIND_DRIVER
:
561 pr_debug("%s: Device %s, group %d binding to driver\n",
562 __func__
, dev_name(dev
),
563 iommu_group_id(group
->iommu_group
));
565 case IOMMU_GROUP_NOTIFY_BOUND_DRIVER
:
566 pr_debug("%s: Device %s, group %d bound to driver %s\n",
567 __func__
, dev_name(dev
),
568 iommu_group_id(group
->iommu_group
), dev
->driver
->name
);
569 BUG_ON(vfio_group_nb_verify(group
, dev
));
571 case IOMMU_GROUP_NOTIFY_UNBIND_DRIVER
:
572 pr_debug("%s: Device %s, group %d unbinding from driver %s\n",
573 __func__
, dev_name(dev
),
574 iommu_group_id(group
->iommu_group
), dev
->driver
->name
);
576 case IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER
:
577 pr_debug("%s: Device %s, group %d unbound from driver\n",
578 __func__
, dev_name(dev
),
579 iommu_group_id(group
->iommu_group
));
581 * XXX An unbound device in a live group is ok, but we'd
582 * really like to avoid the above BUG_ON by preventing other
583 * drivers from binding to it. Once that occurs, we have to
584 * stop the system to maintain isolation. At a minimum, we'd
585 * want a toggle to disable driver auto probe for this device.
590 vfio_group_put(group
);
597 int vfio_add_group_dev(struct device
*dev
,
598 const struct vfio_device_ops
*ops
, void *device_data
)
600 struct iommu_group
*iommu_group
;
601 struct vfio_group
*group
;
602 struct vfio_device
*device
;
604 iommu_group
= iommu_group_get(dev
);
608 group
= vfio_group_get_from_iommu(iommu_group
);
610 group
= vfio_create_group(iommu_group
);
612 iommu_group_put(iommu_group
);
613 return PTR_ERR(group
);
617 device
= vfio_group_get_device(group
, dev
);
619 WARN(1, "Device %s already exists on group %d\n",
620 dev_name(dev
), iommu_group_id(iommu_group
));
621 vfio_device_put(device
);
622 vfio_group_put(group
);
623 iommu_group_put(iommu_group
);
627 device
= vfio_group_create_device(group
, dev
, ops
, device_data
);
628 if (IS_ERR(device
)) {
629 vfio_group_put(group
);
630 iommu_group_put(iommu_group
);
631 return PTR_ERR(device
);
635 * Added device holds reference to iommu_group and vfio_device
636 * (which in turn holds reference to vfio_group). Drop extra
637 * group reference used while acquiring device.
639 vfio_group_put(group
);
643 EXPORT_SYMBOL_GPL(vfio_add_group_dev
);
645 /* Test whether a struct device is present in our tracking */
646 static bool vfio_dev_present(struct device
*dev
)
648 struct iommu_group
*iommu_group
;
649 struct vfio_group
*group
;
650 struct vfio_device
*device
;
652 iommu_group
= iommu_group_get(dev
);
656 group
= vfio_group_get_from_iommu(iommu_group
);
658 iommu_group_put(iommu_group
);
662 device
= vfio_group_get_device(group
, dev
);
664 vfio_group_put(group
);
665 iommu_group_put(iommu_group
);
669 vfio_device_put(device
);
670 vfio_group_put(group
);
671 iommu_group_put(iommu_group
);
676 * Decrement the device reference count and wait for the device to be
677 * removed. Open file descriptors for the device... */
678 void *vfio_del_group_dev(struct device
*dev
)
680 struct vfio_device
*device
= dev_get_drvdata(dev
);
681 struct vfio_group
*group
= device
->group
;
682 struct iommu_group
*iommu_group
= group
->iommu_group
;
683 void *device_data
= device
->device_data
;
685 vfio_device_put(device
);
687 /* TODO send a signal to encourage this to be released */
688 wait_event(vfio
.release_q
, !vfio_dev_present(dev
));
690 iommu_group_put(iommu_group
);
694 EXPORT_SYMBOL_GPL(vfio_del_group_dev
);
697 * VFIO base fd, /dev/vfio/vfio
699 static long vfio_ioctl_check_extension(struct vfio_container
*container
,
702 struct vfio_iommu_driver
*driver
= container
->iommu_driver
;
706 /* No base extensions yet */
709 * If no driver is set, poll all registered drivers for
710 * extensions and return the first positive result. If
711 * a driver is already set, further queries will be passed
712 * only to that driver.
715 mutex_lock(&vfio
.iommu_drivers_lock
);
716 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
,
718 if (!try_module_get(driver
->ops
->owner
))
721 ret
= driver
->ops
->ioctl(NULL
,
722 VFIO_CHECK_EXTENSION
,
724 module_put(driver
->ops
->owner
);
728 mutex_unlock(&vfio
.iommu_drivers_lock
);
730 ret
= driver
->ops
->ioctl(container
->iommu_data
,
731 VFIO_CHECK_EXTENSION
, arg
);
737 /* hold container->group_lock */
738 static int __vfio_container_attach_groups(struct vfio_container
*container
,
739 struct vfio_iommu_driver
*driver
,
742 struct vfio_group
*group
;
745 list_for_each_entry(group
, &container
->group_list
, container_next
) {
746 ret
= driver
->ops
->attach_group(data
, group
->iommu_group
);
754 list_for_each_entry_continue_reverse(group
, &container
->group_list
,
756 driver
->ops
->detach_group(data
, group
->iommu_group
);
762 static long vfio_ioctl_set_iommu(struct vfio_container
*container
,
765 struct vfio_iommu_driver
*driver
;
768 mutex_lock(&container
->group_lock
);
771 * The container is designed to be an unprivileged interface while
772 * the group can be assigned to specific users. Therefore, only by
773 * adding a group to a container does the user get the privilege of
774 * enabling the iommu, which may allocate finite resources. There
775 * is no unset_iommu, but by removing all the groups from a container,
776 * the container is deprivileged and returns to an unset state.
778 if (list_empty(&container
->group_list
) || container
->iommu_driver
) {
779 mutex_unlock(&container
->group_lock
);
783 mutex_lock(&vfio
.iommu_drivers_lock
);
784 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
, vfio_next
) {
787 if (!try_module_get(driver
->ops
->owner
))
791 * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
792 * so test which iommu driver reported support for this
793 * extension and call open on them. We also pass them the
794 * magic, allowing a single driver to support multiple
795 * interfaces if they'd like.
797 if (driver
->ops
->ioctl(NULL
, VFIO_CHECK_EXTENSION
, arg
) <= 0) {
798 module_put(driver
->ops
->owner
);
802 /* module reference holds the driver we're working on */
803 mutex_unlock(&vfio
.iommu_drivers_lock
);
805 data
= driver
->ops
->open(arg
);
808 module_put(driver
->ops
->owner
);
809 goto skip_drivers_unlock
;
812 ret
= __vfio_container_attach_groups(container
, driver
, data
);
814 container
->iommu_driver
= driver
;
815 container
->iommu_data
= data
;
817 driver
->ops
->release(data
);
818 module_put(driver
->ops
->owner
);
821 goto skip_drivers_unlock
;
824 mutex_unlock(&vfio
.iommu_drivers_lock
);
826 mutex_unlock(&container
->group_lock
);
831 static long vfio_fops_unl_ioctl(struct file
*filep
,
832 unsigned int cmd
, unsigned long arg
)
834 struct vfio_container
*container
= filep
->private_data
;
835 struct vfio_iommu_driver
*driver
;
842 driver
= container
->iommu_driver
;
843 data
= container
->iommu_data
;
846 case VFIO_GET_API_VERSION
:
847 ret
= VFIO_API_VERSION
;
849 case VFIO_CHECK_EXTENSION
:
850 ret
= vfio_ioctl_check_extension(container
, arg
);
853 ret
= vfio_ioctl_set_iommu(container
, arg
);
856 if (driver
) /* passthrough all unrecognized ioctls */
857 ret
= driver
->ops
->ioctl(data
, cmd
, arg
);
864 static long vfio_fops_compat_ioctl(struct file
*filep
,
865 unsigned int cmd
, unsigned long arg
)
867 arg
= (unsigned long)compat_ptr(arg
);
868 return vfio_fops_unl_ioctl(filep
, cmd
, arg
);
870 #endif /* CONFIG_COMPAT */
872 static int vfio_fops_open(struct inode
*inode
, struct file
*filep
)
874 struct vfio_container
*container
;
876 container
= kzalloc(sizeof(*container
), GFP_KERNEL
);
880 INIT_LIST_HEAD(&container
->group_list
);
881 mutex_init(&container
->group_lock
);
882 kref_init(&container
->kref
);
884 filep
->private_data
= container
;
889 static int vfio_fops_release(struct inode
*inode
, struct file
*filep
)
891 struct vfio_container
*container
= filep
->private_data
;
893 filep
->private_data
= NULL
;
895 vfio_container_put(container
);
901 * Once an iommu driver is set, we optionally pass read/write/mmap
902 * on to the driver, allowing management interfaces beyond ioctl.
904 static ssize_t
vfio_fops_read(struct file
*filep
, char __user
*buf
,
905 size_t count
, loff_t
*ppos
)
907 struct vfio_container
*container
= filep
->private_data
;
908 struct vfio_iommu_driver
*driver
= container
->iommu_driver
;
910 if (unlikely(!driver
|| !driver
->ops
->read
))
913 return driver
->ops
->read(container
->iommu_data
, buf
, count
, ppos
);
916 static ssize_t
vfio_fops_write(struct file
*filep
, const char __user
*buf
,
917 size_t count
, loff_t
*ppos
)
919 struct vfio_container
*container
= filep
->private_data
;
920 struct vfio_iommu_driver
*driver
= container
->iommu_driver
;
922 if (unlikely(!driver
|| !driver
->ops
->write
))
925 return driver
->ops
->write(container
->iommu_data
, buf
, count
, ppos
);
928 static int vfio_fops_mmap(struct file
*filep
, struct vm_area_struct
*vma
)
930 struct vfio_container
*container
= filep
->private_data
;
931 struct vfio_iommu_driver
*driver
= container
->iommu_driver
;
933 if (unlikely(!driver
|| !driver
->ops
->mmap
))
936 return driver
->ops
->mmap(container
->iommu_data
, vma
);
939 static const struct file_operations vfio_fops
= {
940 .owner
= THIS_MODULE
,
941 .open
= vfio_fops_open
,
942 .release
= vfio_fops_release
,
943 .read
= vfio_fops_read
,
944 .write
= vfio_fops_write
,
945 .unlocked_ioctl
= vfio_fops_unl_ioctl
,
947 .compat_ioctl
= vfio_fops_compat_ioctl
,
949 .mmap
= vfio_fops_mmap
,
953 * VFIO Group fd, /dev/vfio/$GROUP
955 static void __vfio_group_unset_container(struct vfio_group
*group
)
957 struct vfio_container
*container
= group
->container
;
958 struct vfio_iommu_driver
*driver
;
960 mutex_lock(&container
->group_lock
);
962 driver
= container
->iommu_driver
;
964 driver
->ops
->detach_group(container
->iommu_data
,
967 group
->container
= NULL
;
968 list_del(&group
->container_next
);
970 /* Detaching the last group deprivileges a container, remove iommu */
971 if (driver
&& list_empty(&container
->group_list
)) {
972 driver
->ops
->release(container
->iommu_data
);
973 module_put(driver
->ops
->owner
);
974 container
->iommu_driver
= NULL
;
975 container
->iommu_data
= NULL
;
978 mutex_unlock(&container
->group_lock
);
980 vfio_container_put(container
);
984 * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
985 * if there was no container to unset. Since the ioctl is called on
986 * the group, we know that still exists, therefore the only valid
987 * transition here is 1->0.
989 static int vfio_group_unset_container(struct vfio_group
*group
)
991 int users
= atomic_cmpxchg(&group
->container_users
, 1, 0);
998 __vfio_group_unset_container(group
);
1004 * When removing container users, anything that removes the last user
1005 * implicitly removes the group from the container. That is, if the
1006 * group file descriptor is closed, as well as any device file descriptors,
1007 * the group is free.
1009 static void vfio_group_try_dissolve_container(struct vfio_group
*group
)
1011 if (0 == atomic_dec_if_positive(&group
->container_users
))
1012 __vfio_group_unset_container(group
);
1015 static int vfio_group_set_container(struct vfio_group
*group
, int container_fd
)
1018 struct vfio_container
*container
;
1019 struct vfio_iommu_driver
*driver
;
1022 if (atomic_read(&group
->container_users
))
1025 f
= fdget(container_fd
);
1029 /* Sanity check, is this really our fd? */
1030 if (f
.file
->f_op
!= &vfio_fops
) {
1035 container
= f
.file
->private_data
;
1036 WARN_ON(!container
); /* fget ensures we don't race vfio_release */
1038 mutex_lock(&container
->group_lock
);
1040 driver
= container
->iommu_driver
;
1042 ret
= driver
->ops
->attach_group(container
->iommu_data
,
1043 group
->iommu_group
);
1048 group
->container
= container
;
1049 list_add(&group
->container_next
, &container
->group_list
);
1051 /* Get a reference on the container and mark a user within the group */
1052 vfio_container_get(container
);
1053 atomic_inc(&group
->container_users
);
1056 mutex_unlock(&container
->group_lock
);
1061 static bool vfio_group_viable(struct vfio_group
*group
)
1063 return (iommu_group_for_each_dev(group
->iommu_group
,
1064 group
, vfio_dev_viable
) == 0);
1067 static const struct file_operations vfio_device_fops
;
1069 static int vfio_group_get_device_fd(struct vfio_group
*group
, char *buf
)
1071 struct vfio_device
*device
;
1075 if (0 == atomic_read(&group
->container_users
) ||
1076 !group
->container
->iommu_driver
|| !vfio_group_viable(group
))
1079 mutex_lock(&group
->device_lock
);
1080 list_for_each_entry(device
, &group
->device_list
, group_next
) {
1081 if (strcmp(dev_name(device
->dev
), buf
))
1084 ret
= device
->ops
->open(device
->device_data
);
1088 * We can't use anon_inode_getfd() because we need to modify
1089 * the f_mode flags directly to allow more than just ioctls
1091 ret
= get_unused_fd();
1093 device
->ops
->release(device
->device_data
);
1097 filep
= anon_inode_getfile("[vfio-device]", &vfio_device_fops
,
1099 if (IS_ERR(filep
)) {
1101 ret
= PTR_ERR(filep
);
1102 device
->ops
->release(device
->device_data
);
1107 * TODO: add an anon_inode interface to do this.
1108 * Appears to be missing by lack of need rather than
1109 * explicitly prevented. Now there's need.
1111 filep
->f_mode
|= (FMODE_LSEEK
| FMODE_PREAD
| FMODE_PWRITE
);
1113 vfio_device_get(device
);
1114 atomic_inc(&group
->container_users
);
1116 fd_install(ret
, filep
);
1119 mutex_unlock(&group
->device_lock
);
1124 static long vfio_group_fops_unl_ioctl(struct file
*filep
,
1125 unsigned int cmd
, unsigned long arg
)
1127 struct vfio_group
*group
= filep
->private_data
;
1131 case VFIO_GROUP_GET_STATUS
:
1133 struct vfio_group_status status
;
1134 unsigned long minsz
;
1136 minsz
= offsetofend(struct vfio_group_status
, flags
);
1138 if (copy_from_user(&status
, (void __user
*)arg
, minsz
))
1141 if (status
.argsz
< minsz
)
1146 if (vfio_group_viable(group
))
1147 status
.flags
|= VFIO_GROUP_FLAGS_VIABLE
;
1149 if (group
->container
)
1150 status
.flags
|= VFIO_GROUP_FLAGS_CONTAINER_SET
;
1152 if (copy_to_user((void __user
*)arg
, &status
, minsz
))
1158 case VFIO_GROUP_SET_CONTAINER
:
1162 if (get_user(fd
, (int __user
*)arg
))
1168 ret
= vfio_group_set_container(group
, fd
);
1171 case VFIO_GROUP_UNSET_CONTAINER
:
1172 ret
= vfio_group_unset_container(group
);
1174 case VFIO_GROUP_GET_DEVICE_FD
:
1178 buf
= strndup_user((const char __user
*)arg
, PAGE_SIZE
);
1180 return PTR_ERR(buf
);
1182 ret
= vfio_group_get_device_fd(group
, buf
);
1191 #ifdef CONFIG_COMPAT
1192 static long vfio_group_fops_compat_ioctl(struct file
*filep
,
1193 unsigned int cmd
, unsigned long arg
)
1195 arg
= (unsigned long)compat_ptr(arg
);
1196 return vfio_group_fops_unl_ioctl(filep
, cmd
, arg
);
1198 #endif /* CONFIG_COMPAT */
1200 static int vfio_group_fops_open(struct inode
*inode
, struct file
*filep
)
1202 struct vfio_group
*group
;
1204 group
= vfio_group_get_from_minor(iminor(inode
));
1208 if (group
->container
) {
1209 vfio_group_put(group
);
1213 filep
->private_data
= group
;
1218 static int vfio_group_fops_release(struct inode
*inode
, struct file
*filep
)
1220 struct vfio_group
*group
= filep
->private_data
;
1222 filep
->private_data
= NULL
;
1224 vfio_group_try_dissolve_container(group
);
1226 vfio_group_put(group
);
1231 static const struct file_operations vfio_group_fops
= {
1232 .owner
= THIS_MODULE
,
1233 .unlocked_ioctl
= vfio_group_fops_unl_ioctl
,
1234 #ifdef CONFIG_COMPAT
1235 .compat_ioctl
= vfio_group_fops_compat_ioctl
,
1237 .open
= vfio_group_fops_open
,
1238 .release
= vfio_group_fops_release
,
1244 static int vfio_device_fops_release(struct inode
*inode
, struct file
*filep
)
1246 struct vfio_device
*device
= filep
->private_data
;
1248 device
->ops
->release(device
->device_data
);
1250 vfio_group_try_dissolve_container(device
->group
);
1252 vfio_device_put(device
);
1257 static long vfio_device_fops_unl_ioctl(struct file
*filep
,
1258 unsigned int cmd
, unsigned long arg
)
1260 struct vfio_device
*device
= filep
->private_data
;
1262 if (unlikely(!device
->ops
->ioctl
))
1265 return device
->ops
->ioctl(device
->device_data
, cmd
, arg
);
1268 static ssize_t
vfio_device_fops_read(struct file
*filep
, char __user
*buf
,
1269 size_t count
, loff_t
*ppos
)
1271 struct vfio_device
*device
= filep
->private_data
;
1273 if (unlikely(!device
->ops
->read
))
1276 return device
->ops
->read(device
->device_data
, buf
, count
, ppos
);
1279 static ssize_t
vfio_device_fops_write(struct file
*filep
,
1280 const char __user
*buf
,
1281 size_t count
, loff_t
*ppos
)
1283 struct vfio_device
*device
= filep
->private_data
;
1285 if (unlikely(!device
->ops
->write
))
1288 return device
->ops
->write(device
->device_data
, buf
, count
, ppos
);
1291 static int vfio_device_fops_mmap(struct file
*filep
, struct vm_area_struct
*vma
)
1293 struct vfio_device
*device
= filep
->private_data
;
1295 if (unlikely(!device
->ops
->mmap
))
1298 return device
->ops
->mmap(device
->device_data
, vma
);
1301 #ifdef CONFIG_COMPAT
1302 static long vfio_device_fops_compat_ioctl(struct file
*filep
,
1303 unsigned int cmd
, unsigned long arg
)
1305 arg
= (unsigned long)compat_ptr(arg
);
1306 return vfio_device_fops_unl_ioctl(filep
, cmd
, arg
);
1308 #endif /* CONFIG_COMPAT */
1310 static const struct file_operations vfio_device_fops
= {
1311 .owner
= THIS_MODULE
,
1312 .release
= vfio_device_fops_release
,
1313 .read
= vfio_device_fops_read
,
1314 .write
= vfio_device_fops_write
,
1315 .unlocked_ioctl
= vfio_device_fops_unl_ioctl
,
1316 #ifdef CONFIG_COMPAT
1317 .compat_ioctl
= vfio_device_fops_compat_ioctl
,
1319 .mmap
= vfio_device_fops_mmap
,
1323 * Module/class support
1325 static char *vfio_devnode(struct device
*dev
, umode_t
*mode
)
1327 return kasprintf(GFP_KERNEL
, "vfio/%s", dev_name(dev
));
1330 static int __init
vfio_init(void)
1334 idr_init(&vfio
.group_idr
);
1335 mutex_init(&vfio
.group_lock
);
1336 mutex_init(&vfio
.iommu_drivers_lock
);
1337 INIT_LIST_HEAD(&vfio
.group_list
);
1338 INIT_LIST_HEAD(&vfio
.iommu_drivers_list
);
1339 init_waitqueue_head(&vfio
.release_q
);
1341 vfio
.class = class_create(THIS_MODULE
, "vfio");
1342 if (IS_ERR(vfio
.class)) {
1343 ret
= PTR_ERR(vfio
.class);
1347 vfio
.class->devnode
= vfio_devnode
;
1349 ret
= alloc_chrdev_region(&vfio
.devt
, 0, MINORMASK
, "vfio");
1351 goto err_base_chrdev
;
1353 cdev_init(&vfio
.cdev
, &vfio_fops
);
1354 ret
= cdev_add(&vfio
.cdev
, vfio
.devt
, 1);
1358 vfio
.dev
= device_create(vfio
.class, NULL
, vfio
.devt
, NULL
, "vfio");
1359 if (IS_ERR(vfio
.dev
)) {
1360 ret
= PTR_ERR(vfio
.dev
);
1364 /* /dev/vfio/$GROUP */
1365 cdev_init(&vfio
.group_cdev
, &vfio_group_fops
);
1366 ret
= cdev_add(&vfio
.group_cdev
,
1367 MKDEV(MAJOR(vfio
.devt
), 1), MINORMASK
- 1);
1369 goto err_groups_cdev
;
1371 pr_info(DRIVER_DESC
" version: " DRIVER_VERSION
"\n");
1374 * Attempt to load known iommu-drivers. This gives us a working
1375 * environment without the user needing to explicitly load iommu
1378 request_module_nowait("vfio_iommu_type1");
1383 device_destroy(vfio
.class, vfio
.devt
);
1385 cdev_del(&vfio
.cdev
);
1387 unregister_chrdev_region(vfio
.devt
, MINORMASK
);
1389 class_destroy(vfio
.class);
1395 static void __exit
vfio_cleanup(void)
1397 WARN_ON(!list_empty(&vfio
.group_list
));
1399 idr_destroy(&vfio
.group_idr
);
1400 cdev_del(&vfio
.group_cdev
);
1401 device_destroy(vfio
.class, vfio
.devt
);
1402 cdev_del(&vfio
.cdev
);
1403 unregister_chrdev_region(vfio
.devt
, MINORMASK
);
1404 class_destroy(vfio
.class);
1408 module_init(vfio_init
);
1409 module_exit(vfio_cleanup
);
1411 MODULE_VERSION(DRIVER_VERSION
);
1412 MODULE_LICENSE("GPL v2");
1413 MODULE_AUTHOR(DRIVER_AUTHOR
);
1414 MODULE_DESCRIPTION(DRIVER_DESC
);