2 * kvm eventfd support - use eventfd objects to signal various KVM events
4 * Copyright 2009 Novell. All Rights Reserved.
7 * Gregory Haskins <ghaskins@novell.com>
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License
11 * as published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software Foundation,
20 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
23 #include <linux/kvm_host.h>
24 #include <linux/kvm.h>
25 #include <linux/workqueue.h>
26 #include <linux/syscalls.h>
27 #include <linux/wait.h>
28 #include <linux/poll.h>
29 #include <linux/file.h>
30 #include <linux/list.h>
31 #include <linux/eventfd.h>
32 #include <linux/kernel.h>
37 * --------------------------------------------------------------------
38 * irqfd: Allows an fd to be used to inject an interrupt to the guest
40 * Credit goes to Avi Kivity for the original idea.
41 * --------------------------------------------------------------------
46 struct eventfd_ctx
*eventfd
;
48 struct list_head list
;
50 wait_queue_head_t
*wqh
;
52 struct work_struct inject
;
53 struct work_struct shutdown
;
56 static struct workqueue_struct
*irqfd_cleanup_wq
;
59 irqfd_inject(struct work_struct
*work
)
61 struct _irqfd
*irqfd
= container_of(work
, struct _irqfd
, inject
);
62 struct kvm
*kvm
= irqfd
->kvm
;
64 kvm_set_irq(kvm
, KVM_USERSPACE_IRQ_SOURCE_ID
, irqfd
->gsi
, 1);
65 kvm_set_irq(kvm
, KVM_USERSPACE_IRQ_SOURCE_ID
, irqfd
->gsi
, 0);
69 * Race-free decouple logic (ordering is critical)
72 irqfd_shutdown(struct work_struct
*work
)
74 struct _irqfd
*irqfd
= container_of(work
, struct _irqfd
, shutdown
);
78 * Synchronize with the wait-queue and unhook ourselves to prevent
81 eventfd_ctx_remove_wait_queue(irqfd
->eventfd
, &irqfd
->wait
, &cnt
);
84 * We know no new events will be scheduled at this point, so block
85 * until all previously outstanding events have completed
87 flush_work(&irqfd
->inject
);
90 * It is now safe to release the object's resources
92 eventfd_ctx_put(irqfd
->eventfd
);
97 /* assumes kvm->irqfds.lock is held */
99 irqfd_is_active(struct _irqfd
*irqfd
)
101 return list_empty(&irqfd
->list
) ? false : true;
105 * Mark the irqfd as inactive and schedule it for removal
107 * assumes kvm->irqfds.lock is held
110 irqfd_deactivate(struct _irqfd
*irqfd
)
112 BUG_ON(!irqfd_is_active(irqfd
));
114 list_del_init(&irqfd
->list
);
116 queue_work(irqfd_cleanup_wq
, &irqfd
->shutdown
);
120 * Called with wqh->lock held and interrupts disabled
123 irqfd_wakeup(wait_queue_t
*wait
, unsigned mode
, int sync
, void *key
)
125 struct _irqfd
*irqfd
= container_of(wait
, struct _irqfd
, wait
);
126 unsigned long flags
= (unsigned long)key
;
129 /* An event has been signaled, inject an interrupt */
130 schedule_work(&irqfd
->inject
);
132 if (flags
& POLLHUP
) {
133 /* The eventfd is closing, detach from KVM */
134 struct kvm
*kvm
= irqfd
->kvm
;
137 spin_lock_irqsave(&kvm
->irqfds
.lock
, flags
);
140 * We must check if someone deactivated the irqfd before
141 * we could acquire the irqfds.lock since the item is
142 * deactivated from the KVM side before it is unhooked from
143 * the wait-queue. If it is already deactivated, we can
144 * simply return knowing the other side will cleanup for us.
145 * We cannot race against the irqfd going away since the
146 * other side is required to acquire wqh->lock, which we hold
148 if (irqfd_is_active(irqfd
))
149 irqfd_deactivate(irqfd
);
151 spin_unlock_irqrestore(&kvm
->irqfds
.lock
, flags
);
158 irqfd_ptable_queue_proc(struct file
*file
, wait_queue_head_t
*wqh
,
161 struct _irqfd
*irqfd
= container_of(pt
, struct _irqfd
, pt
);
164 add_wait_queue(wqh
, &irqfd
->wait
);
168 kvm_irqfd_assign(struct kvm
*kvm
, int fd
, int gsi
)
170 struct _irqfd
*irqfd
, *tmp
;
171 struct file
*file
= NULL
;
172 struct eventfd_ctx
*eventfd
= NULL
;
176 irqfd
= kzalloc(sizeof(*irqfd
), GFP_KERNEL
);
182 INIT_LIST_HEAD(&irqfd
->list
);
183 INIT_WORK(&irqfd
->inject
, irqfd_inject
);
184 INIT_WORK(&irqfd
->shutdown
, irqfd_shutdown
);
186 file
= eventfd_fget(fd
);
192 eventfd
= eventfd_ctx_fileget(file
);
193 if (IS_ERR(eventfd
)) {
194 ret
= PTR_ERR(eventfd
);
198 irqfd
->eventfd
= eventfd
;
201 * Install our own custom wake-up handling so we are notified via
202 * a callback whenever someone signals the underlying eventfd
204 init_waitqueue_func_entry(&irqfd
->wait
, irqfd_wakeup
);
205 init_poll_funcptr(&irqfd
->pt
, irqfd_ptable_queue_proc
);
207 spin_lock_irq(&kvm
->irqfds
.lock
);
210 list_for_each_entry(tmp
, &kvm
->irqfds
.items
, list
) {
211 if (irqfd
->eventfd
!= tmp
->eventfd
)
213 /* This fd is used for another irq already. */
215 spin_unlock_irq(&kvm
->irqfds
.lock
);
219 events
= file
->f_op
->poll(file
, &irqfd
->pt
);
221 list_add_tail(&irqfd
->list
, &kvm
->irqfds
.items
);
222 spin_unlock_irq(&kvm
->irqfds
.lock
);
225 * Check if there was an event already pending on the eventfd
226 * before we registered, and trigger it as if we didn't miss it.
229 schedule_work(&irqfd
->inject
);
232 * do not drop the file until the irqfd is fully initialized, otherwise
233 * we might race against the POLLHUP
240 if (eventfd
&& !IS_ERR(eventfd
))
241 eventfd_ctx_put(eventfd
);
251 kvm_eventfd_init(struct kvm
*kvm
)
253 spin_lock_init(&kvm
->irqfds
.lock
);
254 INIT_LIST_HEAD(&kvm
->irqfds
.items
);
255 INIT_LIST_HEAD(&kvm
->ioeventfds
);
259 * shutdown any irqfd's that match fd+gsi
262 kvm_irqfd_deassign(struct kvm
*kvm
, int fd
, int gsi
)
264 struct _irqfd
*irqfd
, *tmp
;
265 struct eventfd_ctx
*eventfd
;
267 eventfd
= eventfd_ctx_fdget(fd
);
269 return PTR_ERR(eventfd
);
271 spin_lock_irq(&kvm
->irqfds
.lock
);
273 list_for_each_entry_safe(irqfd
, tmp
, &kvm
->irqfds
.items
, list
) {
274 if (irqfd
->eventfd
== eventfd
&& irqfd
->gsi
== gsi
)
275 irqfd_deactivate(irqfd
);
278 spin_unlock_irq(&kvm
->irqfds
.lock
);
279 eventfd_ctx_put(eventfd
);
282 * Block until we know all outstanding shutdown jobs have completed
283 * so that we guarantee there will not be any more interrupts on this
284 * gsi once this deassign function returns.
286 flush_workqueue(irqfd_cleanup_wq
);
292 kvm_irqfd(struct kvm
*kvm
, int fd
, int gsi
, int flags
)
294 if (flags
& KVM_IRQFD_FLAG_DEASSIGN
)
295 return kvm_irqfd_deassign(kvm
, fd
, gsi
);
297 return kvm_irqfd_assign(kvm
, fd
, gsi
);
301 * This function is called as the kvm VM fd is being released. Shutdown all
302 * irqfds that still remain open
305 kvm_irqfd_release(struct kvm
*kvm
)
307 struct _irqfd
*irqfd
, *tmp
;
309 spin_lock_irq(&kvm
->irqfds
.lock
);
311 list_for_each_entry_safe(irqfd
, tmp
, &kvm
->irqfds
.items
, list
)
312 irqfd_deactivate(irqfd
);
314 spin_unlock_irq(&kvm
->irqfds
.lock
);
317 * Block until we know all outstanding shutdown jobs have completed
318 * since we do not take a kvm* reference.
320 flush_workqueue(irqfd_cleanup_wq
);
325 * create a host-wide workqueue for issuing deferred shutdown requests
326 * aggregated from all vm* instances. We need our own isolated single-thread
327 * queue to prevent deadlock against flushing the normal work-queue.
329 static int __init
irqfd_module_init(void)
331 irqfd_cleanup_wq
= create_singlethread_workqueue("kvm-irqfd-cleanup");
332 if (!irqfd_cleanup_wq
)
338 static void __exit
irqfd_module_exit(void)
340 destroy_workqueue(irqfd_cleanup_wq
);
343 module_init(irqfd_module_init
);
344 module_exit(irqfd_module_exit
);
347 * --------------------------------------------------------------------
348 * ioeventfd: translate a PIO/MMIO memory write to an eventfd signal.
350 * userspace can register a PIO/MMIO address with an eventfd for receiving
351 * notification when the memory has been touched.
352 * --------------------------------------------------------------------
356 struct list_head list
;
359 struct eventfd_ctx
*eventfd
;
361 struct kvm_io_device dev
;
365 static inline struct _ioeventfd
*
366 to_ioeventfd(struct kvm_io_device
*dev
)
368 return container_of(dev
, struct _ioeventfd
, dev
);
372 ioeventfd_release(struct _ioeventfd
*p
)
374 eventfd_ctx_put(p
->eventfd
);
380 ioeventfd_in_range(struct _ioeventfd
*p
, gpa_t addr
, int len
, const void *val
)
384 if (!(addr
== p
->addr
&& len
== p
->length
))
385 /* address-range must be precise for a hit */
389 /* all else equal, wildcard is always a hit */
392 /* otherwise, we have to actually compare the data */
394 BUG_ON(!IS_ALIGNED((unsigned long)val
, len
));
413 return _val
== p
->datamatch
? true : false;
416 /* MMIO/PIO writes trigger an event if the addr/val match */
418 ioeventfd_write(struct kvm_io_device
*this, gpa_t addr
, int len
,
421 struct _ioeventfd
*p
= to_ioeventfd(this);
423 if (!ioeventfd_in_range(p
, addr
, len
, val
))
426 eventfd_signal(p
->eventfd
, 1);
431 * This function is called as KVM is completely shutting down. We do not
432 * need to worry about locking just nuke anything we have as quickly as possible
435 ioeventfd_destructor(struct kvm_io_device
*this)
437 struct _ioeventfd
*p
= to_ioeventfd(this);
439 ioeventfd_release(p
);
442 static const struct kvm_io_device_ops ioeventfd_ops
= {
443 .write
= ioeventfd_write
,
444 .destructor
= ioeventfd_destructor
,
447 /* assumes kvm->slots_lock held */
449 ioeventfd_check_collision(struct kvm
*kvm
, struct _ioeventfd
*p
)
451 struct _ioeventfd
*_p
;
453 list_for_each_entry(_p
, &kvm
->ioeventfds
, list
)
454 if (_p
->addr
== p
->addr
&& _p
->length
== p
->length
&&
455 (_p
->wildcard
|| p
->wildcard
||
456 _p
->datamatch
== p
->datamatch
))
463 kvm_assign_ioeventfd(struct kvm
*kvm
, struct kvm_ioeventfd
*args
)
465 int pio
= args
->flags
& KVM_IOEVENTFD_FLAG_PIO
;
466 struct kvm_io_bus
*bus
= pio
? &kvm
->pio_bus
: &kvm
->mmio_bus
;
467 struct _ioeventfd
*p
;
468 struct eventfd_ctx
*eventfd
;
471 /* must be natural-word sized */
482 /* check for range overflow */
483 if (args
->addr
+ args
->len
< args
->addr
)
486 /* check for extra flags that we don't understand */
487 if (args
->flags
& ~KVM_IOEVENTFD_VALID_FLAG_MASK
)
490 eventfd
= eventfd_ctx_fdget(args
->fd
);
492 return PTR_ERR(eventfd
);
494 p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
500 INIT_LIST_HEAD(&p
->list
);
501 p
->addr
= args
->addr
;
502 p
->length
= args
->len
;
503 p
->eventfd
= eventfd
;
505 /* The datamatch feature is optional, otherwise this is a wildcard */
506 if (args
->flags
& KVM_IOEVENTFD_FLAG_DATAMATCH
)
507 p
->datamatch
= args
->datamatch
;
511 down_write(&kvm
->slots_lock
);
513 /* Verify that there isnt a match already */
514 if (ioeventfd_check_collision(kvm
, p
)) {
519 kvm_iodevice_init(&p
->dev
, &ioeventfd_ops
);
521 ret
= __kvm_io_bus_register_dev(bus
, &p
->dev
);
525 list_add_tail(&p
->list
, &kvm
->ioeventfds
);
527 up_write(&kvm
->slots_lock
);
532 up_write(&kvm
->slots_lock
);
536 eventfd_ctx_put(eventfd
);
542 kvm_deassign_ioeventfd(struct kvm
*kvm
, struct kvm_ioeventfd
*args
)
544 int pio
= args
->flags
& KVM_IOEVENTFD_FLAG_PIO
;
545 struct kvm_io_bus
*bus
= pio
? &kvm
->pio_bus
: &kvm
->mmio_bus
;
546 struct _ioeventfd
*p
, *tmp
;
547 struct eventfd_ctx
*eventfd
;
550 eventfd
= eventfd_ctx_fdget(args
->fd
);
552 return PTR_ERR(eventfd
);
554 down_write(&kvm
->slots_lock
);
556 list_for_each_entry_safe(p
, tmp
, &kvm
->ioeventfds
, list
) {
557 bool wildcard
= !(args
->flags
& KVM_IOEVENTFD_FLAG_DATAMATCH
);
559 if (p
->eventfd
!= eventfd
||
560 p
->addr
!= args
->addr
||
561 p
->length
!= args
->len
||
562 p
->wildcard
!= wildcard
)
565 if (!p
->wildcard
&& p
->datamatch
!= args
->datamatch
)
568 __kvm_io_bus_unregister_dev(bus
, &p
->dev
);
569 ioeventfd_release(p
);
574 up_write(&kvm
->slots_lock
);
576 eventfd_ctx_put(eventfd
);
582 kvm_ioeventfd(struct kvm
*kvm
, struct kvm_ioeventfd
*args
)
584 if (args
->flags
& KVM_IOEVENTFD_FLAG_DEASSIGN
)
585 return kvm_deassign_ioeventfd(kvm
, args
);
587 return kvm_assign_ioeventfd(kvm
, args
);