2 * fs/eventpoll.c ( Efficent event polling implementation )
3 * Copyright (C) 2001,...,2003 Davide Libenzi
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * Davide Libenzi <davidel@xmailserver.org>
14 #include <linux/module.h>
15 #include <linux/init.h>
16 #include <linux/kernel.h>
17 #include <linux/sched.h>
19 #include <linux/file.h>
20 #include <linux/signal.h>
21 #include <linux/errno.h>
23 #include <linux/slab.h>
24 #include <linux/poll.h>
25 #include <linux/smp_lock.h>
26 #include <linux/string.h>
27 #include <linux/list.h>
28 #include <linux/hash.h>
29 #include <linux/spinlock.h>
30 #include <linux/rwsem.h>
31 #include <linux/wait.h>
32 #include <linux/eventpoll.h>
33 #include <linux/mount.h>
34 #include <asm/bitops.h>
35 #include <asm/uaccess.h>
36 #include <asm/system.h>
39 #include <asm/atomic.h>
40 #include <asm/semaphore.h>
45 * There are three level of locking required by epoll :
47 * 1) epsem (semaphore)
48 * 2) ep->sem (rw_semaphore)
49 * 3) ep->lock (rw_lock)
51 * The acquire order is the one listed above, from 1 to 3.
52 * We need a spinlock (ep->lock) because we manipulate objects
53 * from inside the poll callback, that might be triggered from
54 * a wake_up() that in turn might be called from IRQ context.
55 * So we can't sleep inside the poll callback and hence we need
56 * a spinlock. During the event transfer loop (from kernel to
57 * user space) we could end up sleeping due a copy_to_user(), so
58 * we need a lock that will allow us to sleep. This lock is a
59 * read-write semaphore (ep->sem). It is acquired on read during
60 * the event transfer loop and in write during epoll_ctl(EPOLL_CTL_DEL)
61 * and during eventpoll_release(). Then we also need a global
62 * semaphore to serialize eventpoll_release() and ep_free().
63 * This semaphore is acquired by ep_free() during the epoll file
64 * cleanup path and it is also acquired by eventpoll_release()
65 * if a file has been pushed inside an epoll set and it is then
66 * close()d without a previous call toepoll_ctl(EPOLL_CTL_DEL).
67 * It is possible to drop the "ep->sem" and to use the global
68 * semaphore "epsem" (together with "ep->lock") to have it working,
69 * but having "ep->sem" will make the interface more scalable.
70 * Events that require holding "epsem" are very rare, while for
71 * normal operations the epoll private "ep->sem" will guarantee
72 * a greater scalability.
76 #define EVENTPOLLFS_MAGIC 0x03111965 /* My birthday should work for this :) */
81 #define DPRINTK(x) printk x
82 #define DNPRINTK(n, x) do { if ((n) <= DEBUG_EPOLL) printk x; } while (0)
83 #else /* #if DEBUG_EPOLL > 0 */
84 #define DPRINTK(x) (void) 0
85 #define DNPRINTK(n, x) (void) 0
86 #endif /* #if DEBUG_EPOLL > 0 */
91 #define EPI_SLAB_DEBUG (SLAB_DEBUG_FREE | SLAB_RED_ZONE /* | SLAB_POISON */)
92 #else /* #if DEBUG_EPI != 0 */
93 #define EPI_SLAB_DEBUG 0
94 #endif /* #if DEBUG_EPI != 0 */
97 /* Maximum number of poll wake up nests we are allowing */
98 #define EP_MAX_POLLWAKE_NESTS 4
100 /* Maximum size of the hash in bits ( 2^N ) */
101 #define EP_MAX_HASH_BITS 17
103 /* Minimum size of the hash in bits ( 2^N ) */
104 #define EP_MIN_HASH_BITS 9
106 /* Number of hash entries ( "struct list_head" ) inside a page */
107 #define EP_HENTRY_X_PAGE (PAGE_SIZE / sizeof(struct list_head))
109 /* Maximum size of the hash in pages */
110 #define EP_MAX_HPAGES ((1 << EP_MAX_HASH_BITS) / EP_HENTRY_X_PAGE + 1)
112 /* Number of pages allocated for an "hbits" sized hash table */
113 #define EP_HASH_PAGES(hbits) ((int) ((1 << (hbits)) / EP_HENTRY_X_PAGE + \
114 ((1 << (hbits)) % EP_HENTRY_X_PAGE ? 1: 0)))
116 /* Macro to allocate a "struct epitem" from the slab cache */
117 #define EPI_MEM_ALLOC() (struct epitem *) kmem_cache_alloc(epi_cache, SLAB_KERNEL)
119 /* Macro to free a "struct epitem" to the slab cache */
120 #define EPI_MEM_FREE(p) kmem_cache_free(epi_cache, p)
122 /* Macro to allocate a "struct eppoll_entry" from the slab cache */
123 #define PWQ_MEM_ALLOC() (struct eppoll_entry *) kmem_cache_alloc(pwq_cache, SLAB_KERNEL)
125 /* Macro to free a "struct eppoll_entry" to the slab cache */
126 #define PWQ_MEM_FREE(p) kmem_cache_free(pwq_cache, p)
128 /* Fast test to see if the file is an evenpoll file */
129 #define IS_FILE_EPOLL(f) ((f)->f_op == &eventpoll_fops)
132 * Remove the item from the list and perform its initialization.
133 * This is useful for us because we can test if the item is linked
134 * using "EP_IS_LINKED(p)".
136 #define EP_LIST_DEL(p) do { list_del(p); INIT_LIST_HEAD(p); } while (0)
138 /* Tells us if the item is currently linked */
139 #define EP_IS_LINKED(p) (!list_empty(p))
141 /* Get the "struct epitem" from a wait queue pointer */
142 #define EP_ITEM_FROM_WAIT(p) ((struct epitem *) container_of(p, struct eppoll_entry, wait)->base)
144 /* Get the "struct epitem" from an epoll queue wrapper */
145 #define EP_ITEM_FROM_EPQUEUE(p) (container_of(p, struct ep_pqueue, pt)->epi)
148 * This is used to optimize the event transfer to userspace. Since this
149 * is kept on stack, it should be pretty small.
151 #define EP_MAX_BUF_EVENTS 32
156 * Node that is linked into the "wake_task_list" member of the "struct poll_safewake".
157 * It is used to keep track on all tasks that are currently inside the wake_up() code
158 * to 1) short-circuit the one coming from the same task and same wait queue head
159 * ( loop ) 2) allow a maximum number of epoll descriptors inclusion nesting
160 * 3) let go the ones coming from other tasks.
162 struct wake_task_node
{
163 struct list_head llink
;
165 wait_queue_head_t
*wq
;
169 * This is used to implement the safe poll wake up avoiding to reenter
170 * the poll callback from inside wake_up().
172 struct poll_safewake
{
173 struct list_head wake_task_list
;
178 * This structure is stored inside the "private_data" member of the file
179 * structure and rapresent the main data sructure for the eventpoll
183 /* Protect the this structure access */
187 * This semaphore is used to ensure that files are not removed
188 * while epoll is using them. This is read-held during the event
189 * collection loop and it is write-held during the file cleanup
190 * path, the epoll file exit code and the ctl operations.
192 struct rw_semaphore sem
;
194 /* Wait queue used by sys_epoll_wait() */
195 wait_queue_head_t wq
;
197 /* Wait queue used by file->poll() */
198 wait_queue_head_t poll_wait
;
200 /* List of ready file descriptors */
201 struct list_head rdllist
;
203 /* Size of the hash */
204 unsigned int hashbits
;
206 /* Pages for the "struct epitem" hash */
207 char *hpages
[EP_MAX_HPAGES
];
210 /* Wait structure used by the poll hooks */
211 struct eppoll_entry
{
212 /* List header used to link this structure to the "struct epitem" */
213 struct list_head llink
;
215 /* The "base" pointer is set to the container "struct epitem" */
219 * Wait queue item that will be linked to the target file wait
224 /* The wait queue head that linked the "wait" wait queue item */
225 wait_queue_head_t
*whead
;
229 * Each file descriptor added to the eventpoll interface will
230 * have an entry of this type linked to the hash.
233 /* List header used to link this structure to the eventpoll hash */
234 struct list_head llink
;
236 /* List header used to link this structure to the eventpoll ready list */
237 struct list_head rdllink
;
239 /* Number of active wait queue attached to poll operations */
242 /* List containing poll wait queues */
243 struct list_head pwqlist
;
245 /* The "container" of this item */
246 struct eventpoll
*ep
;
248 /* The file descriptor this item refers to */
251 /* The file this item refers to */
254 /* The structure that describe the interested events and the source fd */
255 struct epoll_event event
;
258 * Used to keep track of the usage count of the structure. This avoids
259 * that the structure will desappear from underneath our processing.
263 /* List header used to link this item to the "struct file" items list */
264 struct list_head fllink
;
266 /* List header used to link the item to the transfer list */
267 struct list_head txlink
;
270 * This is used during the collection/transfer of events to userspace
271 * to pin items empty events set.
273 unsigned int revents
;
276 /* Wrapper struct used by poll queueing */
284 static void ep_poll_safewake_init(struct poll_safewake
*psw
);
285 static void ep_poll_safewake(struct poll_safewake
*psw
, wait_queue_head_t
*wq
);
286 static unsigned int ep_get_hash_bits(unsigned int hintsize
);
287 static int ep_getfd(int *efd
, struct inode
**einode
, struct file
**efile
);
288 static int ep_alloc_pages(char **pages
, int numpages
);
289 static int ep_free_pages(char **pages
, int numpages
);
290 static int ep_file_init(struct file
*file
, unsigned int hashbits
);
291 static unsigned int ep_hash_index(struct eventpoll
*ep
, struct file
*file
, int fd
);
292 static struct list_head
*ep_hash_entry(struct eventpoll
*ep
, unsigned int index
);
293 static int ep_init(struct eventpoll
*ep
, unsigned int hashbits
);
294 static void ep_free(struct eventpoll
*ep
);
295 static struct epitem
*ep_find(struct eventpoll
*ep
, struct file
*file
, int fd
);
296 static void ep_use_epitem(struct epitem
*epi
);
297 static void ep_release_epitem(struct epitem
*epi
);
298 static void ep_ptable_queue_proc(struct file
*file
, wait_queue_head_t
*whead
,
300 static int ep_insert(struct eventpoll
*ep
, struct epoll_event
*event
,
301 struct file
*tfile
, int fd
);
302 static int ep_modify(struct eventpoll
*ep
, struct epitem
*epi
, struct epoll_event
*event
);
303 static void ep_unregister_pollwait(struct eventpoll
*ep
, struct epitem
*epi
);
304 static int ep_unlink(struct eventpoll
*ep
, struct epitem
*epi
);
305 static int ep_remove(struct eventpoll
*ep
, struct epitem
*epi
);
306 static int ep_poll_callback(wait_queue_t
*wait
, unsigned mode
, int sync
);
307 static int ep_eventpoll_close(struct inode
*inode
, struct file
*file
);
308 static unsigned int ep_eventpoll_poll(struct file
*file
, poll_table
*wait
);
309 static int ep_collect_ready_items(struct eventpoll
*ep
,
310 struct list_head
*txlist
, int maxevents
);
311 static int ep_send_events(struct eventpoll
*ep
, struct list_head
*txlist
,
312 struct epoll_event
*events
);
313 static void ep_reinject_items(struct eventpoll
*ep
, struct list_head
*txlist
);
314 static int ep_events_transfer(struct eventpoll
*ep
,
315 struct epoll_event
*events
, int maxevents
);
316 static int ep_poll(struct eventpoll
*ep
, struct epoll_event
*events
,
317 int maxevents
, long timeout
);
318 static int eventpollfs_delete_dentry(struct dentry
*dentry
);
319 static struct inode
*ep_eventpoll_inode(void);
320 static struct super_block
*eventpollfs_get_sb(struct file_system_type
*fs_type
,
321 int flags
, const char *dev_name
,
325 * This semaphore is used to serialize ep_free() and eventpoll_release().
327 struct semaphore epsem
;
329 /* Safe wake up implementation */
330 static struct poll_safewake psw
;
332 /* Slab cache used to allocate "struct epitem" */
333 static kmem_cache_t
*epi_cache
;
335 /* Slab cache used to allocate "struct eppoll_entry" */
336 static kmem_cache_t
*pwq_cache
;
338 /* Virtual fs used to allocate inodes for eventpoll files */
339 static struct vfsmount
*eventpoll_mnt
;
341 /* File callbacks that implement the eventpoll file behaviour */
342 static struct file_operations eventpoll_fops
= {
343 .release
= ep_eventpoll_close
,
344 .poll
= ep_eventpoll_poll
348 * This is used to register the virtual file system from where
349 * eventpoll inodes are allocated.
351 static struct file_system_type eventpoll_fs_type
= {
352 .name
= "eventpollfs",
353 .get_sb
= eventpollfs_get_sb
,
354 .kill_sb
= kill_anon_super
,
357 /* Very basic directory entry operations for the eventpoll virtual file system */
358 static struct dentry_operations eventpollfs_dentry_operations
= {
359 .d_delete
= eventpollfs_delete_dentry
,
364 /* Initialize the poll safe wake up structure */
365 static void ep_poll_safewake_init(struct poll_safewake
*psw
)
368 INIT_LIST_HEAD(&psw
->wake_task_list
);
369 spin_lock_init(&psw
->lock
);
374 * Perform a safe wake up of the poll wait list. The problem is that
375 * with the new callback'd wake up system, it is possible that the
376 * poll callback is reentered from inside the call to wake_up() done
377 * on the poll wait queue head. The rule is that we cannot reenter the
378 * wake up code from the same task more than EP_MAX_POLLWAKE_NESTS times,
379 * and we cannot reenter the same wait queue head at all. This will
380 * enable to have a hierarchy of epoll file descriptor of no more than
381 * EP_MAX_POLLWAKE_NESTS deep. We need the irq version of the spin lock
382 * because this one gets called by the poll callback, that in turn is called
383 * from inside a wake_up(), that might be called from irq context.
385 static void ep_poll_safewake(struct poll_safewake
*psw
, wait_queue_head_t
*wq
)
389 task_t
*this_task
= current
;
390 struct list_head
*lsthead
= &psw
->wake_task_list
, *lnk
;
391 struct wake_task_node
*tncur
;
392 struct wake_task_node tnode
;
394 spin_lock_irqsave(&psw
->lock
, flags
);
396 /* Try to see if the current task is already inside this wakeup call */
397 list_for_each(lnk
, lsthead
) {
398 tncur
= list_entry(lnk
, struct wake_task_node
, llink
);
400 if (tncur
->wq
== wq
||
401 (tncur
->task
== this_task
&& ++wake_nests
> EP_MAX_POLLWAKE_NESTS
)) {
403 * Ops ... loop detected or maximum nest level reached.
404 * We abort this wake by breaking the cycle itself.
406 spin_unlock_irqrestore(&psw
->lock
, flags
);
411 /* Add the current task to the list */
412 tnode
.task
= this_task
;
414 list_add(&tnode
.llink
, lsthead
);
416 spin_unlock_irqrestore(&psw
->lock
, flags
);
418 /* Do really wake up now */
421 /* Remove the current task from the list */
422 spin_lock_irqsave(&psw
->lock
, flags
);
423 list_del(&tnode
.llink
);
424 spin_unlock_irqrestore(&psw
->lock
, flags
);
429 * Calculate the size of the hash in bits. The returned size will be
430 * bounded between EP_MIN_HASH_BITS and EP_MAX_HASH_BITS.
432 static unsigned int ep_get_hash_bits(unsigned int hintsize
)
436 for (i
= 0, val
= 1; val
< hintsize
&& i
< EP_MAX_HASH_BITS
; i
++, val
<<= 1);
437 return i
< EP_MIN_HASH_BITS
? EP_MIN_HASH_BITS
: i
;
441 /* Used to initialize the epoll bits inside the "struct file" */
442 void eventpoll_init_file(struct file
*file
)
445 INIT_LIST_HEAD(&file
->f_ep_links
);
446 spin_lock_init(&file
->f_ep_lock
);
451 * This is called from eventpoll_release() to unlink files from the eventpoll
452 * interface. We need to have this facility to cleanup correctly files that are
453 * closed without being removed from the eventpoll interface.
455 void eventpoll_release_file(struct file
*file
)
457 struct list_head
*lsthead
= &file
->f_ep_links
;
458 struct eventpoll
*ep
;
462 * We don't want to get "file->f_ep_lock" because it is not
463 * necessary. It is not necessary because we're in the "struct file"
464 * cleanup path, and this means that noone is using this file anymore.
465 * The only hit might come from ep_free() but by holding the semaphore
466 * will correctly serialize the operation. We do need to acquire
467 * "ep->sem" after "epsem" because ep_remove() requires it when called
468 * from anywhere but ep_free().
472 while (!list_empty(lsthead
)) {
473 epi
= list_entry(lsthead
->next
, struct epitem
, fllink
);
476 EP_LIST_DEL(&epi
->fllink
);
477 down_write(&ep
->sem
);
487 * It opens an eventpoll file descriptor by suggesting a storage of "size"
488 * file descriptors. The size parameter is just an hint about how to size
489 * data structures. It won't prevent the user to store more than "size"
490 * file descriptors inside the epoll interface. It is the kernel part of
491 * the userspace epoll_create(2).
493 asmlinkage
long sys_epoll_create(int size
)
496 unsigned int hashbits
;
500 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: sys_epoll_create(%d)\n",
503 /* Correctly size the hash */
504 hashbits
= ep_get_hash_bits((unsigned int) size
);
507 * Creates all the items needed to setup an eventpoll file. That is,
508 * a file structure, and inode and a free file descriptor.
510 error
= ep_getfd(&fd
, &inode
, &file
);
514 /* Setup the file internal data structure ( "struct eventpoll" ) */
515 error
= ep_file_init(file
, hashbits
);
520 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: sys_epoll_create(%d) = %d\n",
528 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: sys_epoll_create(%d) = %d\n",
529 current
, size
, error
));
535 * The following function implement the controller interface for the eventpoll
536 * file that enable the insertion/removal/change of file descriptors inside
537 * the interest set. It rapresents the kernel part of the user space epoll_ctl(2).
539 asmlinkage
long sys_epoll_ctl(int epfd
, int op
, int fd
, struct epoll_event
*event
)
542 struct file
*file
, *tfile
;
543 struct eventpoll
*ep
;
545 struct epoll_event epds
;
547 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: sys_epoll_ctl(%d, %d, %d, %p)\n",
548 current
, epfd
, op
, fd
, event
));
551 if (copy_from_user(&epds
, event
, sizeof(struct epoll_event
)))
554 /* Get the "struct file *" for the eventpoll file */
560 /* Get the "struct file *" for the target file */
565 /* The target file descriptor must support poll */
567 if (!tfile
->f_op
|| !tfile
->f_op
->poll
)
571 * We have to check that the file structure underneath the file descriptor
572 * the user passed to us _is_ an eventpoll file. And also we do not permit
573 * adding an epoll file descriptor inside itself.
576 if (file
== tfile
|| !IS_FILE_EPOLL(file
))
580 * At this point it is safe to assume that the "private_data" contains
581 * our own data structure.
583 ep
= file
->private_data
;
585 down_write(&ep
->sem
);
587 /* Try to lookup the file inside our hash table */
588 epi
= ep_find(ep
, tfile
, fd
);
594 epds
.events
|= POLLERR
| POLLHUP
;
596 error
= ep_insert(ep
, &epds
, tfile
, fd
);
602 error
= ep_remove(ep
, epi
);
608 epds
.events
|= POLLERR
| POLLHUP
;
609 error
= ep_modify(ep
, epi
, &epds
);
616 * The function ep_find() increments the usage count of the structure
617 * so, if this is not NULL, we need to release it.
620 ep_release_epitem(epi
);
629 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: sys_epoll_ctl(%d, %d, %d, %p) = %d\n",
630 current
, epfd
, op
, fd
, event
, error
));
637 * Implement the event wait interface for the eventpoll file. It is the kernel
638 * part of the user space epoll_wait(2).
640 asmlinkage
long sys_epoll_wait(int epfd
, struct epoll_event
*events
, int maxevents
,
645 struct eventpoll
*ep
;
647 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: sys_epoll_wait(%d, %p, %d, %d)\n",
648 current
, epfd
, events
, maxevents
, timeout
));
650 /* The maximum number of event must be greater than zero */
654 /* Verify that the area passed by the user is writeable */
655 if ((error
= verify_area(VERIFY_WRITE
, events
, maxevents
* sizeof(struct epoll_event
))))
658 /* Get the "struct file *" for the eventpoll file */
665 * We have to check that the file structure underneath the file descriptor
666 * the user passed to us _is_ an eventpoll file.
669 if (!IS_FILE_EPOLL(file
))
673 * At this point it is safe to assume that the "private_data" contains
674 * our own data structure.
676 ep
= file
->private_data
;
678 /* Time to fish for events ... */
679 error
= ep_poll(ep
, events
, maxevents
, timeout
);
684 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: sys_epoll_wait(%d, %p, %d, %d) = %d\n",
685 current
, epfd
, events
, maxevents
, timeout
, error
));
692 * Creates the file descriptor to be used by the epoll interface.
694 static int ep_getfd(int *efd
, struct inode
**einode
, struct file
**efile
)
698 struct dentry
*dentry
;
703 /* Get an ready to use file */
705 file
= get_empty_filp();
709 /* Allocates an inode from the eventpoll file system */
710 inode
= ep_eventpoll_inode();
711 error
= PTR_ERR(inode
);
715 /* Allocates a free descriptor to plug the file onto */
716 error
= get_unused_fd();
722 * Link the inode to a directory entry by creating a unique name
723 * using the inode number.
726 sprintf(name
, "[%lu]", inode
->i_ino
);
728 this.len
= strlen(name
);
729 this.hash
= inode
->i_ino
;
730 dentry
= d_alloc(eventpoll_mnt
->mnt_sb
->s_root
, &this);
733 dentry
->d_op
= &eventpollfs_dentry_operations
;
734 d_add(dentry
, inode
);
735 file
->f_vfsmnt
= mntget(eventpoll_mnt
);
736 file
->f_dentry
= dget(dentry
);
739 file
->f_flags
= O_RDONLY
;
740 file
->f_op
= &eventpoll_fops
;
741 file
->f_mode
= FMODE_READ
;
743 file
->private_data
= NULL
;
745 /* Install the new setup file into the allocated fd. */
746 fd_install(fd
, file
);
764 static int ep_alloc_pages(char **pages
, int numpages
)
768 for (i
= 0; i
< numpages
; i
++) {
769 pages
[i
] = (char *) __get_free_pages(GFP_KERNEL
, 0);
771 for (--i
; i
>= 0; i
--) {
772 ClearPageReserved(virt_to_page(pages
[i
]));
773 free_pages((unsigned long) pages
[i
], 0);
777 SetPageReserved(virt_to_page(pages
[i
]));
783 static int ep_free_pages(char **pages
, int numpages
)
787 for (i
= 0; i
< numpages
; i
++) {
788 ClearPageReserved(virt_to_page(pages
[i
]));
789 free_pages((unsigned long) pages
[i
], 0);
795 static int ep_file_init(struct file
*file
, unsigned int hashbits
)
798 struct eventpoll
*ep
;
800 if (!(ep
= kmalloc(sizeof(struct eventpoll
), GFP_KERNEL
)))
803 memset(ep
, 0, sizeof(*ep
));
805 error
= ep_init(ep
, hashbits
);
811 file
->private_data
= ep
;
813 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: ep_file_init() ep=%p\n",
820 * Calculate the index of the hash relative to "file".
822 static unsigned int ep_hash_index(struct eventpoll
*ep
, struct file
*file
, int fd
)
824 unsigned long ptr
= (unsigned long) file
^ (fd
<< ep
->hashbits
);
826 return (unsigned int) hash_ptr((void *) ptr
, ep
->hashbits
);
831 * Returns the hash entry ( struct list_head * ) of the passed index.
833 static struct list_head
*ep_hash_entry(struct eventpoll
*ep
, unsigned int index
)
836 return (struct list_head
*) (ep
->hpages
[index
/ EP_HENTRY_X_PAGE
] +
837 (index
% EP_HENTRY_X_PAGE
) * sizeof(struct list_head
));
841 static int ep_init(struct eventpoll
*ep
, unsigned int hashbits
)
844 unsigned int i
, hsize
;
846 rwlock_init(&ep
->lock
);
847 init_rwsem(&ep
->sem
);
848 init_waitqueue_head(&ep
->wq
);
849 init_waitqueue_head(&ep
->poll_wait
);
850 INIT_LIST_HEAD(&ep
->rdllist
);
852 /* Hash allocation and setup */
853 ep
->hashbits
= hashbits
;
854 error
= ep_alloc_pages(ep
->hpages
, EP_HASH_PAGES(ep
->hashbits
));
858 /* Initialize hash buckets */
859 for (i
= 0, hsize
= 1 << hashbits
; i
< hsize
; i
++)
860 INIT_LIST_HEAD(ep_hash_entry(ep
, i
));
868 static void ep_free(struct eventpoll
*ep
)
870 unsigned int i
, hsize
;
871 struct list_head
*lsthead
, *lnk
;
874 /* We need to release all tasks waiting for these file */
875 if (waitqueue_active(&ep
->poll_wait
))
876 ep_poll_safewake(&psw
, &ep
->poll_wait
);
879 * We need to lock this because we could be hit by
880 * eventpoll_release() while we're freeing the "struct eventpoll".
881 * We do not need to hold "ep->sem" here because the epoll file
882 * is on the way to be removed and no one has references to it
883 * anymore. The only hit might come from eventpoll_release() but
884 * holding "epsem" is sufficent here.
889 * Walks through the whole hash by unregistering poll callbacks.
891 for (i
= 0, hsize
= 1 << ep
->hashbits
; i
< hsize
; i
++) {
892 lsthead
= ep_hash_entry(ep
, i
);
894 list_for_each(lnk
, lsthead
) {
895 epi
= list_entry(lnk
, struct epitem
, llink
);
897 ep_unregister_pollwait(ep
, epi
);
902 * Walks through the whole hash by freeing each "struct epitem". At this
903 * point we are sure no poll callbacks will be lingering around, and also by
904 * write-holding "sem" we can be sure that no file cleanup code will hit
905 * us during this operation. So we can avoid the lock on "ep->lock".
907 for (i
= 0, hsize
= 1 << ep
->hashbits
; i
< hsize
; i
++) {
908 lsthead
= ep_hash_entry(ep
, i
);
910 while (!list_empty(lsthead
)) {
911 epi
= list_entry(lsthead
->next
, struct epitem
, llink
);
919 /* Free hash pages */
920 ep_free_pages(ep
->hpages
, EP_HASH_PAGES(ep
->hashbits
));
925 * Search the file inside the eventpoll hash. It add usage count to
926 * the returned item, so the caller must call ep_release_epitem()
927 * after finished using the "struct epitem".
929 static struct epitem
*ep_find(struct eventpoll
*ep
, struct file
*file
, int fd
)
932 struct list_head
*lsthead
, *lnk
;
933 struct epitem
*epi
= NULL
;
935 read_lock_irqsave(&ep
->lock
, flags
);
937 lsthead
= ep_hash_entry(ep
, ep_hash_index(ep
, file
, fd
));
938 list_for_each(lnk
, lsthead
) {
939 epi
= list_entry(lnk
, struct epitem
, llink
);
941 if (epi
->file
== file
&& epi
->fd
== fd
) {
948 read_unlock_irqrestore(&ep
->lock
, flags
);
950 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: ep_find(%p) -> %p\n",
951 current
, file
, epi
));
958 * Increment the usage count of the "struct epitem" making it sure
959 * that the user will have a valid pointer to reference.
961 static void ep_use_epitem(struct epitem
*epi
)
964 atomic_inc(&epi
->usecnt
);
969 * Decrement ( release ) the usage count by signaling that the user
970 * has finished using the structure. It might lead to freeing the
971 * structure itself if the count goes to zero.
973 static void ep_release_epitem(struct epitem
*epi
)
976 if (atomic_dec_and_test(&epi
->usecnt
))
982 * This is the callback that is used to add our wait queue to the
983 * target file wakeup lists.
985 static void ep_ptable_queue_proc(struct file
*file
, wait_queue_head_t
*whead
,
988 struct epitem
*epi
= EP_ITEM_FROM_EPQUEUE(pt
);
989 struct eppoll_entry
*pwq
;
991 if (epi
->nwait
>= 0 && (pwq
= PWQ_MEM_ALLOC())) {
992 init_waitqueue_func_entry(&pwq
->wait
, ep_poll_callback
);
995 add_wait_queue(whead
, &pwq
->wait
);
996 list_add_tail(&pwq
->llink
, &epi
->pwqlist
);
999 /* We have to signal that an error occurred */
1005 static int ep_insert(struct eventpoll
*ep
, struct epoll_event
*event
,
1006 struct file
*tfile
, int fd
)
1008 int error
, revents
, pwake
= 0;
1009 unsigned long flags
;
1011 struct ep_pqueue epq
;
1014 if (!(epi
= EPI_MEM_ALLOC()))
1017 /* Item initialization follow here ... */
1018 INIT_LIST_HEAD(&epi
->llink
);
1019 INIT_LIST_HEAD(&epi
->rdllink
);
1020 INIT_LIST_HEAD(&epi
->fllink
);
1021 INIT_LIST_HEAD(&epi
->txlink
);
1022 INIT_LIST_HEAD(&epi
->pwqlist
);
1026 epi
->event
= *event
;
1027 atomic_set(&epi
->usecnt
, 1);
1030 /* Initialize the poll table using the queue callback */
1032 init_poll_funcptr(&epq
.pt
, ep_ptable_queue_proc
);
1035 * Attach the item to the poll hooks and get current event bits.
1036 * We can safely use the file* here because its usage count has
1037 * been increased by the caller of this function.
1039 revents
= tfile
->f_op
->poll(tfile
, &epq
.pt
);
1042 * We have to check if something went wrong during the poll wait queue
1043 * install process. Namely an allocation for a wait queue failed due
1044 * high memory pressure.
1049 /* Add the current item to the list of active epoll hook for this file */
1050 spin_lock(&tfile
->f_ep_lock
);
1051 list_add_tail(&epi
->fllink
, &tfile
->f_ep_links
);
1052 spin_unlock(&tfile
->f_ep_lock
);
1054 /* We have to drop the new item inside our item list to keep track of it */
1055 write_lock_irqsave(&ep
->lock
, flags
);
1057 /* Add the current item to the hash table */
1058 list_add(&epi
->llink
, ep_hash_entry(ep
, ep_hash_index(ep
, tfile
, fd
)));
1060 /* If the file is already "ready" we drop it inside the ready list */
1061 if ((revents
& event
->events
) && !EP_IS_LINKED(&epi
->rdllink
)) {
1062 list_add_tail(&epi
->rdllink
, &ep
->rdllist
);
1064 /* Notify waiting tasks that events are available */
1065 if (waitqueue_active(&ep
->wq
))
1067 if (waitqueue_active(&ep
->poll_wait
))
1071 write_unlock_irqrestore(&ep
->lock
, flags
);
1073 /* We have to call this outside the lock */
1075 ep_poll_safewake(&psw
, &ep
->poll_wait
);
1077 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: ep_insert(%p, %p, %d)\n",
1078 current
, ep
, tfile
, fd
));
1083 ep_unregister_pollwait(ep
, epi
);
1086 * We need to do this because an event could have been arrived on some
1087 * allocated wait queue.
1089 write_lock_irqsave(&ep
->lock
, flags
);
1090 if (EP_IS_LINKED(&epi
->rdllink
))
1091 EP_LIST_DEL(&epi
->rdllink
);
1092 write_unlock_irqrestore(&ep
->lock
, flags
);
1101 * Modify the interest event mask by dropping an event if the new mask
1102 * has a match in the current file status.
1104 static int ep_modify(struct eventpoll
*ep
, struct epitem
*epi
, struct epoll_event
*event
)
1107 unsigned int revents
;
1108 unsigned long flags
;
1111 * Set the new event interest mask before calling f_op->poll(), otherwise
1112 * a potential race might occur. In fact if we do this operation inside
1113 * the lock, an event might happen between the f_op->poll() call and the
1114 * new event set registering.
1116 epi
->event
.events
= event
->events
;
1119 * Get current event bits. We can safely use the file* here because
1120 * its usage count has been increased by the caller of this function.
1122 revents
= epi
->file
->f_op
->poll(epi
->file
, NULL
);
1124 write_lock_irqsave(&ep
->lock
, flags
);
1126 /* Copy the data member from inside the lock */
1127 epi
->event
.data
= event
->data
;
1130 * If the item is not linked to the hash it means that it's on its
1131 * way toward the removal. Do nothing in this case.
1133 if (EP_IS_LINKED(&epi
->llink
)) {
1135 * If the item is "hot" and it is not registered inside the ready
1136 * list, push it inside. If the item is not "hot" and it is currently
1137 * registered inside the ready list, unlink it.
1139 if (revents
& event
->events
) {
1140 if (!EP_IS_LINKED(&epi
->rdllink
)) {
1141 list_add_tail(&epi
->rdllink
, &ep
->rdllist
);
1143 /* Notify waiting tasks that events are available */
1144 if (waitqueue_active(&ep
->wq
))
1146 if (waitqueue_active(&ep
->poll_wait
))
1149 } else if (EP_IS_LINKED(&epi
->rdllink
))
1150 EP_LIST_DEL(&epi
->rdllink
);
1153 write_unlock_irqrestore(&ep
->lock
, flags
);
1155 /* We have to call this outside the lock */
1157 ep_poll_safewake(&psw
, &ep
->poll_wait
);
1164 * This function unregister poll callbacks from the associated file descriptor.
1165 * Since this must be called without holding "ep->lock" the atomic exchange trick
1166 * will protect us from multiple unregister.
1168 static void ep_unregister_pollwait(struct eventpoll
*ep
, struct epitem
*epi
)
1171 struct list_head
*lsthead
= &epi
->pwqlist
;
1172 struct eppoll_entry
*pwq
;
1174 /* This is called without locks, so we need the atomic exchange */
1175 nwait
= xchg(&epi
->nwait
, 0);
1178 while (!list_empty(lsthead
)) {
1179 pwq
= list_entry(lsthead
->next
, struct eppoll_entry
, llink
);
1181 EP_LIST_DEL(&pwq
->llink
);
1182 remove_wait_queue(pwq
->whead
, &pwq
->wait
);
1190 * Unlink the "struct epitem" from all places it might have been hooked up.
1191 * This function must be called with write IRQ lock on "ep->lock".
1193 static int ep_unlink(struct eventpoll
*ep
, struct epitem
*epi
)
1198 * It can happen that this one is called for an item already unlinked.
1199 * The check protect us from doing a double unlink ( crash ).
1202 if (!EP_IS_LINKED(&epi
->llink
))
1206 * Clear the event mask for the unlinked item. This will avoid item
1207 * notifications to be sent after the unlink operation from inside
1208 * the kernel->userspace event transfer loop.
1210 epi
->event
.events
= 0;
1213 * At this point is safe to do the job, unlink the item from our list.
1214 * This operation togheter with the above check closes the door to
1217 EP_LIST_DEL(&epi
->llink
);
1220 * If the item we are going to remove is inside the ready file descriptors
1221 * we want to remove it from this list to avoid stale events.
1223 if (EP_IS_LINKED(&epi
->rdllink
))
1224 EP_LIST_DEL(&epi
->rdllink
);
1229 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: ep_unlink(%p, %p) = %d\n",
1230 current
, ep
, epi
->file
, error
));
1237 * Removes a "struct epitem" from the eventpoll hash and deallocates
1238 * all the associated resources.
1240 static int ep_remove(struct eventpoll
*ep
, struct epitem
*epi
)
1243 unsigned long flags
;
1244 struct file
*file
= epi
->file
;
1247 * Removes poll wait queue hooks. We _have_ to do this without holding
1248 * the "ep->lock" otherwise a deadlock might occur. This because of the
1249 * sequence of the lock acquisition. Here we do "ep->lock" then the wait
1250 * queue head lock when unregistering the wait queue. The wakeup callback
1251 * will run by holding the wait queue head lock and will call our callback
1252 * that will try to get "ep->lock".
1254 ep_unregister_pollwait(ep
, epi
);
1256 /* Remove the current item from the list of epoll hooks */
1257 spin_lock(&file
->f_ep_lock
);
1258 if (EP_IS_LINKED(&epi
->fllink
))
1259 EP_LIST_DEL(&epi
->fllink
);
1260 spin_unlock(&file
->f_ep_lock
);
1262 /* We need to acquire the write IRQ lock before calling ep_unlink() */
1263 write_lock_irqsave(&ep
->lock
, flags
);
1265 /* Really unlink the item from the hash */
1266 error
= ep_unlink(ep
, epi
);
1268 write_unlock_irqrestore(&ep
->lock
, flags
);
1273 /* At this point it is safe to free the eventpoll item */
1274 ep_release_epitem(epi
);
1278 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: ep_remove(%p, %p) = %d\n",
1279 current
, ep
, file
, error
));
1286 * This is the callback that is passed to the wait queue wakeup
1287 * machanism. It is called by the stored file descriptors when they
1288 * have events to report.
1290 static int ep_poll_callback(wait_queue_t
*wait
, unsigned mode
, int sync
)
1293 unsigned long flags
;
1294 struct epitem
*epi
= EP_ITEM_FROM_WAIT(wait
);
1295 struct eventpoll
*ep
= epi
->ep
;
1297 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: poll_callback(%p) epi=%p ep=%p\n",
1298 current
, epi
->file
, epi
, ep
));
1300 write_lock_irqsave(&ep
->lock
, flags
);
1302 /* If this file is already in the ready list we exit soon */
1303 if (EP_IS_LINKED(&epi
->rdllink
))
1306 list_add_tail(&epi
->rdllink
, &ep
->rdllist
);
1310 * Wake up ( if active ) both the eventpoll wait list and the ->poll()
1313 if (waitqueue_active(&ep
->wq
))
1315 if (waitqueue_active(&ep
->poll_wait
))
1318 write_unlock_irqrestore(&ep
->lock
, flags
);
1320 /* We have to call this outside the lock */
1322 ep_poll_safewake(&psw
, &ep
->poll_wait
);
1328 static int ep_eventpoll_close(struct inode
*inode
, struct file
*file
)
1330 struct eventpoll
*ep
= file
->private_data
;
1337 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: close() ep=%p\n", current
, ep
));
1342 static unsigned int ep_eventpoll_poll(struct file
*file
, poll_table
*wait
)
1344 unsigned int pollflags
= 0;
1345 unsigned long flags
;
1346 struct eventpoll
*ep
= file
->private_data
;
1348 /* Insert inside our poll wait queue */
1349 poll_wait(file
, &ep
->poll_wait
, wait
);
1351 /* Check our condition */
1352 read_lock_irqsave(&ep
->lock
, flags
);
1353 if (!list_empty(&ep
->rdllist
))
1354 pollflags
= POLLIN
| POLLRDNORM
;
1355 read_unlock_irqrestore(&ep
->lock
, flags
);
1362 * Since we have to release the lock during the __copy_to_user() operation and
1363 * during the f_op->poll() call, we try to collect the maximum number of items
1364 * by reducing the irqlock/irqunlock switching rate.
1366 static int ep_collect_ready_items(struct eventpoll
*ep
, struct list_head
*txlist
, int maxevents
)
1369 unsigned long flags
;
1370 struct list_head
*lsthead
= &ep
->rdllist
, *lnk
;
1373 write_lock_irqsave(&ep
->lock
, flags
);
1375 for (nepi
= 0, lnk
= lsthead
->next
; lnk
!= lsthead
&& nepi
< maxevents
;) {
1376 epi
= list_entry(lnk
, struct epitem
, rdllink
);
1380 /* If this file is already in the ready list we exit soon */
1381 if (!EP_IS_LINKED(&epi
->txlink
)) {
1383 * This is initialized in this way so that the default
1384 * behaviour of the reinjecting code will be to push back
1385 * the item inside the ready list.
1387 epi
->revents
= epi
->event
.events
;
1389 /* Link the ready item into the transfer list */
1390 list_add(&epi
->txlink
, txlist
);
1394 * Unlink the item from the ready list.
1396 EP_LIST_DEL(&epi
->rdllink
);
1400 write_unlock_irqrestore(&ep
->lock
, flags
);
1407 * This function is called without holding the "ep->lock" since the call to
1408 * __copy_to_user() might sleep, and also f_op->poll() might reenable the IRQ
1409 * because of the way poll() is traditionally implemented in Linux.
1411 static int ep_send_events(struct eventpoll
*ep
, struct list_head
*txlist
,
1412 struct epoll_event
*events
)
1414 int eventcnt
= 0, eventbuf
= 0;
1415 unsigned int revents
;
1416 struct list_head
*lnk
;
1418 struct epoll_event event
[EP_MAX_BUF_EVENTS
];
1421 * We can loop without lock because this is a task private list.
1422 * The test done during the collection loop will guarantee us that
1423 * another task will not try to collect this file. Also, items
1424 * cannot vanish during the loop because we are holding "sem".
1426 list_for_each(lnk
, txlist
) {
1427 epi
= list_entry(lnk
, struct epitem
, txlink
);
1430 * Get the ready file event set. We can safely use the file
1431 * because we are holding the "sem" in read and this will
1432 * guarantee that both the file and the item will not vanish.
1434 revents
= epi
->file
->f_op
->poll(epi
->file
, NULL
);
1437 * Set the return event set for the current file descriptor.
1438 * Note that only the task task was successfully able to link
1439 * the item to its "txlist" will write this field.
1441 epi
->revents
= revents
& epi
->event
.events
;
1444 event
[eventbuf
] = epi
->event
;
1445 event
[eventbuf
].events
&= revents
;
1447 if (eventbuf
== EP_MAX_BUF_EVENTS
) {
1448 if (__copy_to_user(&events
[eventcnt
], event
,
1449 eventbuf
* sizeof(struct epoll_event
)))
1451 eventcnt
+= eventbuf
;
1458 if (__copy_to_user(&events
[eventcnt
], event
,
1459 eventbuf
* sizeof(struct epoll_event
)))
1461 eventcnt
+= eventbuf
;
1469 * Walk through the transfer list we collected with ep_collect_ready_items()
1470 * and, if 1) the item is still "alive" 2) its event set is not empty 3) it's
1471 * not already linked, links it to the ready list. Same as above, we are holding
1472 * "sem" so items cannot vanish underneath our nose.
1474 static void ep_reinject_items(struct eventpoll
*ep
, struct list_head
*txlist
)
1476 int ricnt
= 0, pwake
= 0;
1477 unsigned long flags
;
1480 write_lock_irqsave(&ep
->lock
, flags
);
1482 while (!list_empty(txlist
)) {
1483 epi
= list_entry(txlist
->next
, struct epitem
, txlink
);
1485 /* Unlink the current item from the transfer list */
1486 EP_LIST_DEL(&epi
->txlink
);
1489 * If the item is no more linked to the interest set, we don't
1490 * have to push it inside the ready list because the following
1491 * ep_release_epitem() is going to drop it. Also, if the current
1492 * item is set to have an Edge Triggered behaviour, we don't have
1493 * to push it back either.
1495 if (EP_IS_LINKED(&epi
->llink
) && !(epi
->event
.events
& EPOLLET
) &&
1496 (epi
->revents
& epi
->event
.events
) && !EP_IS_LINKED(&epi
->rdllink
)) {
1497 list_add_tail(&epi
->rdllink
, &ep
->rdllist
);
1504 * Wake up ( if active ) both the eventpoll wait list and the ->poll()
1507 if (waitqueue_active(&ep
->wq
))
1509 if (waitqueue_active(&ep
->poll_wait
))
1513 write_unlock_irqrestore(&ep
->lock
, flags
);
1515 /* We have to call this outside the lock */
1517 ep_poll_safewake(&psw
, &ep
->poll_wait
);
1522 * Perform the transfer of events to user space.
1524 static int ep_events_transfer(struct eventpoll
*ep
, struct epoll_event
*events
, int maxevents
)
1527 struct list_head txlist
;
1529 INIT_LIST_HEAD(&txlist
);
1532 * We need to lock this because we could be hit by
1533 * eventpoll_release() and epoll_ctl(EPOLL_CTL_DEL).
1535 down_read(&ep
->sem
);
1537 /* Collect/extract ready items */
1538 if (ep_collect_ready_items(ep
, &txlist
, maxevents
) > 0) {
1539 /* Build result set in userspace */
1540 eventcnt
= ep_send_events(ep
, &txlist
, events
);
1542 /* Reinject ready items into the ready list */
1543 ep_reinject_items(ep
, &txlist
);
1552 static int ep_poll(struct eventpoll
*ep
, struct epoll_event
*events
, int maxevents
,
1556 unsigned long flags
;
1561 * Calculate the timeout by checking for the "infinite" value ( -1 )
1562 * and the overflow condition. The passed timeout is in milliseconds,
1563 * that why (t * HZ) / 1000.
1565 jtimeout
= timeout
== -1 || timeout
> (MAX_SCHEDULE_TIMEOUT
- 1000) / HZ
?
1566 MAX_SCHEDULE_TIMEOUT
: (timeout
* HZ
+ 999) / 1000;
1569 write_lock_irqsave(&ep
->lock
, flags
);
1572 if (list_empty(&ep
->rdllist
)) {
1574 * We don't have any available event to return to the caller.
1575 * We need to sleep here, and we will be wake up by
1576 * ep_poll_callback() when events will become available.
1578 init_waitqueue_entry(&wait
, current
);
1579 add_wait_queue(&ep
->wq
, &wait
);
1583 * We don't want to sleep if the ep_poll_callback() sends us
1584 * a wakeup in between. That's why we set the task state
1585 * to TASK_INTERRUPTIBLE before doing the checks.
1587 set_current_state(TASK_INTERRUPTIBLE
);
1588 if (!list_empty(&ep
->rdllist
) || !jtimeout
)
1590 if (signal_pending(current
)) {
1595 write_unlock_irqrestore(&ep
->lock
, flags
);
1596 jtimeout
= schedule_timeout(jtimeout
);
1597 write_lock_irqsave(&ep
->lock
, flags
);
1599 remove_wait_queue(&ep
->wq
, &wait
);
1601 set_current_state(TASK_RUNNING
);
1604 /* Is it worth to try to dig for events ? */
1605 eavail
= !list_empty(&ep
->rdllist
);
1607 write_unlock_irqrestore(&ep
->lock
, flags
);
1610 * Try to transfer events to user space. In case we get 0 events and
1611 * there's still timeout left over, we go trying again in search of
1614 if (!res
&& eavail
&&
1615 !(res
= ep_events_transfer(ep
, events
, maxevents
)) && jtimeout
)
1622 static int eventpollfs_delete_dentry(struct dentry
*dentry
)
1629 static struct inode
*ep_eventpoll_inode(void)
1631 int error
= -ENOMEM
;
1632 struct inode
*inode
= new_inode(eventpoll_mnt
->mnt_sb
);
1637 inode
->i_fop
= &eventpoll_fops
;
1640 * Mark the inode dirty from the very beginning,
1641 * that way it will never be moved to the dirty
1642 * list because mark_inode_dirty() will think
1643 * that it already _is_ on the dirty list.
1645 inode
->i_state
= I_DIRTY
;
1646 inode
->i_mode
= S_IRUSR
| S_IWUSR
;
1647 inode
->i_uid
= current
->fsuid
;
1648 inode
->i_gid
= current
->fsgid
;
1649 inode
->i_atime
= inode
->i_mtime
= inode
->i_ctime
= CURRENT_TIME
;
1650 inode
->i_blksize
= PAGE_SIZE
;
1654 return ERR_PTR(error
);
1658 static struct super_block
*
1659 eventpollfs_get_sb(struct file_system_type
*fs_type
, int flags
,
1660 const char *dev_name
, void *data
)
1662 return get_sb_pseudo(fs_type
, "eventpoll:", NULL
, EVENTPOLLFS_MAGIC
);
1666 static int __init
eventpoll_init(void)
1672 /* Initialize the structure used to perform safe poll wait head wake ups */
1673 ep_poll_safewake_init(&psw
);
1675 /* Allocates slab cache used to allocate "struct epitem" items */
1677 epi_cache
= kmem_cache_create("eventpoll_epi",
1678 sizeof(struct epitem
),
1680 SLAB_HWCACHE_ALIGN
| EPI_SLAB_DEBUG
, NULL
, NULL
);
1684 /* Allocates slab cache used to allocate "struct eppoll_entry" */
1686 pwq_cache
= kmem_cache_create("eventpoll_pwq",
1687 sizeof(struct eppoll_entry
),
1689 EPI_SLAB_DEBUG
, NULL
, NULL
);
1694 * Register the virtual file system that will be the source of inodes
1695 * for the eventpoll files
1697 error
= register_filesystem(&eventpoll_fs_type
);
1701 /* Mount the above commented virtual file system */
1702 eventpoll_mnt
= kern_mount(&eventpoll_fs_type
);
1703 error
= PTR_ERR(eventpoll_mnt
);
1704 if (IS_ERR(eventpoll_mnt
))
1707 DNPRINTK(3, (KERN_INFO
"[%p] eventpoll: successfully initialized.\n", current
));
1712 unregister_filesystem(&eventpoll_fs_type
);
1714 kmem_cache_destroy(pwq_cache
);
1716 kmem_cache_destroy(epi_cache
);
1723 static void __exit
eventpoll_exit(void)
1725 /* Undo all operations done inside eventpoll_init() */
1726 unregister_filesystem(&eventpoll_fs_type
);
1727 mntput(eventpoll_mnt
);
1728 kmem_cache_destroy(pwq_cache
);
1729 kmem_cache_destroy(epi_cache
);
1732 module_init(eventpoll_init
);
1733 module_exit(eventpoll_exit
);
1735 MODULE_LICENSE("GPL");