2 * This file contains the procedures for the handling of select and poll
4 * Created for Linux based loosely upon Mathius Lattner's minix
5 * patches by Peter MacDonald. Heavily edited by Linus.
8 * COFF/ELF binary emulation. If the process has the STICKY_TIMEOUTS
9 * flag set in its personality we do *not* modify the given timeout
10 * parameter to reflect time remaining.
13 * Changed sys_poll()/do_poll() to use PAGE_SIZE chunk-based allocation
14 * of fds to overcome nfds < 16390 descriptors limit (Tigran Aivazian).
17 #include <linux/kernel.h>
18 #include <linux/syscalls.h>
19 #include <linux/module.h>
20 #include <linux/slab.h>
21 #include <linux/poll.h>
22 #include <linux/personality.h> /* for STICKY_TIMEOUTS */
23 #include <linux/file.h>
24 #include <linux/fdtable.h>
26 #include <linux/rcupdate.h>
28 #include <asm/uaccess.h>
30 struct poll_table_page
{
31 struct poll_table_page
* next
;
32 struct poll_table_entry
* entry
;
33 struct poll_table_entry entries
[0];
36 #define POLL_TABLE_FULL(table) \
37 ((unsigned long)((table)->entry+1) > PAGE_SIZE + (unsigned long)(table))
40 * Ok, Peter made a complicated, but straightforward multiple_wait() function.
41 * I have rewritten this, taking some shortcuts: This code may not be easy to
42 * follow, but it should be free of race-conditions, and it's practical. If you
43 * understand what I'm doing here, then you understand how the linux
44 * sleep/wakeup mechanism works.
46 * Two very simple procedures, poll_wait() and poll_freewait() make all the
47 * work. poll_wait() is an inline-function defined in <linux/poll.h>,
48 * as all select/poll functions have to call it to add an entry to the
51 static void __pollwait(struct file
*filp
, wait_queue_head_t
*wait_address
,
54 void poll_initwait(struct poll_wqueues
*pwq
)
56 init_poll_funcptr(&pwq
->pt
, __pollwait
);
59 pwq
->inline_index
= 0;
62 EXPORT_SYMBOL(poll_initwait
);
64 static void free_poll_entry(struct poll_table_entry
*entry
)
66 remove_wait_queue(entry
->wait_address
, &entry
->wait
);
70 void poll_freewait(struct poll_wqueues
*pwq
)
72 struct poll_table_page
* p
= pwq
->table
;
74 for (i
= 0; i
< pwq
->inline_index
; i
++)
75 free_poll_entry(pwq
->inline_entries
+ i
);
77 struct poll_table_entry
* entry
;
78 struct poll_table_page
*old
;
83 free_poll_entry(entry
);
84 } while (entry
> p
->entries
);
87 free_page((unsigned long) old
);
91 EXPORT_SYMBOL(poll_freewait
);
93 static struct poll_table_entry
*poll_get_entry(poll_table
*_p
)
95 struct poll_wqueues
*p
= container_of(_p
, struct poll_wqueues
, pt
);
96 struct poll_table_page
*table
= p
->table
;
98 if (p
->inline_index
< N_INLINE_POLL_ENTRIES
)
99 return p
->inline_entries
+ p
->inline_index
++;
101 if (!table
|| POLL_TABLE_FULL(table
)) {
102 struct poll_table_page
*new_table
;
104 new_table
= (struct poll_table_page
*) __get_free_page(GFP_KERNEL
);
107 __set_current_state(TASK_RUNNING
);
110 new_table
->entry
= new_table
->entries
;
111 new_table
->next
= table
;
112 p
->table
= new_table
;
116 return table
->entry
++;
119 /* Add a new entry */
120 static void __pollwait(struct file
*filp
, wait_queue_head_t
*wait_address
,
123 struct poll_table_entry
*entry
= poll_get_entry(p
);
128 entry
->wait_address
= wait_address
;
129 init_waitqueue_entry(&entry
->wait
, current
);
130 add_wait_queue(wait_address
, &entry
->wait
);
133 #define FDS_IN(fds, n) (fds->in + n)
134 #define FDS_OUT(fds, n) (fds->out + n)
135 #define FDS_EX(fds, n) (fds->ex + n)
137 #define BITS(fds, n) (*FDS_IN(fds, n)|*FDS_OUT(fds, n)|*FDS_EX(fds, n))
139 static int max_select_fd(unsigned long n
, fd_set_bits
*fds
)
141 unsigned long *open_fds
;
146 /* handle last in-complete long-word first */
147 set
= ~(~0UL << (n
& (__NFDBITS
-1)));
149 fdt
= files_fdtable(current
->files
);
150 open_fds
= fdt
->open_fds
->fds_bits
+n
;
155 if (!(set
& ~*open_fds
))
166 if (set
& ~*open_fds
)
175 max
+= n
* __NFDBITS
;
181 #define POLLIN_SET (POLLRDNORM | POLLRDBAND | POLLIN | POLLHUP | POLLERR)
182 #define POLLOUT_SET (POLLWRBAND | POLLWRNORM | POLLOUT | POLLERR)
183 #define POLLEX_SET (POLLPRI)
185 int do_select(int n
, fd_set_bits
*fds
, s64
*timeout
)
187 struct poll_wqueues table
;
192 retval
= max_select_fd(n
, fds
);
199 poll_initwait(&table
);
205 unsigned long *rinp
, *routp
, *rexp
, *inp
, *outp
, *exp
;
208 set_current_state(TASK_INTERRUPTIBLE
);
210 inp
= fds
->in
; outp
= fds
->out
; exp
= fds
->ex
;
211 rinp
= fds
->res_in
; routp
= fds
->res_out
; rexp
= fds
->res_ex
;
213 for (i
= 0; i
< n
; ++rinp
, ++routp
, ++rexp
) {
214 unsigned long in
, out
, ex
, all_bits
, bit
= 1, mask
, j
;
215 unsigned long res_in
= 0, res_out
= 0, res_ex
= 0;
216 const struct file_operations
*f_op
= NULL
;
217 struct file
*file
= NULL
;
219 in
= *inp
++; out
= *outp
++; ex
= *exp
++;
220 all_bits
= in
| out
| ex
;
226 for (j
= 0; j
< __NFDBITS
; ++j
, ++i
, bit
<<= 1) {
230 if (!(bit
& all_bits
))
232 file
= fget_light(i
, &fput_needed
);
235 mask
= DEFAULT_POLLMASK
;
236 if (f_op
&& f_op
->poll
)
237 mask
= (*f_op
->poll
)(file
, retval
? NULL
: wait
);
238 fput_light(file
, fput_needed
);
239 if ((mask
& POLLIN_SET
) && (in
& bit
)) {
243 if ((mask
& POLLOUT_SET
) && (out
& bit
)) {
247 if ((mask
& POLLEX_SET
) && (ex
& bit
)) {
262 if (retval
|| !*timeout
|| signal_pending(current
))
265 retval
= table
.error
;
270 /* Wait indefinitely */
271 __timeout
= MAX_SCHEDULE_TIMEOUT
;
272 } else if (unlikely(*timeout
>= (s64
)MAX_SCHEDULE_TIMEOUT
- 1)) {
273 /* Wait for longer than MAX_SCHEDULE_TIMEOUT. Do it in a loop */
274 __timeout
= MAX_SCHEDULE_TIMEOUT
- 1;
275 *timeout
-= __timeout
;
277 __timeout
= *timeout
;
280 __timeout
= schedule_timeout(__timeout
);
282 *timeout
+= __timeout
;
284 __set_current_state(TASK_RUNNING
);
286 poll_freewait(&table
);
292 * We can actually return ERESTARTSYS instead of EINTR, but I'd
293 * like to be certain this leads to no problems. So I return
294 * EINTR just for safety.
296 * Update: ERESTARTSYS breaks at least the xview clock binary, so
297 * I'm trying ERESTARTNOHAND which restart only when you want to.
299 #define MAX_SELECT_SECONDS \
300 ((unsigned long) (MAX_SCHEDULE_TIMEOUT / HZ)-1)
302 int core_sys_select(int n
, fd_set __user
*inp
, fd_set __user
*outp
,
303 fd_set __user
*exp
, s64
*timeout
)
310 /* Allocate small arguments on the stack to save memory and be faster */
311 long stack_fds
[SELECT_STACK_ALLOC
/sizeof(long)];
317 /* max_fds can increase, so grab it once to avoid race */
319 fdt
= files_fdtable(current
->files
);
320 max_fds
= fdt
->max_fds
;
326 * We need 6 bitmaps (in/out/ex for both incoming and outgoing),
327 * since we used fdset we need to allocate memory in units of
332 if (size
> sizeof(stack_fds
) / 6) {
333 /* Not enough space in on-stack array; must use kmalloc */
335 bits
= kmalloc(6 * size
, GFP_KERNEL
);
340 fds
.out
= bits
+ size
;
341 fds
.ex
= bits
+ 2*size
;
342 fds
.res_in
= bits
+ 3*size
;
343 fds
.res_out
= bits
+ 4*size
;
344 fds
.res_ex
= bits
+ 5*size
;
346 if ((ret
= get_fd_set(n
, inp
, fds
.in
)) ||
347 (ret
= get_fd_set(n
, outp
, fds
.out
)) ||
348 (ret
= get_fd_set(n
, exp
, fds
.ex
)))
350 zero_fd_set(n
, fds
.res_in
);
351 zero_fd_set(n
, fds
.res_out
);
352 zero_fd_set(n
, fds
.res_ex
);
354 ret
= do_select(n
, &fds
, timeout
);
359 ret
= -ERESTARTNOHAND
;
360 if (signal_pending(current
))
365 if (set_fd_set(n
, inp
, fds
.res_in
) ||
366 set_fd_set(n
, outp
, fds
.res_out
) ||
367 set_fd_set(n
, exp
, fds
.res_ex
))
371 if (bits
!= stack_fds
)
377 asmlinkage
long sys_select(int n
, fd_set __user
*inp
, fd_set __user
*outp
,
378 fd_set __user
*exp
, struct timeval __user
*tvp
)
385 if (copy_from_user(&tv
, tvp
, sizeof(tv
)))
388 if (tv
.tv_sec
< 0 || tv
.tv_usec
< 0)
391 /* Cast to u64 to make GCC stop complaining */
392 if ((u64
)tv
.tv_sec
>= (u64
)MAX_INT64_SECONDS
)
393 timeout
= -1; /* infinite */
395 timeout
= DIV_ROUND_UP(tv
.tv_usec
, USEC_PER_SEC
/HZ
);
396 timeout
+= tv
.tv_sec
* HZ
;
400 ret
= core_sys_select(n
, inp
, outp
, exp
, &timeout
);
405 if (current
->personality
& STICKY_TIMEOUTS
)
407 rtv
.tv_usec
= jiffies_to_usecs(do_div((*(u64
*)&timeout
), HZ
));
408 rtv
.tv_sec
= timeout
;
409 if (timeval_compare(&rtv
, &tv
) >= 0)
411 if (copy_to_user(tvp
, &rtv
, sizeof(rtv
))) {
414 * If an application puts its timeval in read-only
415 * memory, we don't want the Linux-specific update to
416 * the timeval to cause a fault after the select has
417 * completed successfully. However, because we're not
418 * updating the timeval, we can't restart the system
421 if (ret
== -ERESTARTNOHAND
)
429 #ifdef HAVE_SET_RESTORE_SIGMASK
430 asmlinkage
long sys_pselect7(int n
, fd_set __user
*inp
, fd_set __user
*outp
,
431 fd_set __user
*exp
, struct timespec __user
*tsp
,
432 const sigset_t __user
*sigmask
, size_t sigsetsize
)
434 s64 timeout
= MAX_SCHEDULE_TIMEOUT
;
435 sigset_t ksigmask
, sigsaved
;
440 if (copy_from_user(&ts
, tsp
, sizeof(ts
)))
443 if (ts
.tv_sec
< 0 || ts
.tv_nsec
< 0)
446 /* Cast to u64 to make GCC stop complaining */
447 if ((u64
)ts
.tv_sec
>= (u64
)MAX_INT64_SECONDS
)
448 timeout
= -1; /* infinite */
450 timeout
= DIV_ROUND_UP(ts
.tv_nsec
, NSEC_PER_SEC
/HZ
);
451 timeout
+= ts
.tv_sec
* HZ
;
456 /* XXX: Don't preclude handling different sized sigset_t's. */
457 if (sigsetsize
!= sizeof(sigset_t
))
459 if (copy_from_user(&ksigmask
, sigmask
, sizeof(ksigmask
)))
462 sigdelsetmask(&ksigmask
, sigmask(SIGKILL
)|sigmask(SIGSTOP
));
463 sigprocmask(SIG_SETMASK
, &ksigmask
, &sigsaved
);
466 ret
= core_sys_select(n
, inp
, outp
, exp
, &timeout
);
471 if (current
->personality
& STICKY_TIMEOUTS
)
473 rts
.tv_nsec
= jiffies_to_usecs(do_div((*(u64
*)&timeout
), HZ
)) *
475 rts
.tv_sec
= timeout
;
476 if (timespec_compare(&rts
, &ts
) >= 0)
478 if (copy_to_user(tsp
, &rts
, sizeof(rts
))) {
481 * If an application puts its timeval in read-only
482 * memory, we don't want the Linux-specific update to
483 * the timeval to cause a fault after the select has
484 * completed successfully. However, because we're not
485 * updating the timeval, we can't restart the system
488 if (ret
== -ERESTARTNOHAND
)
493 if (ret
== -ERESTARTNOHAND
) {
495 * Don't restore the signal mask yet. Let do_signal() deliver
496 * the signal on the way back to userspace, before the signal
500 memcpy(¤t
->saved_sigmask
, &sigsaved
,
502 set_restore_sigmask();
505 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
511 * Most architectures can't handle 7-argument syscalls. So we provide a
512 * 6-argument version where the sixth argument is a pointer to a structure
513 * which has a pointer to the sigset_t itself followed by a size_t containing
516 asmlinkage
long sys_pselect6(int n
, fd_set __user
*inp
, fd_set __user
*outp
,
517 fd_set __user
*exp
, struct timespec __user
*tsp
, void __user
*sig
)
519 size_t sigsetsize
= 0;
520 sigset_t __user
*up
= NULL
;
523 if (!access_ok(VERIFY_READ
, sig
, sizeof(void *)+sizeof(size_t))
524 || __get_user(up
, (sigset_t __user
* __user
*)sig
)
525 || __get_user(sigsetsize
,
526 (size_t __user
*)(sig
+sizeof(void *))))
530 return sys_pselect7(n
, inp
, outp
, exp
, tsp
, up
, sigsetsize
);
532 #endif /* HAVE_SET_RESTORE_SIGMASK */
535 struct poll_list
*next
;
537 struct pollfd entries
[0];
540 #define POLLFD_PER_PAGE ((PAGE_SIZE-sizeof(struct poll_list)) / sizeof(struct pollfd))
543 * Fish for pollable events on the pollfd->fd file descriptor. We're only
544 * interested in events matching the pollfd->events mask, and the result
545 * matching that mask is both recorded in pollfd->revents and returned. The
546 * pwait poll_table will be used by the fd-provided poll handler for waiting,
549 static inline unsigned int do_pollfd(struct pollfd
*pollfd
, poll_table
*pwait
)
560 file
= fget_light(fd
, &fput_needed
);
563 mask
= DEFAULT_POLLMASK
;
564 if (file
->f_op
&& file
->f_op
->poll
)
565 mask
= file
->f_op
->poll(file
, pwait
);
566 /* Mask out unneeded events. */
567 mask
&= pollfd
->events
| POLLERR
| POLLHUP
;
568 fput_light(file
, fput_needed
);
571 pollfd
->revents
= mask
;
576 static int do_poll(unsigned int nfds
, struct poll_list
*list
,
577 struct poll_wqueues
*wait
, s64
*timeout
)
580 poll_table
* pt
= &wait
->pt
;
582 /* Optimise the no-wait case */
587 struct poll_list
*walk
;
590 set_current_state(TASK_INTERRUPTIBLE
);
591 for (walk
= list
; walk
!= NULL
; walk
= walk
->next
) {
592 struct pollfd
* pfd
, * pfd_end
;
595 pfd_end
= pfd
+ walk
->len
;
596 for (; pfd
!= pfd_end
; pfd
++) {
598 * Fish for events. If we found one, record it
599 * and kill the poll_table, so we don't
600 * needlessly register any other waiters after
601 * this. They'll get immediately deregistered
602 * when we break out and return.
604 if (do_pollfd(pfd
, pt
)) {
611 * All waiters have already been registered, so don't provide
612 * a poll_table to them on the next loop iteration.
617 if (signal_pending(current
))
620 if (count
|| !*timeout
)
624 /* Wait indefinitely */
625 __timeout
= MAX_SCHEDULE_TIMEOUT
;
626 } else if (unlikely(*timeout
>= (s64
)MAX_SCHEDULE_TIMEOUT
-1)) {
628 * Wait for longer than MAX_SCHEDULE_TIMEOUT. Do it in
631 __timeout
= MAX_SCHEDULE_TIMEOUT
- 1;
632 *timeout
-= __timeout
;
634 __timeout
= *timeout
;
638 __timeout
= schedule_timeout(__timeout
);
640 *timeout
+= __timeout
;
642 __set_current_state(TASK_RUNNING
);
646 #define N_STACK_PPS ((sizeof(stack_pps) - sizeof(struct poll_list)) / \
647 sizeof(struct pollfd))
649 int do_sys_poll(struct pollfd __user
*ufds
, unsigned int nfds
, s64
*timeout
)
651 struct poll_wqueues table
;
652 int err
= -EFAULT
, fdcount
, len
, size
;
653 /* Allocate small arguments on the stack to save memory and be
654 faster - use long to make sure the buffer is aligned properly
655 on 64 bit archs to avoid unaligned access */
656 long stack_pps
[POLL_STACK_ALLOC
/sizeof(long)];
657 struct poll_list
*const head
= (struct poll_list
*)stack_pps
;
658 struct poll_list
*walk
= head
;
659 unsigned long todo
= nfds
;
661 if (nfds
> current
->signal
->rlim
[RLIMIT_NOFILE
].rlim_cur
)
664 len
= min_t(unsigned int, nfds
, N_STACK_PPS
);
671 if (copy_from_user(walk
->entries
, ufds
+ nfds
-todo
,
672 sizeof(struct pollfd
) * walk
->len
))
679 len
= min(todo
, POLLFD_PER_PAGE
);
680 size
= sizeof(struct poll_list
) + sizeof(struct pollfd
) * len
;
681 walk
= walk
->next
= kmalloc(size
, GFP_KERNEL
);
688 poll_initwait(&table
);
689 fdcount
= do_poll(nfds
, head
, &table
, timeout
);
690 poll_freewait(&table
);
692 for (walk
= head
; walk
; walk
= walk
->next
) {
693 struct pollfd
*fds
= walk
->entries
;
696 for (j
= 0; j
< walk
->len
; j
++, ufds
++)
697 if (__put_user(fds
[j
].revents
, &ufds
->revents
))
705 struct poll_list
*pos
= walk
;
713 static long do_restart_poll(struct restart_block
*restart_block
)
715 struct pollfd __user
*ufds
= (struct pollfd __user
*)restart_block
->arg0
;
716 int nfds
= restart_block
->arg1
;
717 s64 timeout
= ((s64
)restart_block
->arg3
<<32) | (s64
)restart_block
->arg2
;
720 ret
= do_sys_poll(ufds
, nfds
, &timeout
);
722 restart_block
->fn
= do_restart_poll
;
723 restart_block
->arg2
= timeout
& 0xFFFFFFFF;
724 restart_block
->arg3
= (u64
)timeout
>> 32;
725 ret
= -ERESTART_RESTARTBLOCK
;
730 asmlinkage
long sys_poll(struct pollfd __user
*ufds
, unsigned int nfds
,
736 if (timeout_msecs
> 0) {
738 /* We can only overflow if HZ > 1000 */
739 if (timeout_msecs
/ 1000 > (s64
)0x7fffffffffffffffULL
/ (s64
)HZ
)
740 timeout_jiffies
= -1;
743 timeout_jiffies
= msecs_to_jiffies(timeout_msecs
) + 1;
745 /* Infinite (< 0) or no (0) timeout */
746 timeout_jiffies
= timeout_msecs
;
749 ret
= do_sys_poll(ufds
, nfds
, &timeout_jiffies
);
751 struct restart_block
*restart_block
;
752 restart_block
= ¤t_thread_info()->restart_block
;
753 restart_block
->fn
= do_restart_poll
;
754 restart_block
->arg0
= (unsigned long)ufds
;
755 restart_block
->arg1
= nfds
;
756 restart_block
->arg2
= timeout_jiffies
& 0xFFFFFFFF;
757 restart_block
->arg3
= (u64
)timeout_jiffies
>> 32;
758 ret
= -ERESTART_RESTARTBLOCK
;
763 #ifdef HAVE_SET_RESTORE_SIGMASK
764 asmlinkage
long sys_ppoll(struct pollfd __user
*ufds
, unsigned int nfds
,
765 struct timespec __user
*tsp
, const sigset_t __user
*sigmask
,
768 sigset_t ksigmask
, sigsaved
;
774 if (copy_from_user(&ts
, tsp
, sizeof(ts
)))
777 /* Cast to u64 to make GCC stop complaining */
778 if ((u64
)ts
.tv_sec
>= (u64
)MAX_INT64_SECONDS
)
779 timeout
= -1; /* infinite */
781 timeout
= DIV_ROUND_UP(ts
.tv_nsec
, NSEC_PER_SEC
/HZ
);
782 timeout
+= ts
.tv_sec
* HZ
;
787 /* XXX: Don't preclude handling different sized sigset_t's. */
788 if (sigsetsize
!= sizeof(sigset_t
))
790 if (copy_from_user(&ksigmask
, sigmask
, sizeof(ksigmask
)))
793 sigdelsetmask(&ksigmask
, sigmask(SIGKILL
)|sigmask(SIGSTOP
));
794 sigprocmask(SIG_SETMASK
, &ksigmask
, &sigsaved
);
797 ret
= do_sys_poll(ufds
, nfds
, &timeout
);
799 /* We can restart this syscall, usually */
802 * Don't restore the signal mask yet. Let do_signal() deliver
803 * the signal on the way back to userspace, before the signal
807 memcpy(¤t
->saved_sigmask
, &sigsaved
,
809 set_restore_sigmask();
811 ret
= -ERESTARTNOHAND
;
813 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
815 if (tsp
&& timeout
>= 0) {
818 if (current
->personality
& STICKY_TIMEOUTS
)
820 /* Yes, we know it's actually an s64, but it's also positive. */
821 rts
.tv_nsec
= jiffies_to_usecs(do_div((*(u64
*)&timeout
), HZ
)) *
823 rts
.tv_sec
= timeout
;
824 if (timespec_compare(&rts
, &ts
) >= 0)
826 if (copy_to_user(tsp
, &rts
, sizeof(rts
))) {
829 * If an application puts its timeval in read-only
830 * memory, we don't want the Linux-specific update to
831 * the timeval to cause a fault after the select has
832 * completed successfully. However, because we're not
833 * updating the timeval, we can't restart the system
836 if (ret
== -ERESTARTNOHAND
&& timeout
>= 0)
843 #endif /* HAVE_SET_RESTORE_SIGMASK */