1 /* Copyright (C) 2002-2013 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
20 #include <shlib-compat.h>
21 #include <lowlevellock.h>
22 #include <lowlevelcond.h>
23 #include <pthread-errnos.h>
24 #include <pthread-pi-defines.h>
25 #include <kernel-features.h>
26 #include <stap-probe.h>
30 /* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
31 const struct timespec *abstime) */
32 .globl __pthread_cond_timedwait
33 .type __pthread_cond_timedwait, @function
35 __pthread_cond_timedwait:
39 cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
40 DW.ref.__gcc_personality_v0)
41 cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
43 cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
44 cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
48 cfi_adjust_cfa_offset(4)
49 cfi_rel_offset(%ebp, 0)
51 cfi_adjust_cfa_offset(4)
52 cfi_rel_offset(%edi, 0)
54 cfi_adjust_cfa_offset(4)
55 cfi_rel_offset(%esi, 0)
57 cfi_adjust_cfa_offset(4)
58 cfi_rel_offset(%ebx, 0)
63 LIBC_PROBE (cond_timedwait, 3, %ebx, 24(%esp), %ebp)
65 cmpl $1000000000, 4(%ebp)
72 +--------------------------+
73 esp + 24 | timeout value |
74 +--------------------------+
75 esp + 20 | futex pointer |
76 +--------------------------+
77 esp + 16 | pi-requeued flag |
78 +--------------------------+
79 esp + 12 | old broadcast_seq value |
80 +--------------------------+
81 esp + 4 | old wake_seq value |
82 +--------------------------+
83 esp + 0 | old cancellation mode |
84 +--------------------------+
87 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
90 cmpl $0, __have_futex_clock_realtime@GOTOFF(%ecx)
92 cmpl $0, __have_futex_clock_realtime
97 /* Get internal lock. */
102 cmpxchgl %edx, (%ebx)
104 cmpxchgl %edx, cond_lock(%ebx)
108 /* Store the reference to the mutex. If there is already a
109 different value in there this is a bad user bug. */
110 2: cmpl $-1, dep_mutex(%ebx)
113 movl %eax, dep_mutex(%ebx)
115 /* Unlock the mutex. */
117 call __pthread_mutex_unlock_usercnt
122 addl $1, total_seq(%ebx)
123 adcl $0, total_seq+4(%ebx)
124 addl $1, cond_futex(%ebx)
125 addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
127 #ifdef __ASSUME_FUTEX_CLOCK_REALTIME
128 # define FRAME_SIZE 24
130 # define FRAME_SIZE 32
132 subl $FRAME_SIZE, %esp
133 cfi_adjust_cfa_offset(FRAME_SIZE)
136 /* Get and store current wakeup_seq value. */
137 movl wakeup_seq(%ebx), %edi
138 movl wakeup_seq+4(%ebx), %edx
139 movl broadcast_seq(%ebx), %eax
144 /* Reset the pi-requeued flag. */
148 movl $-ETIMEDOUT, %esi
151 8: movl cond_futex(%ebx), %edi
159 subl $1, cond_lock(%ebx)
164 4: call __pthread_enable_asynccancel
168 #if FUTEX_PRIVATE_FLAG > 255
171 cmpl $-1, dep_mutex(%ebx)
175 movl dep_mutex(%ebx), %edi
176 /* Requeue to a non-robust PI mutex if the PI bit is set and
177 the robust bit is not set. */
178 movl MUTEX_KIND(%edi), %eax
179 andl $(ROBUST_BIT|PI_BIT), %eax
183 movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
184 /* The following only works like this because we only support
185 two clocks, represented using a single bit. */
186 testl $1, cond_nwaiters(%ebx)
187 /* XXX Need to implement using sete instead of a jump. */
189 orl $FUTEX_CLOCK_REALTIME, %ecx
191 42: movl 20(%esp), %edx
192 addl $cond_futex, %ebx
194 movl $SYS_futex, %eax
196 subl $cond_futex, %ebx
199 /* Set the pi-requeued flag only if the kernel has returned 0. The
200 kernel does not hold the mutex on ETIMEDOUT or any other error. */
205 /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns
206 successfully, it has already locked the mutex for us and the
207 pi_flag (16(%esp)) is set to denote that fact. However, if another
208 thread changed the futex value before we entered the wait, the
209 syscall may return an EAGAIN and the mutex is not locked. We go
210 ahead with a success anyway since later we look at the pi_flag to
211 decide if we got the mutex or not. The sequence numbers then make
212 sure that only one of the threads actually wake up. We retry using
213 normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal
214 and PI futexes don't mix.
216 Note that we don't check for EAGAIN specifically; we assume that the
217 only other error the futex function could return is EAGAIN (barring
218 the ETIMEOUT of course, for the timeout case in futex) since
219 anything else would mean an error in our function. It is too
220 expensive to do that check for every call (which is quite common in
221 case of a large number of threads), so it has been skipped. */
228 #ifdef __ASSUME_PRIVATE_FUTEX
229 andl $FUTEX_PRIVATE_FLAG, %ecx
231 andl %gs:PRIVATE_FUTEX, %ecx
233 addl $FUTEX_WAIT_BITSET, %ecx
234 /* The following only works like this because we only support
235 two clocks, represented using a single bit. */
236 testl $1, cond_nwaiters(%ebx)
238 orl $FUTEX_CLOCK_REALTIME, %ecx
241 movl $0xffffffff, %ebp
242 addl $cond_futex, %ebx
244 movl $SYS_futex, %eax
246 subl $cond_futex, %ebx
248 movl 28+FRAME_SIZE(%esp), %ebp
251 41: movl (%esp), %eax
252 call __pthread_disable_asynccancel
260 cmpxchgl %edx, (%ebx)
262 cmpxchgl %edx, cond_lock(%ebx)
266 6: movl broadcast_seq(%ebx), %eax
270 movl woken_seq(%ebx), %eax
271 movl woken_seq+4(%ebx), %ecx
273 movl wakeup_seq(%ebx), %edi
274 movl wakeup_seq+4(%ebx), %edx
286 15: cmpl $-ETIMEDOUT, %esi
289 /* We need to go back to futex_wait. If we're using requeue_pi, then
290 release the mutex we had acquired and go back. */
295 /* Adjust the mutex values first and then unlock it. The unlock
296 should always succeed or else the kernel did not lock the mutex
298 movl dep_mutex(%ebx), %eax
299 call __pthread_mutex_cond_lock_adjust
301 call __pthread_mutex_unlock_usercnt
304 28: addl $1, wakeup_seq(%ebx)
305 adcl $0, wakeup_seq+4(%ebx)
306 addl $1, cond_futex(%ebx)
307 movl $ETIMEDOUT, %esi
314 14: addl $1, woken_seq(%ebx)
315 adcl $0, woken_seq+4(%ebx)
317 24: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
319 /* Wake up a thread which wants to destroy the condvar object. */
320 movl total_seq(%ebx), %eax
321 andl total_seq+4(%ebx), %eax
322 cmpl $0xffffffff, %eax
324 movl cond_nwaiters(%ebx), %eax
325 andl $~((1 << nwaiters_shift) - 1), %eax
328 addl $cond_nwaiters, %ebx
329 movl $SYS_futex, %eax
330 #if FUTEX_PRIVATE_FLAG > 255
333 cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
336 #ifdef __ASSUME_PRIVATE_FUTEX
337 andl $FUTEX_PRIVATE_FLAG, %ecx
339 andl %gs:PRIVATE_FUTEX, %ecx
341 addl $FUTEX_WAKE, %ecx
344 subl $cond_nwaiters, %ebx
350 subl $1, cond_lock(%ebx)
354 11: movl 24+FRAME_SIZE(%esp), %eax
355 /* With requeue_pi, the mutex lock is held in the kernel. */
360 call __pthread_mutex_cond_lock
361 26: addl $FRAME_SIZE, %esp
362 cfi_adjust_cfa_offset(-FRAME_SIZE)
364 /* We return the result of the mutex_lock operation if it failed. */
375 cfi_adjust_cfa_offset(-4)
378 cfi_adjust_cfa_offset(-4)
381 cfi_adjust_cfa_offset(-4)
384 cfi_adjust_cfa_offset(-4)
391 27: call __pthread_mutex_cond_lock_adjust
395 cfi_adjust_cfa_offset(-FRAME_SIZE);
396 /* Initial locking failed. */
401 leal cond_lock(%ebx), %edx
403 #if (LLL_SHARED-LLL_PRIVATE) > 255
406 cmpl $-1, dep_mutex(%ebx)
409 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
411 addl $LLL_PRIVATE, %ecx
416 /* The initial unlocking of the mutex failed. */
422 subl $1, cond_lock(%ebx)
430 leal cond_lock(%ebx), %eax
432 #if (LLL_SHARED-LLL_PRIVATE) > 255
435 cmpl $-1, dep_mutex(%ebx)
438 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
440 addl $LLL_PRIVATE, %ecx
442 call __lll_unlock_wake
447 cfi_adjust_cfa_offset(FRAME_SIZE)
449 /* Unlock in loop requires wakeup. */
454 leal cond_lock(%ebx), %eax
456 #if (LLL_SHARED-LLL_PRIVATE) > 255
459 cmpl $-1, dep_mutex(%ebx)
462 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
464 addl $LLL_PRIVATE, %ecx
466 call __lll_unlock_wake
469 /* Locking in loop failed. */
474 leal cond_lock(%ebx), %edx
476 #if (LLL_SHARED-LLL_PRIVATE) > 255
479 cmpl $-1, dep_mutex(%ebx)
482 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
484 addl $LLL_PRIVATE, %ecx
489 /* Unlock after loop requires wakeup. */
494 leal cond_lock(%ebx), %eax
496 #if (LLL_SHARED-LLL_PRIVATE) > 255
499 cmpl $-1, dep_mutex(%ebx)
502 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
504 addl $LLL_PRIVATE, %ecx
506 call __lll_unlock_wake
509 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
510 cfi_adjust_cfa_offset(-FRAME_SIZE)
512 /* Get internal lock. */
517 cmpxchgl %edx, (%ebx)
519 cmpxchgl %edx, cond_lock(%ebx)
523 /* Store the reference to the mutex. If there is already a
524 different value in there this is a bad user bug. */
525 102: cmpl $-1, dep_mutex(%ebx)
528 movl %eax, dep_mutex(%ebx)
530 /* Unlock the mutex. */
532 call __pthread_mutex_unlock_usercnt
537 addl $1, total_seq(%ebx)
538 adcl $0, total_seq+4(%ebx)
539 addl $1, cond_futex(%ebx)
540 addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
542 subl $FRAME_SIZE, %esp
543 cfi_adjust_cfa_offset(FRAME_SIZE)
545 /* Get and store current wakeup_seq value. */
546 movl wakeup_seq(%ebx), %edi
547 movl wakeup_seq+4(%ebx), %edx
548 movl broadcast_seq(%ebx), %eax
553 /* Reset the pi-requeued flag. */
556 /* Get the current time. */
558 # ifdef __NR_clock_gettime
559 /* Get the clock number. */
560 movl cond_nwaiters(%ebx), %ebx
561 andl $((1 << nwaiters_shift) - 1), %ebx
562 /* Only clocks 0 and 1 are allowed so far. Both are handled in the
565 movl $__NR_clock_gettime, %eax
569 /* Compute relative timeout. */
575 /* Get the current time. */
578 movl $__NR_gettimeofday, %eax
582 /* Compute relative timeout. */
585 mul %edx /* Milli seconds to nano seconds. */
592 addl $1000000000, %edx
594 112: testl %ecx, %ecx
595 movl $-ETIMEDOUT, %esi
598 /* Store relative timeout. */
599 121: movl %ecx, 24(%esp)
602 movl cond_futex(%ebx), %edi
610 subl $1, cond_lock(%ebx)
615 104: call __pthread_enable_asynccancel
619 # if FUTEX_PRIVATE_FLAG > 255
622 cmpl $-1, dep_mutex(%ebx)
625 # ifdef __ASSUME_PRIVATE_FUTEX
626 andl $FUTEX_PRIVATE_FLAG, %ecx
628 andl %gs:PRIVATE_FUTEX, %ecx
631 addl $FUTEX_WAIT, %ecx
634 addl $cond_futex, %ebx
636 movl $SYS_futex, %eax
638 subl $cond_futex, %ebx
642 141: movl (%esp), %eax
643 call __pthread_disable_asynccancel
652 cmpxchgl %edx, (%ebx)
654 cmpxchgl %edx, cond_lock(%ebx)
658 106: movl broadcast_seq(%ebx), %eax
662 movl woken_seq(%ebx), %eax
663 movl woken_seq+4(%ebx), %ecx
665 movl wakeup_seq(%ebx), %edi
666 movl wakeup_seq+4(%ebx), %edx
678 115: cmpl $-ETIMEDOUT, %esi
683 cfi_adjust_cfa_offset(-FRAME_SIZE)
684 /* Initial locking failed. */
689 leal cond_lock(%ebx), %edx
691 # if (LLL_SHARED-LLL_PRIVATE) > 255
694 cmpl $-1, dep_mutex(%ebx)
697 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
698 # if LLL_PRIVATE != 0
699 addl $LLL_PRIVATE, %ecx
704 cfi_adjust_cfa_offset(FRAME_SIZE)
706 /* Unlock in loop requires wakeup. */
711 leal cond_lock(%ebx), %eax
713 # if (LLL_SHARED-LLL_PRIVATE) > 255
716 cmpl $-1, dep_mutex(%ebx)
719 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
720 # if LLL_PRIVATE != 0
721 addl $LLL_PRIVATE, %ecx
723 call __lll_unlock_wake
726 /* Locking in loop failed. */
731 leal cond_lock(%ebx), %edx
733 # if (LLL_SHARED-LLL_PRIVATE) > 255
736 cmpl $-1, dep_mutex(%ebx)
739 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
740 # if LLL_PRIVATE != 0
741 addl $LLL_PRIVATE, %ecx
747 .size __pthread_cond_timedwait, .-__pthread_cond_timedwait
748 versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
752 .type __condvar_tw_cleanup2, @function
753 __condvar_tw_cleanup2:
754 subl $cond_futex, %ebx
755 .size __condvar_tw_cleanup2, .-__condvar_tw_cleanup2
756 .type __condvar_tw_cleanup, @function
757 __condvar_tw_cleanup:
760 /* Get internal lock. */
765 cmpxchgl %edx, (%ebx)
767 cmpxchgl %edx, cond_lock(%ebx)
774 leal cond_lock(%ebx), %edx
776 #if (LLL_SHARED-LLL_PRIVATE) > 255
779 cmpl $-1, dep_mutex(%ebx)
782 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
784 addl $LLL_PRIVATE, %ecx
788 1: movl broadcast_seq(%ebx), %eax
792 /* We increment the wakeup_seq counter only if it is lower than
793 total_seq. If this is not the case the thread was woken and
794 then canceled. In this case we ignore the signal. */
795 movl total_seq(%ebx), %eax
796 movl total_seq+4(%ebx), %edi
797 cmpl wakeup_seq+4(%ebx), %edi
800 cmpl wakeup_seq(%ebx), %eax
803 6: addl $1, wakeup_seq(%ebx)
804 adcl $0, wakeup_seq+4(%ebx)
805 addl $1, cond_futex(%ebx)
807 7: addl $1, woken_seq(%ebx)
808 adcl $0, woken_seq+4(%ebx)
810 3: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
812 /* Wake up a thread which wants to destroy the condvar object. */
814 movl total_seq(%ebx), %eax
815 andl total_seq+4(%ebx), %eax
816 cmpl $0xffffffff, %eax
818 movl cond_nwaiters(%ebx), %eax
819 andl $~((1 << nwaiters_shift) - 1), %eax
822 addl $cond_nwaiters, %ebx
823 movl $SYS_futex, %eax
824 #if FUTEX_PRIVATE_FLAG > 255
827 cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
830 #ifdef __ASSUME_PRIVATE_FUTEX
831 andl $FUTEX_PRIVATE_FLAG, %ecx
833 andl %gs:PRIVATE_FUTEX, %ecx
835 addl $FUTEX_WAKE, %ecx
838 subl $cond_nwaiters, %ebx
845 subl $1, cond_lock(%ebx)
852 leal cond_lock(%ebx), %eax
854 #if (LLL_SHARED-LLL_PRIVATE) > 255
857 cmpl $-1, dep_mutex(%ebx)
860 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
862 addl $LLL_PRIVATE, %ecx
864 call __lll_unlock_wake
866 /* Wake up all waiters to make sure no signal gets lost. */
869 addl $cond_futex, %ebx
870 #if FUTEX_PRIVATE_FLAG > 255
873 cmpl $-1, dep_mutex-cond_futex(%ebx)
876 #ifdef __ASSUME_PRIVATE_FUTEX
877 andl $FUTEX_PRIVATE_FLAG, %ecx
879 andl %gs:PRIVATE_FUTEX, %ecx
881 addl $FUTEX_WAKE, %ecx
882 movl $SYS_futex, %eax
883 movl $0x7fffffff, %edx
886 /* Lock the mutex only if we don't own it already. This only happens
887 in case of PI mutexes, if we got cancelled after a successful
888 return of the futex syscall and before disabling async
890 5: movl 24+FRAME_SIZE(%esp), %eax
891 movl MUTEX_KIND(%eax), %ebx
892 andl $(ROBUST_BIT|PI_BIT), %ebx
900 /* We managed to get the lock. Fix it up before returning. */
901 call __pthread_mutex_cond_lock_adjust
904 8: call __pthread_mutex_cond_lock
912 .size __condvar_tw_cleanup, .-__condvar_tw_cleanup
915 .section .gcc_except_table,"a",@progbits
917 .byte DW_EH_PE_omit # @LPStart format (omit)
918 .byte DW_EH_PE_omit # @TType format (omit)
919 .byte DW_EH_PE_sdata4 # call-site format
921 .uleb128 .Lcstend-.Lcstbegin
923 .long .LcleanupSTART-.LSTARTCODE
924 .long .Ladd_cond_futex_pi-.LcleanupSTART
925 .long __condvar_tw_cleanup-.LSTARTCODE
927 .long .Ladd_cond_futex_pi-.LSTARTCODE
928 .long .Lsub_cond_futex_pi-.Ladd_cond_futex_pi
929 .long __condvar_tw_cleanup2-.LSTARTCODE
931 .long .Lsub_cond_futex_pi-.LSTARTCODE
932 .long .Ladd_cond_futex-.Lsub_cond_futex_pi
933 .long __condvar_tw_cleanup-.LSTARTCODE
935 .long .Ladd_cond_futex-.LSTARTCODE
936 .long .Lsub_cond_futex-.Ladd_cond_futex
937 .long __condvar_tw_cleanup2-.LSTARTCODE
939 .long .Lsub_cond_futex-.LSTARTCODE
940 .long .LcleanupEND-.Lsub_cond_futex
941 .long __condvar_tw_cleanup-.LSTARTCODE
943 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
944 .long .LcleanupSTART2-.LSTARTCODE
945 .long .Ladd_cond_futex2-.LcleanupSTART2
946 .long __condvar_tw_cleanup-.LSTARTCODE
948 .long .Ladd_cond_futex2-.LSTARTCODE
949 .long .Lsub_cond_futex2-.Ladd_cond_futex2
950 .long __condvar_tw_cleanup2-.LSTARTCODE
952 .long .Lsub_cond_futex2-.LSTARTCODE
953 .long .LcleanupEND2-.Lsub_cond_futex2
954 .long __condvar_tw_cleanup-.LSTARTCODE
957 .long .LcallUR-.LSTARTCODE
958 .long .LENDCODE-.LcallUR
965 .hidden DW.ref.__gcc_personality_v0
966 .weak DW.ref.__gcc_personality_v0
967 .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
969 .type DW.ref.__gcc_personality_v0, @object
970 .size DW.ref.__gcc_personality_v0, 4
971 DW.ref.__gcc_personality_v0:
972 .long __gcc_personality_v0