1 /* Copyright (C) 2002-2012 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
20 #include <shlib-compat.h>
21 #include <lowlevellock.h>
22 #include <lowlevelcond.h>
23 #include <tcb-offsets.h>
24 #include <pthread-errnos.h>
25 #include <pthread-pi-defines.h>
26 #include <kernel-features.h>
27 #include <stap-probe.h>
32 /* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) */
33 .globl __pthread_cond_wait
34 .type __pthread_cond_wait, @function
40 cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
41 DW.ref.__gcc_personality_v0)
42 cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
44 cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
45 cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
49 cfi_adjust_cfa_offset(4)
50 cfi_rel_offset(%ebp, 0)
52 cfi_adjust_cfa_offset(4)
53 cfi_rel_offset(%edi, 0)
55 cfi_adjust_cfa_offset(4)
56 cfi_rel_offset(%esi, 0)
58 cfi_adjust_cfa_offset(4)
59 cfi_rel_offset(%ebx, 0)
64 LIBC_PROBE (cond_wait, 2, 24(%esp), %ebx)
66 /* Get internal lock. */
73 cmpxchgl %edx, cond_lock(%ebx)
77 /* Store the reference to the mutex. If there is already a
78 different value in there this is a bad user bug. */
79 2: cmpl $-1, dep_mutex(%ebx)
82 movl %eax, dep_mutex(%ebx)
84 /* Unlock the mutex. */
86 call __pthread_mutex_unlock_usercnt
91 addl $1, total_seq(%ebx)
92 adcl $0, total_seq+4(%ebx)
93 addl $1, cond_futex(%ebx)
94 addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
97 subl $FRAME_SIZE, %esp
98 cfi_adjust_cfa_offset(FRAME_SIZE)
101 /* Get and store current wakeup_seq value. */
102 movl wakeup_seq(%ebx), %edi
103 movl wakeup_seq+4(%ebx), %edx
104 movl broadcast_seq(%ebx), %eax
109 /* Reset the pi-requeued flag. */
111 movl cond_futex(%ebx), %ebp
118 subl $1, cond_lock(%ebx)
123 4: call __pthread_enable_asynccancel
127 cmpl $-1, dep_mutex(%ebx)
131 movl dep_mutex(%ebx), %edi
132 /* Requeue to a non-robust PI mutex if the PI bit is set and
133 the robust bit is not set. */
134 movl MUTEX_KIND(%edi), %eax
135 andl $(ROBUST_BIT|PI_BIT), %eax
139 movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
142 addl $cond_futex, %ebx
144 movl $SYS_futex, %eax
146 subl $cond_futex, %ebx
148 /* Set the pi-requeued flag only if the kernel has returned 0. The
149 kernel does not hold the mutex on error. */
154 /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns
155 successfully, it has already locked the mutex for us and the
156 pi_flag (16(%esp)) is set to denote that fact. However, if another
157 thread changed the futex value before we entered the wait, the
158 syscall may return an EAGAIN and the mutex is not locked. We go
159 ahead with a success anyway since later we look at the pi_flag to
160 decide if we got the mutex or not. The sequence numbers then make
161 sure that only one of the threads actually wake up. We retry using
162 normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal
163 and PI futexes don't mix.
165 Note that we don't check for EAGAIN specifically; we assume that the
166 only other error the futex function could return is EAGAIN since
167 anything else would mean an error in our function. It is too
168 expensive to do that check for every call (which is quite common in
169 case of a large number of threads), so it has been skipped. */
175 #ifdef __ASSUME_PRIVATE_FUTEX
176 andl $FUTEX_PRIVATE_FLAG, %ecx
178 andl %gs:PRIVATE_FUTEX, %ecx
181 addl $FUTEX_WAIT, %ecx
184 addl $cond_futex, %ebx
186 movl $SYS_futex, %eax
188 subl $cond_futex, %ebx
191 19: movl (%esp), %eax
192 call __pthread_disable_asynccancel
200 cmpxchgl %edx, (%ebx)
202 cmpxchgl %edx, cond_lock(%ebx)
206 6: movl broadcast_seq(%ebx), %eax
210 movl woken_seq(%ebx), %eax
211 movl woken_seq+4(%ebx), %ecx
213 movl wakeup_seq(%ebx), %edi
214 movl wakeup_seq+4(%ebx), %edx
226 9: addl $1, woken_seq(%ebx)
227 adcl $0, woken_seq+4(%ebx)
230 16: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
232 /* Wake up a thread which wants to destroy the condvar object. */
233 movl total_seq(%ebx), %eax
234 andl total_seq+4(%ebx), %eax
235 cmpl $0xffffffff, %eax
237 movl cond_nwaiters(%ebx), %eax
238 andl $~((1 << nwaiters_shift) - 1), %eax
241 addl $cond_nwaiters, %ebx
242 movl $SYS_futex, %eax
243 #if FUTEX_PRIVATE_FLAG > 255
246 cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
249 #ifdef __ASSUME_PRIVATE_FUTEX
250 andl $FUTEX_PRIVATE_FLAG, %ecx
252 andl %gs:PRIVATE_FUTEX, %ecx
254 addl $FUTEX_WAKE, %ecx
257 subl $cond_nwaiters, %ebx
263 subl $1, cond_lock(%ebx)
267 /* With requeue_pi, the mutex lock is held in the kernel. */
268 11: movl 24+FRAME_SIZE(%esp), %eax
273 call __pthread_mutex_cond_lock
274 20: addl $FRAME_SIZE, %esp
275 cfi_adjust_cfa_offset(-FRAME_SIZE);
278 cfi_adjust_cfa_offset(-4)
281 cfi_adjust_cfa_offset(-4)
284 cfi_adjust_cfa_offset(-4)
287 cfi_adjust_cfa_offset(-4)
290 /* We return the result of the mutex_lock operation. */
295 21: call __pthread_mutex_cond_lock_adjust
299 cfi_adjust_cfa_offset(-FRAME_SIZE);
301 /* We need to go back to futex_wait. If we're using requeue_pi, then
302 release the mutex we had acquired and go back. */
303 22: movl 16(%esp), %edx
307 /* Adjust the mutex values first and then unlock it. The unlock
308 should always succeed or else the kernel did not lock the mutex
310 movl dep_mutex(%ebx), %eax
311 call __pthread_mutex_cond_lock_adjust
313 call __pthread_mutex_unlock_usercnt
316 /* Initial locking failed. */
321 leal cond_lock(%ebx), %edx
323 #if (LLL_SHARED-LLL_PRIVATE) > 255
326 cmpl $-1, dep_mutex(%ebx)
329 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
331 addl $LLL_PRIVATE, %ecx
336 /* The initial unlocking of the mutex failed. */
342 subl $1, cond_lock(%ebx)
350 leal cond_lock(%ebx), %eax
352 #if (LLL_SHARED-LLL_PRIVATE) > 255
355 cmpl $-1, dep_mutex(%ebx)
358 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
360 addl $LLL_PRIVATE, %ecx
362 call __lll_unlock_wake
367 cfi_adjust_cfa_offset(FRAME_SIZE)
369 /* Unlock in loop requires wakeup. */
374 leal cond_lock(%ebx), %eax
376 #if (LLL_SHARED-LLL_PRIVATE) > 255
379 cmpl $-1, dep_mutex(%ebx)
382 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
384 addl $LLL_PRIVATE, %ecx
386 call __lll_unlock_wake
389 /* Locking in loop failed. */
394 leal cond_lock(%ebx), %edx
396 #if (LLL_SHARED-LLL_PRIVATE) > 255
399 cmpl $-1, dep_mutex(%ebx)
402 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
404 addl $LLL_PRIVATE, %ecx
409 /* Unlock after loop requires wakeup. */
414 leal cond_lock(%ebx), %eax
416 #if (LLL_SHARED-LLL_PRIVATE) > 255
419 cmpl $-1, dep_mutex(%ebx)
422 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
424 addl $LLL_PRIVATE, %ecx
426 call __lll_unlock_wake
429 .size __pthread_cond_wait, .-__pthread_cond_wait
430 versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
434 .type __condvar_w_cleanup2, @function
435 __condvar_w_cleanup2:
436 subl $cond_futex, %ebx
437 .size __condvar_w_cleanup2, .-__condvar_w_cleanup2
439 .type __condvar_w_cleanup, @function
443 /* Get internal lock. */
448 cmpxchgl %edx, (%ebx)
450 cmpxchgl %edx, cond_lock(%ebx)
457 leal cond_lock(%ebx), %edx
459 #if (LLL_SHARED-LLL_PRIVATE) > 255
462 cmpl $-1, dep_mutex(%ebx)
465 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
467 addl $LLL_PRIVATE, %ecx
471 1: movl broadcast_seq(%ebx), %eax
475 /* We increment the wakeup_seq counter only if it is lower than
476 total_seq. If this is not the case the thread was woken and
477 then canceled. In this case we ignore the signal. */
478 movl total_seq(%ebx), %eax
479 movl total_seq+4(%ebx), %edi
480 cmpl wakeup_seq+4(%ebx), %edi
483 cmpl wakeup_seq(%ebx), %eax
486 6: addl $1, wakeup_seq(%ebx)
487 adcl $0, wakeup_seq+4(%ebx)
488 addl $1, cond_futex(%ebx)
490 7: addl $1, woken_seq(%ebx)
491 adcl $0, woken_seq+4(%ebx)
493 3: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
495 /* Wake up a thread which wants to destroy the condvar object. */
497 movl total_seq(%ebx), %eax
498 andl total_seq+4(%ebx), %eax
499 cmpl $0xffffffff, %eax
501 movl cond_nwaiters(%ebx), %eax
502 andl $~((1 << nwaiters_shift) - 1), %eax
505 addl $cond_nwaiters, %ebx
506 movl $SYS_futex, %eax
507 #if FUTEX_PRIVATE_FLAG > 255
510 cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
513 #ifdef __ASSUME_PRIVATE_FUTEX
514 andl $FUTEX_PRIVATE_FLAG, %ecx
516 andl %gs:PRIVATE_FUTEX, %ecx
518 addl $FUTEX_WAKE, %ecx
521 subl $cond_nwaiters, %ebx
528 subl $1, cond_lock(%ebx)
535 leal cond_lock(%ebx), %eax
537 #if (LLL_SHARED-LLL_PRIVATE) > 255
540 cmpl $-1, dep_mutex(%ebx)
543 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
545 addl $LLL_PRIVATE, %ecx
547 call __lll_unlock_wake
549 /* Wake up all waiters to make sure no signal gets lost. */
552 addl $cond_futex, %ebx
553 #if FUTEX_PRIVATE_FLAG > 255
556 cmpl $-1, dep_mutex-cond_futex(%ebx)
559 #ifdef __ASSUME_PRIVATE_FUTEX
560 andl $FUTEX_PRIVATE_FLAG, %ecx
562 andl %gs:PRIVATE_FUTEX, %ecx
564 addl $FUTEX_WAKE, %ecx
565 movl $SYS_futex, %eax
566 movl $0x7fffffff, %edx
569 /* Lock the mutex only if we don't own it already. This only happens
570 in case of PI mutexes, if we got cancelled after a successful
571 return of the futex syscall and before disabling async
573 5: movl 24+FRAME_SIZE(%esp), %eax
574 movl MUTEX_KIND(%eax), %ebx
575 andl $(ROBUST_BIT|PI_BIT), %ebx
583 /* We managed to get the lock. Fix it up before returning. */
584 call __pthread_mutex_cond_lock_adjust
587 8: call __pthread_mutex_cond_lock
595 .size __condvar_w_cleanup, .-__condvar_w_cleanup
598 .section .gcc_except_table,"a",@progbits
600 .byte DW_EH_PE_omit # @LPStart format (omit)
601 .byte DW_EH_PE_omit # @TType format (omit)
602 .byte DW_EH_PE_sdata4 # call-site format
604 .uleb128 .Lcstend-.Lcstbegin
606 .long .LcleanupSTART-.LSTARTCODE
607 .long .Ladd_cond_futex_pi-.LcleanupSTART
608 .long __condvar_w_cleanup-.LSTARTCODE
610 .long .Ladd_cond_futex_pi-.LSTARTCODE
611 .long .Lsub_cond_futex_pi-.Ladd_cond_futex_pi
612 .long __condvar_w_cleanup2-.LSTARTCODE
614 .long .Lsub_cond_futex_pi-.LSTARTCODE
615 .long .Ladd_cond_futex-.Lsub_cond_futex_pi
616 .long __condvar_w_cleanup-.LSTARTCODE
618 .long .Ladd_cond_futex-.LSTARTCODE
619 .long .Lsub_cond_futex-.Ladd_cond_futex
620 .long __condvar_w_cleanup2-.LSTARTCODE
622 .long .Lsub_cond_futex-.LSTARTCODE
623 .long .LcleanupEND-.Lsub_cond_futex
624 .long __condvar_w_cleanup-.LSTARTCODE
626 .long .LcallUR-.LSTARTCODE
627 .long .LENDCODE-.LcallUR
633 .hidden DW.ref.__gcc_personality_v0
634 .weak DW.ref.__gcc_personality_v0
635 .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
637 .type DW.ref.__gcc_personality_v0, @object
638 .size DW.ref.__gcc_personality_v0, 4
639 DW.ref.__gcc_personality_v0:
640 .long __gcc_personality_v0