1 /* Copyright (C) 2002-2016 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
20 #include <shlib-compat.h>
21 #include <lowlevellock.h>
22 #include <lowlevelcond.h>
23 #include <tcb-offsets.h>
24 #include <pthread-errnos.h>
25 #include <pthread-pi-defines.h>
26 #include <kernel-features.h>
27 #include <stap-probe.h>
32 /* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) */
33 .globl __pthread_cond_wait
34 .type __pthread_cond_wait, @function
40 cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
41 DW.ref.__gcc_personality_v0)
42 cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
44 cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
45 cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
49 cfi_adjust_cfa_offset(4)
50 cfi_rel_offset(%ebp, 0)
52 cfi_adjust_cfa_offset(4)
53 cfi_rel_offset(%edi, 0)
55 cfi_adjust_cfa_offset(4)
56 cfi_rel_offset(%esi, 0)
58 cfi_adjust_cfa_offset(4)
59 cfi_rel_offset(%ebx, 0)
64 LIBC_PROBE (cond_wait, 2, 24(%esp), %ebx)
66 /* Get internal lock. */
73 cmpxchgl %edx, cond_lock(%ebx)
77 /* Store the reference to the mutex. If there is already a
78 different value in there this is a bad user bug. */
79 2: cmpl $-1, dep_mutex(%ebx)
82 movl %eax, dep_mutex(%ebx)
84 /* Unlock the mutex. */
86 call __pthread_mutex_unlock_usercnt
91 addl $1, total_seq(%ebx)
92 adcl $0, total_seq+4(%ebx)
93 addl $1, cond_futex(%ebx)
94 addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
97 subl $FRAME_SIZE, %esp
98 cfi_adjust_cfa_offset(FRAME_SIZE)
101 /* Get and store current wakeup_seq value. */
102 movl wakeup_seq(%ebx), %edi
103 movl wakeup_seq+4(%ebx), %edx
104 movl broadcast_seq(%ebx), %eax
109 /* Reset the pi-requeued flag. */
111 movl cond_futex(%ebx), %ebp
118 subl $1, cond_lock(%ebx)
123 4: call __pthread_enable_asynccancel
127 cmpl $-1, dep_mutex(%ebx)
131 movl dep_mutex(%ebx), %edi
132 /* Requeue to a non-robust PI mutex if the PI bit is set and
133 the robust bit is not set. */
134 movl MUTEX_KIND(%edi), %eax
135 andl $(ROBUST_BIT|PI_BIT), %eax
139 movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
142 addl $cond_futex, %ebx
144 movl $SYS_futex, %eax
146 subl $cond_futex, %ebx
148 /* Set the pi-requeued flag only if the kernel has returned 0. The
149 kernel does not hold the mutex on error. */
154 /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns
155 successfully, it has already locked the mutex for us and the
156 pi_flag (16(%esp)) is set to denote that fact. However, if another
157 thread changed the futex value before we entered the wait, the
158 syscall may return an EAGAIN and the mutex is not locked. We go
159 ahead with a success anyway since later we look at the pi_flag to
160 decide if we got the mutex or not. The sequence numbers then make
161 sure that only one of the threads actually wake up. We retry using
162 normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal
163 and PI futexes don't mix.
165 Note that we don't check for EAGAIN specifically; we assume that the
166 only other error the futex function could return is EAGAIN since
167 anything else would mean an error in our function. It is too
168 expensive to do that check for every call (which is quite common in
169 case of a large number of threads), so it has been skipped. */
175 #ifdef __ASSUME_PRIVATE_FUTEX
176 andl $FUTEX_PRIVATE_FLAG, %ecx
178 andl %gs:PRIVATE_FUTEX, %ecx
181 addl $FUTEX_WAIT, %ecx
184 addl $cond_futex, %ebx
186 movl $SYS_futex, %eax
188 subl $cond_futex, %ebx
191 19: movl (%esp), %eax
192 call __pthread_disable_asynccancel
200 cmpxchgl %edx, (%ebx)
202 cmpxchgl %edx, cond_lock(%ebx)
206 6: movl broadcast_seq(%ebx), %eax
210 movl woken_seq(%ebx), %eax
211 movl woken_seq+4(%ebx), %ecx
213 movl wakeup_seq(%ebx), %edi
214 movl wakeup_seq+4(%ebx), %edx
226 9: addl $1, woken_seq(%ebx)
227 adcl $0, woken_seq+4(%ebx)
230 16: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
232 /* Wake up a thread which wants to destroy the condvar object. */
233 movl total_seq(%ebx), %eax
234 andl total_seq+4(%ebx), %eax
235 cmpl $0xffffffff, %eax
237 movl cond_nwaiters(%ebx), %eax
238 andl $~((1 << nwaiters_shift) - 1), %eax
241 addl $cond_nwaiters, %ebx
242 movl $SYS_futex, %eax
243 #if FUTEX_PRIVATE_FLAG > 255
246 cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
249 #ifdef __ASSUME_PRIVATE_FUTEX
250 andl $FUTEX_PRIVATE_FLAG, %ecx
252 andl %gs:PRIVATE_FUTEX, %ecx
254 addl $FUTEX_WAKE, %ecx
257 subl $cond_nwaiters, %ebx
263 subl $1, cond_lock(%ebx)
267 /* With requeue_pi, the mutex lock is held in the kernel. */
268 11: movl 24+FRAME_SIZE(%esp), %eax
273 call __pthread_mutex_cond_lock
274 20: addl $FRAME_SIZE, %esp
275 cfi_adjust_cfa_offset(-FRAME_SIZE);
278 cfi_adjust_cfa_offset(-4)
281 cfi_adjust_cfa_offset(-4)
284 cfi_adjust_cfa_offset(-4)
287 cfi_adjust_cfa_offset(-4)
290 /* We return the result of the mutex_lock operation. */
295 21: call __pthread_mutex_cond_lock_adjust
299 cfi_adjust_cfa_offset(-FRAME_SIZE);
301 /* We need to go back to futex_wait. If we're using requeue_pi, then
302 release the mutex we had acquired and go back. */
303 22: movl 16(%esp), %edx
307 /* Adjust the mutex values first and then unlock it. The unlock
308 should always succeed or else the kernel did not lock the mutex
310 movl dep_mutex(%ebx), %eax
311 call __pthread_mutex_cond_lock_adjust
312 movl dep_mutex(%ebx), %eax
314 call __pthread_mutex_unlock_usercnt
317 /* Initial locking failed. */
322 leal cond_lock(%ebx), %edx
324 #if (LLL_SHARED-LLL_PRIVATE) > 255
327 cmpl $-1, dep_mutex(%ebx)
330 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
332 addl $LLL_PRIVATE, %ecx
337 /* The initial unlocking of the mutex failed. */
343 subl $1, cond_lock(%ebx)
351 leal cond_lock(%ebx), %eax
353 #if (LLL_SHARED-LLL_PRIVATE) > 255
356 cmpl $-1, dep_mutex(%ebx)
359 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
361 addl $LLL_PRIVATE, %ecx
363 call __lll_unlock_wake
368 cfi_adjust_cfa_offset(FRAME_SIZE)
370 /* Unlock in loop requires wakeup. */
375 leal cond_lock(%ebx), %eax
377 #if (LLL_SHARED-LLL_PRIVATE) > 255
380 cmpl $-1, dep_mutex(%ebx)
383 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
385 addl $LLL_PRIVATE, %ecx
387 call __lll_unlock_wake
390 /* Locking in loop failed. */
395 leal cond_lock(%ebx), %edx
397 #if (LLL_SHARED-LLL_PRIVATE) > 255
400 cmpl $-1, dep_mutex(%ebx)
403 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
405 addl $LLL_PRIVATE, %ecx
410 /* Unlock after loop requires wakeup. */
415 leal cond_lock(%ebx), %eax
417 #if (LLL_SHARED-LLL_PRIVATE) > 255
420 cmpl $-1, dep_mutex(%ebx)
423 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
425 addl $LLL_PRIVATE, %ecx
427 call __lll_unlock_wake
430 .size __pthread_cond_wait, .-__pthread_cond_wait
431 versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
435 .type __condvar_w_cleanup2, @function
436 __condvar_w_cleanup2:
437 subl $cond_futex, %ebx
438 .size __condvar_w_cleanup2, .-__condvar_w_cleanup2
440 .type __condvar_w_cleanup, @function
444 /* Get internal lock. */
449 cmpxchgl %edx, (%ebx)
451 cmpxchgl %edx, cond_lock(%ebx)
458 leal cond_lock(%ebx), %edx
460 #if (LLL_SHARED-LLL_PRIVATE) > 255
463 cmpl $-1, dep_mutex(%ebx)
466 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
468 addl $LLL_PRIVATE, %ecx
472 1: movl broadcast_seq(%ebx), %eax
476 /* We increment the wakeup_seq counter only if it is lower than
477 total_seq. If this is not the case the thread was woken and
478 then canceled. In this case we ignore the signal. */
479 movl total_seq(%ebx), %eax
480 movl total_seq+4(%ebx), %edi
481 cmpl wakeup_seq+4(%ebx), %edi
484 cmpl wakeup_seq(%ebx), %eax
487 6: addl $1, wakeup_seq(%ebx)
488 adcl $0, wakeup_seq+4(%ebx)
489 addl $1, cond_futex(%ebx)
491 7: addl $1, woken_seq(%ebx)
492 adcl $0, woken_seq+4(%ebx)
494 3: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
496 /* Wake up a thread which wants to destroy the condvar object. */
498 movl total_seq(%ebx), %eax
499 andl total_seq+4(%ebx), %eax
500 cmpl $0xffffffff, %eax
502 movl cond_nwaiters(%ebx), %eax
503 andl $~((1 << nwaiters_shift) - 1), %eax
506 addl $cond_nwaiters, %ebx
507 movl $SYS_futex, %eax
508 #if FUTEX_PRIVATE_FLAG > 255
511 cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
514 #ifdef __ASSUME_PRIVATE_FUTEX
515 andl $FUTEX_PRIVATE_FLAG, %ecx
517 andl %gs:PRIVATE_FUTEX, %ecx
519 addl $FUTEX_WAKE, %ecx
522 subl $cond_nwaiters, %ebx
529 subl $1, cond_lock(%ebx)
536 leal cond_lock(%ebx), %eax
538 #if (LLL_SHARED-LLL_PRIVATE) > 255
541 cmpl $-1, dep_mutex(%ebx)
544 andl $(LLL_SHARED-LLL_PRIVATE), %ecx
546 addl $LLL_PRIVATE, %ecx
548 call __lll_unlock_wake
550 /* Wake up all waiters to make sure no signal gets lost. */
553 addl $cond_futex, %ebx
554 #if FUTEX_PRIVATE_FLAG > 255
557 cmpl $-1, dep_mutex-cond_futex(%ebx)
560 #ifdef __ASSUME_PRIVATE_FUTEX
561 andl $FUTEX_PRIVATE_FLAG, %ecx
563 andl %gs:PRIVATE_FUTEX, %ecx
565 addl $FUTEX_WAKE, %ecx
566 movl $SYS_futex, %eax
567 movl $0x7fffffff, %edx
570 /* Lock the mutex only if we don't own it already. This only happens
571 in case of PI mutexes, if we got cancelled after a successful
572 return of the futex syscall and before disabling async
574 5: movl 24+FRAME_SIZE(%esp), %eax
575 movl MUTEX_KIND(%eax), %ebx
576 andl $(ROBUST_BIT|PI_BIT), %ebx
584 /* We managed to get the lock. Fix it up before returning. */
585 call __pthread_mutex_cond_lock_adjust
588 8: call __pthread_mutex_cond_lock
596 .size __condvar_w_cleanup, .-__condvar_w_cleanup
599 .section .gcc_except_table,"a",@progbits
601 .byte DW_EH_PE_omit # @LPStart format (omit)
602 .byte DW_EH_PE_omit # @TType format (omit)
603 .byte DW_EH_PE_sdata4 # call-site format
605 .uleb128 .Lcstend-.Lcstbegin
607 .long .LcleanupSTART-.LSTARTCODE
608 .long .Ladd_cond_futex_pi-.LcleanupSTART
609 .long __condvar_w_cleanup-.LSTARTCODE
611 .long .Ladd_cond_futex_pi-.LSTARTCODE
612 .long .Lsub_cond_futex_pi-.Ladd_cond_futex_pi
613 .long __condvar_w_cleanup2-.LSTARTCODE
615 .long .Lsub_cond_futex_pi-.LSTARTCODE
616 .long .Ladd_cond_futex-.Lsub_cond_futex_pi
617 .long __condvar_w_cleanup-.LSTARTCODE
619 .long .Ladd_cond_futex-.LSTARTCODE
620 .long .Lsub_cond_futex-.Ladd_cond_futex
621 .long __condvar_w_cleanup2-.LSTARTCODE
623 .long .Lsub_cond_futex-.LSTARTCODE
624 .long .LcleanupEND-.Lsub_cond_futex
625 .long __condvar_w_cleanup-.LSTARTCODE
627 .long .LcallUR-.LSTARTCODE
628 .long .LENDCODE-.LcallUR
634 .hidden DW.ref.__gcc_personality_v0
635 .weak DW.ref.__gcc_personality_v0
636 .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
638 .type DW.ref.__gcc_personality_v0, @object
639 .size DW.ref.__gcc_personality_v0, 4
640 DW.ref.__gcc_personality_v0:
641 .long __gcc_personality_v0