1 /* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
21 #include <shlib-compat.h>
22 #include <lowlevellock.h>
23 #include <lowlevelcond.h>
24 #include <pthread-pi-defines.h>
25 #include <pthread-errnos.h>
27 #include <kernel-features.h>
29 /* For the calculation see asm/vsyscall.h. */
30 #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000
36 /* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
37 const struct timespec *abstime) */
38 .globl __pthread_cond_timedwait
39 .type __pthread_cond_timedwait, @function
41 __pthread_cond_timedwait:
45 cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
46 DW.ref.__gcc_personality_v0)
47 cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
49 cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
50 cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
54 cfi_adjust_cfa_offset(8)
55 cfi_rel_offset(%r12, 0)
57 cfi_adjust_cfa_offset(8)
58 cfi_rel_offset(%r13, 0)
60 cfi_adjust_cfa_offset(8)
61 cfi_rel_offset(%r14, 0)
63 cfi_adjust_cfa_offset(8)
64 cfi_rel_offset(%r15, 0)
65 #ifdef __ASSUME_FUTEX_CLOCK_REALTIME
66 # define FRAME_SIZE 32
68 # define FRAME_SIZE 48
70 subq $FRAME_SIZE, %rsp
71 cfi_adjust_cfa_offset(FRAME_SIZE)
73 cmpq $1000000000, 8(%rdx)
80 +--------------------------+
81 rsp + 32 | timeout value |
82 +--------------------------+
83 rsp + 24 | old wake_seq value |
84 +--------------------------+
85 rsp + 16 | mutex pointer |
86 +--------------------------+
87 rsp + 8 | condvar pointer |
88 +--------------------------+
89 rsp + 4 | old broadcast_seq value |
90 +--------------------------+
91 rsp + 0 | old cancellation mode |
92 +--------------------------+
95 cmpq $-1, dep_mutex(%rdi)
97 /* Prepare structure passed to cancellation handler. */
103 movq %rsi, dep_mutex(%rdi)
106 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
108 cmpl $0, __have_futex_clock_realtime(%rip)
110 cmpl $0, __have_futex_clock_realtime
115 /* Get internal lock. */
120 cmpxchgl %esi, (%rdi)
122 cmpxchgl %esi, cond_lock(%rdi)
126 /* Unlock the mutex. */
127 32: movq 16(%rsp), %rdi
129 callq __pthread_mutex_unlock_usercnt
136 incl cond_futex(%rdi)
137 addl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
139 /* Get and store current wakeup_seq value. */
141 movq wakeup_seq(%rdi), %r9
142 movl broadcast_seq(%rdi), %edx
146 38: movl cond_futex(%rdi), %r12d
158 34: callq __pthread_enable_asynccancel
163 cmpq $-1, dep_mutex(%rdi)
164 movl $FUTEX_WAIT_BITSET, %eax
165 movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
169 movq dep_mutex(%rdi), %r8
170 /* Requeue to a PI mutex if the PI bit is set. */
171 testl $PI_BIT, MUTEX_KIND(%r8)
174 movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
176 /* The following only works like this because we only support
177 two clocks, represented using a single bit. */
178 testl $1, cond_nwaiters(%rdi)
179 movl $FUTEX_CLOCK_REALTIME, %edx
183 addq $cond_futex, %rdi
184 movl $SYS_futex, %eax
188 #ifdef __ASSUME_REQUEUE_PI
194 movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
195 subq $cond_futex, %rdi
198 60: xorl %r15d, %r15d
200 /* The following only works like this because we only support
201 two clocks, represented using a single bit. */
202 testl $1, cond_nwaiters(%rdi)
203 movl $FUTEX_CLOCK_REALTIME, %edx
204 movl $0xffffffff, %r9d
208 addq $cond_futex, %rdi
209 movl $SYS_futex, %eax
214 callq __pthread_disable_asynccancel
223 cmpxchgl %esi, (%rdi)
225 cmpxchgl %esi, cond_lock(%rdi)
229 36: movl broadcast_seq(%rdi), %edx
231 movq woken_seq(%rdi), %rax
233 movq wakeup_seq(%rdi), %r9
244 45: cmpq $-ETIMEDOUT, %r14
247 99: incq wakeup_seq(%rdi)
248 incl cond_futex(%rdi)
249 movl $ETIMEDOUT, %r14d
256 44: incq woken_seq(%rdi)
258 54: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
260 /* Wake up a thread which wants to destroy the condvar object. */
261 cmpq $0xffffffffffffffff, total_seq(%rdi)
263 movl cond_nwaiters(%rdi), %eax
264 andl $~((1 << nwaiters_shift) - 1), %eax
267 addq $cond_nwaiters, %rdi
268 cmpq $-1, dep_mutex-cond_nwaiters(%rdi)
270 #ifdef __ASSUME_PRIVATE_FUTEX
271 movl $FUTEX_WAKE, %eax
272 movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
276 movl %fs:PRIVATE_FUTEX, %esi
278 orl $FUTEX_WAKE, %esi
280 movl $SYS_futex, %eax
282 subq $cond_nwaiters, %rdi
292 /* If requeue_pi is used the kernel performs the locking of the
299 callq __pthread_mutex_cond_lock
304 48: addq $FRAME_SIZE, %rsp
305 cfi_adjust_cfa_offset(-FRAME_SIZE)
307 cfi_adjust_cfa_offset(-8)
310 cfi_adjust_cfa_offset(-8)
313 cfi_adjust_cfa_offset(-8)
316 cfi_adjust_cfa_offset(-8)
321 /* Initial locking failed. */
322 31: cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE)
323 cfi_rel_offset(%r12, FRAME_SIZE + 24)
324 cfi_rel_offset(%r13, FRAME_SIZE + 16)
325 cfi_rel_offset(%r14, FRAME_SIZE + 8)
326 cfi_rel_offset(%r15, FRAME_SIZE)
328 addq $cond_lock, %rdi
330 cmpq $-1, dep_mutex-cond_lock(%rdi)
331 movl $LLL_PRIVATE, %eax
332 movl $LLL_SHARED, %esi
334 callq __lll_lock_wait
337 /* Unlock in loop requires wakeup. */
340 addq $cond_lock, %rdi
342 cmpq $-1, dep_mutex-cond_lock(%rdi)
343 movl $LLL_PRIVATE, %eax
344 movl $LLL_SHARED, %esi
346 callq __lll_unlock_wake
349 /* Locking in loop failed. */
352 addq $cond_lock, %rdi
354 cmpq $-1, dep_mutex-cond_lock(%rdi)
355 movl $LLL_PRIVATE, %eax
356 movl $LLL_SHARED, %esi
358 callq __lll_lock_wait
360 subq $cond_lock, %rdi
364 /* Unlock after loop requires wakeup. */
367 addq $cond_lock, %rdi
369 cmpq $-1, dep_mutex-cond_lock(%rdi)
370 movl $LLL_PRIVATE, %eax
371 movl $LLL_SHARED, %esi
373 callq __lll_unlock_wake
376 /* The initial unlocking of the mutex failed. */
377 46: movq 8(%rsp), %rdi
388 addq $cond_lock, %rdi
390 cmpq $-1, dep_mutex-cond_lock(%rdi)
391 movl $LLL_PRIVATE, %eax
392 movl $LLL_SHARED, %esi
394 callq __lll_unlock_wake
396 47: movq (%rsp), %rax
400 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
404 /* Get internal lock. */
409 cmpxchgl %esi, (%rdi)
411 cmpxchgl %esi, cond_lock(%rdi)
415 /* Unlock the mutex. */
416 2: movq 16(%rsp), %rdi
418 callq __pthread_mutex_unlock_usercnt
425 incl cond_futex(%rdi)
426 addl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
428 /* Get and store current wakeup_seq value. */
430 movq wakeup_seq(%rdi), %r9
431 movl broadcast_seq(%rdi), %edx
435 /* Get the current time. */
437 # ifdef __NR_clock_gettime
438 /* Get the clock number. Note that the field in the condvar
439 structure stores the number minus 1. */
441 movl cond_nwaiters(%rdi), %edi
442 andl $((1 << nwaiters_shift) - 1), %edi
443 /* Only clocks 0 and 1 are allowed so far. Both are handled in the
447 movq __vdso_clock_gettime@GOTPCREL(%rip), %rax
454 26: movl $__NR_clock_gettime, %eax
457 # ifndef __ASSUME_POSIX_TIMERS
462 /* Compute relative timeout. */
470 movq $VSYSCALL_ADDR_vgettimeofday, %rax
473 /* Compute relative timeout. */
476 mul %rdx /* Milli seconds to nano seconds. */
483 addq $1000000000, %rdx
487 movq $-ETIMEDOUT, %r14
490 /* Store relative timeout. */
491 21: movq %rcx, 32(%rsp)
494 movl cond_futex(%rdi), %r12d
506 4: callq __pthread_enable_asynccancel
511 cmpq $-1, dep_mutex(%rdi)
513 # ifdef __ASSUME_PRIVATE_FUTEX
514 movl $FUTEX_WAIT, %eax
515 movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
519 movl %fs:PRIVATE_FUTEX, %esi
522 orl $FUTEX_WAIT, %esi
525 addq $cond_futex, %rdi
526 movl $SYS_futex, %eax
531 callq __pthread_disable_asynccancel
540 cmpxchgl %esi, (%rdi)
542 cmpxchgl %esi, cond_lock(%rdi)
546 6: movl broadcast_seq(%rdi), %edx
548 movq woken_seq(%rdi), %rax
550 movq wakeup_seq(%rdi), %r9
561 cmpq $-ETIMEDOUT, %r14
566 /* Initial locking failed. */
569 addq $cond_lock, %rdi
571 cmpq $-1, dep_mutex-cond_lock(%rdi)
572 movl $LLL_PRIVATE, %eax
573 movl $LLL_SHARED, %esi
575 callq __lll_lock_wait
578 /* Unlock in loop requires wakeup. */
581 addq $cond_lock, %rdi
583 cmpq $-1, dep_mutex-cond_lock(%rdi)
584 movl $LLL_PRIVATE, %eax
585 movl $LLL_SHARED, %esi
587 callq __lll_unlock_wake
590 /* Locking in loop failed. */
593 addq $cond_lock, %rdi
595 cmpq $-1, dep_mutex-cond_lock(%rdi)
596 movl $LLL_PRIVATE, %eax
597 movl $LLL_SHARED, %esi
599 callq __lll_lock_wait
601 subq $cond_lock, %rdi
605 # if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS
606 /* clock_gettime not available. */
607 19: leaq 32(%rsp), %rdi
609 movq $VSYSCALL_ADDR_vgettimeofday, %rax
612 /* Compute relative timeout. */
615 mul %rdx /* Milli seconds to nano seconds. */
621 addq $1000000000, %rdx
625 movq $-ETIMEDOUT, %r14
630 .size __pthread_cond_timedwait, .-__pthread_cond_timedwait
631 versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
636 .type __condvar_cleanup2, @function
641 +--------------------------+
643 +--------------------------+
645 +--------------------------+
647 +--------------------------+
649 +--------------------------+
650 rsp + 16 | mutex pointer |
651 +--------------------------+
652 rsp + 8 | condvar pointer |
653 +--------------------------+
654 rsp + 4 | old broadcast_seq value |
655 +--------------------------+
656 rsp + 0 | old cancellation mode |
657 +--------------------------+
662 /* Get internal lock. */
668 cmpxchgl %esi, (%rdi)
670 cmpxchgl %esi, cond_lock(%rdi)
675 addq $cond_lock, %rdi
677 cmpq $-1, dep_mutex-cond_lock(%rdi)
678 movl $LLL_PRIVATE, %eax
679 movl $LLL_SHARED, %esi
681 callq __lll_lock_wait
683 subq $cond_lock, %rdi
686 1: movl broadcast_seq(%rdi), %edx
690 /* We increment the wakeup_seq counter only if it is lower than
691 total_seq. If this is not the case the thread was woken and
692 then canceled. In this case we ignore the signal. */
693 movq total_seq(%rdi), %rax
694 cmpq wakeup_seq(%rdi), %rax
696 incq wakeup_seq(%rdi)
697 incl cond_futex(%rdi)
698 6: incq woken_seq(%rdi)
700 3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
702 /* Wake up a thread which wants to destroy the condvar object. */
704 cmpq $0xffffffffffffffff, total_seq(%rdi)
706 movl cond_nwaiters(%rdi), %eax
707 andl $~((1 << nwaiters_shift) - 1), %eax
710 cmpq $-1, dep_mutex(%rdi)
711 leaq cond_nwaiters(%rdi), %rdi
713 #ifdef __ASSUME_PRIVATE_FUTEX
714 movl $FUTEX_WAKE, %eax
715 movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
719 movl %fs:PRIVATE_FUTEX, %esi
721 orl $FUTEX_WAKE, %esi
723 movl $SYS_futex, %eax
725 subq $cond_nwaiters, %rdi
736 addq $cond_lock, %rdi
738 cmpq $-1, dep_mutex-cond_lock(%rdi)
739 movl $LLL_PRIVATE, %eax
740 movl $LLL_SHARED, %esi
742 callq __lll_unlock_wake
744 /* Wake up all waiters to make sure no signal gets lost. */
747 addq $cond_futex, %rdi
748 cmpq $-1, dep_mutex-cond_futex(%rdi)
749 movl $0x7fffffff, %edx
750 #ifdef __ASSUME_PRIVATE_FUTEX
751 movl $FUTEX_WAKE, %eax
752 movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
756 movl %fs:PRIVATE_FUTEX, %esi
758 orl $FUTEX_WAKE, %esi
760 movl $SYS_futex, %eax
763 5: movq 16(%rsp), %rdi
764 callq __pthread_mutex_cond_lock
767 movq FRAME_SIZE(%rsp), %r15
768 movq FRAME_SIZE+8(%rsp), %r14
769 movq FRAME_SIZE+16(%rsp), %r13
770 movq FRAME_SIZE+24(%rsp), %r12
772 call _Unwind_Resume@PLT
776 .size __condvar_cleanup2, .-__condvar_cleanup2
779 .section .gcc_except_table,"a",@progbits
781 .byte DW_EH_PE_omit # @LPStart format
782 .byte DW_EH_PE_omit # @TType format
783 .byte DW_EH_PE_uleb128 # call-site format
784 .uleb128 .Lcstend-.Lcstbegin
786 .uleb128 .LcleanupSTART1-.LSTARTCODE
787 .uleb128 .LcleanupEND1-.LcleanupSTART1
788 .uleb128 __condvar_cleanup2-.LSTARTCODE
790 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
791 .uleb128 .LcleanupSTART2-.LSTARTCODE
792 .uleb128 .LcleanupEND2-.LcleanupSTART2
793 .uleb128 __condvar_cleanup2-.LSTARTCODE
796 .uleb128 .LcallUR-.LSTARTCODE
797 .uleb128 .LENDCODE-.LcallUR
804 .hidden DW.ref.__gcc_personality_v0
805 .weak DW.ref.__gcc_personality_v0
806 .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
808 .type DW.ref.__gcc_personality_v0, @object
809 .size DW.ref.__gcc_personality_v0, 8
810 DW.ref.__gcc_personality_v0:
811 .quad __gcc_personality_v0