1 /* Copyright (C) 2002-2015 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
20 #include <pthread-errnos.h>
21 #include <kernel-features.h>
22 #include <lowlevellock.h>
24 #include <stap-probe.h>
28 #ifdef __ASSUME_PRIVATE_FUTEX
29 # define LOAD_PRIVATE_FUTEX_WAIT(reg) \
30 movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
31 # define LOAD_PRIVATE_FUTEX_WAKE(reg) \
32 movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
33 # define LOAD_FUTEX_WAIT(reg) \
34 xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
35 # define LOAD_FUTEX_WAIT_ABS(reg) \
36 xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
37 # define LOAD_FUTEX_WAKE(reg) \
38 xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
41 # define LOAD_PRIVATE_FUTEX_WAIT(reg) \
42 movl %fs:PRIVATE_FUTEX, reg
44 # define LOAD_PRIVATE_FUTEX_WAIT(reg) \
45 movl %fs:PRIVATE_FUTEX, reg ; \
48 # define LOAD_PRIVATE_FUTEX_WAKE(reg) \
49 movl %fs:PRIVATE_FUTEX, reg ; \
52 # define LOAD_FUTEX_WAIT(reg) \
53 xorl $FUTEX_PRIVATE_FLAG, reg ; \
54 andl %fs:PRIVATE_FUTEX, reg
56 # define LOAD_FUTEX_WAIT(reg) \
57 xorl $FUTEX_PRIVATE_FLAG, reg ; \
58 andl %fs:PRIVATE_FUTEX, reg ; \
61 # define LOAD_FUTEX_WAIT_ABS(reg) \
62 xorl $FUTEX_PRIVATE_FLAG, reg ; \
63 andl %fs:PRIVATE_FUTEX, reg ; \
64 orl $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg
65 # define LOAD_FUTEX_WAKE(reg) \
66 xorl $FUTEX_PRIVATE_FLAG, reg ; \
67 andl %fs:PRIVATE_FUTEX, reg ; \
72 .globl __lll_lock_wait_private
73 .type __lll_lock_wait_private,@function
74 .hidden __lll_lock_wait_private
76 __lll_lock_wait_private:
79 cfi_adjust_cfa_offset(8)
81 cfi_adjust_cfa_offset(8)
84 xorq %r10, %r10 /* No timeout. */
86 LOAD_PRIVATE_FUTEX_WAIT (%esi)
88 cmpl %edx, %eax /* NB: %edx == 2 */
91 1: LIBC_PROBE (lll_lock_wait_private, 1, %rdi)
96 xchgl %eax, (%rdi) /* NB: lock is implied */
102 cfi_adjust_cfa_offset(-8)
105 cfi_adjust_cfa_offset(-8)
109 .size __lll_lock_wait_private,.-__lll_lock_wait_private
112 .globl __lll_lock_wait
113 .type __lll_lock_wait,@function
114 .hidden __lll_lock_wait
119 cfi_adjust_cfa_offset(8)
121 cfi_adjust_cfa_offset(8)
122 cfi_offset(%r10, -16)
123 cfi_offset(%rdx, -24)
124 xorq %r10, %r10 /* No timeout. */
126 LOAD_FUTEX_WAIT (%esi)
128 cmpl %edx, %eax /* NB: %edx == 2 */
131 1: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi)
132 movl $SYS_futex, %eax
136 xchgl %eax, (%rdi) /* NB: lock is implied */
142 cfi_adjust_cfa_offset(-8)
145 cfi_adjust_cfa_offset(-8)
149 .size __lll_lock_wait,.-__lll_lock_wait
156 .globl __lll_timedlock_wait
157 .type __lll_timedlock_wait,@function
158 .hidden __lll_timedlock_wait
160 __lll_timedlock_wait:
162 # ifndef __ASSUME_FUTEX_CLOCK_REALTIME
164 cmpl $0, __have_futex_clock_realtime(%rip)
166 cmpl $0, __have_futex_clock_realtime
175 cfi_adjust_cfa_offset(8)
176 cfi_rel_offset(%r9, 0)
179 movl $0xffffffff, %r9d
180 LOAD_FUTEX_WAIT_ABS (%esi)
186 1: movl $SYS_futex, %eax
190 2: xchgl %edx, (%rdi) /* NB: lock is implied */
195 cmpl $-ETIMEDOUT, %eax
204 cfi_adjust_cfa_offset(-8)
208 5: movl $ETIMEDOUT, %eax
211 # ifndef __ASSUME_FUTEX_CLOCK_REALTIME
213 /* Check for a valid timeout value. */
214 cmpq $1000000000, 8(%rdx)
218 cfi_adjust_cfa_offset(8)
220 cfi_adjust_cfa_offset(8)
222 cfi_adjust_cfa_offset(8)
224 cfi_adjust_cfa_offset(8)
226 cfi_adjust_cfa_offset(8)
229 cfi_offset(%r12, -32)
230 cfi_offset(%r13, -40)
231 cfi_offset(%r14, -48)
233 cfi_adjust_cfa_offset(8)
235 /* Stack frame for the timespec and timeval structs. */
237 cfi_adjust_cfa_offset(24)
249 /* Get current time. */
252 /* This call works because we directly jump to a system call entry
253 which preserves all the registers. */
254 call JUMPTARGET(__gettimeofday)
256 /* Compute relative timeout. */
259 mul %rdi /* Milli seconds to nano seconds. */
265 addq $1000000000, %rsi
268 js 2f /* Time is already up. */
270 /* Store relative timeout. */
279 LOAD_FUTEX_WAIT (%esi)
281 movl $SYS_futex, %eax
290 cmpl $-ETIMEDOUT, %eax
292 2: movl $ETIMEDOUT, %edx
295 cfi_adjust_cfa_offset(-32)
297 cfi_adjust_cfa_offset(-8)
300 cfi_adjust_cfa_offset(-8)
303 cfi_adjust_cfa_offset(-8)
306 cfi_adjust_cfa_offset(-8)
309 cfi_adjust_cfa_offset(-8)
314 3: movl $EINVAL, %eax
318 .size __lll_timedlock_wait,.-__lll_timedlock_wait
322 .globl __lll_unlock_wake_private
323 .type __lll_unlock_wake_private,@function
324 .hidden __lll_unlock_wake_private
326 __lll_unlock_wake_private:
329 cfi_adjust_cfa_offset(8)
331 cfi_adjust_cfa_offset(8)
332 cfi_offset(%rsi, -16)
333 cfi_offset(%rdx, -24)
336 LOAD_PRIVATE_FUTEX_WAKE (%esi)
337 movl $1, %edx /* Wake one thread. */
338 movl $SYS_futex, %eax
342 cfi_adjust_cfa_offset(-8)
345 cfi_adjust_cfa_offset(-8)
349 .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
352 .globl __lll_unlock_wake
353 .type __lll_unlock_wake,@function
354 .hidden __lll_unlock_wake
359 cfi_adjust_cfa_offset(8)
361 cfi_adjust_cfa_offset(8)
362 cfi_offset(%rsi, -16)
363 cfi_offset(%rdx, -24)
366 LOAD_FUTEX_WAKE (%esi)
367 movl $1, %edx /* Wake one thread. */
368 movl $SYS_futex, %eax
372 cfi_adjust_cfa_offset(-8)
375 cfi_adjust_cfa_offset(-8)
379 .size __lll_unlock_wake,.-__lll_unlock_wake
381 .globl __lll_timedwait_tid
382 .type __lll_timedwait_tid,@function
383 .hidden __lll_timedwait_tid
388 cfi_adjust_cfa_offset(8)
390 cfi_adjust_cfa_offset(8)
391 cfi_offset(%r12, -16)
392 cfi_offset(%r13, -24)
398 cfi_adjust_cfa_offset(16)
400 /* Get current time. */
403 /* This call works because we directly jump to a system call entry
404 which preserves all the registers. */
405 call JUMPTARGET(__gettimeofday)
407 /* Compute relative timeout. */
410 mul %rdi /* Milli seconds to nano seconds. */
416 addq $1000000000, %rsi
419 js 6f /* Time is already up. */
421 movq %rdi, (%rsp) /* Store relative timeout. */
429 /* XXX The kernel so far uses global futex for the wakeup at
434 movl $FUTEX_WAIT, %esi
437 movl $SYS_futex, %eax
445 cfi_adjust_cfa_offset(-16)
447 cfi_adjust_cfa_offset(-8)
450 cfi_adjust_cfa_offset(-8)
454 cfi_adjust_cfa_offset(32)
455 1: cmpq $-ETIMEDOUT, %rax
458 6: movl $ETIMEDOUT, %eax
461 .size __lll_timedwait_tid,.-__lll_timedwait_tid