1 /* Copyright (C) 2002-2018 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
20 #include <pthread-errnos.h>
21 #include <kernel-features.h>
22 #include <lowlevellock.h>
24 #include <stap-probe.h>
28 #define LOAD_PRIVATE_FUTEX_WAIT(reg) \
29 movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
30 #define LOAD_PRIVATE_FUTEX_WAKE(reg) \
31 movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
32 #define LOAD_FUTEX_WAIT(reg) \
33 xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
34 #define LOAD_FUTEX_WAIT_ABS(reg) \
35 xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
36 #define LOAD_FUTEX_WAKE(reg) \
37 xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
40 .globl __lll_lock_wait_private
41 .type __lll_lock_wait_private,@function
42 .hidden __lll_lock_wait_private
44 __lll_lock_wait_private:
47 cfi_adjust_cfa_offset(8)
49 cfi_adjust_cfa_offset(8)
52 xorq %r10, %r10 /* No timeout. */
54 LOAD_PRIVATE_FUTEX_WAIT (%esi)
56 cmpl %edx, %eax /* NB: %edx == 2 */
59 1: LIBC_PROBE (lll_lock_wait_private, 1, %rdi)
64 xchgl %eax, (%rdi) /* NB: lock is implied */
70 cfi_adjust_cfa_offset(-8)
73 cfi_adjust_cfa_offset(-8)
77 .size __lll_lock_wait_private,.-__lll_lock_wait_private
80 .globl __lll_lock_wait
81 .type __lll_lock_wait,@function
82 .hidden __lll_lock_wait
87 cfi_adjust_cfa_offset(8)
89 cfi_adjust_cfa_offset(8)
92 xorq %r10, %r10 /* No timeout. */
94 LOAD_FUTEX_WAIT (%esi)
96 cmpl %edx, %eax /* NB: %edx == 2 */
99 1: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi)
100 movl $SYS_futex, %eax
104 xchgl %eax, (%rdi) /* NB: lock is implied */
110 cfi_adjust_cfa_offset(-8)
113 cfi_adjust_cfa_offset(-8)
117 .size __lll_lock_wait,.-__lll_lock_wait
124 .globl __lll_timedlock_wait
125 .type __lll_timedlock_wait,@function
126 .hidden __lll_timedlock_wait
128 __lll_timedlock_wait:
130 # ifndef __ASSUME_FUTEX_CLOCK_REALTIME
132 cmpl $0, __have_futex_clock_realtime(%rip)
134 cmpl $0, __have_futex_clock_realtime
143 cfi_adjust_cfa_offset(8)
144 cfi_rel_offset(%r9, 0)
147 movl $0xffffffff, %r9d
148 LOAD_FUTEX_WAIT_ABS (%esi)
154 1: movl $SYS_futex, %eax
158 2: xchgl %edx, (%rdi) /* NB: lock is implied */
163 cmpl $-ETIMEDOUT, %eax
172 cfi_adjust_cfa_offset(-8)
176 5: movl $ETIMEDOUT, %eax
179 # ifndef __ASSUME_FUTEX_CLOCK_REALTIME
181 /* Check for a valid timeout value. */
182 cmpq $1000000000, 8(%rdx)
186 cfi_adjust_cfa_offset(8)
188 cfi_adjust_cfa_offset(8)
190 cfi_adjust_cfa_offset(8)
192 cfi_adjust_cfa_offset(8)
194 cfi_adjust_cfa_offset(8)
197 cfi_offset(%r12, -32)
198 cfi_offset(%r13, -40)
199 cfi_offset(%r14, -48)
201 cfi_adjust_cfa_offset(8)
203 /* Stack frame for the timespec and timeval structs. */
205 cfi_adjust_cfa_offset(24)
217 /* Get current time. */
220 /* This call works because we directly jump to a system call entry
221 which preserves all the registers. */
222 call JUMPTARGET(__gettimeofday)
224 /* Compute relative timeout. */
227 mul %rdi /* Milli seconds to nano seconds. */
233 addq $1000000000, %rsi
236 js 2f /* Time is already up. */
238 /* Store relative timeout. */
247 LOAD_FUTEX_WAIT (%esi)
249 movl $SYS_futex, %eax
258 cmpl $-ETIMEDOUT, %eax
260 2: movl $ETIMEDOUT, %edx
263 cfi_adjust_cfa_offset(-32)
265 cfi_adjust_cfa_offset(-8)
268 cfi_adjust_cfa_offset(-8)
271 cfi_adjust_cfa_offset(-8)
274 cfi_adjust_cfa_offset(-8)
277 cfi_adjust_cfa_offset(-8)
282 3: movl $EINVAL, %eax
286 .size __lll_timedlock_wait,.-__lll_timedlock_wait
290 .globl __lll_unlock_wake_private
291 .type __lll_unlock_wake_private,@function
292 .hidden __lll_unlock_wake_private
294 __lll_unlock_wake_private:
297 cfi_adjust_cfa_offset(8)
299 cfi_adjust_cfa_offset(8)
300 cfi_offset(%rsi, -16)
301 cfi_offset(%rdx, -24)
304 LOAD_PRIVATE_FUTEX_WAKE (%esi)
305 movl $1, %edx /* Wake one thread. */
306 movl $SYS_futex, %eax
310 cfi_adjust_cfa_offset(-8)
313 cfi_adjust_cfa_offset(-8)
317 .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
320 .globl __lll_unlock_wake
321 .type __lll_unlock_wake,@function
322 .hidden __lll_unlock_wake
327 cfi_adjust_cfa_offset(8)
329 cfi_adjust_cfa_offset(8)
330 cfi_offset(%rsi, -16)
331 cfi_offset(%rdx, -24)
334 LOAD_FUTEX_WAKE (%esi)
335 movl $1, %edx /* Wake one thread. */
336 movl $SYS_futex, %eax
340 cfi_adjust_cfa_offset(-8)
343 cfi_adjust_cfa_offset(-8)
347 .size __lll_unlock_wake,.-__lll_unlock_wake
349 .globl __lll_timedwait_tid
350 .type __lll_timedwait_tid,@function
351 .hidden __lll_timedwait_tid
356 cfi_adjust_cfa_offset(8)
358 cfi_adjust_cfa_offset(8)
359 cfi_offset(%r12, -16)
360 cfi_offset(%r13, -24)
365 /* Align stack to 16 bytes when calling __gettimeofday. */
367 cfi_adjust_cfa_offset(24)
369 /* Get current time. */
372 /* This call works because we directly jump to a system call entry
373 which preserves all the registers. */
374 call JUMPTARGET(__gettimeofday)
376 /* Compute relative timeout. */
379 mul %rdi /* Milli seconds to nano seconds. */
385 addq $1000000000, %rsi
388 js 6f /* Time is already up. */
390 movq %rdi, (%rsp) /* Store relative timeout. */
398 /* XXX The kernel so far uses global futex for the wakeup at
403 movl $FUTEX_WAIT, %esi
406 movl $SYS_futex, %eax
414 cfi_adjust_cfa_offset(-24)
416 cfi_adjust_cfa_offset(-8)
419 cfi_adjust_cfa_offset(-8)
423 cfi_adjust_cfa_offset(32)
424 1: cmpq $-ETIMEDOUT, %rax
427 6: movl $ETIMEDOUT, %eax
430 .size __lll_timedwait_tid,.-__lll_timedwait_tid