1 /* Copyright (C) 2002-2019 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
22 #include <sys/param.h>
26 #include <lowlevellock.h>
27 #include <not-cancel.h>
29 #include <stap-probe.h>
31 #ifndef lll_clocklock_elision
32 #define lll_clocklock_elision(futex, adapt_count, clockid, abstime, private) \
33 lll_clocklock (futex, clockid, abstime, private)
36 #ifndef lll_trylock_elision
37 #define lll_trylock_elision(a,t) lll_trylock(a)
41 #define FORCE_ELISION(m, s)
45 __pthread_mutex_clocklock_common (pthread_mutex_t
*mutex
,
47 const struct timespec
*abstime
)
50 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
53 /* We must not check ABSTIME here. If the thread does not block
54 abstime must not be checked for a valid value. */
56 /* See concurrency notes regarding mutex type which is loaded from __kind
57 in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */
58 switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex
),
59 PTHREAD_MUTEX_TIMED_NP
))
61 /* Recursive mutex. */
62 case PTHREAD_MUTEX_RECURSIVE_NP
|PTHREAD_MUTEX_ELISION_NP
:
63 case PTHREAD_MUTEX_RECURSIVE_NP
:
64 /* Check whether we already hold the mutex. */
65 if (mutex
->__data
.__owner
== id
)
67 /* Just bump the counter. */
68 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
69 /* Overflow of the counter. */
72 ++mutex
->__data
.__count
;
77 /* We have to get the mutex. */
78 result
= lll_clocklock (mutex
->__data
.__lock
, clockid
, abstime
,
79 PTHREAD_MUTEX_PSHARED (mutex
));
84 /* Only locked once so far. */
85 mutex
->__data
.__count
= 1;
88 /* Error checking mutex. */
89 case PTHREAD_MUTEX_ERRORCHECK_NP
:
90 /* Check whether we already hold the mutex. */
91 if (__glibc_unlikely (mutex
->__data
.__owner
== id
))
94 /* Don't do lock elision on an error checking mutex. */
97 case PTHREAD_MUTEX_TIMED_NP
:
98 FORCE_ELISION (mutex
, goto elision
);
101 result
= lll_clocklock (mutex
->__data
.__lock
, clockid
, abstime
,
102 PTHREAD_MUTEX_PSHARED (mutex
));
105 case PTHREAD_MUTEX_TIMED_ELISION_NP
:
106 elision
: __attribute__((unused
))
107 /* Don't record ownership */
108 return lll_clocklock_elision (mutex
->__data
.__lock
,
109 mutex
->__data
.__spins
,
111 PTHREAD_MUTEX_PSHARED (mutex
));
114 case PTHREAD_MUTEX_ADAPTIVE_NP
:
118 if (lll_trylock (mutex
->__data
.__lock
) != 0)
121 int max_cnt
= MIN (max_adaptive_count (),
122 mutex
->__data
.__spins
* 2 + 10);
125 if (cnt
++ >= max_cnt
)
127 result
= lll_clocklock (mutex
->__data
.__lock
,
129 PTHREAD_MUTEX_PSHARED (mutex
));
134 while (lll_trylock (mutex
->__data
.__lock
) != 0);
136 mutex
->__data
.__spins
+= (cnt
- mutex
->__data
.__spins
) / 8;
140 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
:
141 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
:
142 case PTHREAD_MUTEX_ROBUST_NORMAL_NP
:
143 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP
:
144 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
145 &mutex
->__data
.__list
.__next
);
146 /* We need to set op_pending before starting the operation. Also
147 see comments at ENQUEUE_MUTEX. */
148 __asm ("" ::: "memory");
150 oldval
= mutex
->__data
.__lock
;
151 /* This is set to FUTEX_WAITERS iff we might have shared the
152 FUTEX_WAITERS flag with other threads, and therefore need to keep it
153 set to avoid lost wake-ups. We have the same requirement in the
154 simple mutex algorithm. */
155 unsigned int assume_other_futex_waiters
= 0;
158 /* Try to acquire the lock through a CAS from 0 (not acquired) to
159 our TID | assume_other_futex_waiters. */
160 if (__glibc_likely (oldval
== 0))
163 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
164 id
| assume_other_futex_waiters
, 0);
165 if (__glibc_likely (oldval
== 0))
169 if ((oldval
& FUTEX_OWNER_DIED
) != 0)
171 /* The previous owner died. Try locking the mutex. */
172 int newval
= id
| (oldval
& FUTEX_WAITERS
)
173 | assume_other_futex_waiters
;
176 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
178 if (newval
!= oldval
)
184 /* We got the mutex. */
185 mutex
->__data
.__count
= 1;
186 /* But it is inconsistent unless marked otherwise. */
187 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
189 /* We must not enqueue the mutex before we have acquired it.
190 Also see comments at ENQUEUE_MUTEX. */
191 __asm ("" ::: "memory");
192 ENQUEUE_MUTEX (mutex
);
193 /* We need to clear op_pending after we enqueue the mutex. */
194 __asm ("" ::: "memory");
195 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
197 /* Note that we deliberately exit here. If we fall
198 through to the end of the function __nusers would be
199 incremented which is not correct because the old
200 owner has to be discounted. */
204 /* Check whether we already hold the mutex. */
205 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
207 int kind
= PTHREAD_MUTEX_TYPE (mutex
);
208 if (kind
== PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
)
210 /* We do not need to ensure ordering wrt another memory
211 access. Also see comments at ENQUEUE_MUTEX. */
212 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
217 if (kind
== PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
)
219 /* We do not need to ensure ordering wrt another memory
221 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
224 /* Just bump the counter. */
225 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
226 /* Overflow of the counter. */
229 ++mutex
->__data
.__count
;
231 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
237 /* We are about to block; check whether the timeout is invalid. */
238 if (abstime
->tv_nsec
< 0 || abstime
->tv_nsec
>= 1000000000)
240 /* Work around the fact that the kernel rejects negative timeout
241 values despite them being valid. */
242 if (__glibc_unlikely (abstime
->tv_sec
< 0))
245 /* We cannot acquire the mutex nor has its owner died. Thus, try
246 to block using futexes. Set FUTEX_WAITERS if necessary so that
247 other threads are aware that there are potentially threads
248 blocked on the futex. Restart if oldval changed in the
250 if ((oldval
& FUTEX_WAITERS
) == 0)
252 if (atomic_compare_and_exchange_bool_acq (&mutex
->__data
.__lock
,
253 oldval
| FUTEX_WAITERS
,
257 oldval
= mutex
->__data
.__lock
;
260 oldval
|= FUTEX_WAITERS
;
263 /* It is now possible that we share the FUTEX_WAITERS flag with
264 another thread; therefore, update assume_other_futex_waiters so
265 that we do not forget about this when handling other cases
266 above and thus do not cause lost wake-ups. */
267 assume_other_futex_waiters
|= FUTEX_WAITERS
;
269 /* Block using the futex. */
270 int err
= lll_futex_clock_wait_bitset (&mutex
->__data
.__lock
,
271 oldval
, clockid
, abstime
,
272 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
));
273 /* The futex call timed out. */
274 if (err
== -ETIMEDOUT
)
276 /* Reload current lock value. */
277 oldval
= mutex
->__data
.__lock
;
280 /* We have acquired the mutex; check if it is still consistent. */
281 if (__builtin_expect (mutex
->__data
.__owner
282 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
284 /* This mutex is now not recoverable. */
285 mutex
->__data
.__count
= 0;
286 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex
);
287 lll_unlock (mutex
->__data
.__lock
, private);
288 /* FIXME This violates the mutex destruction requirements. See
289 __pthread_mutex_unlock_full. */
290 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
291 return ENOTRECOVERABLE
;
294 mutex
->__data
.__count
= 1;
295 /* We must not enqueue the mutex before we have acquired it.
296 Also see comments at ENQUEUE_MUTEX. */
297 __asm ("" ::: "memory");
298 ENQUEUE_MUTEX (mutex
);
299 /* We need to clear op_pending after we enqueue the mutex. */
300 __asm ("" ::: "memory");
301 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
304 /* The PI support requires the Linux futex system call. If that's not
305 available, pthread_mutex_init should never have allowed the type to
306 be set. So it will get the default case for an invalid type. */
308 case PTHREAD_MUTEX_PI_RECURSIVE_NP
:
309 case PTHREAD_MUTEX_PI_ERRORCHECK_NP
:
310 case PTHREAD_MUTEX_PI_NORMAL_NP
:
311 case PTHREAD_MUTEX_PI_ADAPTIVE_NP
:
312 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP
:
313 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP
:
314 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP
:
315 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP
:
319 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
320 in sysdeps/nptl/bits/thread-shared-types.h. */
321 int mutex_kind
= atomic_load_relaxed (&(mutex
->__data
.__kind
));
322 kind
= mutex_kind
& PTHREAD_MUTEX_KIND_MASK_NP
;
323 robust
= mutex_kind
& PTHREAD_MUTEX_ROBUST_NORMAL_NP
;
328 /* Note: robust PI futexes are signaled by setting bit 0. */
329 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
330 (void *) (((uintptr_t) &mutex
->__data
.__list
.__next
)
332 /* We need to set op_pending before starting the operation. Also
333 see comments at ENQUEUE_MUTEX. */
334 __asm ("" ::: "memory");
337 oldval
= mutex
->__data
.__lock
;
339 /* Check whether we already hold the mutex. */
340 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
342 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
344 /* We do not need to ensure ordering wrt another memory
346 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
350 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
352 /* We do not need to ensure ordering wrt another memory
354 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
356 /* Just bump the counter. */
357 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
358 /* Overflow of the counter. */
361 ++mutex
->__data
.__count
;
363 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
369 oldval
= atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
374 /* The mutex is locked. The kernel will now take care of
375 everything. The timeout value must be a relative value.
377 int private = (robust
378 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)
379 : PTHREAD_MUTEX_PSHARED (mutex
));
380 INTERNAL_SYSCALL_DECL (__err
);
382 int e
= INTERNAL_SYSCALL (futex
, __err
, 4, &mutex
->__data
.__lock
,
383 __lll_private_flag (FUTEX_LOCK_PI
,
386 if (INTERNAL_SYSCALL_ERROR_P (e
, __err
))
388 if (INTERNAL_SYSCALL_ERRNO (e
, __err
) == ETIMEDOUT
)
391 if (INTERNAL_SYSCALL_ERRNO (e
, __err
) == ESRCH
392 || INTERNAL_SYSCALL_ERRNO (e
, __err
) == EDEADLK
)
394 assert (INTERNAL_SYSCALL_ERRNO (e
, __err
) != EDEADLK
395 || (kind
!= PTHREAD_MUTEX_ERRORCHECK_NP
396 && kind
!= PTHREAD_MUTEX_RECURSIVE_NP
));
397 /* ESRCH can happen only for non-robust PI mutexes where
398 the owner of the lock died. */
399 assert (INTERNAL_SYSCALL_ERRNO (e
, __err
) != ESRCH
402 /* Delay the thread until the timeout is reached.
403 Then return ETIMEDOUT. */
404 struct timespec reltime
;
407 INTERNAL_SYSCALL (clock_gettime
, __err
, 2, clockid
,
409 reltime
.tv_sec
= abstime
->tv_sec
- now
.tv_sec
;
410 reltime
.tv_nsec
= abstime
->tv_nsec
- now
.tv_nsec
;
411 if (reltime
.tv_nsec
< 0)
413 reltime
.tv_nsec
+= 1000000000;
416 if (reltime
.tv_sec
>= 0)
417 while (__nanosleep_nocancel (&reltime
, &reltime
) != 0)
423 return INTERNAL_SYSCALL_ERRNO (e
, __err
);
426 oldval
= mutex
->__data
.__lock
;
428 assert (robust
|| (oldval
& FUTEX_OWNER_DIED
) == 0);
431 if (__glibc_unlikely (oldval
& FUTEX_OWNER_DIED
))
433 atomic_and (&mutex
->__data
.__lock
, ~FUTEX_OWNER_DIED
);
435 /* We got the mutex. */
436 mutex
->__data
.__count
= 1;
437 /* But it is inconsistent unless marked otherwise. */
438 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
440 /* We must not enqueue the mutex before we have acquired it.
441 Also see comments at ENQUEUE_MUTEX. */
442 __asm ("" ::: "memory");
443 ENQUEUE_MUTEX_PI (mutex
);
444 /* We need to clear op_pending after we enqueue the mutex. */
445 __asm ("" ::: "memory");
446 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
448 /* Note that we deliberately exit here. If we fall
449 through to the end of the function __nusers would be
450 incremented which is not correct because the old owner
451 has to be discounted. */
456 && __builtin_expect (mutex
->__data
.__owner
457 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
459 /* This mutex is now not recoverable. */
460 mutex
->__data
.__count
= 0;
462 INTERNAL_SYSCALL_DECL (__err
);
463 INTERNAL_SYSCALL (futex
, __err
, 4, &mutex
->__data
.__lock
,
464 __lll_private_flag (FUTEX_UNLOCK_PI
,
465 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)),
468 /* To the kernel, this will be visible after the kernel has
469 acquired the mutex in the syscall. */
470 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
471 return ENOTRECOVERABLE
;
474 mutex
->__data
.__count
= 1;
477 /* We must not enqueue the mutex before we have acquired it.
478 Also see comments at ENQUEUE_MUTEX. */
479 __asm ("" ::: "memory");
480 ENQUEUE_MUTEX_PI (mutex
);
481 /* We need to clear op_pending after we enqueue the mutex. */
482 __asm ("" ::: "memory");
483 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
487 #endif /* __NR_futex. */
489 case PTHREAD_MUTEX_PP_RECURSIVE_NP
:
490 case PTHREAD_MUTEX_PP_ERRORCHECK_NP
:
491 case PTHREAD_MUTEX_PP_NORMAL_NP
:
492 case PTHREAD_MUTEX_PP_ADAPTIVE_NP
:
494 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
495 in sysdeps/nptl/bits/thread-shared-types.h. */
496 int kind
= atomic_load_relaxed (&(mutex
->__data
.__kind
))
497 & PTHREAD_MUTEX_KIND_MASK_NP
;
499 oldval
= mutex
->__data
.__lock
;
501 /* Check whether we already hold the mutex. */
502 if (mutex
->__data
.__owner
== id
)
504 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
507 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
509 /* Just bump the counter. */
510 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
511 /* Overflow of the counter. */
514 ++mutex
->__data
.__count
;
516 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
522 int oldprio
= -1, ceilval
;
525 int ceiling
= (oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
)
526 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
528 if (__pthread_current_priority () > ceiling
)
533 __pthread_tpp_change_priority (oldprio
, -1);
537 result
= __pthread_tpp_change_priority (oldprio
, ceiling
);
541 ceilval
= ceiling
<< PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
545 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
546 ceilval
| 1, ceilval
);
548 if (oldval
== ceilval
)
554 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
558 if ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
)
561 if (oldval
!= ceilval
)
563 /* Reject invalid timeouts. */
564 if (abstime
->tv_nsec
< 0 || abstime
->tv_nsec
>= 1000000000)
573 /* Get the current time. */
574 (void) __gettimeofday (&tv
, NULL
);
576 /* Compute relative timeout. */
577 rt
.tv_sec
= abstime
->tv_sec
- tv
.tv_sec
;
578 rt
.tv_nsec
= abstime
->tv_nsec
- tv
.tv_usec
* 1000;
581 rt
.tv_nsec
+= 1000000000;
585 /* Already timed out? */
592 lll_futex_timed_wait (&mutex
->__data
.__lock
,
594 PTHREAD_MUTEX_PSHARED (mutex
));
597 while (atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
598 ceilval
| 2, ceilval
)
601 while ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
);
603 assert (mutex
->__data
.__owner
== 0);
604 mutex
->__data
.__count
= 1;
609 /* Correct code cannot set any other type. */
615 /* Record the ownership. */
616 mutex
->__data
.__owner
= id
;
617 ++mutex
->__data
.__nusers
;
619 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
627 __pthread_mutex_clocklock (pthread_mutex_t
*mutex
,
629 const struct timespec
*abstime
)
631 if (__glibc_unlikely (!lll_futex_supported_clockid (clockid
)))
634 LIBC_PROBE (mutex_clocklock_entry
, 3, mutex
, clockid
, abstime
);
635 return __pthread_mutex_clocklock_common (mutex
, clockid
, abstime
);
637 weak_alias (__pthread_mutex_clocklock
, pthread_mutex_clocklock
)
640 __pthread_mutex_timedlock (pthread_mutex_t
*mutex
,
641 const struct timespec
*abstime
)
643 LIBC_PROBE (mutex_timedlock_entry
, 2, mutex
, abstime
);
644 return __pthread_mutex_clocklock_common (mutex
, CLOCK_REALTIME
, abstime
);
646 weak_alias (__pthread_mutex_timedlock
, pthread_mutex_timedlock
)