1 /* Copyright (C) 2002-2021 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
4 The GNU C Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
9 The GNU C Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
14 You should have received a copy of the GNU Lesser General Public
15 License along with the GNU C Library; if not, see
16 <https://www.gnu.org/licenses/>. */
21 #include <sys/param.h>
25 #include <lowlevellock.h>
26 #include <not-cancel.h>
27 #include <futex-internal.h>
29 #include <stap-probe.h>
32 __pthread_mutex_clocklock_common (pthread_mutex_t
*mutex
,
34 const struct __timespec64
*abstime
)
37 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
40 /* We must not check ABSTIME here. If the thread does not block
41 abstime must not be checked for a valid value. */
43 /* See concurrency notes regarding mutex type which is loaded from __kind
44 in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */
45 switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex
),
46 PTHREAD_MUTEX_TIMED_NP
))
48 /* Recursive mutex. */
49 case PTHREAD_MUTEX_RECURSIVE_NP
|PTHREAD_MUTEX_ELISION_NP
:
50 case PTHREAD_MUTEX_RECURSIVE_NP
:
51 /* Check whether we already hold the mutex. */
52 if (mutex
->__data
.__owner
== id
)
54 /* Just bump the counter. */
55 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
56 /* Overflow of the counter. */
59 ++mutex
->__data
.__count
;
64 /* We have to get the mutex. */
65 result
= __futex_clocklock64 (&mutex
->__data
.__lock
, clockid
, abstime
,
66 PTHREAD_MUTEX_PSHARED (mutex
));
71 /* Only locked once so far. */
72 mutex
->__data
.__count
= 1;
75 /* Error checking mutex. */
76 case PTHREAD_MUTEX_ERRORCHECK_NP
:
77 /* Check whether we already hold the mutex. */
78 if (__glibc_unlikely (mutex
->__data
.__owner
== id
))
81 /* Don't do lock elision on an error checking mutex. */
84 case PTHREAD_MUTEX_TIMED_NP
:
85 FORCE_ELISION (mutex
, goto elision
);
88 result
= __futex_clocklock64 (&mutex
->__data
.__lock
, clockid
, abstime
,
89 PTHREAD_MUTEX_PSHARED (mutex
));
92 case PTHREAD_MUTEX_TIMED_ELISION_NP
:
93 elision
: __attribute__((unused
))
94 /* Don't record ownership */
95 return lll_clocklock_elision (mutex
->__data
.__lock
,
96 mutex
->__data
.__spins
,
98 PTHREAD_MUTEX_PSHARED (mutex
));
101 case PTHREAD_MUTEX_ADAPTIVE_NP
:
102 if (lll_trylock (mutex
->__data
.__lock
) != 0)
105 int max_cnt
= MIN (max_adaptive_count (),
106 mutex
->__data
.__spins
* 2 + 10);
109 if (cnt
++ >= max_cnt
)
111 result
= __futex_clocklock64 (&mutex
->__data
.__lock
,
113 PTHREAD_MUTEX_PSHARED (mutex
));
118 while (lll_trylock (mutex
->__data
.__lock
) != 0);
120 mutex
->__data
.__spins
+= (cnt
- mutex
->__data
.__spins
) / 8;
124 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
:
125 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
:
126 case PTHREAD_MUTEX_ROBUST_NORMAL_NP
:
127 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP
:
128 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
129 &mutex
->__data
.__list
.__next
);
130 /* We need to set op_pending before starting the operation. Also
131 see comments at ENQUEUE_MUTEX. */
132 __asm ("" ::: "memory");
134 oldval
= mutex
->__data
.__lock
;
135 /* This is set to FUTEX_WAITERS iff we might have shared the
136 FUTEX_WAITERS flag with other threads, and therefore need to keep it
137 set to avoid lost wake-ups. We have the same requirement in the
138 simple mutex algorithm. */
139 unsigned int assume_other_futex_waiters
= 0;
142 /* Try to acquire the lock through a CAS from 0 (not acquired) to
143 our TID | assume_other_futex_waiters. */
144 if (__glibc_likely (oldval
== 0))
147 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
148 id
| assume_other_futex_waiters
, 0);
149 if (__glibc_likely (oldval
== 0))
153 if ((oldval
& FUTEX_OWNER_DIED
) != 0)
155 /* The previous owner died. Try locking the mutex. */
156 int newval
= id
| (oldval
& FUTEX_WAITERS
)
157 | assume_other_futex_waiters
;
160 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
162 if (newval
!= oldval
)
168 /* We got the mutex. */
169 mutex
->__data
.__count
= 1;
170 /* But it is inconsistent unless marked otherwise. */
171 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
173 /* We must not enqueue the mutex before we have acquired it.
174 Also see comments at ENQUEUE_MUTEX. */
175 __asm ("" ::: "memory");
176 ENQUEUE_MUTEX (mutex
);
177 /* We need to clear op_pending after we enqueue the mutex. */
178 __asm ("" ::: "memory");
179 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
181 /* Note that we deliberately exit here. If we fall
182 through to the end of the function __nusers would be
183 incremented which is not correct because the old
184 owner has to be discounted. */
188 /* Check whether we already hold the mutex. */
189 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
191 int kind
= PTHREAD_MUTEX_TYPE (mutex
);
192 if (kind
== PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
)
194 /* We do not need to ensure ordering wrt another memory
195 access. Also see comments at ENQUEUE_MUTEX. */
196 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
201 if (kind
== PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
)
203 /* We do not need to ensure ordering wrt another memory
205 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
208 /* Just bump the counter. */
209 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
210 /* Overflow of the counter. */
213 ++mutex
->__data
.__count
;
215 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
221 /* We are about to block; check whether the timeout is invalid. */
222 if (! valid_nanoseconds (abstime
->tv_nsec
))
224 /* Work around the fact that the kernel rejects negative timeout
225 values despite them being valid. */
226 if (__glibc_unlikely (abstime
->tv_sec
< 0))
229 /* We cannot acquire the mutex nor has its owner died. Thus, try
230 to block using futexes. Set FUTEX_WAITERS if necessary so that
231 other threads are aware that there are potentially threads
232 blocked on the futex. Restart if oldval changed in the
234 if ((oldval
& FUTEX_WAITERS
) == 0)
236 if (atomic_compare_and_exchange_bool_acq (&mutex
->__data
.__lock
,
237 oldval
| FUTEX_WAITERS
,
241 oldval
= mutex
->__data
.__lock
;
244 oldval
|= FUTEX_WAITERS
;
247 /* It is now possible that we share the FUTEX_WAITERS flag with
248 another thread; therefore, update assume_other_futex_waiters so
249 that we do not forget about this when handling other cases
250 above and thus do not cause lost wake-ups. */
251 assume_other_futex_waiters
|= FUTEX_WAITERS
;
253 /* Block using the futex. */
254 int err
= __futex_abstimed_wait64 (
255 (unsigned int *) &mutex
->__data
.__lock
,
256 oldval
, clockid
, abstime
,
257 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
));
258 /* The futex call timed out. */
259 if (err
== ETIMEDOUT
|| err
== EOVERFLOW
)
261 /* Reload current lock value. */
262 oldval
= mutex
->__data
.__lock
;
265 /* We have acquired the mutex; check if it is still consistent. */
266 if (__builtin_expect (mutex
->__data
.__owner
267 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
269 /* This mutex is now not recoverable. */
270 mutex
->__data
.__count
= 0;
271 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex
);
272 lll_unlock (mutex
->__data
.__lock
, private);
273 /* FIXME This violates the mutex destruction requirements. See
274 __pthread_mutex_unlock_full. */
275 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
276 return ENOTRECOVERABLE
;
279 mutex
->__data
.__count
= 1;
280 /* We must not enqueue the mutex before we have acquired it.
281 Also see comments at ENQUEUE_MUTEX. */
282 __asm ("" ::: "memory");
283 ENQUEUE_MUTEX (mutex
);
284 /* We need to clear op_pending after we enqueue the mutex. */
285 __asm ("" ::: "memory");
286 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
289 /* The PI support requires the Linux futex system call. If that's not
290 available, pthread_mutex_init should never have allowed the type to
291 be set. So it will get the default case for an invalid type. */
293 case PTHREAD_MUTEX_PI_RECURSIVE_NP
:
294 case PTHREAD_MUTEX_PI_ERRORCHECK_NP
:
295 case PTHREAD_MUTEX_PI_NORMAL_NP
:
296 case PTHREAD_MUTEX_PI_ADAPTIVE_NP
:
297 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP
:
298 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP
:
299 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP
:
300 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP
:
304 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
305 in sysdeps/nptl/bits/thread-shared-types.h. */
306 int mutex_kind
= atomic_load_relaxed (&(mutex
->__data
.__kind
));
307 kind
= mutex_kind
& PTHREAD_MUTEX_KIND_MASK_NP
;
308 robust
= mutex_kind
& PTHREAD_MUTEX_ROBUST_NORMAL_NP
;
313 /* Note: robust PI futexes are signaled by setting bit 0. */
314 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
315 (void *) (((uintptr_t) &mutex
->__data
.__list
.__next
)
317 /* We need to set op_pending before starting the operation. Also
318 see comments at ENQUEUE_MUTEX. */
319 __asm ("" ::: "memory");
322 oldval
= mutex
->__data
.__lock
;
324 /* Check whether we already hold the mutex. */
325 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
327 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
329 /* We do not need to ensure ordering wrt another memory
331 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
335 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
337 /* We do not need to ensure ordering wrt another memory
339 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
341 /* Just bump the counter. */
342 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
343 /* Overflow of the counter. */
346 ++mutex
->__data
.__count
;
348 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
354 oldval
= atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
359 /* The mutex is locked. The kernel will now take care of
360 everything. The timeout value must be a relative value.
362 int private = (robust
363 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)
364 : PTHREAD_MUTEX_PSHARED (mutex
));
365 int e
= __futex_lock_pi64 (&mutex
->__data
.__lock
, clockid
, abstime
,
369 else if (e
== ESRCH
|| e
== EDEADLK
)
372 || (kind
!= PTHREAD_MUTEX_ERRORCHECK_NP
373 && kind
!= PTHREAD_MUTEX_RECURSIVE_NP
));
374 /* ESRCH can happen only for non-robust PI mutexes where
375 the owner of the lock died. */
376 assert (e
!= ESRCH
|| !robust
);
378 /* Delay the thread until the timeout is reached. Then return
381 e
= __futex_abstimed_wait64 (&(unsigned int){0}, 0, clockid
,
383 while (e
!= ETIMEDOUT
);
389 oldval
= mutex
->__data
.__lock
;
391 assert (robust
|| (oldval
& FUTEX_OWNER_DIED
) == 0);
394 if (__glibc_unlikely (oldval
& FUTEX_OWNER_DIED
))
396 atomic_and (&mutex
->__data
.__lock
, ~FUTEX_OWNER_DIED
);
398 /* We got the mutex. */
399 mutex
->__data
.__count
= 1;
400 /* But it is inconsistent unless marked otherwise. */
401 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
403 /* We must not enqueue the mutex before we have acquired it.
404 Also see comments at ENQUEUE_MUTEX. */
405 __asm ("" ::: "memory");
406 ENQUEUE_MUTEX_PI (mutex
);
407 /* We need to clear op_pending after we enqueue the mutex. */
408 __asm ("" ::: "memory");
409 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
411 /* Note that we deliberately exit here. If we fall
412 through to the end of the function __nusers would be
413 incremented which is not correct because the old owner
414 has to be discounted. */
419 && __builtin_expect (mutex
->__data
.__owner
420 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
422 /* This mutex is now not recoverable. */
423 mutex
->__data
.__count
= 0;
425 futex_unlock_pi ((unsigned int *) &mutex
->__data
.__lock
,
426 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
));
428 /* To the kernel, this will be visible after the kernel has
429 acquired the mutex in the syscall. */
430 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
431 return ENOTRECOVERABLE
;
434 mutex
->__data
.__count
= 1;
437 /* We must not enqueue the mutex before we have acquired it.
438 Also see comments at ENQUEUE_MUTEX. */
439 __asm ("" ::: "memory");
440 ENQUEUE_MUTEX_PI (mutex
);
441 /* We need to clear op_pending after we enqueue the mutex. */
442 __asm ("" ::: "memory");
443 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
447 #endif /* __NR_futex. */
449 case PTHREAD_MUTEX_PP_RECURSIVE_NP
:
450 case PTHREAD_MUTEX_PP_ERRORCHECK_NP
:
451 case PTHREAD_MUTEX_PP_NORMAL_NP
:
452 case PTHREAD_MUTEX_PP_ADAPTIVE_NP
:
454 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
455 in sysdeps/nptl/bits/thread-shared-types.h. */
456 int kind
= atomic_load_relaxed (&(mutex
->__data
.__kind
))
457 & PTHREAD_MUTEX_KIND_MASK_NP
;
459 oldval
= mutex
->__data
.__lock
;
461 /* Check whether we already hold the mutex. */
462 if (mutex
->__data
.__owner
== id
)
464 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
467 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
469 /* Just bump the counter. */
470 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
471 /* Overflow of the counter. */
474 ++mutex
->__data
.__count
;
476 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
482 int oldprio
= -1, ceilval
;
485 int ceiling
= (oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
)
486 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
488 if (__pthread_current_priority () > ceiling
)
493 __pthread_tpp_change_priority (oldprio
, -1);
497 result
= __pthread_tpp_change_priority (oldprio
, ceiling
);
501 ceilval
= ceiling
<< PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
505 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
506 ceilval
| 1, ceilval
);
508 if (oldval
== ceilval
)
514 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
518 if ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
)
521 if (oldval
!= ceilval
)
523 /* Reject invalid timeouts. */
524 if (! valid_nanoseconds (abstime
->tv_nsec
))
530 int e
= __futex_abstimed_wait64 (
531 (unsigned int *) &mutex
->__data
.__lock
, ceilval
| 2,
532 clockid
, abstime
, PTHREAD_MUTEX_PSHARED (mutex
));
533 if (e
== ETIMEDOUT
|| e
== EOVERFLOW
)
537 while (atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
538 ceilval
| 2, ceilval
)
541 while ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
);
543 assert (mutex
->__data
.__owner
== 0);
544 mutex
->__data
.__count
= 1;
549 /* Correct code cannot set any other type. */
555 /* Record the ownership. */
556 mutex
->__data
.__owner
= id
;
557 ++mutex
->__data
.__nusers
;
559 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
567 ___pthread_mutex_clocklock64 (pthread_mutex_t
*mutex
,
569 const struct __timespec64
*abstime
)
571 if (__glibc_unlikely (!futex_abstimed_supported_clockid (clockid
)))
574 LIBC_PROBE (mutex_clocklock_entry
, 3, mutex
, clockid
, abstime
);
575 return __pthread_mutex_clocklock_common (mutex
, clockid
, abstime
);
579 strong_alias (___pthread_mutex_clocklock64
, ___pthread_mutex_clocklock
)
580 #else /* __TIMESPEC64 != 64 */
581 strong_alias (___pthread_mutex_clocklock64
, __pthread_mutex_clocklock64
)
582 libc_hidden_def (__pthread_mutex_clocklock64
)
585 ___pthread_mutex_clocklock (pthread_mutex_t
*mutex
,
587 const struct timespec
*abstime
)
589 struct __timespec64 ts64
= valid_timespec_to_timespec64 (*abstime
);
591 return ___pthread_mutex_clocklock64 (mutex
, clockid
, &ts64
);
593 #endif /* __TIMESPEC64 != 64 */
594 libc_hidden_ver (___pthread_mutex_clocklock
, __pthread_mutex_clocklock
)
596 strong_alias (___pthread_mutex_clocklock
, __pthread_mutex_clocklock
)
598 versioned_symbol (libc
, ___pthread_mutex_clocklock
,
599 pthread_mutex_clocklock
, GLIBC_2_34
);
600 #if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_30, GLIBC_2_34)
601 compat_symbol (libpthread
, ___pthread_mutex_clocklock
,
602 pthread_mutex_clocklock
, GLIBC_2_30
);
606 ___pthread_mutex_timedlock64 (pthread_mutex_t
*mutex
,
607 const struct __timespec64
*abstime
)
609 LIBC_PROBE (mutex_timedlock_entry
, 2, mutex
, abstime
);
610 return __pthread_mutex_clocklock_common (mutex
, CLOCK_REALTIME
, abstime
);
614 strong_alias (___pthread_mutex_timedlock64
, ___pthread_mutex_timedlock
)
615 #else /* __TIMESPEC64 != 64 */
616 strong_alias (___pthread_mutex_timedlock64
, __pthread_mutex_timedlock64
);
617 libc_hidden_def (__pthread_mutex_timedlock64
)
620 ___pthread_mutex_timedlock (pthread_mutex_t
*mutex
,
621 const struct timespec
*abstime
)
623 struct __timespec64 ts64
= valid_timespec_to_timespec64 (*abstime
);
625 return __pthread_mutex_timedlock64 (mutex
, &ts64
);
627 #endif /* __TIMESPEC64 != 64 */
628 versioned_symbol (libc
, ___pthread_mutex_timedlock
,
629 pthread_mutex_timedlock
, GLIBC_2_34
);
630 libc_hidden_ver (___pthread_mutex_timedlock
, __pthread_mutex_timedlock
)
632 strong_alias (___pthread_mutex_timedlock
, __pthread_mutex_timedlock
)
635 #if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_2, GLIBC_2_34)
636 compat_symbol (libpthread
, ___pthread_mutex_timedlock
,
637 pthread_mutex_timedlock
, GLIBC_2_2
);