1 /* Copyright (C) 2002-2022 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
4 The GNU C Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
9 The GNU C Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
14 You should have received a copy of the GNU Lesser General Public
15 License along with the GNU C Library; if not, see
16 <https://www.gnu.org/licenses/>. */
22 #include <sys/param.h>
23 #include <not-cancel.h>
26 #include <futex-internal.h>
27 #include <stap-probe.h>
28 #include <shlib-compat.h>
30 /* Some of the following definitions differ when pthread_mutex_cond_lock.c
31 includes this file. */
32 #ifndef LLL_MUTEX_LOCK
33 /* lll_lock with single-thread optimization. */
35 lll_mutex_lock_optimized (pthread_mutex_t
*mutex
)
37 /* The single-threaded optimization is only valid for private
38 mutexes. For process-shared mutexes, the mutex could be in a
39 shared mapping, so synchronization with another process is needed
40 even without any threads. If the lock is already marked as
41 acquired, POSIX requires that pthread_mutex_lock deadlocks for
42 normal mutexes, so skip the optimization in that case as
44 int private = PTHREAD_MUTEX_PSHARED (mutex
);
45 if (private == LLL_PRIVATE
&& SINGLE_THREAD_P
&& mutex
->__data
.__lock
== 0)
46 mutex
->__data
.__lock
= 1;
48 lll_lock (mutex
->__data
.__lock
, private);
51 # define LLL_MUTEX_LOCK(mutex) \
52 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
53 # define LLL_MUTEX_LOCK_OPTIMIZED(mutex) lll_mutex_lock_optimized (mutex)
54 # define LLL_MUTEX_TRYLOCK(mutex) \
55 lll_trylock ((mutex)->__data.__lock)
56 # define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0
57 # define LLL_MUTEX_LOCK_ELISION(mutex) \
58 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
59 PTHREAD_MUTEX_PSHARED (mutex))
60 # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
61 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
62 PTHREAD_MUTEX_PSHARED (mutex))
63 # define PTHREAD_MUTEX_LOCK ___pthread_mutex_lock
64 # define PTHREAD_MUTEX_VERSIONS 1
67 #ifndef LLL_MUTEX_READ_LOCK
68 # define LLL_MUTEX_READ_LOCK(mutex) \
69 atomic_load_relaxed (&(mutex)->__data.__lock)
72 static int __pthread_mutex_lock_full (pthread_mutex_t
*mutex
)
73 __attribute_noinline__
;
76 PTHREAD_MUTEX_LOCK (pthread_mutex_t
*mutex
)
78 /* See concurrency notes regarding mutex type which is loaded from __kind
79 in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */
80 unsigned int type
= PTHREAD_MUTEX_TYPE_ELISION (mutex
);
82 LIBC_PROBE (mutex_entry
, 1, mutex
);
84 if (__builtin_expect (type
& ~(PTHREAD_MUTEX_KIND_MASK_NP
85 | PTHREAD_MUTEX_ELISION_FLAGS_NP
), 0))
86 return __pthread_mutex_lock_full (mutex
);
88 if (__glibc_likely (type
== PTHREAD_MUTEX_TIMED_NP
))
90 FORCE_ELISION (mutex
, goto elision
);
93 LLL_MUTEX_LOCK_OPTIMIZED (mutex
);
94 assert (mutex
->__data
.__owner
== 0);
96 #if ENABLE_ELISION_SUPPORT
97 else if (__glibc_likely (type
== PTHREAD_MUTEX_TIMED_ELISION_NP
))
99 elision
: __attribute__((unused
))
100 /* This case can never happen on a system without elision,
101 as the mutex type initialization functions will not
102 allow to set the elision flags. */
103 /* Don't record owner or users for elision case. This is a
105 return LLL_MUTEX_LOCK_ELISION (mutex
);
108 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex
)
109 == PTHREAD_MUTEX_RECURSIVE_NP
, 1))
111 /* Recursive mutex. */
112 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
114 /* Check whether we already hold the mutex. */
115 if (mutex
->__data
.__owner
== id
)
117 /* Just bump the counter. */
118 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
119 /* Overflow of the counter. */
122 ++mutex
->__data
.__count
;
127 /* We have to get the mutex. */
128 LLL_MUTEX_LOCK_OPTIMIZED (mutex
);
130 assert (mutex
->__data
.__owner
== 0);
131 mutex
->__data
.__count
= 1;
133 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex
)
134 == PTHREAD_MUTEX_ADAPTIVE_NP
, 1))
136 if (LLL_MUTEX_TRYLOCK (mutex
) != 0)
139 int max_cnt
= MIN (max_adaptive_count (),
140 mutex
->__data
.__spins
* 2 + 10);
143 if (cnt
++ >= max_cnt
)
145 LLL_MUTEX_LOCK (mutex
);
150 while (LLL_MUTEX_READ_LOCK (mutex
) != 0
151 || LLL_MUTEX_TRYLOCK (mutex
) != 0);
153 mutex
->__data
.__spins
+= (cnt
- mutex
->__data
.__spins
) / 8;
155 assert (mutex
->__data
.__owner
== 0);
159 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
160 assert (PTHREAD_MUTEX_TYPE (mutex
) == PTHREAD_MUTEX_ERRORCHECK_NP
);
161 /* Check whether we already hold the mutex. */
162 if (__glibc_unlikely (mutex
->__data
.__owner
== id
))
167 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
169 /* Record the ownership. */
170 mutex
->__data
.__owner
= id
;
172 ++mutex
->__data
.__nusers
;
175 LIBC_PROBE (mutex_acquired
, 1, mutex
);
181 __pthread_mutex_lock_full (pthread_mutex_t
*mutex
)
184 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
186 switch (PTHREAD_MUTEX_TYPE (mutex
))
188 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
:
189 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
:
190 case PTHREAD_MUTEX_ROBUST_NORMAL_NP
:
191 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP
:
192 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
193 &mutex
->__data
.__list
.__next
);
194 /* We need to set op_pending before starting the operation. Also
195 see comments at ENQUEUE_MUTEX. */
196 __asm ("" ::: "memory");
198 oldval
= mutex
->__data
.__lock
;
199 /* This is set to FUTEX_WAITERS iff we might have shared the
200 FUTEX_WAITERS flag with other threads, and therefore need to keep it
201 set to avoid lost wake-ups. We have the same requirement in the
202 simple mutex algorithm.
203 We start with value zero for a normal mutex, and FUTEX_WAITERS if we
204 are building the special case mutexes for use from within condition
206 unsigned int assume_other_futex_waiters
= LLL_ROBUST_MUTEX_LOCK_MODIFIER
;
209 /* Try to acquire the lock through a CAS from 0 (not acquired) to
210 our TID | assume_other_futex_waiters. */
211 if (__glibc_likely (oldval
== 0))
214 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
215 id
| assume_other_futex_waiters
, 0);
216 if (__glibc_likely (oldval
== 0))
220 if ((oldval
& FUTEX_OWNER_DIED
) != 0)
222 /* The previous owner died. Try locking the mutex. */
225 /* We are not taking assume_other_futex_waiters into accoount
226 here simply because we'll set FUTEX_WAITERS anyway. */
227 newval
|= FUTEX_WAITERS
;
229 newval
|= (oldval
& FUTEX_WAITERS
) | assume_other_futex_waiters
;
233 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
236 if (newval
!= oldval
)
242 /* We got the mutex. */
243 mutex
->__data
.__count
= 1;
244 /* But it is inconsistent unless marked otherwise. */
245 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
247 /* We must not enqueue the mutex before we have acquired it.
248 Also see comments at ENQUEUE_MUTEX. */
249 __asm ("" ::: "memory");
250 ENQUEUE_MUTEX (mutex
);
251 /* We need to clear op_pending after we enqueue the mutex. */
252 __asm ("" ::: "memory");
253 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
255 /* Note that we deliberately exit here. If we fall
256 through to the end of the function __nusers would be
257 incremented which is not correct because the old
258 owner has to be discounted. If we are not supposed
259 to increment __nusers we actually have to decrement
262 --mutex
->__data
.__nusers
;
268 /* Check whether we already hold the mutex. */
269 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
271 int kind
= PTHREAD_MUTEX_TYPE (mutex
);
272 if (kind
== PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
)
274 /* We do not need to ensure ordering wrt another memory
275 access. Also see comments at ENQUEUE_MUTEX. */
276 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
281 if (kind
== PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
)
283 /* We do not need to ensure ordering wrt another memory
285 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
288 /* Just bump the counter. */
289 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
290 /* Overflow of the counter. */
293 ++mutex
->__data
.__count
;
299 /* We cannot acquire the mutex nor has its owner died. Thus, try
300 to block using futexes. Set FUTEX_WAITERS if necessary so that
301 other threads are aware that there are potentially threads
302 blocked on the futex. Restart if oldval changed in the
304 if ((oldval
& FUTEX_WAITERS
) == 0)
306 int val
= atomic_compare_and_exchange_val_acq
307 (&mutex
->__data
.__lock
, oldval
| FUTEX_WAITERS
, oldval
);
313 oldval
|= FUTEX_WAITERS
;
316 /* It is now possible that we share the FUTEX_WAITERS flag with
317 another thread; therefore, update assume_other_futex_waiters so
318 that we do not forget about this when handling other cases
319 above and thus do not cause lost wake-ups. */
320 assume_other_futex_waiters
|= FUTEX_WAITERS
;
322 /* Block using the futex and reload current lock value. */
323 futex_wait ((unsigned int *) &mutex
->__data
.__lock
, oldval
,
324 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
));
325 oldval
= mutex
->__data
.__lock
;
328 /* We have acquired the mutex; check if it is still consistent. */
329 if (__builtin_expect (mutex
->__data
.__owner
330 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
332 /* This mutex is now not recoverable. */
333 mutex
->__data
.__count
= 0;
334 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex
);
335 lll_unlock (mutex
->__data
.__lock
, private);
336 /* FIXME This violates the mutex destruction requirements. See
337 __pthread_mutex_unlock_full. */
338 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
339 return ENOTRECOVERABLE
;
342 mutex
->__data
.__count
= 1;
343 /* We must not enqueue the mutex before we have acquired it.
344 Also see comments at ENQUEUE_MUTEX. */
345 __asm ("" ::: "memory");
346 ENQUEUE_MUTEX (mutex
);
347 /* We need to clear op_pending after we enqueue the mutex. */
348 __asm ("" ::: "memory");
349 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
352 /* The PI support requires the Linux futex system call. If that's not
353 available, pthread_mutex_init should never have allowed the type to
354 be set. So it will get the default case for an invalid type. */
356 case PTHREAD_MUTEX_PI_RECURSIVE_NP
:
357 case PTHREAD_MUTEX_PI_ERRORCHECK_NP
:
358 case PTHREAD_MUTEX_PI_NORMAL_NP
:
359 case PTHREAD_MUTEX_PI_ADAPTIVE_NP
:
360 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP
:
361 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP
:
362 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP
:
363 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP
:
367 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
368 in sysdeps/nptl/bits/thread-shared-types.h. */
369 int mutex_kind
= atomic_load_relaxed (&(mutex
->__data
.__kind
));
370 kind
= mutex_kind
& PTHREAD_MUTEX_KIND_MASK_NP
;
371 robust
= mutex_kind
& PTHREAD_MUTEX_ROBUST_NORMAL_NP
;
376 /* Note: robust PI futexes are signaled by setting bit 0. */
377 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
378 (void *) (((uintptr_t) &mutex
->__data
.__list
.__next
)
380 /* We need to set op_pending before starting the operation. Also
381 see comments at ENQUEUE_MUTEX. */
382 __asm ("" ::: "memory");
385 oldval
= mutex
->__data
.__lock
;
387 /* Check whether we already hold the mutex. */
388 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
390 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
392 /* We do not need to ensure ordering wrt another memory
394 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
398 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
400 /* We do not need to ensure ordering wrt another memory
402 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
404 /* Just bump the counter. */
405 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
406 /* Overflow of the counter. */
409 ++mutex
->__data
.__count
;
417 newval
|= FUTEX_WAITERS
;
419 oldval
= atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
424 /* The mutex is locked. The kernel will now take care of
426 int private = (robust
427 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)
428 : PTHREAD_MUTEX_PSHARED (mutex
));
429 int e
= __futex_lock_pi64 (&mutex
->__data
.__lock
, 0 /* ununsed */,
431 if (e
== ESRCH
|| e
== EDEADLK
)
434 || (kind
!= PTHREAD_MUTEX_ERRORCHECK_NP
435 && kind
!= PTHREAD_MUTEX_RECURSIVE_NP
));
436 /* ESRCH can happen only for non-robust PI mutexes where
437 the owner of the lock died. */
438 assert (e
!= ESRCH
|| !robust
);
440 /* Delay the thread indefinitely. */
442 __futex_abstimed_wait64 (&(unsigned int){0}, 0,
443 0 /* ignored */, NULL
, private);
446 oldval
= mutex
->__data
.__lock
;
448 assert (robust
|| (oldval
& FUTEX_OWNER_DIED
) == 0);
451 if (__glibc_unlikely (oldval
& FUTEX_OWNER_DIED
))
453 atomic_and (&mutex
->__data
.__lock
, ~FUTEX_OWNER_DIED
);
455 /* We got the mutex. */
456 mutex
->__data
.__count
= 1;
457 /* But it is inconsistent unless marked otherwise. */
458 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
460 /* We must not enqueue the mutex before we have acquired it.
461 Also see comments at ENQUEUE_MUTEX. */
462 __asm ("" ::: "memory");
463 ENQUEUE_MUTEX_PI (mutex
);
464 /* We need to clear op_pending after we enqueue the mutex. */
465 __asm ("" ::: "memory");
466 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
468 /* Note that we deliberately exit here. If we fall
469 through to the end of the function __nusers would be
470 incremented which is not correct because the old owner
471 has to be discounted. If we are not supposed to
472 increment __nusers we actually have to decrement it here. */
474 --mutex
->__data
.__nusers
;
481 && __builtin_expect (mutex
->__data
.__owner
482 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
484 /* This mutex is now not recoverable. */
485 mutex
->__data
.__count
= 0;
487 futex_unlock_pi ((unsigned int *) &mutex
->__data
.__lock
,
488 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
));
490 /* To the kernel, this will be visible after the kernel has
491 acquired the mutex in the syscall. */
492 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
493 return ENOTRECOVERABLE
;
496 mutex
->__data
.__count
= 1;
499 /* We must not enqueue the mutex before we have acquired it.
500 Also see comments at ENQUEUE_MUTEX. */
501 __asm ("" ::: "memory");
502 ENQUEUE_MUTEX_PI (mutex
);
503 /* We need to clear op_pending after we enqueue the mutex. */
504 __asm ("" ::: "memory");
505 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
509 #endif /* __NR_futex. */
511 case PTHREAD_MUTEX_PP_RECURSIVE_NP
:
512 case PTHREAD_MUTEX_PP_ERRORCHECK_NP
:
513 case PTHREAD_MUTEX_PP_NORMAL_NP
:
514 case PTHREAD_MUTEX_PP_ADAPTIVE_NP
:
516 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
517 in sysdeps/nptl/bits/thread-shared-types.h. */
518 int kind
= atomic_load_relaxed (&(mutex
->__data
.__kind
))
519 & PTHREAD_MUTEX_KIND_MASK_NP
;
521 oldval
= mutex
->__data
.__lock
;
523 /* Check whether we already hold the mutex. */
524 if (mutex
->__data
.__owner
== id
)
526 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
529 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
531 /* Just bump the counter. */
532 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
533 /* Overflow of the counter. */
536 ++mutex
->__data
.__count
;
542 int oldprio
= -1, ceilval
;
545 int ceiling
= (oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
)
546 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
548 if (__pthread_current_priority () > ceiling
)
551 __pthread_tpp_change_priority (oldprio
, -1);
555 int retval
= __pthread_tpp_change_priority (oldprio
, ceiling
);
559 ceilval
= ceiling
<< PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
563 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
571 if (oldval
== ceilval
)
577 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
581 if ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
)
584 if (oldval
!= ceilval
)
585 futex_wait ((unsigned int * ) &mutex
->__data
.__lock
,
587 PTHREAD_MUTEX_PSHARED (mutex
));
589 while (atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
590 ceilval
| 2, ceilval
)
593 while ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
);
595 assert (mutex
->__data
.__owner
== 0);
596 mutex
->__data
.__count
= 1;
601 /* Correct code cannot set any other type. */
605 /* Record the ownership. */
606 mutex
->__data
.__owner
= id
;
608 ++mutex
->__data
.__nusers
;
611 LIBC_PROBE (mutex_acquired
, 1, mutex
);
616 #if PTHREAD_MUTEX_VERSIONS
617 libc_hidden_ver (___pthread_mutex_lock
, __pthread_mutex_lock
)
619 strong_alias (___pthread_mutex_lock
, __pthread_mutex_lock
)
621 versioned_symbol (libpthread
, ___pthread_mutex_lock
, pthread_mutex_lock
,
624 # if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_0, GLIBC_2_34)
625 compat_symbol (libpthread
, ___pthread_mutex_lock
, __pthread_mutex_lock
,
628 #endif /* PTHREAD_MUTEX_VERSIONS */
633 __pthread_mutex_cond_lock_adjust (pthread_mutex_t
*mutex
)
635 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
636 in sysdeps/nptl/bits/thread-shared-types.h. */
637 int mutex_kind
= atomic_load_relaxed (&(mutex
->__data
.__kind
));
638 assert ((mutex_kind
& PTHREAD_MUTEX_PRIO_INHERIT_NP
) != 0);
639 assert ((mutex_kind
& PTHREAD_MUTEX_ROBUST_NORMAL_NP
) == 0);
640 assert ((mutex_kind
& PTHREAD_MUTEX_PSHARED_BIT
) == 0);
642 /* Record the ownership. */
643 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
644 mutex
->__data
.__owner
= id
;
646 if (mutex_kind
== PTHREAD_MUTEX_PI_RECURSIVE_NP
)
647 ++mutex
->__data
.__count
;