1 /* Copyright (C) 2002-2014 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
23 #include <sys/param.h>
24 #include <not-cancel.h>
26 #include <lowlevellock.h>
27 #include <stap-probe.h>
29 #ifndef lll_lock_elision
30 #define lll_lock_elision(lock, try_lock, private) ({ \
31 lll_lock (lock, private); 0; })
34 #ifndef lll_trylock_elision
35 #define lll_trylock_elision(a,t) lll_trylock(a)
38 #ifndef LLL_MUTEX_LOCK
39 # define LLL_MUTEX_LOCK(mutex) \
40 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
41 # define LLL_MUTEX_TRYLOCK(mutex) \
42 lll_trylock ((mutex)->__data.__lock)
43 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
44 lll_robust_lock ((mutex)->__data.__lock, id, \
45 PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
46 # define LLL_MUTEX_LOCK_ELISION(mutex) \
47 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
48 PTHREAD_MUTEX_PSHARED (mutex))
49 # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
50 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
51 PTHREAD_MUTEX_PSHARED (mutex))
55 #define FORCE_ELISION(m, s)
58 static int __pthread_mutex_lock_full (pthread_mutex_t
*mutex
)
59 __attribute_noinline__
;
62 __pthread_mutex_lock (mutex
)
63 pthread_mutex_t
*mutex
;
65 assert (sizeof (mutex
->__size
) >= sizeof (mutex
->__data
));
67 unsigned int type
= PTHREAD_MUTEX_TYPE_ELISION (mutex
);
69 LIBC_PROBE (mutex_entry
, 1, mutex
);
71 if (__builtin_expect (type
& ~(PTHREAD_MUTEX_KIND_MASK_NP
72 | PTHREAD_MUTEX_ELISION_FLAGS_NP
), 0))
73 return __pthread_mutex_lock_full (mutex
);
75 if (__glibc_likely (type
== PTHREAD_MUTEX_TIMED_NP
))
77 FORCE_ELISION (mutex
, goto elision
);
80 LLL_MUTEX_LOCK (mutex
);
81 assert (mutex
->__data
.__owner
== 0);
84 else if (__glibc_likely (type
== PTHREAD_MUTEX_TIMED_ELISION_NP
))
86 elision
: __attribute__((unused
))
87 /* This case can never happen on a system without elision,
88 as the mutex type initialization functions will not
89 allow to set the elision flags. */
90 /* Don't record owner or users for elision case. This is a
92 return LLL_MUTEX_LOCK_ELISION (mutex
);
95 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex
)
96 == PTHREAD_MUTEX_RECURSIVE_NP
, 1))
98 /* Recursive mutex. */
99 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
101 /* Check whether we already hold the mutex. */
102 if (mutex
->__data
.__owner
== id
)
104 /* Just bump the counter. */
105 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
106 /* Overflow of the counter. */
109 ++mutex
->__data
.__count
;
114 /* We have to get the mutex. */
115 LLL_MUTEX_LOCK (mutex
);
117 assert (mutex
->__data
.__owner
== 0);
118 mutex
->__data
.__count
= 1;
120 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex
)
121 == PTHREAD_MUTEX_ADAPTIVE_NP
, 1))
126 if (LLL_MUTEX_TRYLOCK (mutex
) != 0)
129 int max_cnt
= MIN (MAX_ADAPTIVE_COUNT
,
130 mutex
->__data
.__spins
* 2 + 10);
133 if (cnt
++ >= max_cnt
)
135 LLL_MUTEX_LOCK (mutex
);
143 while (LLL_MUTEX_TRYLOCK (mutex
) != 0);
145 mutex
->__data
.__spins
+= (cnt
- mutex
->__data
.__spins
) / 8;
147 assert (mutex
->__data
.__owner
== 0);
151 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
152 assert (PTHREAD_MUTEX_TYPE (mutex
) == PTHREAD_MUTEX_ERRORCHECK_NP
);
153 /* Check whether we already hold the mutex. */
154 if (__glibc_unlikely (mutex
->__data
.__owner
== id
))
159 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
161 /* Record the ownership. */
162 mutex
->__data
.__owner
= id
;
164 ++mutex
->__data
.__nusers
;
167 LIBC_PROBE (mutex_acquired
, 1, mutex
);
173 __pthread_mutex_lock_full (pthread_mutex_t
*mutex
)
176 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
178 switch (PTHREAD_MUTEX_TYPE (mutex
))
180 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
:
181 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
:
182 case PTHREAD_MUTEX_ROBUST_NORMAL_NP
:
183 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP
:
184 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
185 &mutex
->__data
.__list
.__next
);
187 oldval
= mutex
->__data
.__lock
;
191 if ((oldval
& FUTEX_OWNER_DIED
) != 0)
193 /* The previous owner died. Try locking the mutex. */
196 newval
|= FUTEX_WAITERS
;
198 newval
|= (oldval
& FUTEX_WAITERS
);
202 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
205 if (newval
!= oldval
)
211 /* We got the mutex. */
212 mutex
->__data
.__count
= 1;
213 /* But it is inconsistent unless marked otherwise. */
214 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
216 ENQUEUE_MUTEX (mutex
);
217 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
219 /* Note that we deliberately exit here. If we fall
220 through to the end of the function __nusers would be
221 incremented which is not correct because the old
222 owner has to be discounted. If we are not supposed
223 to increment __nusers we actually have to decrement
226 --mutex
->__data
.__nusers
;
232 /* Check whether we already hold the mutex. */
233 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
235 int kind
= PTHREAD_MUTEX_TYPE (mutex
);
236 if (kind
== PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
)
238 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
243 if (kind
== PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
)
245 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
248 /* Just bump the counter. */
249 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
250 /* Overflow of the counter. */
253 ++mutex
->__data
.__count
;
259 oldval
= LLL_ROBUST_MUTEX_LOCK (mutex
, id
);
261 if (__builtin_expect (mutex
->__data
.__owner
262 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
264 /* This mutex is now not recoverable. */
265 mutex
->__data
.__count
= 0;
266 lll_unlock (mutex
->__data
.__lock
,
267 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
));
268 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
269 return ENOTRECOVERABLE
;
272 while ((oldval
& FUTEX_OWNER_DIED
) != 0);
274 mutex
->__data
.__count
= 1;
275 ENQUEUE_MUTEX (mutex
);
276 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
279 case PTHREAD_MUTEX_PI_RECURSIVE_NP
:
280 case PTHREAD_MUTEX_PI_ERRORCHECK_NP
:
281 case PTHREAD_MUTEX_PI_NORMAL_NP
:
282 case PTHREAD_MUTEX_PI_ADAPTIVE_NP
:
283 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP
:
284 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP
:
285 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP
:
286 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP
:
288 int kind
= mutex
->__data
.__kind
& PTHREAD_MUTEX_KIND_MASK_NP
;
289 int robust
= mutex
->__data
.__kind
& PTHREAD_MUTEX_ROBUST_NORMAL_NP
;
292 /* Note: robust PI futexes are signaled by setting bit 0. */
293 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
294 (void *) (((uintptr_t) &mutex
->__data
.__list
.__next
)
297 oldval
= mutex
->__data
.__lock
;
299 /* Check whether we already hold the mutex. */
300 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
302 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
304 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
308 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
310 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
312 /* Just bump the counter. */
313 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
314 /* Overflow of the counter. */
317 ++mutex
->__data
.__count
;
325 newval
|= FUTEX_WAITERS
;
327 oldval
= atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
332 /* The mutex is locked. The kernel will now take care of
334 int private = (robust
335 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)
336 : PTHREAD_MUTEX_PSHARED (mutex
));
337 INTERNAL_SYSCALL_DECL (__err
);
338 int e
= INTERNAL_SYSCALL (futex
, __err
, 4, &mutex
->__data
.__lock
,
339 __lll_private_flag (FUTEX_LOCK_PI
,
342 if (INTERNAL_SYSCALL_ERROR_P (e
, __err
)
343 && (INTERNAL_SYSCALL_ERRNO (e
, __err
) == ESRCH
344 || INTERNAL_SYSCALL_ERRNO (e
, __err
) == EDEADLK
))
346 assert (INTERNAL_SYSCALL_ERRNO (e
, __err
) != EDEADLK
347 || (kind
!= PTHREAD_MUTEX_ERRORCHECK_NP
348 && kind
!= PTHREAD_MUTEX_RECURSIVE_NP
));
349 /* ESRCH can happen only for non-robust PI mutexes where
350 the owner of the lock died. */
351 assert (INTERNAL_SYSCALL_ERRNO (e
, __err
) != ESRCH
|| !robust
);
353 /* Delay the thread indefinitely. */
358 oldval
= mutex
->__data
.__lock
;
360 assert (robust
|| (oldval
& FUTEX_OWNER_DIED
) == 0);
363 if (__glibc_unlikely (oldval
& FUTEX_OWNER_DIED
))
365 atomic_and (&mutex
->__data
.__lock
, ~FUTEX_OWNER_DIED
);
367 /* We got the mutex. */
368 mutex
->__data
.__count
= 1;
369 /* But it is inconsistent unless marked otherwise. */
370 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
372 ENQUEUE_MUTEX_PI (mutex
);
373 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
375 /* Note that we deliberately exit here. If we fall
376 through to the end of the function __nusers would be
377 incremented which is not correct because the old owner
378 has to be discounted. If we are not supposed to
379 increment __nusers we actually have to decrement it here. */
381 --mutex
->__data
.__nusers
;
388 && __builtin_expect (mutex
->__data
.__owner
389 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
391 /* This mutex is now not recoverable. */
392 mutex
->__data
.__count
= 0;
394 INTERNAL_SYSCALL_DECL (__err
);
395 INTERNAL_SYSCALL (futex
, __err
, 4, &mutex
->__data
.__lock
,
396 __lll_private_flag (FUTEX_UNLOCK_PI
,
397 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)),
400 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
401 return ENOTRECOVERABLE
;
404 mutex
->__data
.__count
= 1;
407 ENQUEUE_MUTEX_PI (mutex
);
408 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
413 case PTHREAD_MUTEX_PP_RECURSIVE_NP
:
414 case PTHREAD_MUTEX_PP_ERRORCHECK_NP
:
415 case PTHREAD_MUTEX_PP_NORMAL_NP
:
416 case PTHREAD_MUTEX_PP_ADAPTIVE_NP
:
418 int kind
= mutex
->__data
.__kind
& PTHREAD_MUTEX_KIND_MASK_NP
;
420 oldval
= mutex
->__data
.__lock
;
422 /* Check whether we already hold the mutex. */
423 if (mutex
->__data
.__owner
== id
)
425 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
428 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
430 /* Just bump the counter. */
431 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
432 /* Overflow of the counter. */
435 ++mutex
->__data
.__count
;
441 int oldprio
= -1, ceilval
;
444 int ceiling
= (oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
)
445 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
447 if (__pthread_current_priority () > ceiling
)
450 __pthread_tpp_change_priority (oldprio
, -1);
454 int retval
= __pthread_tpp_change_priority (oldprio
, ceiling
);
458 ceilval
= ceiling
<< PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
462 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
470 if (oldval
== ceilval
)
476 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
480 if ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
)
483 if (oldval
!= ceilval
)
484 lll_futex_wait (&mutex
->__data
.__lock
, ceilval
| 2,
485 PTHREAD_MUTEX_PSHARED (mutex
));
487 while (atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
488 ceilval
| 2, ceilval
)
491 while ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
);
493 assert (mutex
->__data
.__owner
== 0);
494 mutex
->__data
.__count
= 1;
499 /* Correct code cannot set any other type. */
503 /* Record the ownership. */
504 mutex
->__data
.__owner
= id
;
506 ++mutex
->__data
.__nusers
;
509 LIBC_PROBE (mutex_acquired
, 1, mutex
);
513 #ifndef __pthread_mutex_lock
514 strong_alias (__pthread_mutex_lock
, pthread_mutex_lock
)
515 hidden_def (__pthread_mutex_lock
)
521 __pthread_mutex_cond_lock_adjust (mutex
)
522 pthread_mutex_t
*mutex
;
524 assert ((mutex
->__data
.__kind
& PTHREAD_MUTEX_PRIO_INHERIT_NP
) != 0);
525 assert ((mutex
->__data
.__kind
& PTHREAD_MUTEX_ROBUST_NORMAL_NP
) == 0);
526 assert ((mutex
->__data
.__kind
& PTHREAD_MUTEX_PSHARED_BIT
) == 0);
528 /* Record the ownership. */
529 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
530 mutex
->__data
.__owner
= id
;
532 if (mutex
->__data
.__kind
== PTHREAD_MUTEX_PI_RECURSIVE_NP
)
533 ++mutex
->__data
.__count
;