1 /* Copyright (C) 2002-2014 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
23 #include <not-cancel.h>
25 #include <lowlevellock.h>
26 #include <stap-probe.h>
28 #ifndef lll_lock_elision
29 #define lll_lock_elision(lock, try_lock, private) ({ \
30 lll_lock (lock, private); 0; })
33 #ifndef lll_trylock_elision
34 #define lll_trylock_elision(a,t) lll_trylock(a)
37 #ifndef LLL_MUTEX_LOCK
38 # define LLL_MUTEX_LOCK(mutex) \
39 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
40 # define LLL_MUTEX_TRYLOCK(mutex) \
41 lll_trylock ((mutex)->__data.__lock)
42 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
43 lll_robust_lock ((mutex)->__data.__lock, id, \
44 PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
45 # define LLL_MUTEX_LOCK_ELISION(mutex) \
46 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
47 PTHREAD_MUTEX_PSHARED (mutex))
48 # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
49 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
50 PTHREAD_MUTEX_PSHARED (mutex))
54 #define FORCE_ELISION(m, s)
57 static int __pthread_mutex_lock_full (pthread_mutex_t
*mutex
)
58 __attribute_noinline__
;
61 __pthread_mutex_lock (mutex
)
62 pthread_mutex_t
*mutex
;
64 assert (sizeof (mutex
->__size
) >= sizeof (mutex
->__data
));
66 unsigned int type
= PTHREAD_MUTEX_TYPE_ELISION (mutex
);
68 LIBC_PROBE (mutex_entry
, 1, mutex
);
70 if (__builtin_expect (type
& ~(PTHREAD_MUTEX_KIND_MASK_NP
71 | PTHREAD_MUTEX_ELISION_FLAGS_NP
), 0))
72 return __pthread_mutex_lock_full (mutex
);
74 if (__glibc_likely (type
== PTHREAD_MUTEX_TIMED_NP
))
76 FORCE_ELISION (mutex
, goto elision
);
79 LLL_MUTEX_LOCK (mutex
);
80 assert (mutex
->__data
.__owner
== 0);
83 else if (__glibc_likely (type
== PTHREAD_MUTEX_TIMED_ELISION_NP
))
85 elision
: __attribute__((unused
))
86 /* This case can never happen on a system without elision,
87 as the mutex type initialization functions will not
88 allow to set the elision flags. */
89 /* Don't record owner or users for elision case. This is a
91 return LLL_MUTEX_LOCK_ELISION (mutex
);
94 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex
)
95 == PTHREAD_MUTEX_RECURSIVE_NP
, 1))
97 /* Recursive mutex. */
98 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
100 /* Check whether we already hold the mutex. */
101 if (mutex
->__data
.__owner
== id
)
103 /* Just bump the counter. */
104 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
105 /* Overflow of the counter. */
108 ++mutex
->__data
.__count
;
113 /* We have to get the mutex. */
114 LLL_MUTEX_LOCK (mutex
);
116 assert (mutex
->__data
.__owner
== 0);
117 mutex
->__data
.__count
= 1;
119 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex
)
120 == PTHREAD_MUTEX_ADAPTIVE_NP
, 1))
125 if (LLL_MUTEX_TRYLOCK (mutex
) != 0)
128 int max_cnt
= MIN (MAX_ADAPTIVE_COUNT
,
129 mutex
->__data
.__spins
* 2 + 10);
132 if (cnt
++ >= max_cnt
)
134 LLL_MUTEX_LOCK (mutex
);
142 while (LLL_MUTEX_TRYLOCK (mutex
) != 0);
144 mutex
->__data
.__spins
+= (cnt
- mutex
->__data
.__spins
) / 8;
146 assert (mutex
->__data
.__owner
== 0);
150 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
151 assert (PTHREAD_MUTEX_TYPE (mutex
) == PTHREAD_MUTEX_ERRORCHECK_NP
);
152 /* Check whether we already hold the mutex. */
153 if (__glibc_unlikely (mutex
->__data
.__owner
== id
))
158 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
160 /* Record the ownership. */
161 mutex
->__data
.__owner
= id
;
163 ++mutex
->__data
.__nusers
;
166 LIBC_PROBE (mutex_acquired
, 1, mutex
);
172 __pthread_mutex_lock_full (pthread_mutex_t
*mutex
)
175 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
177 switch (PTHREAD_MUTEX_TYPE (mutex
))
179 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
:
180 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
:
181 case PTHREAD_MUTEX_ROBUST_NORMAL_NP
:
182 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP
:
183 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
184 &mutex
->__data
.__list
.__next
);
186 oldval
= mutex
->__data
.__lock
;
190 if ((oldval
& FUTEX_OWNER_DIED
) != 0)
192 /* The previous owner died. Try locking the mutex. */
195 newval
|= FUTEX_WAITERS
;
197 newval
|= (oldval
& FUTEX_WAITERS
);
201 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
204 if (newval
!= oldval
)
210 /* We got the mutex. */
211 mutex
->__data
.__count
= 1;
212 /* But it is inconsistent unless marked otherwise. */
213 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
215 ENQUEUE_MUTEX (mutex
);
216 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
218 /* Note that we deliberately exit here. If we fall
219 through to the end of the function __nusers would be
220 incremented which is not correct because the old
221 owner has to be discounted. If we are not supposed
222 to increment __nusers we actually have to decrement
225 --mutex
->__data
.__nusers
;
231 /* Check whether we already hold the mutex. */
232 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
234 int kind
= PTHREAD_MUTEX_TYPE (mutex
);
235 if (kind
== PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
)
237 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
242 if (kind
== PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
)
244 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
247 /* Just bump the counter. */
248 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
249 /* Overflow of the counter. */
252 ++mutex
->__data
.__count
;
258 oldval
= LLL_ROBUST_MUTEX_LOCK (mutex
, id
);
260 if (__builtin_expect (mutex
->__data
.__owner
261 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
263 /* This mutex is now not recoverable. */
264 mutex
->__data
.__count
= 0;
265 lll_unlock (mutex
->__data
.__lock
,
266 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
));
267 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
268 return ENOTRECOVERABLE
;
271 while ((oldval
& FUTEX_OWNER_DIED
) != 0);
273 mutex
->__data
.__count
= 1;
274 ENQUEUE_MUTEX (mutex
);
275 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
278 case PTHREAD_MUTEX_PI_RECURSIVE_NP
:
279 case PTHREAD_MUTEX_PI_ERRORCHECK_NP
:
280 case PTHREAD_MUTEX_PI_NORMAL_NP
:
281 case PTHREAD_MUTEX_PI_ADAPTIVE_NP
:
282 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP
:
283 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP
:
284 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP
:
285 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP
:
287 int kind
= mutex
->__data
.__kind
& PTHREAD_MUTEX_KIND_MASK_NP
;
288 int robust
= mutex
->__data
.__kind
& PTHREAD_MUTEX_ROBUST_NORMAL_NP
;
291 /* Note: robust PI futexes are signaled by setting bit 0. */
292 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
293 (void *) (((uintptr_t) &mutex
->__data
.__list
.__next
)
296 oldval
= mutex
->__data
.__lock
;
298 /* Check whether we already hold the mutex. */
299 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
301 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
303 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
307 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
309 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
311 /* Just bump the counter. */
312 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
313 /* Overflow of the counter. */
316 ++mutex
->__data
.__count
;
324 newval
|= FUTEX_WAITERS
;
326 oldval
= atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
331 /* The mutex is locked. The kernel will now take care of
333 int private = (robust
334 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)
335 : PTHREAD_MUTEX_PSHARED (mutex
));
336 INTERNAL_SYSCALL_DECL (__err
);
337 int e
= INTERNAL_SYSCALL (futex
, __err
, 4, &mutex
->__data
.__lock
,
338 __lll_private_flag (FUTEX_LOCK_PI
,
341 if (INTERNAL_SYSCALL_ERROR_P (e
, __err
)
342 && (INTERNAL_SYSCALL_ERRNO (e
, __err
) == ESRCH
343 || INTERNAL_SYSCALL_ERRNO (e
, __err
) == EDEADLK
))
345 assert (INTERNAL_SYSCALL_ERRNO (e
, __err
) != EDEADLK
346 || (kind
!= PTHREAD_MUTEX_ERRORCHECK_NP
347 && kind
!= PTHREAD_MUTEX_RECURSIVE_NP
));
348 /* ESRCH can happen only for non-robust PI mutexes where
349 the owner of the lock died. */
350 assert (INTERNAL_SYSCALL_ERRNO (e
, __err
) != ESRCH
|| !robust
);
352 /* Delay the thread indefinitely. */
357 oldval
= mutex
->__data
.__lock
;
359 assert (robust
|| (oldval
& FUTEX_OWNER_DIED
) == 0);
362 if (__glibc_unlikely (oldval
& FUTEX_OWNER_DIED
))
364 atomic_and (&mutex
->__data
.__lock
, ~FUTEX_OWNER_DIED
);
366 /* We got the mutex. */
367 mutex
->__data
.__count
= 1;
368 /* But it is inconsistent unless marked otherwise. */
369 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
371 ENQUEUE_MUTEX_PI (mutex
);
372 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
374 /* Note that we deliberately exit here. If we fall
375 through to the end of the function __nusers would be
376 incremented which is not correct because the old owner
377 has to be discounted. If we are not supposed to
378 increment __nusers we actually have to decrement it here. */
380 --mutex
->__data
.__nusers
;
387 && __builtin_expect (mutex
->__data
.__owner
388 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
390 /* This mutex is now not recoverable. */
391 mutex
->__data
.__count
= 0;
393 INTERNAL_SYSCALL_DECL (__err
);
394 INTERNAL_SYSCALL (futex
, __err
, 4, &mutex
->__data
.__lock
,
395 __lll_private_flag (FUTEX_UNLOCK_PI
,
396 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)),
399 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
400 return ENOTRECOVERABLE
;
403 mutex
->__data
.__count
= 1;
406 ENQUEUE_MUTEX_PI (mutex
);
407 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
412 case PTHREAD_MUTEX_PP_RECURSIVE_NP
:
413 case PTHREAD_MUTEX_PP_ERRORCHECK_NP
:
414 case PTHREAD_MUTEX_PP_NORMAL_NP
:
415 case PTHREAD_MUTEX_PP_ADAPTIVE_NP
:
417 int kind
= mutex
->__data
.__kind
& PTHREAD_MUTEX_KIND_MASK_NP
;
419 oldval
= mutex
->__data
.__lock
;
421 /* Check whether we already hold the mutex. */
422 if (mutex
->__data
.__owner
== id
)
424 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
427 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
429 /* Just bump the counter. */
430 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
431 /* Overflow of the counter. */
434 ++mutex
->__data
.__count
;
440 int oldprio
= -1, ceilval
;
443 int ceiling
= (oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
)
444 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
446 if (__pthread_current_priority () > ceiling
)
449 __pthread_tpp_change_priority (oldprio
, -1);
453 int retval
= __pthread_tpp_change_priority (oldprio
, ceiling
);
457 ceilval
= ceiling
<< PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
461 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
469 if (oldval
== ceilval
)
475 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
479 if ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
)
482 if (oldval
!= ceilval
)
483 lll_futex_wait (&mutex
->__data
.__lock
, ceilval
| 2,
484 PTHREAD_MUTEX_PSHARED (mutex
));
486 while (atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
487 ceilval
| 2, ceilval
)
490 while ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
);
492 assert (mutex
->__data
.__owner
== 0);
493 mutex
->__data
.__count
= 1;
498 /* Correct code cannot set any other type. */
502 /* Record the ownership. */
503 mutex
->__data
.__owner
= id
;
505 ++mutex
->__data
.__nusers
;
508 LIBC_PROBE (mutex_acquired
, 1, mutex
);
512 #ifndef __pthread_mutex_lock
513 strong_alias (__pthread_mutex_lock
, pthread_mutex_lock
)
514 hidden_def (__pthread_mutex_lock
)
520 __pthread_mutex_cond_lock_adjust (mutex
)
521 pthread_mutex_t
*mutex
;
523 assert ((mutex
->__data
.__kind
& PTHREAD_MUTEX_PRIO_INHERIT_NP
) != 0);
524 assert ((mutex
->__data
.__kind
& PTHREAD_MUTEX_ROBUST_NORMAL_NP
) == 0);
525 assert ((mutex
->__data
.__kind
& PTHREAD_MUTEX_PSHARED_BIT
) == 0);
527 /* Record the ownership. */
528 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
529 mutex
->__data
.__owner
= id
;
531 if (mutex
->__data
.__kind
== PTHREAD_MUTEX_PI_RECURSIVE_NP
)
532 ++mutex
->__data
.__count
;