1 /* Copyright (C) 2002-2015 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
22 #include <sys/param.h>
26 #include <lowlevellock.h>
27 #include <not-cancel.h>
29 #include <stap-probe.h>
31 #ifndef lll_timedlock_elision
32 #define lll_timedlock_elision(a,dummy,b,c) lll_timedlock(a, b, c)
35 #ifndef lll_trylock_elision
36 #define lll_trylock_elision(a,t) lll_trylock(a)
40 #define FORCE_ELISION(m, s)
44 pthread_mutex_timedlock (pthread_mutex_t
*mutex
,
45 const struct timespec
*abstime
)
48 pid_t id
= THREAD_GETMEM (THREAD_SELF
, tid
);
51 LIBC_PROBE (mutex_timedlock_entry
, 2, mutex
, abstime
);
53 /* We must not check ABSTIME here. If the thread does not block
54 abstime must not be checked for a valid value. */
56 switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex
),
57 PTHREAD_MUTEX_TIMED_NP
))
59 /* Recursive mutex. */
60 case PTHREAD_MUTEX_RECURSIVE_NP
|PTHREAD_MUTEX_ELISION_NP
:
61 case PTHREAD_MUTEX_RECURSIVE_NP
:
62 /* Check whether we already hold the mutex. */
63 if (mutex
->__data
.__owner
== id
)
65 /* Just bump the counter. */
66 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
67 /* Overflow of the counter. */
70 ++mutex
->__data
.__count
;
75 /* We have to get the mutex. */
76 result
= lll_timedlock (mutex
->__data
.__lock
, abstime
,
77 PTHREAD_MUTEX_PSHARED (mutex
));
82 /* Only locked once so far. */
83 mutex
->__data
.__count
= 1;
86 /* Error checking mutex. */
87 case PTHREAD_MUTEX_ERRORCHECK_NP
:
88 /* Check whether we already hold the mutex. */
89 if (__glibc_unlikely (mutex
->__data
.__owner
== id
))
94 case PTHREAD_MUTEX_TIMED_NP
:
95 FORCE_ELISION (mutex
, goto elision
);
98 result
= lll_timedlock (mutex
->__data
.__lock
, abstime
,
99 PTHREAD_MUTEX_PSHARED (mutex
));
102 case PTHREAD_MUTEX_TIMED_ELISION_NP
:
103 elision
: __attribute__((unused
))
104 /* Don't record ownership */
105 return lll_timedlock_elision (mutex
->__data
.__lock
,
106 mutex
->__data
.__spins
,
108 PTHREAD_MUTEX_PSHARED (mutex
));
111 case PTHREAD_MUTEX_ADAPTIVE_NP
:
115 if (lll_trylock (mutex
->__data
.__lock
) != 0)
118 int max_cnt
= MIN (MAX_ADAPTIVE_COUNT
,
119 mutex
->__data
.__spins
* 2 + 10);
122 if (cnt
++ >= max_cnt
)
124 result
= lll_timedlock (mutex
->__data
.__lock
, abstime
,
125 PTHREAD_MUTEX_PSHARED (mutex
));
130 while (lll_trylock (mutex
->__data
.__lock
) != 0);
132 mutex
->__data
.__spins
+= (cnt
- mutex
->__data
.__spins
) / 8;
136 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
:
137 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
:
138 case PTHREAD_MUTEX_ROBUST_NORMAL_NP
:
139 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP
:
140 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
141 &mutex
->__data
.__list
.__next
);
143 oldval
= mutex
->__data
.__lock
;
147 if ((oldval
& FUTEX_OWNER_DIED
) != 0)
149 /* The previous owner died. Try locking the mutex. */
150 int newval
= id
| (oldval
& FUTEX_WAITERS
);
153 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
155 if (newval
!= oldval
)
161 /* We got the mutex. */
162 mutex
->__data
.__count
= 1;
163 /* But it is inconsistent unless marked otherwise. */
164 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
166 ENQUEUE_MUTEX (mutex
);
167 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
169 /* Note that we deliberately exit here. If we fall
170 through to the end of the function __nusers would be
171 incremented which is not correct because the old
172 owner has to be discounted. */
176 /* Check whether we already hold the mutex. */
177 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
179 int kind
= PTHREAD_MUTEX_TYPE (mutex
);
180 if (kind
== PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP
)
182 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
187 if (kind
== PTHREAD_MUTEX_ROBUST_RECURSIVE_NP
)
189 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
192 /* Just bump the counter. */
193 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
194 /* Overflow of the counter. */
197 ++mutex
->__data
.__count
;
199 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
205 result
= lll_robust_timedlock (mutex
->__data
.__lock
, abstime
, id
,
206 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
));
208 if (__builtin_expect (mutex
->__data
.__owner
209 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
211 /* This mutex is now not recoverable. */
212 mutex
->__data
.__count
= 0;
213 lll_unlock (mutex
->__data
.__lock
,
214 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
));
215 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
216 return ENOTRECOVERABLE
;
219 if (result
== ETIMEDOUT
|| result
== EINVAL
)
224 while ((oldval
& FUTEX_OWNER_DIED
) != 0);
226 mutex
->__data
.__count
= 1;
227 ENQUEUE_MUTEX (mutex
);
228 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
231 /* The PI support requires the Linux futex system call. If that's not
232 available, pthread_mutex_init should never have allowed the type to
233 be set. So it will get the default case for an invalid type. */
235 case PTHREAD_MUTEX_PI_RECURSIVE_NP
:
236 case PTHREAD_MUTEX_PI_ERRORCHECK_NP
:
237 case PTHREAD_MUTEX_PI_NORMAL_NP
:
238 case PTHREAD_MUTEX_PI_ADAPTIVE_NP
:
239 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP
:
240 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP
:
241 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP
:
242 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP
:
244 int kind
= mutex
->__data
.__kind
& PTHREAD_MUTEX_KIND_MASK_NP
;
245 int robust
= mutex
->__data
.__kind
& PTHREAD_MUTEX_ROBUST_NORMAL_NP
;
248 /* Note: robust PI futexes are signaled by setting bit 0. */
249 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
,
250 (void *) (((uintptr_t) &mutex
->__data
.__list
.__next
)
253 oldval
= mutex
->__data
.__lock
;
255 /* Check whether we already hold the mutex. */
256 if (__glibc_unlikely ((oldval
& FUTEX_TID_MASK
) == id
))
258 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
260 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
264 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
266 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
268 /* Just bump the counter. */
269 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
270 /* Overflow of the counter. */
273 ++mutex
->__data
.__count
;
275 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
281 oldval
= atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
286 /* The mutex is locked. The kernel will now take care of
287 everything. The timeout value must be a relative value.
289 int private = (robust
290 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)
291 : PTHREAD_MUTEX_PSHARED (mutex
));
292 INTERNAL_SYSCALL_DECL (__err
);
294 int e
= INTERNAL_SYSCALL (futex
, __err
, 4, &mutex
->__data
.__lock
,
295 __lll_private_flag (FUTEX_LOCK_PI
,
298 if (INTERNAL_SYSCALL_ERROR_P (e
, __err
))
300 if (INTERNAL_SYSCALL_ERRNO (e
, __err
) == ETIMEDOUT
)
303 if (INTERNAL_SYSCALL_ERRNO (e
, __err
) == ESRCH
304 || INTERNAL_SYSCALL_ERRNO (e
, __err
) == EDEADLK
)
306 assert (INTERNAL_SYSCALL_ERRNO (e
, __err
) != EDEADLK
307 || (kind
!= PTHREAD_MUTEX_ERRORCHECK_NP
308 && kind
!= PTHREAD_MUTEX_RECURSIVE_NP
));
309 /* ESRCH can happen only for non-robust PI mutexes where
310 the owner of the lock died. */
311 assert (INTERNAL_SYSCALL_ERRNO (e
, __err
) != ESRCH
314 /* Delay the thread until the timeout is reached.
315 Then return ETIMEDOUT. */
316 struct timespec reltime
;
319 INTERNAL_SYSCALL (clock_gettime
, __err
, 2, CLOCK_REALTIME
,
321 reltime
.tv_sec
= abstime
->tv_sec
- now
.tv_sec
;
322 reltime
.tv_nsec
= abstime
->tv_nsec
- now
.tv_nsec
;
323 if (reltime
.tv_nsec
< 0)
325 reltime
.tv_nsec
+= 1000000000;
328 if (reltime
.tv_sec
>= 0)
329 while (nanosleep_not_cancel (&reltime
, &reltime
) != 0)
335 return INTERNAL_SYSCALL_ERRNO (e
, __err
);
338 oldval
= mutex
->__data
.__lock
;
340 assert (robust
|| (oldval
& FUTEX_OWNER_DIED
) == 0);
343 if (__glibc_unlikely (oldval
& FUTEX_OWNER_DIED
))
345 atomic_and (&mutex
->__data
.__lock
, ~FUTEX_OWNER_DIED
);
347 /* We got the mutex. */
348 mutex
->__data
.__count
= 1;
349 /* But it is inconsistent unless marked otherwise. */
350 mutex
->__data
.__owner
= PTHREAD_MUTEX_INCONSISTENT
;
352 ENQUEUE_MUTEX_PI (mutex
);
353 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
355 /* Note that we deliberately exit here. If we fall
356 through to the end of the function __nusers would be
357 incremented which is not correct because the old owner
358 has to be discounted. */
363 && __builtin_expect (mutex
->__data
.__owner
364 == PTHREAD_MUTEX_NOTRECOVERABLE
, 0))
366 /* This mutex is now not recoverable. */
367 mutex
->__data
.__count
= 0;
369 INTERNAL_SYSCALL_DECL (__err
);
370 INTERNAL_SYSCALL (futex
, __err
, 4, &mutex
->__data
.__lock
,
371 __lll_private_flag (FUTEX_UNLOCK_PI
,
372 PTHREAD_ROBUST_MUTEX_PSHARED (mutex
)),
375 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
376 return ENOTRECOVERABLE
;
379 mutex
->__data
.__count
= 1;
382 ENQUEUE_MUTEX_PI (mutex
);
383 THREAD_SETMEM (THREAD_SELF
, robust_head
.list_op_pending
, NULL
);
387 #endif /* __NR_futex. */
389 case PTHREAD_MUTEX_PP_RECURSIVE_NP
:
390 case PTHREAD_MUTEX_PP_ERRORCHECK_NP
:
391 case PTHREAD_MUTEX_PP_NORMAL_NP
:
392 case PTHREAD_MUTEX_PP_ADAPTIVE_NP
:
394 int kind
= mutex
->__data
.__kind
& PTHREAD_MUTEX_KIND_MASK_NP
;
396 oldval
= mutex
->__data
.__lock
;
398 /* Check whether we already hold the mutex. */
399 if (mutex
->__data
.__owner
== id
)
401 if (kind
== PTHREAD_MUTEX_ERRORCHECK_NP
)
404 if (kind
== PTHREAD_MUTEX_RECURSIVE_NP
)
406 /* Just bump the counter. */
407 if (__glibc_unlikely (mutex
->__data
.__count
+ 1 == 0))
408 /* Overflow of the counter. */
411 ++mutex
->__data
.__count
;
413 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);
419 int oldprio
= -1, ceilval
;
422 int ceiling
= (oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
)
423 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
425 if (__pthread_current_priority () > ceiling
)
430 __pthread_tpp_change_priority (oldprio
, -1);
434 result
= __pthread_tpp_change_priority (oldprio
, ceiling
);
438 ceilval
= ceiling
<< PTHREAD_MUTEX_PRIO_CEILING_SHIFT
;
442 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
443 ceilval
| 1, ceilval
);
445 if (oldval
== ceilval
)
451 = atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
455 if ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
)
458 if (oldval
!= ceilval
)
460 /* Reject invalid timeouts. */
461 if (abstime
->tv_nsec
< 0 || abstime
->tv_nsec
>= 1000000000)
470 /* Get the current time. */
471 (void) __gettimeofday (&tv
, NULL
);
473 /* Compute relative timeout. */
474 rt
.tv_sec
= abstime
->tv_sec
- tv
.tv_sec
;
475 rt
.tv_nsec
= abstime
->tv_nsec
- tv
.tv_usec
* 1000;
478 rt
.tv_nsec
+= 1000000000;
482 /* Already timed out? */
489 lll_futex_timed_wait (&mutex
->__data
.__lock
,
491 PTHREAD_MUTEX_PSHARED (mutex
));
494 while (atomic_compare_and_exchange_val_acq (&mutex
->__data
.__lock
,
495 ceilval
| 2, ceilval
)
498 while ((oldval
& PTHREAD_MUTEX_PRIO_CEILING_MASK
) != ceilval
);
500 assert (mutex
->__data
.__owner
== 0);
501 mutex
->__data
.__count
= 1;
506 /* Correct code cannot set any other type. */
512 /* Record the ownership. */
513 mutex
->__data
.__owner
= id
;
514 ++mutex
->__data
.__nusers
;
516 LIBC_PROBE (mutex_timedlock_acquired
, 1, mutex
);