Fix posix/tst-glob_lstat_compat on alpha [BZ #22818]
[glibc.git] / nptl / pthread_mutex_timedlock.c
blob66efd3989f18831c79e12eb8085affe67ffc9dd9
1 /* Copyright (C) 2002-2018 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
19 #include <assert.h>
20 #include <errno.h>
21 #include <time.h>
22 #include <sys/param.h>
23 #include <sys/time.h>
24 #include "pthreadP.h"
25 #include <atomic.h>
26 #include <lowlevellock.h>
27 #include <not-cancel.h>
29 #include <stap-probe.h>
31 #ifndef lll_timedlock_elision
32 #define lll_timedlock_elision(a,dummy,b,c) lll_timedlock(a, b, c)
33 #endif
35 #ifndef lll_trylock_elision
36 #define lll_trylock_elision(a,t) lll_trylock(a)
37 #endif
39 #ifndef FORCE_ELISION
40 #define FORCE_ELISION(m, s)
41 #endif
43 int
44 __pthread_mutex_timedlock (pthread_mutex_t *mutex,
45 const struct timespec *abstime)
47 int oldval;
48 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
49 int result = 0;
51 LIBC_PROBE (mutex_timedlock_entry, 2, mutex, abstime);
53 /* We must not check ABSTIME here. If the thread does not block
54 abstime must not be checked for a valid value. */
56 switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
57 PTHREAD_MUTEX_TIMED_NP))
59 /* Recursive mutex. */
60 case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
61 case PTHREAD_MUTEX_RECURSIVE_NP:
62 /* Check whether we already hold the mutex. */
63 if (mutex->__data.__owner == id)
65 /* Just bump the counter. */
66 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
67 /* Overflow of the counter. */
68 return EAGAIN;
70 ++mutex->__data.__count;
72 goto out;
75 /* We have to get the mutex. */
76 result = lll_timedlock (mutex->__data.__lock, abstime,
77 PTHREAD_MUTEX_PSHARED (mutex));
79 if (result != 0)
80 goto out;
82 /* Only locked once so far. */
83 mutex->__data.__count = 1;
84 break;
86 /* Error checking mutex. */
87 case PTHREAD_MUTEX_ERRORCHECK_NP:
88 /* Check whether we already hold the mutex. */
89 if (__glibc_unlikely (mutex->__data.__owner == id))
90 return EDEADLK;
92 /* Don't do lock elision on an error checking mutex. */
93 goto simple;
95 case PTHREAD_MUTEX_TIMED_NP:
96 FORCE_ELISION (mutex, goto elision);
97 simple:
98 /* Normal mutex. */
99 result = lll_timedlock (mutex->__data.__lock, abstime,
100 PTHREAD_MUTEX_PSHARED (mutex));
101 break;
103 case PTHREAD_MUTEX_TIMED_ELISION_NP:
104 elision: __attribute__((unused))
105 /* Don't record ownership */
106 return lll_timedlock_elision (mutex->__data.__lock,
107 mutex->__data.__spins,
108 abstime,
109 PTHREAD_MUTEX_PSHARED (mutex));
112 case PTHREAD_MUTEX_ADAPTIVE_NP:
113 if (! __is_smp)
114 goto simple;
116 if (lll_trylock (mutex->__data.__lock) != 0)
118 int cnt = 0;
119 int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
120 mutex->__data.__spins * 2 + 10);
123 if (cnt++ >= max_cnt)
125 result = lll_timedlock (mutex->__data.__lock, abstime,
126 PTHREAD_MUTEX_PSHARED (mutex));
127 break;
129 atomic_spin_nop ();
131 while (lll_trylock (mutex->__data.__lock) != 0);
133 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
135 break;
137 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
138 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
139 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
140 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
141 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
142 &mutex->__data.__list.__next);
143 /* We need to set op_pending before starting the operation. Also
144 see comments at ENQUEUE_MUTEX. */
145 __asm ("" ::: "memory");
147 oldval = mutex->__data.__lock;
148 /* This is set to FUTEX_WAITERS iff we might have shared the
149 FUTEX_WAITERS flag with other threads, and therefore need to keep it
150 set to avoid lost wake-ups. We have the same requirement in the
151 simple mutex algorithm. */
152 unsigned int assume_other_futex_waiters = 0;
153 while (1)
155 /* Try to acquire the lock through a CAS from 0 (not acquired) to
156 our TID | assume_other_futex_waiters. */
157 if (__glibc_likely (oldval == 0))
159 oldval
160 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
161 id | assume_other_futex_waiters, 0);
162 if (__glibc_likely (oldval == 0))
163 break;
166 if ((oldval & FUTEX_OWNER_DIED) != 0)
168 /* The previous owner died. Try locking the mutex. */
169 int newval = id | (oldval & FUTEX_WAITERS)
170 | assume_other_futex_waiters;
172 newval
173 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
174 newval, oldval);
175 if (newval != oldval)
177 oldval = newval;
178 continue;
181 /* We got the mutex. */
182 mutex->__data.__count = 1;
183 /* But it is inconsistent unless marked otherwise. */
184 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
186 /* We must not enqueue the mutex before we have acquired it.
187 Also see comments at ENQUEUE_MUTEX. */
188 __asm ("" ::: "memory");
189 ENQUEUE_MUTEX (mutex);
190 /* We need to clear op_pending after we enqueue the mutex. */
191 __asm ("" ::: "memory");
192 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
194 /* Note that we deliberately exit here. If we fall
195 through to the end of the function __nusers would be
196 incremented which is not correct because the old
197 owner has to be discounted. */
198 return EOWNERDEAD;
201 /* Check whether we already hold the mutex. */
202 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
204 int kind = PTHREAD_MUTEX_TYPE (mutex);
205 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
207 /* We do not need to ensure ordering wrt another memory
208 access. Also see comments at ENQUEUE_MUTEX. */
209 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
210 NULL);
211 return EDEADLK;
214 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
216 /* We do not need to ensure ordering wrt another memory
217 access. */
218 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
219 NULL);
221 /* Just bump the counter. */
222 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
223 /* Overflow of the counter. */
224 return EAGAIN;
226 ++mutex->__data.__count;
228 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
230 return 0;
234 /* We are about to block; check whether the timeout is invalid. */
235 if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
236 return EINVAL;
237 /* Work around the fact that the kernel rejects negative timeout
238 values despite them being valid. */
239 if (__glibc_unlikely (abstime->tv_sec < 0))
240 return ETIMEDOUT;
241 #if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
242 || !defined lll_futex_timed_wait_bitset)
243 struct timeval tv;
244 struct timespec rt;
246 /* Get the current time. */
247 (void) __gettimeofday (&tv, NULL);
249 /* Compute relative timeout. */
250 rt.tv_sec = abstime->tv_sec - tv.tv_sec;
251 rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
252 if (rt.tv_nsec < 0)
254 rt.tv_nsec += 1000000000;
255 --rt.tv_sec;
258 /* Already timed out? */
259 if (rt.tv_sec < 0)
260 return ETIMEDOUT;
261 #endif
263 /* We cannot acquire the mutex nor has its owner died. Thus, try
264 to block using futexes. Set FUTEX_WAITERS if necessary so that
265 other threads are aware that there are potentially threads
266 blocked on the futex. Restart if oldval changed in the
267 meantime. */
268 if ((oldval & FUTEX_WAITERS) == 0)
270 if (atomic_compare_and_exchange_bool_acq (&mutex->__data.__lock,
271 oldval | FUTEX_WAITERS,
272 oldval)
273 != 0)
275 oldval = mutex->__data.__lock;
276 continue;
278 oldval |= FUTEX_WAITERS;
281 /* It is now possible that we share the FUTEX_WAITERS flag with
282 another thread; therefore, update assume_other_futex_waiters so
283 that we do not forget about this when handling other cases
284 above and thus do not cause lost wake-ups. */
285 assume_other_futex_waiters |= FUTEX_WAITERS;
287 /* Block using the futex. */
288 #if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
289 || !defined lll_futex_timed_wait_bitset)
290 lll_futex_timed wait (&mutex->__data.__lock, oldval,
291 &rt, PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
292 #else
293 int err = lll_futex_timed_wait_bitset (&mutex->__data.__lock,
294 oldval, abstime, FUTEX_CLOCK_REALTIME,
295 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
296 /* The futex call timed out. */
297 if (err == -ETIMEDOUT)
298 return -err;
299 #endif
300 /* Reload current lock value. */
301 oldval = mutex->__data.__lock;
304 /* We have acquired the mutex; check if it is still consistent. */
305 if (__builtin_expect (mutex->__data.__owner
306 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
308 /* This mutex is now not recoverable. */
309 mutex->__data.__count = 0;
310 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
311 lll_unlock (mutex->__data.__lock, private);
312 /* FIXME This violates the mutex destruction requirements. See
313 __pthread_mutex_unlock_full. */
314 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
315 return ENOTRECOVERABLE;
318 mutex->__data.__count = 1;
319 /* We must not enqueue the mutex before we have acquired it.
320 Also see comments at ENQUEUE_MUTEX. */
321 __asm ("" ::: "memory");
322 ENQUEUE_MUTEX (mutex);
323 /* We need to clear op_pending after we enqueue the mutex. */
324 __asm ("" ::: "memory");
325 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
326 break;
328 /* The PI support requires the Linux futex system call. If that's not
329 available, pthread_mutex_init should never have allowed the type to
330 be set. So it will get the default case for an invalid type. */
331 #ifdef __NR_futex
332 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
333 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
334 case PTHREAD_MUTEX_PI_NORMAL_NP:
335 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
336 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
337 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
338 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
339 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
341 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
342 int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
344 if (robust)
346 /* Note: robust PI futexes are signaled by setting bit 0. */
347 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
348 (void *) (((uintptr_t) &mutex->__data.__list.__next)
349 | 1));
350 /* We need to set op_pending before starting the operation. Also
351 see comments at ENQUEUE_MUTEX. */
352 __asm ("" ::: "memory");
355 oldval = mutex->__data.__lock;
357 /* Check whether we already hold the mutex. */
358 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
360 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
362 /* We do not need to ensure ordering wrt another memory
363 access. */
364 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
365 return EDEADLK;
368 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
370 /* We do not need to ensure ordering wrt another memory
371 access. */
372 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
374 /* Just bump the counter. */
375 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
376 /* Overflow of the counter. */
377 return EAGAIN;
379 ++mutex->__data.__count;
381 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
383 return 0;
387 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
388 id, 0);
390 if (oldval != 0)
392 /* The mutex is locked. The kernel will now take care of
393 everything. The timeout value must be a relative value.
394 Convert it. */
395 int private = (robust
396 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
397 : PTHREAD_MUTEX_PSHARED (mutex));
398 INTERNAL_SYSCALL_DECL (__err);
400 int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
401 __lll_private_flag (FUTEX_LOCK_PI,
402 private), 1,
403 abstime);
404 if (INTERNAL_SYSCALL_ERROR_P (e, __err))
406 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
407 return ETIMEDOUT;
409 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
410 || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
412 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
413 || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
414 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
415 /* ESRCH can happen only for non-robust PI mutexes where
416 the owner of the lock died. */
417 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
418 || !robust);
420 /* Delay the thread until the timeout is reached.
421 Then return ETIMEDOUT. */
422 struct timespec reltime;
423 struct timespec now;
425 INTERNAL_SYSCALL (clock_gettime, __err, 2, CLOCK_REALTIME,
426 &now);
427 reltime.tv_sec = abstime->tv_sec - now.tv_sec;
428 reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec;
429 if (reltime.tv_nsec < 0)
431 reltime.tv_nsec += 1000000000;
432 --reltime.tv_sec;
434 if (reltime.tv_sec >= 0)
435 while (__nanosleep_nocancel (&reltime, &reltime) != 0)
436 continue;
438 return ETIMEDOUT;
441 return INTERNAL_SYSCALL_ERRNO (e, __err);
444 oldval = mutex->__data.__lock;
446 assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
449 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
451 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
453 /* We got the mutex. */
454 mutex->__data.__count = 1;
455 /* But it is inconsistent unless marked otherwise. */
456 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
458 /* We must not enqueue the mutex before we have acquired it.
459 Also see comments at ENQUEUE_MUTEX. */
460 __asm ("" ::: "memory");
461 ENQUEUE_MUTEX_PI (mutex);
462 /* We need to clear op_pending after we enqueue the mutex. */
463 __asm ("" ::: "memory");
464 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
466 /* Note that we deliberately exit here. If we fall
467 through to the end of the function __nusers would be
468 incremented which is not correct because the old owner
469 has to be discounted. */
470 return EOWNERDEAD;
473 if (robust
474 && __builtin_expect (mutex->__data.__owner
475 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
477 /* This mutex is now not recoverable. */
478 mutex->__data.__count = 0;
480 INTERNAL_SYSCALL_DECL (__err);
481 INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
482 __lll_private_flag (FUTEX_UNLOCK_PI,
483 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
484 0, 0);
486 /* To the kernel, this will be visible after the kernel has
487 acquired the mutex in the syscall. */
488 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
489 return ENOTRECOVERABLE;
492 mutex->__data.__count = 1;
493 if (robust)
495 /* We must not enqueue the mutex before we have acquired it.
496 Also see comments at ENQUEUE_MUTEX. */
497 __asm ("" ::: "memory");
498 ENQUEUE_MUTEX_PI (mutex);
499 /* We need to clear op_pending after we enqueue the mutex. */
500 __asm ("" ::: "memory");
501 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
504 break;
505 #endif /* __NR_futex. */
507 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
508 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
509 case PTHREAD_MUTEX_PP_NORMAL_NP:
510 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
512 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
514 oldval = mutex->__data.__lock;
516 /* Check whether we already hold the mutex. */
517 if (mutex->__data.__owner == id)
519 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
520 return EDEADLK;
522 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
524 /* Just bump the counter. */
525 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
526 /* Overflow of the counter. */
527 return EAGAIN;
529 ++mutex->__data.__count;
531 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
533 return 0;
537 int oldprio = -1, ceilval;
540 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
541 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
543 if (__pthread_current_priority () > ceiling)
545 result = EINVAL;
546 failpp:
547 if (oldprio != -1)
548 __pthread_tpp_change_priority (oldprio, -1);
549 return result;
552 result = __pthread_tpp_change_priority (oldprio, ceiling);
553 if (result)
554 return result;
556 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
557 oldprio = ceiling;
559 oldval
560 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
561 ceilval | 1, ceilval);
563 if (oldval == ceilval)
564 break;
568 oldval
569 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
570 ceilval | 2,
571 ceilval | 1);
573 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
574 break;
576 if (oldval != ceilval)
578 /* Reject invalid timeouts. */
579 if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
581 result = EINVAL;
582 goto failpp;
585 struct timeval tv;
586 struct timespec rt;
588 /* Get the current time. */
589 (void) __gettimeofday (&tv, NULL);
591 /* Compute relative timeout. */
592 rt.tv_sec = abstime->tv_sec - tv.tv_sec;
593 rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
594 if (rt.tv_nsec < 0)
596 rt.tv_nsec += 1000000000;
597 --rt.tv_sec;
600 /* Already timed out? */
601 if (rt.tv_sec < 0)
603 result = ETIMEDOUT;
604 goto failpp;
607 lll_futex_timed_wait (&mutex->__data.__lock,
608 ceilval | 2, &rt,
609 PTHREAD_MUTEX_PSHARED (mutex));
612 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
613 ceilval | 2, ceilval)
614 != ceilval);
616 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
618 assert (mutex->__data.__owner == 0);
619 mutex->__data.__count = 1;
621 break;
623 default:
624 /* Correct code cannot set any other type. */
625 return EINVAL;
628 if (result == 0)
630 /* Record the ownership. */
631 mutex->__data.__owner = id;
632 ++mutex->__data.__nusers;
634 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
637 out:
638 return result;
640 weak_alias (__pthread_mutex_timedlock, pthread_mutex_timedlock)