libpthread/nptl/pthread_mutex_timedlock.c

   1 /* Copyright (C) 2002-2007, 2008 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <time.h>
  22 #include "pthreadP.h"
  23 #include <lowlevellock.h>
  24 #include <not-cancel.h>
  25
  26 #if defined(__UCLIBC_USE_TIME64__)
  27 #include "internal/time64_helpers.h"
  28 #endif
  29
  30 /* We need to build this function with optimization to avoid
  31  * lll_timedlock erroring out with
  32  * error: can't find a register in class ‘GENERAL_REGS’ while reloading ‘asm’
  33  */
  34 int
  35 #ifndef  __OPTIMIZE__
  36 attribute_optimize("Os")
  37 #endif
  38 pthread_mutex_timedlock (
  39      pthread_mutex_t *mutex,
  40      const struct timespec *abstime)
  41 {
  42   int oldval;
  43   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  44   int result = 0;
  45
  46   /* We must not check ABSTIME here.  If the thread does not block
  47      abstime must not be checked for a valid value.  */
  48
  49   switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex),
  50                             PTHREAD_MUTEX_TIMED_NP))
  51     {
  52       /* Recursive mutex.  */
  53     case PTHREAD_MUTEX_RECURSIVE_NP:
  54       /* Check whether we already hold the mutex.  */
  55       if (mutex->__data.__owner == id)
  56         {
  57           /* Just bump the counter.  */
  58           if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
  59             /* Overflow of the counter.  */
  60             return EAGAIN;
  61
  62           ++mutex->__data.__count;
  63
  64           goto out;
  65         }
  66
  67       /* We have to get the mutex.  */
  68       result = lll_timedlock (mutex->__data.__lock, abstime,
  69                               PTHREAD_MUTEX_PSHARED (mutex));
  70
  71       if (result != 0)
  72         goto out;
  73
  74       /* Only locked once so far.  */
  75       mutex->__data.__count = 1;
  76       break;
  77
  78       /* Error checking mutex.  */
  79     case PTHREAD_MUTEX_ERRORCHECK_NP:
  80       /* Check whether we already hold the mutex.  */
  81       if (__builtin_expect (mutex->__data.__owner == id, 0))
  82         return EDEADLK;
  83
  84       /* FALLTHROUGH */
  85
  86     case PTHREAD_MUTEX_TIMED_NP:
  87     simple:
  88       /* Normal mutex.  */
  89       result = lll_timedlock (mutex->__data.__lock, abstime,
  90                               PTHREAD_MUTEX_PSHARED (mutex));
  91       break;
  92
  93     case PTHREAD_MUTEX_ADAPTIVE_NP:
  94       if (! __is_smp)
  95         goto simple;
  96
  97       if (lll_trylock (mutex->__data.__lock) != 0)
  98         {
  99           int cnt = 0;
 100           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
 101                              mutex->__data.__spins * 2 + 10);
 102           do
 103             {
 104               if (cnt++ >= max_cnt)
 105                 {
 106                   result = lll_timedlock (mutex->__data.__lock, abstime,
 107                                           PTHREAD_MUTEX_PSHARED (mutex));
 108                   break;
 109                 }
 110
 111 #ifdef BUSY_WAIT_NOP
 112               BUSY_WAIT_NOP;
 113 #endif
 114             }
 115           while (lll_trylock (mutex->__data.__lock) != 0);
 116
 117           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 118         }
 119       break;
 120
 121     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 122     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 123     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 124     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 125       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 126                      &mutex->__data.__list.__next);
 127
 128       oldval = mutex->__data.__lock;
 129       do
 130         {
 131         again:
 132           if ((oldval & FUTEX_OWNER_DIED) != 0)
 133             {
 134               /* The previous owner died.  Try locking the mutex.  */
 135               int newval = id | (oldval & FUTEX_WAITERS);
 136
 137               newval
 138                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 139                                                        newval, oldval);
 140               if (newval != oldval)
 141                 {
 142                   oldval = newval;
 143                   goto again;
 144                 }
 145
 146               /* We got the mutex.  */
 147               mutex->__data.__count = 1;
 148               /* But it is inconsistent unless marked otherwise.  */
 149               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 150
 151               ENQUEUE_MUTEX (mutex);
 152               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 153
 154               /* Note that we deliberately exit here.  If we fall
 155                  through to the end of the function __nusers would be
 156                  incremented which is not correct because the old
 157                  owner has to be discounted.  */
 158               return EOWNERDEAD;
 159             }
 160
 161           /* Check whether we already hold the mutex.  */
 162           if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 163             {
 164               int kind = PTHREAD_MUTEX_TYPE (mutex);
 165               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 166                 {
 167                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 168                                  NULL);
 169                   return EDEADLK;
 170                 }
 171
 172               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 173                 {
 174                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 175                                  NULL);
 176
 177                   /* Just bump the counter.  */
 178                   if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 179                     /* Overflow of the counter.  */
 180                     return EAGAIN;
 181
 182                   ++mutex->__data.__count;
 183
 184                   return 0;
 185                 }
 186             }
 187
 188           result = lll_robust_timedlock (mutex->__data.__lock, abstime, id,
 189                                          PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 190
 191           if (__builtin_expect (mutex->__data.__owner
 192                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 193             {
 194               /* This mutex is now not recoverable.  */
 195               mutex->__data.__count = 0;
 196               lll_unlock (mutex->__data.__lock,
 197                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 198               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 199               return ENOTRECOVERABLE;
 200             }
 201
 202           if (result == ETIMEDOUT || result == EINVAL)
 203             goto out;
 204
 205           oldval = result;
 206         }
 207       while ((oldval & FUTEX_OWNER_DIED) != 0);
 208
 209       mutex->__data.__count = 1;
 210       ENQUEUE_MUTEX (mutex);
 211       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 212       break;
 213
 214     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 215     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 216     case PTHREAD_MUTEX_PI_NORMAL_NP:
 217     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 218     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 219     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 220     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 221     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 222       {
 223         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 224         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 225
 226         if (robust)
 227           /* Note: robust PI futexes are signaled by setting bit 0.  */
 228           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 229                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 230                                    | 1));
 231
 232         oldval = mutex->__data.__lock;
 233
 234         /* Check whether we already hold the mutex.  */
 235         if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 236           {
 237             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 238               {
 239                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 240                 return EDEADLK;
 241               }
 242
 243             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 244               {
 245                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 246
 247                 /* Just bump the counter.  */
 248                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 249                   /* Overflow of the counter.  */
 250                   return EAGAIN;
 251
 252                 ++mutex->__data.__count;
 253
 254                 return 0;
 255               }
 256           }
 257
 258         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 259                                                       id, 0);
 260
 261         if (oldval != 0)
 262           {
 263             /* The mutex is locked.  The kernel will now take care of
 264                everything.  The timeout value must be a relative value.
 265                Convert it.  */
 266             int private = (robust
 267                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 268                            : PTHREAD_MUTEX_PSHARED (mutex));
 269             INTERNAL_SYSCALL_DECL (__err);
 270
 271 #if defined(__UCLIBC_USE_TIME64__) && defined(__NR_futex_time64)
 272             int e = INTERNAL_SYSCALL (futex_time64, __err, 4, &mutex->__data.__lock,
 273                                       __lll_private_flag (FUTEX_LOCK_PI,
 274                                                           private), 1,
 275                                       TO_TS64_P(abstime));
 276 #else
 277             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 278                                       __lll_private_flag (FUTEX_LOCK_PI,
 279                                                           private), 1,
 280                                       abstime);
 281 #endif
 282             if (INTERNAL_SYSCALL_ERROR_P (e, __err))
 283               {
 284                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
 285                   return ETIMEDOUT;
 286
 287                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 288                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
 289                   {
 290                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 291                             || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 292                                 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 293                     /* ESRCH can happen only for non-robust PI mutexes where
 294                        the owner of the lock died.  */
 295                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
 296                             || !robust);
 297
 298                     /* Delay the thread until the timeout is reached.
 299                        Then return ETIMEDOUT.  */
 300                     struct timespec reltime;
 301 #if defined(__UCLIBC_USE_TIME64__)
 302                     struct __ts64_struct __now64;
 303 #endif
 304                     struct timespec now = {.tv_sec = 0, .tv_nsec = 0};
 305
 306 #if defined(__UCLIBC_USE_TIME64__) && defined(__NR_clock_gettime64)
 307                     int __r = INTERNAL_SYSCALL (clock_gettime64, __err, 2, CLOCK_REALTIME,
 308                                       &__now64);
 309
 310                     if (__r == 0) {
 311                         now.tv_sec = __now64.tv_sec;
 312                         now.tv_nsec = __now64.tv_nsec;
 313                     }
 314 #else
 315                     INTERNAL_SYSCALL (clock_gettime, __err, 2, CLOCK_REALTIME,
 316                                       &now);
 317 #endif
 318                     reltime.tv_sec = abstime->tv_sec - now.tv_sec;
 319                     reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec;
 320                     if (reltime.tv_nsec < 0)
 321                       {
 322                         reltime.tv_nsec += 1000000000;
 323                         --reltime.tv_sec;
 324                       }
 325                     if (reltime.tv_sec >= 0)
 326                       while (nanosleep_not_cancel (&reltime, &reltime) != 0)
 327                         continue;
 328
 329                     return ETIMEDOUT;
 330                   }
 331
 332                 return INTERNAL_SYSCALL_ERRNO (e, __err);
 333               }
 334
 335             oldval = mutex->__data.__lock;
 336
 337             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 338           }
 339
 340         if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
 341           {
 342             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 343
 344             /* We got the mutex.  */
 345             mutex->__data.__count = 1;
 346             /* But it is inconsistent unless marked otherwise.  */
 347             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 348
 349             ENQUEUE_MUTEX_PI (mutex);
 350             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 351
 352             /* Note that we deliberately exit here.  If we fall
 353                through to the end of the function __nusers would be
 354                incremented which is not correct because the old owner
 355                has to be discounted.  */
 356             return EOWNERDEAD;
 357           }
 358
 359         if (robust
 360             && __builtin_expect (mutex->__data.__owner
 361                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 362           {
 363             /* This mutex is now not recoverable.  */
 364             mutex->__data.__count = 0;
 365
 366             INTERNAL_SYSCALL_DECL (__err);
 367 #if defined(__UCLIBC_USE_TIME64__) && defined(__NR_futex_time64)
 368             INTERNAL_SYSCALL (futex_time64, __err, 4, &mutex->__data.__lock,
 369                               __lll_private_flag (FUTEX_UNLOCK_PI,
 370                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 371                               0, 0);
 372 #else
 373             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 374                               __lll_private_flag (FUTEX_UNLOCK_PI,
 375                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 376                               0, 0);
 377 #endif
 378
 379             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 380             return ENOTRECOVERABLE;
 381           }
 382
 383         mutex->__data.__count = 1;
 384         if (robust)
 385           {
 386             ENQUEUE_MUTEX_PI (mutex);
 387             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 388           }
 389         }
 390       break;
 391
 392     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 393     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 394     case PTHREAD_MUTEX_PP_NORMAL_NP:
 395     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 396       {
 397         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 398
 399         oldval = mutex->__data.__lock;
 400
 401         /* Check whether we already hold the mutex.  */
 402         if (mutex->__data.__owner == id)
 403           {
 404             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 405               return EDEADLK;
 406
 407             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 408               {
 409                 /* Just bump the counter.  */
 410                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 411                   /* Overflow of the counter.  */
 412                   return EAGAIN;
 413
 414                 ++mutex->__data.__count;
 415
 416                 return 0;
 417               }
 418           }
 419
 420         int oldprio = -1, ceilval;
 421         do
 422           {
 423             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 424                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 425
 426             if (__pthread_current_priority () > ceiling)
 427               {
 428                 result = EINVAL;
 429               failpp:
 430                 if (oldprio != -1)
 431                   __pthread_tpp_change_priority (oldprio, -1);
 432                 return result;
 433               }
 434
 435             result = __pthread_tpp_change_priority (oldprio, ceiling);
 436             if (result)
 437               return result;
 438
 439             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 440             oldprio = ceiling;
 441
 442             oldval
 443               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 444                                                      ceilval | 1, ceilval);
 445
 446             if (oldval == ceilval)
 447               break;
 448
 449             do
 450               {
 451                 oldval
 452                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 453                                                          ceilval | 2,
 454                                                          ceilval | 1);
 455
 456                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 457                   break;
 458
 459                 if (oldval != ceilval)
 460                   {
 461                     /* Reject invalid timeouts.  */
 462                     if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
 463                       {
 464                         result = EINVAL;
 465                         goto failpp;
 466                       }
 467
 468                     struct timeval tv;
 469                     struct timespec rt;
 470
 471                     /* Get the current time.  */
 472                     (void) gettimeofday (&tv, NULL);
 473
 474                     /* Compute relative timeout.  */
 475                     rt.tv_sec = abstime->tv_sec - tv.tv_sec;
 476                     rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
 477                     if (rt.tv_nsec < 0)
 478                       {
 479                         rt.tv_nsec += 1000000000;
 480                         --rt.tv_sec;
 481                       }
 482
 483                     /* Already timed out?  */
 484                     if (rt.tv_sec < 0)
 485                       {
 486                         result = ETIMEDOUT;
 487                         goto failpp;
 488                       }
 489
 490                     lll_futex_timed_wait (&mutex->__data.__lock,
 491                                           ceilval | 2, &rt,
 492                                           PTHREAD_MUTEX_PSHARED (mutex));
 493                   }
 494               }
 495             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 496                                                         ceilval | 2, ceilval)
 497                    != ceilval);
 498           }
 499         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 500
 501         assert (mutex->__data.__owner == 0);
 502         mutex->__data.__count = 1;
 503       }
 504       break;
 505
 506     default:
 507       /* Correct code cannot set any other type.  */
 508       return EINVAL;
 509     }
 510
 511   if (result == 0)
 512     {
 513       /* Record the ownership.  */
 514       mutex->__data.__owner = id;
 515       ++mutex->__data.__nusers;
 516     }
 517
 518  out:
 519   return result;
 520 }