libpthread/nptl/pthread_mutex_timedlock.c

   1 /* Copyright (C) 2002-2007, 2008 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <time.h>
  22 #include "pthreadP.h"
  23 #include <lowlevellock.h>
  24 #include <not-cancel.h>
  25
  26 /* We need to build this function with optimization to avoid
  27  * lll_timedlock erroring out with
  28  * error: can't find a register in class ‘GENERAL_REGS’ while reloading ‘asm’
  29  */
  30 int
  31 #ifndef  __OPTIMIZE__
  32 attribute_optimize("Os")
  33 #endif
  34 pthread_mutex_timedlock (
  35      pthread_mutex_t *mutex,
  36      const struct timespec *abstime)
  37 {
  38   int oldval;
  39   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  40   int result = 0;
  41
  42   /* We must not check ABSTIME here.  If the thread does not block
  43      abstime must not be checked for a valid value.  */
  44
  45   switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex),
  46                             PTHREAD_MUTEX_TIMED_NP))
  47     {
  48       /* Recursive mutex.  */
  49     case PTHREAD_MUTEX_RECURSIVE_NP:
  50       /* Check whether we already hold the mutex.  */
  51       if (mutex->__data.__owner == id)
  52         {
  53           /* Just bump the counter.  */
  54           if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
  55             /* Overflow of the counter.  */
  56             return EAGAIN;
  57
  58           ++mutex->__data.__count;
  59
  60           goto out;
  61         }
  62
  63       /* We have to get the mutex.  */
  64       result = lll_timedlock (mutex->__data.__lock, abstime,
  65                               PTHREAD_MUTEX_PSHARED (mutex));
  66
  67       if (result != 0)
  68         goto out;
  69
  70       /* Only locked once so far.  */
  71       mutex->__data.__count = 1;
  72       break;
  73
  74       /* Error checking mutex.  */
  75     case PTHREAD_MUTEX_ERRORCHECK_NP:
  76       /* Check whether we already hold the mutex.  */
  77       if (__builtin_expect (mutex->__data.__owner == id, 0))
  78         return EDEADLK;
  79
  80       /* FALLTHROUGH */
  81
  82     case PTHREAD_MUTEX_TIMED_NP:
  83     simple:
  84       /* Normal mutex.  */
  85       result = lll_timedlock (mutex->__data.__lock, abstime,
  86                               PTHREAD_MUTEX_PSHARED (mutex));
  87       break;
  88
  89     case PTHREAD_MUTEX_ADAPTIVE_NP:
  90       if (! __is_smp)
  91         goto simple;
  92
  93       if (lll_trylock (mutex->__data.__lock) != 0)
  94         {
  95           int cnt = 0;
  96           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
  97                              mutex->__data.__spins * 2 + 10);
  98           do
  99             {
 100               if (cnt++ >= max_cnt)
 101                 {
 102                   result = lll_timedlock (mutex->__data.__lock, abstime,
 103                                           PTHREAD_MUTEX_PSHARED (mutex));
 104                   break;
 105                 }
 106
 107 #ifdef BUSY_WAIT_NOP
 108               BUSY_WAIT_NOP;
 109 #endif
 110             }
 111           while (lll_trylock (mutex->__data.__lock) != 0);
 112
 113           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 114         }
 115       break;
 116
 117     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 118     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 119     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 120     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 121       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 122                      &mutex->__data.__list.__next);
 123
 124       oldval = mutex->__data.__lock;
 125       do
 126         {
 127         again:
 128           if ((oldval & FUTEX_OWNER_DIED) != 0)
 129             {
 130               /* The previous owner died.  Try locking the mutex.  */
 131               int newval = id | (oldval & FUTEX_WAITERS);
 132
 133               newval
 134                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 135                                                        newval, oldval);
 136               if (newval != oldval)
 137                 {
 138                   oldval = newval;
 139                   goto again;
 140                 }
 141
 142               /* We got the mutex.  */
 143               mutex->__data.__count = 1;
 144               /* But it is inconsistent unless marked otherwise.  */
 145               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 146
 147               ENQUEUE_MUTEX (mutex);
 148               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 149
 150               /* Note that we deliberately exit here.  If we fall
 151                  through to the end of the function __nusers would be
 152                  incremented which is not correct because the old
 153                  owner has to be discounted.  */
 154               return EOWNERDEAD;
 155             }
 156
 157           /* Check whether we already hold the mutex.  */
 158           if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 159             {
 160               int kind = PTHREAD_MUTEX_TYPE (mutex);
 161               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 162                 {
 163                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 164                                  NULL);
 165                   return EDEADLK;
 166                 }
 167
 168               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 169                 {
 170                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 171                                  NULL);
 172
 173                   /* Just bump the counter.  */
 174                   if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 175                     /* Overflow of the counter.  */
 176                     return EAGAIN;
 177
 178                   ++mutex->__data.__count;
 179
 180                   return 0;
 181                 }
 182             }
 183
 184           result = lll_robust_timedlock (mutex->__data.__lock, abstime, id,
 185                                          PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 186
 187           if (__builtin_expect (mutex->__data.__owner
 188                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 189             {
 190               /* This mutex is now not recoverable.  */
 191               mutex->__data.__count = 0;
 192               lll_unlock (mutex->__data.__lock,
 193                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 194               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 195               return ENOTRECOVERABLE;
 196             }
 197
 198           if (result == ETIMEDOUT || result == EINVAL)
 199             goto out;
 200
 201           oldval = result;
 202         }
 203       while ((oldval & FUTEX_OWNER_DIED) != 0);
 204
 205       mutex->__data.__count = 1;
 206       ENQUEUE_MUTEX (mutex);
 207       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 208       break;
 209
 210     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 211     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 212     case PTHREAD_MUTEX_PI_NORMAL_NP:
 213     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 214     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 215     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 216     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 217     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 218       {
 219         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 220         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 221
 222         if (robust)
 223           /* Note: robust PI futexes are signaled by setting bit 0.  */
 224           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 225                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 226                                    | 1));
 227
 228         oldval = mutex->__data.__lock;
 229
 230         /* Check whether we already hold the mutex.  */
 231         if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 232           {
 233             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 234               {
 235                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 236                 return EDEADLK;
 237               }
 238
 239             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 240               {
 241                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 242
 243                 /* Just bump the counter.  */
 244                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 245                   /* Overflow of the counter.  */
 246                   return EAGAIN;
 247
 248                 ++mutex->__data.__count;
 249
 250                 return 0;
 251               }
 252           }
 253
 254         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 255                                                       id, 0);
 256
 257         if (oldval != 0)
 258           {
 259             /* The mutex is locked.  The kernel will now take care of
 260                everything.  The timeout value must be a relative value.
 261                Convert it.  */
 262             int private = (robust
 263                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 264                            : PTHREAD_MUTEX_PSHARED (mutex));
 265             INTERNAL_SYSCALL_DECL (__err);
 266
 267             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 268                                       __lll_private_flag (FUTEX_LOCK_PI,
 269                                                           private), 1,
 270                                       abstime);
 271             if (INTERNAL_SYSCALL_ERROR_P (e, __err))
 272               {
 273                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
 274                   return ETIMEDOUT;
 275
 276                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 277                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
 278                   {
 279                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 280                             || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 281                                 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 282                     /* ESRCH can happen only for non-robust PI mutexes where
 283                        the owner of the lock died.  */
 284                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
 285                             || !robust);
 286
 287                     /* Delay the thread until the timeout is reached.
 288                        Then return ETIMEDOUT.  */
 289                     struct timespec reltime;
 290                     struct timespec now;
 291
 292                     INTERNAL_SYSCALL (clock_gettime, __err, 2, CLOCK_REALTIME,
 293                                       &now);
 294                     reltime.tv_sec = abstime->tv_sec - now.tv_sec;
 295                     reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec;
 296                     if (reltime.tv_nsec < 0)
 297                       {
 298                         reltime.tv_nsec += 1000000000;
 299                         --reltime.tv_sec;
 300                       }
 301                     if (reltime.tv_sec >= 0)
 302                       while (nanosleep_not_cancel (&reltime, &reltime) != 0)
 303                         continue;
 304
 305                     return ETIMEDOUT;
 306                   }
 307
 308                 return INTERNAL_SYSCALL_ERRNO (e, __err);
 309               }
 310
 311             oldval = mutex->__data.__lock;
 312
 313             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 314           }
 315
 316         if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
 317           {
 318             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 319
 320             /* We got the mutex.  */
 321             mutex->__data.__count = 1;
 322             /* But it is inconsistent unless marked otherwise.  */
 323             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 324
 325             ENQUEUE_MUTEX_PI (mutex);
 326             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 327
 328             /* Note that we deliberately exit here.  If we fall
 329                through to the end of the function __nusers would be
 330                incremented which is not correct because the old owner
 331                has to be discounted.  */
 332             return EOWNERDEAD;
 333           }
 334
 335         if (robust
 336             && __builtin_expect (mutex->__data.__owner
 337                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 338           {
 339             /* This mutex is now not recoverable.  */
 340             mutex->__data.__count = 0;
 341
 342             INTERNAL_SYSCALL_DECL (__err);
 343             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 344                               __lll_private_flag (FUTEX_UNLOCK_PI,
 345                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 346                               0, 0);
 347
 348             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 349             return ENOTRECOVERABLE;
 350           }
 351
 352         mutex->__data.__count = 1;
 353         if (robust)
 354           {
 355             ENQUEUE_MUTEX_PI (mutex);
 356             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 357           }
 358         }
 359       break;
 360
 361     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 362     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 363     case PTHREAD_MUTEX_PP_NORMAL_NP:
 364     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 365       {
 366         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 367
 368         oldval = mutex->__data.__lock;
 369
 370         /* Check whether we already hold the mutex.  */
 371         if (mutex->__data.__owner == id)
 372           {
 373             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 374               return EDEADLK;
 375
 376             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 377               {
 378                 /* Just bump the counter.  */
 379                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 380                   /* Overflow of the counter.  */
 381                   return EAGAIN;
 382
 383                 ++mutex->__data.__count;
 384
 385                 return 0;
 386               }
 387           }
 388
 389         int oldprio = -1, ceilval;
 390         do
 391           {
 392             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 393                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 394
 395             if (__pthread_current_priority () > ceiling)
 396               {
 397                 result = EINVAL;
 398               failpp:
 399                 if (oldprio != -1)
 400                   __pthread_tpp_change_priority (oldprio, -1);
 401                 return result;
 402               }
 403
 404             result = __pthread_tpp_change_priority (oldprio, ceiling);
 405             if (result)
 406               return result;
 407
 408             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 409             oldprio = ceiling;
 410
 411             oldval
 412               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 413                                                      ceilval | 1, ceilval);
 414
 415             if (oldval == ceilval)
 416               break;
 417
 418             do
 419               {
 420                 oldval
 421                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 422                                                          ceilval | 2,
 423                                                          ceilval | 1);
 424
 425                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 426                   break;
 427
 428                 if (oldval != ceilval)
 429                   {
 430                     /* Reject invalid timeouts.  */
 431                     if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
 432                       {
 433                         result = EINVAL;
 434                         goto failpp;
 435                       }
 436
 437                     struct timeval tv;
 438                     struct timespec rt;
 439
 440                     /* Get the current time.  */
 441                     (void) gettimeofday (&tv, NULL);
 442
 443                     /* Compute relative timeout.  */
 444                     rt.tv_sec = abstime->tv_sec - tv.tv_sec;
 445                     rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
 446                     if (rt.tv_nsec < 0)
 447                       {
 448                         rt.tv_nsec += 1000000000;
 449                         --rt.tv_sec;
 450                       }
 451
 452                     /* Already timed out?  */
 453                     if (rt.tv_sec < 0)
 454                       {
 455                         result = ETIMEDOUT;
 456                         goto failpp;
 457                       }
 458
 459                     lll_futex_timed_wait (&mutex->__data.__lock,
 460                                           ceilval | 2, &rt,
 461                                           PTHREAD_MUTEX_PSHARED (mutex));
 462                   }
 463               }
 464             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 465                                                         ceilval | 2, ceilval)
 466                    != ceilval);
 467           }
 468         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 469
 470         assert (mutex->__data.__owner == 0);
 471         mutex->__data.__count = 1;
 472       }
 473       break;
 474
 475     default:
 476       /* Correct code cannot set any other type.  */
 477       return EINVAL;
 478     }
 479
 480   if (result == 0)
 481     {
 482       /* Record the ownership.  */
 483       mutex->__data.__owner = id;
 484       ++mutex->__data.__nusers;
 485     }
 486
 487  out:
 488   return result;
 489 }