libpthread/nptl/pthread_mutex_timedlock.c

   1 /* Copyright (C) 2002-2007, 2008 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <time.h>
  22 #include "pthreadP.h"
  23 #include <lowlevellock.h>
  24 #include <not-cancel.h>
  25
  26 /* We need to build this function with optimization to avoid
  27  * lll_timedlock erroring out with
  28  * error: can't find a register in class ‘GENERAL_REGS’ while reloading ‘asm’
  29  */
  30 int
  31 attribute_optimize("Os")
  32 pthread_mutex_timedlock (
  33      pthread_mutex_t *mutex,
  34      const struct timespec *abstime)
  35 {
  36   int oldval;
  37   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  38   int result = 0;
  39
  40   /* We must not check ABSTIME here.  If the thread does not block
  41      abstime must not be checked for a valid value.  */
  42
  43   switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex),
  44                             PTHREAD_MUTEX_TIMED_NP))
  45     {
  46       /* Recursive mutex.  */
  47     case PTHREAD_MUTEX_RECURSIVE_NP:
  48       /* Check whether we already hold the mutex.  */
  49       if (mutex->__data.__owner == id)
  50         {
  51           /* Just bump the counter.  */
  52           if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
  53             /* Overflow of the counter.  */
  54             return EAGAIN;
  55
  56           ++mutex->__data.__count;
  57
  58           goto out;
  59         }
  60
  61       /* We have to get the mutex.  */
  62       result = lll_timedlock (mutex->__data.__lock, abstime,
  63                               PTHREAD_MUTEX_PSHARED (mutex));
  64
  65       if (result != 0)
  66         goto out;
  67
  68       /* Only locked once so far.  */
  69       mutex->__data.__count = 1;
  70       break;
  71
  72       /* Error checking mutex.  */
  73     case PTHREAD_MUTEX_ERRORCHECK_NP:
  74       /* Check whether we already hold the mutex.  */
  75       if (__builtin_expect (mutex->__data.__owner == id, 0))
  76         return EDEADLK;
  77
  78       /* FALLTHROUGH */
  79
  80     case PTHREAD_MUTEX_TIMED_NP:
  81     simple:
  82       /* Normal mutex.  */
  83       result = lll_timedlock (mutex->__data.__lock, abstime,
  84                               PTHREAD_MUTEX_PSHARED (mutex));
  85       break;
  86
  87     case PTHREAD_MUTEX_ADAPTIVE_NP:
  88       if (! __is_smp)
  89         goto simple;
  90
  91       if (lll_trylock (mutex->__data.__lock) != 0)
  92         {
  93           int cnt = 0;
  94           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
  95                              mutex->__data.__spins * 2 + 10);
  96           do
  97             {
  98               if (cnt++ >= max_cnt)
  99                 {
 100                   result = lll_timedlock (mutex->__data.__lock, abstime,
 101                                           PTHREAD_MUTEX_PSHARED (mutex));
 102                   break;
 103                 }
 104
 105 #ifdef BUSY_WAIT_NOP
 106               BUSY_WAIT_NOP;
 107 #endif
 108             }
 109           while (lll_trylock (mutex->__data.__lock) != 0);
 110
 111           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 112         }
 113       break;
 114
 115     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 116     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 117     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 118     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 119       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 120                      &mutex->__data.__list.__next);
 121
 122       oldval = mutex->__data.__lock;
 123       do
 124         {
 125         again:
 126           if ((oldval & FUTEX_OWNER_DIED) != 0)
 127             {
 128               /* The previous owner died.  Try locking the mutex.  */
 129               int newval = id | (oldval & FUTEX_WAITERS);
 130
 131               newval
 132                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 133                                                        newval, oldval);
 134               if (newval != oldval)
 135                 {
 136                   oldval = newval;
 137                   goto again;
 138                 }
 139
 140               /* We got the mutex.  */
 141               mutex->__data.__count = 1;
 142               /* But it is inconsistent unless marked otherwise.  */
 143               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 144
 145               ENQUEUE_MUTEX (mutex);
 146               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 147
 148               /* Note that we deliberately exit here.  If we fall
 149                  through to the end of the function __nusers would be
 150                  incremented which is not correct because the old
 151                  owner has to be discounted.  */
 152               return EOWNERDEAD;
 153             }
 154
 155           /* Check whether we already hold the mutex.  */
 156           if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 157             {
 158               int kind = PTHREAD_MUTEX_TYPE (mutex);
 159               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 160                 {
 161                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 162                                  NULL);
 163                   return EDEADLK;
 164                 }
 165
 166               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 167                 {
 168                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 169                                  NULL);
 170
 171                   /* Just bump the counter.  */
 172                   if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 173                     /* Overflow of the counter.  */
 174                     return EAGAIN;
 175
 176                   ++mutex->__data.__count;
 177
 178                   return 0;
 179                 }
 180             }
 181
 182           result = lll_robust_timedlock (mutex->__data.__lock, abstime, id,
 183                                          PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 184
 185           if (__builtin_expect (mutex->__data.__owner
 186                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 187             {
 188               /* This mutex is now not recoverable.  */
 189               mutex->__data.__count = 0;
 190               lll_unlock (mutex->__data.__lock,
 191                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 192               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 193               return ENOTRECOVERABLE;
 194             }
 195
 196           if (result == ETIMEDOUT || result == EINVAL)
 197             goto out;
 198
 199           oldval = result;
 200         }
 201       while ((oldval & FUTEX_OWNER_DIED) != 0);
 202
 203       mutex->__data.__count = 1;
 204       ENQUEUE_MUTEX (mutex);
 205       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 206       break;
 207
 208     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 209     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 210     case PTHREAD_MUTEX_PI_NORMAL_NP:
 211     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 212     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 213     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 214     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 215     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 216       {
 217         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 218         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 219
 220         if (robust)
 221           /* Note: robust PI futexes are signaled by setting bit 0.  */
 222           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 223                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 224                                    | 1));
 225
 226         oldval = mutex->__data.__lock;
 227
 228         /* Check whether we already hold the mutex.  */
 229         if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 230           {
 231             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 232               {
 233                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 234                 return EDEADLK;
 235               }
 236
 237             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 238               {
 239                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 240
 241                 /* Just bump the counter.  */
 242                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 243                   /* Overflow of the counter.  */
 244                   return EAGAIN;
 245
 246                 ++mutex->__data.__count;
 247
 248                 return 0;
 249               }
 250           }
 251
 252         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 253                                                       id, 0);
 254
 255         if (oldval != 0)
 256           {
 257             /* The mutex is locked.  The kernel will now take care of
 258                everything.  The timeout value must be a relative value.
 259                Convert it.  */
 260             int private = (robust
 261                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 262                            : PTHREAD_MUTEX_PSHARED (mutex));
 263             INTERNAL_SYSCALL_DECL (__err);
 264
 265             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 266                                       __lll_private_flag (FUTEX_LOCK_PI,
 267                                                           private), 1,
 268                                       abstime);
 269             if (INTERNAL_SYSCALL_ERROR_P (e, __err))
 270               {
 271                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
 272                   return ETIMEDOUT;
 273
 274                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 275                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
 276                   {
 277                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 278                             || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 279                                 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 280                     /* ESRCH can happen only for non-robust PI mutexes where
 281                        the owner of the lock died.  */
 282                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
 283                             || !robust);
 284
 285                     /* Delay the thread until the timeout is reached.
 286                        Then return ETIMEDOUT.  */
 287                     struct timespec reltime;
 288                     struct timespec now;
 289
 290                     INTERNAL_SYSCALL (clock_gettime, __err, 2, CLOCK_REALTIME,
 291                                       &now);
 292                     reltime.tv_sec = abstime->tv_sec - now.tv_sec;
 293                     reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec;
 294                     if (reltime.tv_nsec < 0)
 295                       {
 296                         reltime.tv_nsec += 1000000000;
 297                         --reltime.tv_sec;
 298                       }
 299                     if (reltime.tv_sec >= 0)
 300                       while (nanosleep_not_cancel (&reltime, &reltime) != 0)
 301                         continue;
 302
 303                     return ETIMEDOUT;
 304                   }
 305
 306                 return INTERNAL_SYSCALL_ERRNO (e, __err);
 307               }
 308
 309             oldval = mutex->__data.__lock;
 310
 311             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 312           }
 313
 314         if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
 315           {
 316             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 317
 318             /* We got the mutex.  */
 319             mutex->__data.__count = 1;
 320             /* But it is inconsistent unless marked otherwise.  */
 321             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 322
 323             ENQUEUE_MUTEX_PI (mutex);
 324             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 325
 326             /* Note that we deliberately exit here.  If we fall
 327                through to the end of the function __nusers would be
 328                incremented which is not correct because the old owner
 329                has to be discounted.  */
 330             return EOWNERDEAD;
 331           }
 332
 333         if (robust
 334             && __builtin_expect (mutex->__data.__owner
 335                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 336           {
 337             /* This mutex is now not recoverable.  */
 338             mutex->__data.__count = 0;
 339
 340             INTERNAL_SYSCALL_DECL (__err);
 341             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 342                               __lll_private_flag (FUTEX_UNLOCK_PI,
 343                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 344                               0, 0);
 345
 346             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 347             return ENOTRECOVERABLE;
 348           }
 349
 350         mutex->__data.__count = 1;
 351         if (robust)
 352           {
 353             ENQUEUE_MUTEX_PI (mutex);
 354             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 355           }
 356         }
 357       break;
 358
 359     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 360     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 361     case PTHREAD_MUTEX_PP_NORMAL_NP:
 362     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 363       {
 364         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 365
 366         oldval = mutex->__data.__lock;
 367
 368         /* Check whether we already hold the mutex.  */
 369         if (mutex->__data.__owner == id)
 370           {
 371             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 372               return EDEADLK;
 373
 374             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 375               {
 376                 /* Just bump the counter.  */
 377                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 378                   /* Overflow of the counter.  */
 379                   return EAGAIN;
 380
 381                 ++mutex->__data.__count;
 382
 383                 return 0;
 384               }
 385           }
 386
 387         int oldprio = -1, ceilval;
 388         do
 389           {
 390             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 391                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 392
 393             if (__pthread_current_priority () > ceiling)
 394               {
 395                 result = EINVAL;
 396               failpp:
 397                 if (oldprio != -1)
 398                   __pthread_tpp_change_priority (oldprio, -1);
 399                 return result;
 400               }
 401
 402             result = __pthread_tpp_change_priority (oldprio, ceiling);
 403             if (result)
 404               return result;
 405
 406             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 407             oldprio = ceiling;
 408
 409             oldval
 410               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 411                                                      ceilval | 1, ceilval);
 412
 413             if (oldval == ceilval)
 414               break;
 415
 416             do
 417               {
 418                 oldval
 419                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 420                                                          ceilval | 2,
 421                                                          ceilval | 1);
 422
 423                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 424                   break;
 425
 426                 if (oldval != ceilval)
 427                   {
 428                     /* Reject invalid timeouts.  */
 429                     if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
 430                       {
 431                         result = EINVAL;
 432                         goto failpp;
 433                       }
 434
 435                     struct timeval tv;
 436                     struct timespec rt;
 437
 438                     /* Get the current time.  */
 439                     (void) gettimeofday (&tv, NULL);
 440
 441                     /* Compute relative timeout.  */
 442                     rt.tv_sec = abstime->tv_sec - tv.tv_sec;
 443                     rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
 444                     if (rt.tv_nsec < 0)
 445                       {
 446                         rt.tv_nsec += 1000000000;
 447                         --rt.tv_sec;
 448                       }
 449
 450                     /* Already timed out?  */
 451                     if (rt.tv_sec < 0)
 452                       {
 453                         result = ETIMEDOUT;
 454                         goto failpp;
 455                       }
 456
 457                     lll_futex_timed_wait (&mutex->__data.__lock,
 458                                           ceilval | 2, &rt,
 459                                           PTHREAD_MUTEX_PSHARED (mutex));
 460                   }
 461               }
 462             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 463                                                         ceilval | 2, ceilval)
 464                    != ceilval);
 465           }
 466         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 467
 468         assert (mutex->__data.__owner == 0);
 469         mutex->__data.__count = 1;
 470       }
 471       break;
 472
 473     default:
 474       /* Correct code cannot set any other type.  */
 475       return EINVAL;
 476     }
 477
 478   if (result == 0)
 479     {
 480       /* Record the ownership.  */
 481       mutex->__data.__owner = id;
 482       ++mutex->__data.__nusers;
 483     }
 484
 485  out:
 486   return result;
 487 }