nptl/pthread_mutex_timedlock.c

   1 /* Copyright (C) 2002-2013 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <time.h>
  22 #include "pthreadP.h"
  23 #include <lowlevellock.h>
  24 #include <not-cancel.h>
  25
  26 #include <stap-probe.h>
  27
  28 #ifndef lll_timedlock_elision
  29 #define lll_timedlock_elision(a,dummy,b,c) lll_timedlock(a, b, c)
  30 #endif
  31
  32 #ifndef lll_trylock_elision
  33 #define lll_trylock_elision(a,t) lll_trylock(a)
  34 #endif
  35
  36 #ifndef FORCE_ELISION
  37 #define FORCE_ELISION(m, s)
  38 #endif
  39
  40 int
  41 pthread_mutex_timedlock (mutex, abstime)
  42      pthread_mutex_t *mutex;
  43      const struct timespec *abstime;
  44 {
  45   int oldval;
  46   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  47   int result = 0;
  48
  49   LIBC_PROBE (mutex_timedlock_entry, 2, mutex, abstime);
  50
  51   /* We must not check ABSTIME here.  If the thread does not block
  52      abstime must not be checked for a valid value.  */
  53
  54   switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
  55                             PTHREAD_MUTEX_TIMED_NP))
  56     {
  57       /* Recursive mutex.  */
  58     case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
  59     case PTHREAD_MUTEX_RECURSIVE_NP:
  60       /* Check whether we already hold the mutex.  */
  61       if (mutex->__data.__owner == id)
  62         {
  63           /* Just bump the counter.  */
  64           if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
  65             /* Overflow of the counter.  */
  66             return EAGAIN;
  67
  68           ++mutex->__data.__count;
  69
  70           goto out;
  71         }
  72
  73       /* We have to get the mutex.  */
  74       result = lll_timedlock (mutex->__data.__lock, abstime,
  75                               PTHREAD_MUTEX_PSHARED (mutex));
  76
  77       if (result != 0)
  78         goto out;
  79
  80       /* Only locked once so far.  */
  81       mutex->__data.__count = 1;
  82       break;
  83
  84       /* Error checking mutex.  */
  85     case PTHREAD_MUTEX_ERRORCHECK_NP:
  86       /* Check whether we already hold the mutex.  */
  87       if (__builtin_expect (mutex->__data.__owner == id, 0))
  88         return EDEADLK;
  89
  90       /* FALLTHROUGH */
  91
  92     case PTHREAD_MUTEX_TIMED_NP:
  93       FORCE_ELISION (mutex, goto elision);
  94     simple:
  95       /* Normal mutex.  */
  96       result = lll_timedlock (mutex->__data.__lock, abstime,
  97                               PTHREAD_MUTEX_PSHARED (mutex));
  98       break;
  99
 100     case PTHREAD_MUTEX_TIMED_ELISION_NP:
 101     elision: __attribute__((unused))
 102       /* Don't record ownership */
 103       return lll_timedlock_elision (mutex->__data.__lock,
 104                                     mutex->__data.__spins,
 105                                     abstime,
 106                                     PTHREAD_MUTEX_PSHARED (mutex));
 107
 108
 109     case PTHREAD_MUTEX_ADAPTIVE_NP:
 110       if (! __is_smp)
 111         goto simple;
 112
 113       if (lll_trylock (mutex->__data.__lock) != 0)
 114         {
 115           int cnt = 0;
 116           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
 117                              mutex->__data.__spins * 2 + 10);
 118           do
 119             {
 120               if (cnt++ >= max_cnt)
 121                 {
 122                   result = lll_timedlock (mutex->__data.__lock, abstime,
 123                                           PTHREAD_MUTEX_PSHARED (mutex));
 124                   break;
 125                 }
 126
 127 #ifdef BUSY_WAIT_NOP
 128               BUSY_WAIT_NOP;
 129 #endif
 130             }
 131           while (lll_trylock (mutex->__data.__lock) != 0);
 132
 133           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 134         }
 135       break;
 136
 137     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 138     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 139     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 140     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 141       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 142                      &mutex->__data.__list.__next);
 143
 144       oldval = mutex->__data.__lock;
 145       do
 146         {
 147         again:
 148           if ((oldval & FUTEX_OWNER_DIED) != 0)
 149             {
 150               /* The previous owner died.  Try locking the mutex.  */
 151               int newval = id | (oldval & FUTEX_WAITERS);
 152
 153               newval
 154                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 155                                                        newval, oldval);
 156               if (newval != oldval)
 157                 {
 158                   oldval = newval;
 159                   goto again;
 160                 }
 161
 162               /* We got the mutex.  */
 163               mutex->__data.__count = 1;
 164               /* But it is inconsistent unless marked otherwise.  */
 165               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 166
 167               ENQUEUE_MUTEX (mutex);
 168               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 169
 170               /* Note that we deliberately exit here.  If we fall
 171                  through to the end of the function __nusers would be
 172                  incremented which is not correct because the old
 173                  owner has to be discounted.  */
 174               return EOWNERDEAD;
 175             }
 176
 177           /* Check whether we already hold the mutex.  */
 178           if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 179             {
 180               int kind = PTHREAD_MUTEX_TYPE (mutex);
 181               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 182                 {
 183                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 184                                  NULL);
 185                   return EDEADLK;
 186                 }
 187
 188               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 189                 {
 190                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 191                                  NULL);
 192
 193                   /* Just bump the counter.  */
 194                   if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 195                     /* Overflow of the counter.  */
 196                     return EAGAIN;
 197
 198                   ++mutex->__data.__count;
 199
 200                   LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 201
 202                   return 0;
 203                 }
 204             }
 205
 206           result = lll_robust_timedlock (mutex->__data.__lock, abstime, id,
 207                                          PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 208
 209           if (__builtin_expect (mutex->__data.__owner
 210                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 211             {
 212               /* This mutex is now not recoverable.  */
 213               mutex->__data.__count = 0;
 214               lll_unlock (mutex->__data.__lock,
 215                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 216               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 217               return ENOTRECOVERABLE;
 218             }
 219
 220           if (result == ETIMEDOUT || result == EINVAL)
 221             goto out;
 222
 223           oldval = result;
 224         }
 225       while ((oldval & FUTEX_OWNER_DIED) != 0);
 226
 227       mutex->__data.__count = 1;
 228       ENQUEUE_MUTEX (mutex);
 229       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 230       break;
 231
 232     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 233     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 234     case PTHREAD_MUTEX_PI_NORMAL_NP:
 235     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 236     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 237     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 238     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 239     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 240       {
 241         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 242         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 243
 244         if (robust)
 245           /* Note: robust PI futexes are signaled by setting bit 0.  */
 246           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 247                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 248                                    | 1));
 249
 250         oldval = mutex->__data.__lock;
 251
 252         /* Check whether we already hold the mutex.  */
 253         if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 254           {
 255             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 256               {
 257                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 258                 return EDEADLK;
 259               }
 260
 261             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 262               {
 263                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 264
 265                 /* Just bump the counter.  */
 266                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 267                   /* Overflow of the counter.  */
 268                   return EAGAIN;
 269
 270                 ++mutex->__data.__count;
 271
 272                 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 273
 274                 return 0;
 275               }
 276           }
 277
 278         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 279                                                       id, 0);
 280
 281         if (oldval != 0)
 282           {
 283             /* The mutex is locked.  The kernel will now take care of
 284                everything.  The timeout value must be a relative value.
 285                Convert it.  */
 286             int private = (robust
 287                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 288                            : PTHREAD_MUTEX_PSHARED (mutex));
 289             INTERNAL_SYSCALL_DECL (__err);
 290
 291             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 292                                       __lll_private_flag (FUTEX_LOCK_PI,
 293                                                           private), 1,
 294                                       abstime);
 295             if (INTERNAL_SYSCALL_ERROR_P (e, __err))
 296               {
 297                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
 298                   return ETIMEDOUT;
 299
 300                 if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 301                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
 302                   {
 303                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 304                             || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 305                                 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 306                     /* ESRCH can happen only for non-robust PI mutexes where
 307                        the owner of the lock died.  */
 308                     assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
 309                             || !robust);
 310
 311                     /* Delay the thread until the timeout is reached.
 312                        Then return ETIMEDOUT.  */
 313                     struct timespec reltime;
 314                     struct timespec now;
 315
 316                     INTERNAL_SYSCALL (clock_gettime, __err, 2, CLOCK_REALTIME,
 317                                       &now);
 318                     reltime.tv_sec = abstime->tv_sec - now.tv_sec;
 319                     reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec;
 320                     if (reltime.tv_nsec < 0)
 321                       {
 322                         reltime.tv_nsec += 1000000000;
 323                         --reltime.tv_sec;
 324                       }
 325                     if (reltime.tv_sec >= 0)
 326                       while (nanosleep_not_cancel (&reltime, &reltime) != 0)
 327                         continue;
 328
 329                     return ETIMEDOUT;
 330                   }
 331
 332                 return INTERNAL_SYSCALL_ERRNO (e, __err);
 333               }
 334
 335             oldval = mutex->__data.__lock;
 336
 337             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 338           }
 339
 340         if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
 341           {
 342             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 343
 344             /* We got the mutex.  */
 345             mutex->__data.__count = 1;
 346             /* But it is inconsistent unless marked otherwise.  */
 347             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 348
 349             ENQUEUE_MUTEX_PI (mutex);
 350             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 351
 352             /* Note that we deliberately exit here.  If we fall
 353                through to the end of the function __nusers would be
 354                incremented which is not correct because the old owner
 355                has to be discounted.  */
 356             return EOWNERDEAD;
 357           }
 358
 359         if (robust
 360             && __builtin_expect (mutex->__data.__owner
 361                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 362           {
 363             /* This mutex is now not recoverable.  */
 364             mutex->__data.__count = 0;
 365
 366             INTERNAL_SYSCALL_DECL (__err);
 367             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 368                               __lll_private_flag (FUTEX_UNLOCK_PI,
 369                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 370                               0, 0);
 371
 372             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 373             return ENOTRECOVERABLE;
 374           }
 375
 376         mutex->__data.__count = 1;
 377         if (robust)
 378           {
 379             ENQUEUE_MUTEX_PI (mutex);
 380             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 381           }
 382         }
 383       break;
 384
 385     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 386     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 387     case PTHREAD_MUTEX_PP_NORMAL_NP:
 388     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 389       {
 390         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 391
 392         oldval = mutex->__data.__lock;
 393
 394         /* Check whether we already hold the mutex.  */
 395         if (mutex->__data.__owner == id)
 396           {
 397             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 398               return EDEADLK;
 399
 400             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 401               {
 402                 /* Just bump the counter.  */
 403                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 404                   /* Overflow of the counter.  */
 405                   return EAGAIN;
 406
 407                 ++mutex->__data.__count;
 408
 409                 LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 410
 411                 return 0;
 412               }
 413           }
 414
 415         int oldprio = -1, ceilval;
 416         do
 417           {
 418             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 419                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 420
 421             if (__pthread_current_priority () > ceiling)
 422               {
 423                 result = EINVAL;
 424               failpp:
 425                 if (oldprio != -1)
 426                   __pthread_tpp_change_priority (oldprio, -1);
 427                 return result;
 428               }
 429
 430             result = __pthread_tpp_change_priority (oldprio, ceiling);
 431             if (result)
 432               return result;
 433
 434             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 435             oldprio = ceiling;
 436
 437             oldval
 438               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 439                                                      ceilval | 1, ceilval);
 440
 441             if (oldval == ceilval)
 442               break;
 443
 444             do
 445               {
 446                 oldval
 447                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 448                                                          ceilval | 2,
 449                                                          ceilval | 1);
 450
 451                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 452                   break;
 453
 454                 if (oldval != ceilval)
 455                   {
 456                     /* Reject invalid timeouts.  */
 457                     if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
 458                       {
 459                         result = EINVAL;
 460                         goto failpp;
 461                       }
 462
 463                     struct timeval tv;
 464                     struct timespec rt;
 465
 466                     /* Get the current time.  */
 467                     (void) __gettimeofday (&tv, NULL);
 468
 469                     /* Compute relative timeout.  */
 470                     rt.tv_sec = abstime->tv_sec - tv.tv_sec;
 471                     rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
 472                     if (rt.tv_nsec < 0)
 473                       {
 474                         rt.tv_nsec += 1000000000;
 475                         --rt.tv_sec;
 476                       }
 477
 478                     /* Already timed out?  */
 479                     if (rt.tv_sec < 0)
 480                       {
 481                         result = ETIMEDOUT;
 482                         goto failpp;
 483                       }
 484
 485                     lll_futex_timed_wait (&mutex->__data.__lock,
 486                                           ceilval | 2, &rt,
 487                                           PTHREAD_MUTEX_PSHARED (mutex));
 488                   }
 489               }
 490             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 491                                                         ceilval | 2, ceilval)
 492                    != ceilval);
 493           }
 494         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 495
 496         assert (mutex->__data.__owner == 0);
 497         mutex->__data.__count = 1;
 498       }
 499       break;
 500
 501     default:
 502       /* Correct code cannot set any other type.  */
 503       return EINVAL;
 504     }
 505
 506   if (result == 0)
 507     {
 508       /* Record the ownership.  */
 509       mutex->__data.__owner = id;
 510       ++mutex->__data.__nusers;
 511
 512       LIBC_PROBE (mutex_timedlock_acquired, 1, mutex);
 513     }
 514
 515  out:
 516   return result;
 517 }