nptl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002-2007, 2008, 2009 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <not-cancel.h>
  24 #include "pthreadP.h"
  25 #include <lowlevellock.h>
  26
  27
  28 #ifndef LLL_MUTEX_LOCK
  29 # define LLL_MUTEX_LOCK(mutex) \
  30   lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
  31 # define LLL_MUTEX_TRYLOCK(mutex) \
  32   lll_trylock ((mutex)->__data.__lock)
  33 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
  34   lll_robust_lock ((mutex)->__data.__lock, id, \
  35                    PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
  36 #endif
  37
  38
  39 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
  40      __attribute_noinline__;
  41
  42
  43 int
  44 __pthread_mutex_lock (mutex)
  45      pthread_mutex_t *mutex;
  46 {
  47   assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
  48
  49   unsigned int type = PTHREAD_MUTEX_TYPE (mutex);
  50   if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0))
  51     return __pthread_mutex_lock_full (mutex);
  52
  53   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  54
  55   if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
  56       == PTHREAD_MUTEX_TIMED_NP)
  57     {
  58     simple:
  59       /* Normal mutex.  */
  60       LLL_MUTEX_LOCK (mutex);
  61       assert (mutex->__data.__owner == 0);
  62     }
  63   else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  64     {
  65       /* Recursive mutex.  */
  66
  67       /* Check whether we already hold the mutex.  */
  68       if (mutex->__data.__owner == id)
  69         {
  70           /* Just bump the counter.  */
  71           if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
  72             /* Overflow of the counter.  */
  73             return EAGAIN;
  74
  75           ++mutex->__data.__count;
  76
  77           return 0;
  78         }
  79
  80       /* We have to get the mutex.  */
  81       LLL_MUTEX_LOCK (mutex);
  82
  83       assert (mutex->__data.__owner == 0);
  84       mutex->__data.__count = 1;
  85     }
  86   else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
  87     {
  88       if (! __is_smp)
  89         goto simple;
  90
  91       if (LLL_MUTEX_TRYLOCK (mutex) != 0)
  92         {
  93           int cnt = 0;
  94           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
  95                              mutex->__data.__spins * 2 + 10);
  96           do
  97             {
  98               if (cnt++ >= max_cnt)
  99                 {
 100                   LLL_MUTEX_LOCK (mutex);
 101                   break;
 102                 }
 103
 104 #ifdef BUSY_WAIT_NOP
 105               BUSY_WAIT_NOP;
 106 #endif
 107             }
 108           while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 109
 110           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 111         }
 112       assert (mutex->__data.__owner == 0);
 113     }
 114   else
 115     {
 116       assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
 117       /* Check whether we already hold the mutex.  */
 118       if (__builtin_expect (mutex->__data.__owner == id, 0))
 119         return EDEADLK;
 120       goto simple;
 121     }
 122
 123   /* Record the ownership.  */
 124   mutex->__data.__owner = id;
 125 #ifndef NO_INCR
 126   ++mutex->__data.__nusers;
 127 #endif
 128
 129   return 0;
 130 }
 131
 132 static int
 133 __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 134 {
 135   int oldval;
 136   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 137
 138   switch (PTHREAD_MUTEX_TYPE (mutex))
 139     {
 140     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 141     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 142     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 143     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 144       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 145                      &mutex->__data.__list.__next);
 146
 147       oldval = mutex->__data.__lock;
 148       do
 149         {
 150         again:
 151           if ((oldval & FUTEX_OWNER_DIED) != 0)
 152             {
 153               /* The previous owner died.  Try locking the mutex.  */
 154               int newval = id;
 155 #ifdef NO_INCR
 156               newval |= FUTEX_WAITERS;
 157 #else
 158               newval |= (oldval & FUTEX_WAITERS);
 159 #endif
 160
 161               newval
 162                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 163                                                        newval, oldval);
 164
 165               if (newval != oldval)
 166                 {
 167                   oldval = newval;
 168                   goto again;
 169                 }
 170
 171               /* We got the mutex.  */
 172               mutex->__data.__count = 1;
 173               /* But it is inconsistent unless marked otherwise.  */
 174               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 175
 176               ENQUEUE_MUTEX (mutex);
 177               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 178
 179               /* Note that we deliberately exit here.  If we fall
 180                  through to the end of the function __nusers would be
 181                  incremented which is not correct because the old
 182                  owner has to be discounted.  If we are not supposed
 183                  to increment __nusers we actually have to decrement
 184                  it here.  */
 185 #ifdef NO_INCR
 186               --mutex->__data.__nusers;
 187 #endif
 188
 189               return EOWNERDEAD;
 190             }
 191
 192           /* Check whether we already hold the mutex.  */
 193           if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 194             {
 195               int kind = PTHREAD_MUTEX_TYPE (mutex);
 196               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 197                 {
 198                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 199                                  NULL);
 200                   return EDEADLK;
 201                 }
 202
 203               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 204                 {
 205                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 206                                  NULL);
 207
 208                   /* Just bump the counter.  */
 209                   if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 210                     /* Overflow of the counter.  */
 211                     return EAGAIN;
 212
 213                   ++mutex->__data.__count;
 214
 215                   return 0;
 216                 }
 217             }
 218
 219           oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id);
 220
 221           if (__builtin_expect (mutex->__data.__owner
 222                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 223             {
 224               /* This mutex is now not recoverable.  */
 225               mutex->__data.__count = 0;
 226               lll_unlock (mutex->__data.__lock,
 227                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 228               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 229               return ENOTRECOVERABLE;
 230             }
 231         }
 232       while ((oldval & FUTEX_OWNER_DIED) != 0);
 233
 234       mutex->__data.__count = 1;
 235       ENQUEUE_MUTEX (mutex);
 236       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 237       break;
 238
 239     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 240     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 241     case PTHREAD_MUTEX_PI_NORMAL_NP:
 242     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 243     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 244     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 245     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 246     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 247       {
 248         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 249         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 250
 251         if (robust)
 252           /* Note: robust PI futexes are signaled by setting bit 0.  */
 253           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 254                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 255                                    | 1));
 256
 257         oldval = mutex->__data.__lock;
 258
 259         /* Check whether we already hold the mutex.  */
 260         if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 261           {
 262             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 263               {
 264                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 265                 return EDEADLK;
 266               }
 267
 268             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 269               {
 270                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 271
 272                 /* Just bump the counter.  */
 273                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 274                   /* Overflow of the counter.  */
 275                   return EAGAIN;
 276
 277                 ++mutex->__data.__count;
 278
 279                 return 0;
 280               }
 281           }
 282
 283         int newval = id;
 284 #ifdef NO_INCR
 285         newval |= FUTEX_WAITERS;
 286 #endif
 287         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 288                                                       newval, 0);
 289
 290         if (oldval != 0)
 291           {
 292             /* The mutex is locked.  The kernel will now take care of
 293                everything.  */
 294             int private = (robust
 295                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 296                            : PTHREAD_MUTEX_PSHARED (mutex));
 297             INTERNAL_SYSCALL_DECL (__err);
 298             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 299                                       __lll_private_flag (FUTEX_LOCK_PI,
 300                                                           private), 1, 0);
 301
 302             if (INTERNAL_SYSCALL_ERROR_P (e, __err)
 303                 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 304                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
 305               {
 306                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 307                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 308                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 309                 /* ESRCH can happen only for non-robust PI mutexes where
 310                    the owner of the lock died.  */
 311                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
 312
 313                 /* Delay the thread indefinitely.  */
 314                 while (1)
 315                   pause_not_cancel ();
 316               }
 317
 318             oldval = mutex->__data.__lock;
 319
 320             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 321           }
 322
 323         if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
 324           {
 325             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 326
 327             /* We got the mutex.  */
 328             mutex->__data.__count = 1;
 329             /* But it is inconsistent unless marked otherwise.  */
 330             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 331
 332             ENQUEUE_MUTEX_PI (mutex);
 333             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 334
 335             /* Note that we deliberately exit here.  If we fall
 336                through to the end of the function __nusers would be
 337                incremented which is not correct because the old owner
 338                has to be discounted.  If we are not supposed to
 339                increment __nusers we actually have to decrement it here.  */
 340 #ifdef NO_INCR
 341             --mutex->__data.__nusers;
 342 #endif
 343
 344             return EOWNERDEAD;
 345           }
 346
 347         if (robust
 348             && __builtin_expect (mutex->__data.__owner
 349                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 350           {
 351             /* This mutex is now not recoverable.  */
 352             mutex->__data.__count = 0;
 353
 354             INTERNAL_SYSCALL_DECL (__err);
 355             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 356                               __lll_private_flag (FUTEX_UNLOCK_PI,
 357                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 358                               0, 0);
 359
 360             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 361             return ENOTRECOVERABLE;
 362           }
 363
 364         mutex->__data.__count = 1;
 365         if (robust)
 366           {
 367             ENQUEUE_MUTEX_PI (mutex);
 368             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 369           }
 370       }
 371       break;
 372
 373     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 374     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 375     case PTHREAD_MUTEX_PP_NORMAL_NP:
 376     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 377       {
 378         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 379
 380         oldval = mutex->__data.__lock;
 381
 382         /* Check whether we already hold the mutex.  */
 383         if (mutex->__data.__owner == id)
 384           {
 385             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 386               return EDEADLK;
 387
 388             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 389               {
 390                 /* Just bump the counter.  */
 391                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 392                   /* Overflow of the counter.  */
 393                   return EAGAIN;
 394
 395                 ++mutex->__data.__count;
 396
 397                 return 0;
 398               }
 399           }
 400
 401         int oldprio = -1, ceilval;
 402         do
 403           {
 404             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 405                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 406
 407             if (__pthread_current_priority () > ceiling)
 408               {
 409                 if (oldprio != -1)
 410                   __pthread_tpp_change_priority (oldprio, -1);
 411                 return EINVAL;
 412               }
 413
 414             int retval = __pthread_tpp_change_priority (oldprio, ceiling);
 415             if (retval)
 416               return retval;
 417
 418             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 419             oldprio = ceiling;
 420
 421             oldval
 422               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 423 #ifdef NO_INCR
 424                                                      ceilval | 2,
 425 #else
 426                                                      ceilval | 1,
 427 #endif
 428                                                      ceilval);
 429
 430             if (oldval == ceilval)
 431               break;
 432
 433             do
 434               {
 435                 oldval
 436                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 437                                                          ceilval | 2,
 438                                                          ceilval | 1);
 439
 440                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 441                   break;
 442
 443                 if (oldval != ceilval)
 444                   lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
 445                                   PTHREAD_MUTEX_PSHARED (mutex));
 446               }
 447             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 448                                                         ceilval | 2, ceilval)
 449                    != ceilval);
 450           }
 451         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 452
 453         assert (mutex->__data.__owner == 0);
 454         mutex->__data.__count = 1;
 455       }
 456       break;
 457
 458     default:
 459       /* Correct code cannot set any other type.  */
 460       return EINVAL;
 461     }
 462
 463   /* Record the ownership.  */
 464   mutex->__data.__owner = id;
 465 #ifndef NO_INCR
 466   ++mutex->__data.__nusers;
 467 #endif
 468
 469   return 0;
 470 }
 471 #ifndef __pthread_mutex_lock
 472 strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
 473 strong_alias (__pthread_mutex_lock, __pthread_mutex_lock_internal)
 474 #endif
 475
 476
 477 #ifdef NO_INCR
 478 void
 479 __pthread_mutex_cond_lock_adjust (mutex)
 480      pthread_mutex_t *mutex;
 481 {
 482   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
 483   assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
 484   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
 485
 486   /* Record the ownership.  */
 487   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 488   mutex->__data.__owner = id;
 489
 490   if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
 491     ++mutex->__data.__count;
 492 }
 493 #endif