nptl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002-2007, 2008, 2009 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <not-cancel.h>
  25 #include "pthreadP.h"
  26 #include <lowlevellock.h>
  27
  28
  29 #ifndef LLL_MUTEX_LOCK
  30 # define LLL_MUTEX_LOCK(mutex) \
  31   lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
  32 # define LLL_MUTEX_TRYLOCK(mutex) \
  33   lll_trylock ((mutex)->__data.__lock)
  34 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
  35   lll_robust_lock ((mutex)->__data.__lock, id, \
  36                    PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
  37 #endif
  38
  39
  40 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
  41      __attribute_noinline__;
  42
  43
  44 int
  45 __pthread_mutex_lock (mutex)
  46      pthread_mutex_t *mutex;
  47 {
  48   assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
  49
  50   unsigned int type = PTHREAD_MUTEX_TYPE (mutex);
  51   if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0))
  52     return __pthread_mutex_lock_full (mutex);
  53
  54   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  55
  56   if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
  57       == PTHREAD_MUTEX_TIMED_NP)
  58     {
  59     simple:
  60       /* Normal mutex.  */
  61       LLL_MUTEX_LOCK (mutex);
  62       assert (mutex->__data.__owner == 0);
  63     }
  64   else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  65     {
  66       /* Recursive mutex.  */
  67
  68       /* Check whether we already hold the mutex.  */
  69       if (mutex->__data.__owner == id)
  70         {
  71           /* Just bump the counter.  */
  72           if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
  73             /* Overflow of the counter.  */
  74             return EAGAIN;
  75
  76           ++mutex->__data.__count;
  77
  78           return 0;
  79         }
  80
  81       /* We have to get the mutex.  */
  82       LLL_MUTEX_LOCK (mutex);
  83
  84       assert (mutex->__data.__owner == 0);
  85       mutex->__data.__count = 1;
  86     }
  87   else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
  88     {
  89       if (! __is_smp)
  90         goto simple;
  91
  92       if (LLL_MUTEX_TRYLOCK (mutex) != 0)
  93         {
  94           int cnt = 0;
  95           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
  96                              mutex->__data.__spins * 2 + 10);
  97           do
  98             {
  99               if (cnt++ >= max_cnt)
 100                 {
 101                   LLL_MUTEX_LOCK (mutex);
 102                   break;
 103                 }
 104
 105 #ifdef BUSY_WAIT_NOP
 106               BUSY_WAIT_NOP;
 107 #endif
 108             }
 109           while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 110
 111           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 112         }
 113       assert (mutex->__data.__owner == 0);
 114     }
 115   else
 116     {
 117       assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
 118       /* Check whether we already hold the mutex.  */
 119       if (__builtin_expect (mutex->__data.__owner == id, 0))
 120         return EDEADLK;
 121       goto simple;
 122     }
 123
 124   /* Record the ownership.  */
 125   mutex->__data.__owner = id;
 126 #ifndef NO_INCR
 127   ++mutex->__data.__nusers;
 128 #endif
 129
 130   return 0;
 131 }
 132
 133 static int
 134 __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 135 {
 136   int oldval;
 137   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 138
 139   switch (PTHREAD_MUTEX_TYPE (mutex))
 140     {
 141     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 142     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 143     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 144     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 145       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 146                      &mutex->__data.__list.__next);
 147
 148       oldval = mutex->__data.__lock;
 149       do
 150         {
 151         again:
 152           if ((oldval & FUTEX_OWNER_DIED) != 0)
 153             {
 154               /* The previous owner died.  Try locking the mutex.  */
 155               int newval = id;
 156 #ifdef NO_INCR
 157               newval |= FUTEX_WAITERS;
 158 #else
 159               newval |= (oldval & FUTEX_WAITERS);
 160 #endif
 161
 162               newval
 163                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 164                                                        newval, oldval);
 165
 166               if (newval != oldval)
 167                 {
 168                   oldval = newval;
 169                   goto again;
 170                 }
 171
 172               /* We got the mutex.  */
 173               mutex->__data.__count = 1;
 174               /* But it is inconsistent unless marked otherwise.  */
 175               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 176
 177               ENQUEUE_MUTEX (mutex);
 178               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 179
 180               /* Note that we deliberately exit here.  If we fall
 181                  through to the end of the function __nusers would be
 182                  incremented which is not correct because the old
 183                  owner has to be discounted.  If we are not supposed
 184                  to increment __nusers we actually have to decrement
 185                  it here.  */
 186 #ifdef NO_INCR
 187               --mutex->__data.__nusers;
 188 #endif
 189
 190               return EOWNERDEAD;
 191             }
 192
 193           /* Check whether we already hold the mutex.  */
 194           if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 195             {
 196               int kind = PTHREAD_MUTEX_TYPE (mutex);
 197               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 198                 {
 199                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 200                                  NULL);
 201                   return EDEADLK;
 202                 }
 203
 204               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 205                 {
 206                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 207                                  NULL);
 208
 209                   /* Just bump the counter.  */
 210                   if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 211                     /* Overflow of the counter.  */
 212                     return EAGAIN;
 213
 214                   ++mutex->__data.__count;
 215
 216                   return 0;
 217                 }
 218             }
 219
 220           oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id);
 221
 222           if (__builtin_expect (mutex->__data.__owner
 223                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 224             {
 225               /* This mutex is now not recoverable.  */
 226               mutex->__data.__count = 0;
 227               lll_unlock (mutex->__data.__lock,
 228                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 229               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 230               return ENOTRECOVERABLE;
 231             }
 232         }
 233       while ((oldval & FUTEX_OWNER_DIED) != 0);
 234
 235       mutex->__data.__count = 1;
 236       ENQUEUE_MUTEX (mutex);
 237       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 238       break;
 239
 240     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 241     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 242     case PTHREAD_MUTEX_PI_NORMAL_NP:
 243     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 244     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 245     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 246     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 247     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 248       {
 249         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 250         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 251
 252         if (robust)
 253           /* Note: robust PI futexes are signaled by setting bit 0.  */
 254           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 255                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 256                                    | 1));
 257
 258         oldval = mutex->__data.__lock;
 259
 260         /* Check whether we already hold the mutex.  */
 261         if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 262           {
 263             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 264               {
 265                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 266                 return EDEADLK;
 267               }
 268
 269             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 270               {
 271                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 272
 273                 /* Just bump the counter.  */
 274                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 275                   /* Overflow of the counter.  */
 276                   return EAGAIN;
 277
 278                 ++mutex->__data.__count;
 279
 280                 return 0;
 281               }
 282           }
 283
 284         int newval = id;
 285 #ifdef NO_INCR
 286         newval |= FUTEX_WAITERS;
 287 #endif
 288         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 289                                                       newval, 0);
 290
 291         if (oldval != 0)
 292           {
 293             /* The mutex is locked.  The kernel will now take care of
 294                everything.  */
 295             int private = (robust
 296                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 297                            : PTHREAD_MUTEX_PSHARED (mutex));
 298             INTERNAL_SYSCALL_DECL (__err);
 299             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 300                                       __lll_private_flag (FUTEX_LOCK_PI,
 301                                                           private), 1, 0);
 302
 303             if (INTERNAL_SYSCALL_ERROR_P (e, __err)
 304                 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 305                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
 306               {
 307                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 308                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 309                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 310                 /* ESRCH can happen only for non-robust PI mutexes where
 311                    the owner of the lock died.  */
 312                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
 313
 314                 /* Delay the thread indefinitely.  */
 315                 while (1)
 316                   pause_not_cancel ();
 317               }
 318
 319             oldval = mutex->__data.__lock;
 320
 321             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 322           }
 323
 324         if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
 325           {
 326             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 327
 328             /* We got the mutex.  */
 329             mutex->__data.__count = 1;
 330             /* But it is inconsistent unless marked otherwise.  */
 331             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 332
 333             ENQUEUE_MUTEX_PI (mutex);
 334             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 335
 336             /* Note that we deliberately exit here.  If we fall
 337                through to the end of the function __nusers would be
 338                incremented which is not correct because the old owner
 339                has to be discounted.  If we are not supposed to
 340                increment __nusers we actually have to decrement it here.  */
 341 #ifdef NO_INCR
 342             --mutex->__data.__nusers;
 343 #endif
 344
 345             return EOWNERDEAD;
 346           }
 347
 348         if (robust
 349             && __builtin_expect (mutex->__data.__owner
 350                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 351           {
 352             /* This mutex is now not recoverable.  */
 353             mutex->__data.__count = 0;
 354
 355             INTERNAL_SYSCALL_DECL (__err);
 356             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 357                               __lll_private_flag (FUTEX_UNLOCK_PI,
 358                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 359                               0, 0);
 360
 361             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 362             return ENOTRECOVERABLE;
 363           }
 364
 365         mutex->__data.__count = 1;
 366         if (robust)
 367           {
 368             ENQUEUE_MUTEX_PI (mutex);
 369             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 370           }
 371       }
 372       break;
 373
 374     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 375     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 376     case PTHREAD_MUTEX_PP_NORMAL_NP:
 377     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 378       {
 379         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 380
 381         oldval = mutex->__data.__lock;
 382
 383         /* Check whether we already hold the mutex.  */
 384         if (mutex->__data.__owner == id)
 385           {
 386             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 387               return EDEADLK;
 388
 389             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 390               {
 391                 /* Just bump the counter.  */
 392                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 393                   /* Overflow of the counter.  */
 394                   return EAGAIN;
 395
 396                 ++mutex->__data.__count;
 397
 398                 return 0;
 399               }
 400           }
 401
 402         int oldprio = -1, ceilval;
 403         do
 404           {
 405             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 406                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 407
 408             if (__pthread_current_priority () > ceiling)
 409               {
 410                 if (oldprio != -1)
 411                   __pthread_tpp_change_priority (oldprio, -1);
 412                 return EINVAL;
 413               }
 414
 415             int retval = __pthread_tpp_change_priority (oldprio, ceiling);
 416             if (retval)
 417               return retval;
 418
 419             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 420             oldprio = ceiling;
 421
 422             oldval
 423               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 424 #ifdef NO_INCR
 425                                                      ceilval | 2,
 426 #else
 427                                                      ceilval | 1,
 428 #endif
 429                                                      ceilval);
 430
 431             if (oldval == ceilval)
 432               break;
 433
 434             do
 435               {
 436                 oldval
 437                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 438                                                          ceilval | 2,
 439                                                          ceilval | 1);
 440
 441                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 442                   break;
 443
 444                 if (oldval != ceilval)
 445                   lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
 446                                   PTHREAD_MUTEX_PSHARED (mutex));
 447               }
 448             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 449                                                         ceilval | 2, ceilval)
 450                    != ceilval);
 451           }
 452         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 453
 454         assert (mutex->__data.__owner == 0);
 455         mutex->__data.__count = 1;
 456       }
 457       break;
 458
 459     default:
 460       /* Correct code cannot set any other type.  */
 461       return EINVAL;
 462     }
 463
 464   /* Record the ownership.  */
 465   mutex->__data.__owner = id;
 466 #ifndef NO_INCR
 467   ++mutex->__data.__nusers;
 468 #endif
 469
 470   return 0;
 471 }
 472 #ifndef __pthread_mutex_lock
 473 strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
 474 strong_alias (__pthread_mutex_lock, __pthread_mutex_lock_internal)
 475 #endif
 476
 477
 478 #ifdef NO_INCR
 479 void
 480 __pthread_mutex_cond_lock_adjust (mutex)
 481      pthread_mutex_t *mutex;
 482 {
 483   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
 484   assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
 485   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
 486
 487   /* Record the ownership.  */
 488   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 489   mutex->__data.__owner = id;
 490
 491   if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
 492     ++mutex->__data.__count;
 493 }
 494 #endif