nptl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002-2012 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <not-cancel.h>
  24 #include "pthreadP.h"
  25 #include <lowlevellock.h>
  26 #include <stap-probe.h>
  27
  28
  29 #ifndef LLL_MUTEX_LOCK
  30 # define LLL_MUTEX_LOCK(mutex) \
  31   lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
  32 # define LLL_MUTEX_TRYLOCK(mutex) \
  33   lll_trylock ((mutex)->__data.__lock)
  34 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
  35   lll_robust_lock ((mutex)->__data.__lock, id, \
  36                    PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
  37 #endif
  38
  39
  40 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
  41      __attribute_noinline__;
  42
  43
  44 int
  45 __pthread_mutex_lock (mutex)
  46      pthread_mutex_t *mutex;
  47 {
  48   assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
  49
  50   unsigned int type = PTHREAD_MUTEX_TYPE (mutex);
  51
  52   LIBC_PROBE (mutex_entry, 1, mutex);
  53
  54   if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0))
  55     return __pthread_mutex_lock_full (mutex);
  56
  57   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  58
  59   if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
  60       == PTHREAD_MUTEX_TIMED_NP)
  61     {
  62     simple:
  63       /* Normal mutex.  */
  64       LLL_MUTEX_LOCK (mutex);
  65       assert (mutex->__data.__owner == 0);
  66     }
  67   else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  68     {
  69       /* Recursive mutex.  */
  70
  71       /* Check whether we already hold the mutex.  */
  72       if (mutex->__data.__owner == id)
  73         {
  74           /* Just bump the counter.  */
  75           if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
  76             /* Overflow of the counter.  */
  77             return EAGAIN;
  78
  79           ++mutex->__data.__count;
  80
  81           return 0;
  82         }
  83
  84       /* We have to get the mutex.  */
  85       LLL_MUTEX_LOCK (mutex);
  86
  87       assert (mutex->__data.__owner == 0);
  88       mutex->__data.__count = 1;
  89     }
  90   else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
  91     {
  92       if (! __is_smp)
  93         goto simple;
  94
  95       if (LLL_MUTEX_TRYLOCK (mutex) != 0)
  96         {
  97           int cnt = 0;
  98           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
  99                              mutex->__data.__spins * 2 + 10);
 100           do
 101             {
 102               if (cnt++ >= max_cnt)
 103                 {
 104                   LLL_MUTEX_LOCK (mutex);
 105                   break;
 106                 }
 107
 108 #ifdef BUSY_WAIT_NOP
 109               BUSY_WAIT_NOP;
 110 #endif
 111             }
 112           while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 113
 114           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 115         }
 116       assert (mutex->__data.__owner == 0);
 117     }
 118   else
 119     {
 120       assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
 121       /* Check whether we already hold the mutex.  */
 122       if (__builtin_expect (mutex->__data.__owner == id, 0))
 123         return EDEADLK;
 124       goto simple;
 125     }
 126
 127   /* Record the ownership.  */
 128   mutex->__data.__owner = id;
 129 #ifndef NO_INCR
 130   ++mutex->__data.__nusers;
 131 #endif
 132
 133   LIBC_PROBE (mutex_acquired, 1, mutex);
 134
 135   return 0;
 136 }
 137
 138 static int
 139 __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 140 {
 141   int oldval;
 142   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 143
 144   switch (PTHREAD_MUTEX_TYPE (mutex))
 145     {
 146     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 147     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 148     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 149     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 150       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 151                      &mutex->__data.__list.__next);
 152
 153       oldval = mutex->__data.__lock;
 154       do
 155         {
 156         again:
 157           if ((oldval & FUTEX_OWNER_DIED) != 0)
 158             {
 159               /* The previous owner died.  Try locking the mutex.  */
 160               int newval = id;
 161 #ifdef NO_INCR
 162               newval |= FUTEX_WAITERS;
 163 #else
 164               newval |= (oldval & FUTEX_WAITERS);
 165 #endif
 166
 167               newval
 168                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 169                                                        newval, oldval);
 170
 171               if (newval != oldval)
 172                 {
 173                   oldval = newval;
 174                   goto again;
 175                 }
 176
 177               /* We got the mutex.  */
 178               mutex->__data.__count = 1;
 179               /* But it is inconsistent unless marked otherwise.  */
 180               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 181
 182               ENQUEUE_MUTEX (mutex);
 183               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 184
 185               /* Note that we deliberately exit here.  If we fall
 186                  through to the end of the function __nusers would be
 187                  incremented which is not correct because the old
 188                  owner has to be discounted.  If we are not supposed
 189                  to increment __nusers we actually have to decrement
 190                  it here.  */
 191 #ifdef NO_INCR
 192               --mutex->__data.__nusers;
 193 #endif
 194
 195               return EOWNERDEAD;
 196             }
 197
 198           /* Check whether we already hold the mutex.  */
 199           if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 200             {
 201               int kind = PTHREAD_MUTEX_TYPE (mutex);
 202               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 203                 {
 204                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 205                                  NULL);
 206                   return EDEADLK;
 207                 }
 208
 209               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 210                 {
 211                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 212                                  NULL);
 213
 214                   /* Just bump the counter.  */
 215                   if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 216                     /* Overflow of the counter.  */
 217                     return EAGAIN;
 218
 219                   ++mutex->__data.__count;
 220
 221                   return 0;
 222                 }
 223             }
 224
 225           oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id);
 226
 227           if (__builtin_expect (mutex->__data.__owner
 228                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 229             {
 230               /* This mutex is now not recoverable.  */
 231               mutex->__data.__count = 0;
 232               lll_unlock (mutex->__data.__lock,
 233                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 234               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 235               return ENOTRECOVERABLE;
 236             }
 237         }
 238       while ((oldval & FUTEX_OWNER_DIED) != 0);
 239
 240       mutex->__data.__count = 1;
 241       ENQUEUE_MUTEX (mutex);
 242       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 243       break;
 244
 245     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 246     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 247     case PTHREAD_MUTEX_PI_NORMAL_NP:
 248     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 249     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 250     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 251     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 252     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 253       {
 254         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 255         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 256
 257         if (robust)
 258           /* Note: robust PI futexes are signaled by setting bit 0.  */
 259           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 260                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 261                                    | 1));
 262
 263         oldval = mutex->__data.__lock;
 264
 265         /* Check whether we already hold the mutex.  */
 266         if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 267           {
 268             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 269               {
 270                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 271                 return EDEADLK;
 272               }
 273
 274             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 275               {
 276                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 277
 278                 /* Just bump the counter.  */
 279                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 280                   /* Overflow of the counter.  */
 281                   return EAGAIN;
 282
 283                 ++mutex->__data.__count;
 284
 285                 return 0;
 286               }
 287           }
 288
 289         int newval = id;
 290 #ifdef NO_INCR
 291         newval |= FUTEX_WAITERS;
 292 #endif
 293         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 294                                                       newval, 0);
 295
 296         if (oldval != 0)
 297           {
 298             /* The mutex is locked.  The kernel will now take care of
 299                everything.  */
 300             int private = (robust
 301                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 302                            : PTHREAD_MUTEX_PSHARED (mutex));
 303             INTERNAL_SYSCALL_DECL (__err);
 304             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 305                                       __lll_private_flag (FUTEX_LOCK_PI,
 306                                                           private), 1, 0);
 307
 308             if (INTERNAL_SYSCALL_ERROR_P (e, __err)
 309                 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 310                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
 311               {
 312                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 313                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 314                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 315                 /* ESRCH can happen only for non-robust PI mutexes where
 316                    the owner of the lock died.  */
 317                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
 318
 319                 /* Delay the thread indefinitely.  */
 320                 while (1)
 321                   pause_not_cancel ();
 322               }
 323
 324             oldval = mutex->__data.__lock;
 325
 326             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 327           }
 328
 329         if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
 330           {
 331             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 332
 333             /* We got the mutex.  */
 334             mutex->__data.__count = 1;
 335             /* But it is inconsistent unless marked otherwise.  */
 336             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 337
 338             ENQUEUE_MUTEX_PI (mutex);
 339             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 340
 341             /* Note that we deliberately exit here.  If we fall
 342                through to the end of the function __nusers would be
 343                incremented which is not correct because the old owner
 344                has to be discounted.  If we are not supposed to
 345                increment __nusers we actually have to decrement it here.  */
 346 #ifdef NO_INCR
 347             --mutex->__data.__nusers;
 348 #endif
 349
 350             return EOWNERDEAD;
 351           }
 352
 353         if (robust
 354             && __builtin_expect (mutex->__data.__owner
 355                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 356           {
 357             /* This mutex is now not recoverable.  */
 358             mutex->__data.__count = 0;
 359
 360             INTERNAL_SYSCALL_DECL (__err);
 361             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 362                               __lll_private_flag (FUTEX_UNLOCK_PI,
 363                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 364                               0, 0);
 365
 366             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 367             return ENOTRECOVERABLE;
 368           }
 369
 370         mutex->__data.__count = 1;
 371         if (robust)
 372           {
 373             ENQUEUE_MUTEX_PI (mutex);
 374             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 375           }
 376       }
 377       break;
 378
 379     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 380     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 381     case PTHREAD_MUTEX_PP_NORMAL_NP:
 382     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 383       {
 384         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 385
 386         oldval = mutex->__data.__lock;
 387
 388         /* Check whether we already hold the mutex.  */
 389         if (mutex->__data.__owner == id)
 390           {
 391             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 392               return EDEADLK;
 393
 394             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 395               {
 396                 /* Just bump the counter.  */
 397                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 398                   /* Overflow of the counter.  */
 399                   return EAGAIN;
 400
 401                 ++mutex->__data.__count;
 402
 403                 return 0;
 404               }
 405           }
 406
 407         int oldprio = -1, ceilval;
 408         do
 409           {
 410             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 411                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 412
 413             if (__pthread_current_priority () > ceiling)
 414               {
 415                 if (oldprio != -1)
 416                   __pthread_tpp_change_priority (oldprio, -1);
 417                 return EINVAL;
 418               }
 419
 420             int retval = __pthread_tpp_change_priority (oldprio, ceiling);
 421             if (retval)
 422               return retval;
 423
 424             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 425             oldprio = ceiling;
 426
 427             oldval
 428               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 429 #ifdef NO_INCR
 430                                                      ceilval | 2,
 431 #else
 432                                                      ceilval | 1,
 433 #endif
 434                                                      ceilval);
 435
 436             if (oldval == ceilval)
 437               break;
 438
 439             do
 440               {
 441                 oldval
 442                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 443                                                          ceilval | 2,
 444                                                          ceilval | 1);
 445
 446                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 447                   break;
 448
 449                 if (oldval != ceilval)
 450                   lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
 451                                   PTHREAD_MUTEX_PSHARED (mutex));
 452               }
 453             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 454                                                         ceilval | 2, ceilval)
 455                    != ceilval);
 456           }
 457         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 458
 459         assert (mutex->__data.__owner == 0);
 460         mutex->__data.__count = 1;
 461       }
 462       break;
 463
 464     default:
 465       /* Correct code cannot set any other type.  */
 466       return EINVAL;
 467     }
 468
 469   /* Record the ownership.  */
 470   mutex->__data.__owner = id;
 471 #ifndef NO_INCR
 472   ++mutex->__data.__nusers;
 473 #endif
 474
 475   LIBC_PROBE (mutex_acquired, 1, mutex);
 476
 477   return 0;
 478 }
 479 #ifndef __pthread_mutex_lock
 480 strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
 481 hidden_def (__pthread_mutex_lock)
 482 #endif
 483
 484
 485 #ifdef NO_INCR
 486 void
 487 __pthread_mutex_cond_lock_adjust (mutex)
 488      pthread_mutex_t *mutex;
 489 {
 490   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
 491   assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
 492   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
 493
 494   /* Record the ownership.  */
 495   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 496   mutex->__data.__owner = id;
 497
 498   if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
 499     ++mutex->__data.__count;
 500 }
 501 #endif