nptl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002-2014 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <sys/param.h>
  24 #include <not-cancel.h>
  25 #include "pthreadP.h"
  26 #include <lowlevellock.h>
  27 #include <stap-probe.h>
  28
  29 #ifndef lll_lock_elision
  30 #define lll_lock_elision(lock, try_lock, private)       ({ \
  31       lll_lock (lock, private); 0; })
  32 #endif
  33
  34 #ifndef lll_trylock_elision
  35 #define lll_trylock_elision(a,t) lll_trylock(a)
  36 #endif
  37
  38 #ifndef LLL_MUTEX_LOCK
  39 # define LLL_MUTEX_LOCK(mutex) \
  40   lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
  41 # define LLL_MUTEX_TRYLOCK(mutex) \
  42   lll_trylock ((mutex)->__data.__lock)
  43 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
  44   lll_robust_lock ((mutex)->__data.__lock, id, \
  45                    PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
  46 # define LLL_MUTEX_LOCK_ELISION(mutex) \
  47   lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
  48                    PTHREAD_MUTEX_PSHARED (mutex))
  49 # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
  50   lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
  51                    PTHREAD_MUTEX_PSHARED (mutex))
  52 #endif
  53
  54 #ifndef FORCE_ELISION
  55 #define FORCE_ELISION(m, s)
  56 #endif
  57
  58 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
  59      __attribute_noinline__;
  60
  61 int
  62 __pthread_mutex_lock (mutex)
  63      pthread_mutex_t *mutex;
  64 {
  65   assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
  66
  67   unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  68
  69   LIBC_PROBE (mutex_entry, 1, mutex);
  70
  71   if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
  72                                  | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
  73     return __pthread_mutex_lock_full (mutex);
  74
  75   if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
  76     {
  77       FORCE_ELISION (mutex, goto elision);
  78     simple:
  79       /* Normal mutex.  */
  80       LLL_MUTEX_LOCK (mutex);
  81       assert (mutex->__data.__owner == 0);
  82     }
  83 #ifdef HAVE_ELISION
  84   else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
  85     {
  86   elision: __attribute__((unused))
  87       /* This case can never happen on a system without elision,
  88          as the mutex type initialization functions will not
  89          allow to set the elision flags.  */
  90       /* Don't record owner or users for elision case.  This is a
  91          tail call.  */
  92       return LLL_MUTEX_LOCK_ELISION (mutex);
  93     }
  94 #endif
  95   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  96                              == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  97     {
  98       /* Recursive mutex.  */
  99       pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 100
 101       /* Check whether we already hold the mutex.  */
 102       if (mutex->__data.__owner == id)
 103         {
 104           /* Just bump the counter.  */
 105           if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 106             /* Overflow of the counter.  */
 107             return EAGAIN;
 108
 109           ++mutex->__data.__count;
 110
 111           return 0;
 112         }
 113
 114       /* We have to get the mutex.  */
 115       LLL_MUTEX_LOCK (mutex);
 116
 117       assert (mutex->__data.__owner == 0);
 118       mutex->__data.__count = 1;
 119     }
 120   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
 121                           == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
 122     {
 123       if (! __is_smp)
 124         goto simple;
 125
 126       if (LLL_MUTEX_TRYLOCK (mutex) != 0)
 127         {
 128           int cnt = 0;
 129           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
 130                              mutex->__data.__spins * 2 + 10);
 131           do
 132             {
 133               if (cnt++ >= max_cnt)
 134                 {
 135                   LLL_MUTEX_LOCK (mutex);
 136                   break;
 137                 }
 138
 139 #ifdef BUSY_WAIT_NOP
 140               BUSY_WAIT_NOP;
 141 #endif
 142             }
 143           while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 144
 145           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 146         }
 147       assert (mutex->__data.__owner == 0);
 148     }
 149   else
 150     {
 151       pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 152       assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
 153       /* Check whether we already hold the mutex.  */
 154       if (__glibc_unlikely (mutex->__data.__owner == id))
 155         return EDEADLK;
 156       goto simple;
 157     }
 158
 159   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 160
 161   /* Record the ownership.  */
 162   mutex->__data.__owner = id;
 163 #ifndef NO_INCR
 164   ++mutex->__data.__nusers;
 165 #endif
 166
 167   LIBC_PROBE (mutex_acquired, 1, mutex);
 168
 169   return 0;
 170 }
 171
 172 static int
 173 __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 174 {
 175   int oldval;
 176   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 177
 178   switch (PTHREAD_MUTEX_TYPE (mutex))
 179     {
 180     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 181     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 182     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 183     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 184       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 185                      &mutex->__data.__list.__next);
 186
 187       oldval = mutex->__data.__lock;
 188       do
 189         {
 190         again:
 191           if ((oldval & FUTEX_OWNER_DIED) != 0)
 192             {
 193               /* The previous owner died.  Try locking the mutex.  */
 194               int newval = id;
 195 #ifdef NO_INCR
 196               newval |= FUTEX_WAITERS;
 197 #else
 198               newval |= (oldval & FUTEX_WAITERS);
 199 #endif
 200
 201               newval
 202                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 203                                                        newval, oldval);
 204
 205               if (newval != oldval)
 206                 {
 207                   oldval = newval;
 208                   goto again;
 209                 }
 210
 211               /* We got the mutex.  */
 212               mutex->__data.__count = 1;
 213               /* But it is inconsistent unless marked otherwise.  */
 214               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 215
 216               ENQUEUE_MUTEX (mutex);
 217               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 218
 219               /* Note that we deliberately exit here.  If we fall
 220                  through to the end of the function __nusers would be
 221                  incremented which is not correct because the old
 222                  owner has to be discounted.  If we are not supposed
 223                  to increment __nusers we actually have to decrement
 224                  it here.  */
 225 #ifdef NO_INCR
 226               --mutex->__data.__nusers;
 227 #endif
 228
 229               return EOWNERDEAD;
 230             }
 231
 232           /* Check whether we already hold the mutex.  */
 233           if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 234             {
 235               int kind = PTHREAD_MUTEX_TYPE (mutex);
 236               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 237                 {
 238                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 239                                  NULL);
 240                   return EDEADLK;
 241                 }
 242
 243               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 244                 {
 245                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 246                                  NULL);
 247
 248                   /* Just bump the counter.  */
 249                   if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 250                     /* Overflow of the counter.  */
 251                     return EAGAIN;
 252
 253                   ++mutex->__data.__count;
 254
 255                   return 0;
 256                 }
 257             }
 258
 259           oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id);
 260
 261           if (__builtin_expect (mutex->__data.__owner
 262                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 263             {
 264               /* This mutex is now not recoverable.  */
 265               mutex->__data.__count = 0;
 266               lll_unlock (mutex->__data.__lock,
 267                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 268               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 269               return ENOTRECOVERABLE;
 270             }
 271         }
 272       while ((oldval & FUTEX_OWNER_DIED) != 0);
 273
 274       mutex->__data.__count = 1;
 275       ENQUEUE_MUTEX (mutex);
 276       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 277       break;
 278
 279     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 280     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 281     case PTHREAD_MUTEX_PI_NORMAL_NP:
 282     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 283     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 284     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 285     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 286     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 287       {
 288         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 289         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 290
 291         if (robust)
 292           /* Note: robust PI futexes are signaled by setting bit 0.  */
 293           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 294                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 295                                    | 1));
 296
 297         oldval = mutex->__data.__lock;
 298
 299         /* Check whether we already hold the mutex.  */
 300         if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 301           {
 302             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 303               {
 304                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 305                 return EDEADLK;
 306               }
 307
 308             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 309               {
 310                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 311
 312                 /* Just bump the counter.  */
 313                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 314                   /* Overflow of the counter.  */
 315                   return EAGAIN;
 316
 317                 ++mutex->__data.__count;
 318
 319                 return 0;
 320               }
 321           }
 322
 323         int newval = id;
 324 #ifdef NO_INCR
 325         newval |= FUTEX_WAITERS;
 326 #endif
 327         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 328                                                       newval, 0);
 329
 330         if (oldval != 0)
 331           {
 332             /* The mutex is locked.  The kernel will now take care of
 333                everything.  */
 334             int private = (robust
 335                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 336                            : PTHREAD_MUTEX_PSHARED (mutex));
 337             INTERNAL_SYSCALL_DECL (__err);
 338             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 339                                       __lll_private_flag (FUTEX_LOCK_PI,
 340                                                           private), 1, 0);
 341
 342             if (INTERNAL_SYSCALL_ERROR_P (e, __err)
 343                 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 344                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
 345               {
 346                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 347                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 348                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 349                 /* ESRCH can happen only for non-robust PI mutexes where
 350                    the owner of the lock died.  */
 351                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
 352
 353                 /* Delay the thread indefinitely.  */
 354                 while (1)
 355                   pause_not_cancel ();
 356               }
 357
 358             oldval = mutex->__data.__lock;
 359
 360             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 361           }
 362
 363         if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
 364           {
 365             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 366
 367             /* We got the mutex.  */
 368             mutex->__data.__count = 1;
 369             /* But it is inconsistent unless marked otherwise.  */
 370             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 371
 372             ENQUEUE_MUTEX_PI (mutex);
 373             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 374
 375             /* Note that we deliberately exit here.  If we fall
 376                through to the end of the function __nusers would be
 377                incremented which is not correct because the old owner
 378                has to be discounted.  If we are not supposed to
 379                increment __nusers we actually have to decrement it here.  */
 380 #ifdef NO_INCR
 381             --mutex->__data.__nusers;
 382 #endif
 383
 384             return EOWNERDEAD;
 385           }
 386
 387         if (robust
 388             && __builtin_expect (mutex->__data.__owner
 389                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 390           {
 391             /* This mutex is now not recoverable.  */
 392             mutex->__data.__count = 0;
 393
 394             INTERNAL_SYSCALL_DECL (__err);
 395             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 396                               __lll_private_flag (FUTEX_UNLOCK_PI,
 397                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 398                               0, 0);
 399
 400             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 401             return ENOTRECOVERABLE;
 402           }
 403
 404         mutex->__data.__count = 1;
 405         if (robust)
 406           {
 407             ENQUEUE_MUTEX_PI (mutex);
 408             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 409           }
 410       }
 411       break;
 412
 413     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 414     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 415     case PTHREAD_MUTEX_PP_NORMAL_NP:
 416     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 417       {
 418         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 419
 420         oldval = mutex->__data.__lock;
 421
 422         /* Check whether we already hold the mutex.  */
 423         if (mutex->__data.__owner == id)
 424           {
 425             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 426               return EDEADLK;
 427
 428             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 429               {
 430                 /* Just bump the counter.  */
 431                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 432                   /* Overflow of the counter.  */
 433                   return EAGAIN;
 434
 435                 ++mutex->__data.__count;
 436
 437                 return 0;
 438               }
 439           }
 440
 441         int oldprio = -1, ceilval;
 442         do
 443           {
 444             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 445                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 446
 447             if (__pthread_current_priority () > ceiling)
 448               {
 449                 if (oldprio != -1)
 450                   __pthread_tpp_change_priority (oldprio, -1);
 451                 return EINVAL;
 452               }
 453
 454             int retval = __pthread_tpp_change_priority (oldprio, ceiling);
 455             if (retval)
 456               return retval;
 457
 458             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 459             oldprio = ceiling;
 460
 461             oldval
 462               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 463 #ifdef NO_INCR
 464                                                      ceilval | 2,
 465 #else
 466                                                      ceilval | 1,
 467 #endif
 468                                                      ceilval);
 469
 470             if (oldval == ceilval)
 471               break;
 472
 473             do
 474               {
 475                 oldval
 476                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 477                                                          ceilval | 2,
 478                                                          ceilval | 1);
 479
 480                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 481                   break;
 482
 483                 if (oldval != ceilval)
 484                   lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
 485                                   PTHREAD_MUTEX_PSHARED (mutex));
 486               }
 487             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 488                                                         ceilval | 2, ceilval)
 489                    != ceilval);
 490           }
 491         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 492
 493         assert (mutex->__data.__owner == 0);
 494         mutex->__data.__count = 1;
 495       }
 496       break;
 497
 498     default:
 499       /* Correct code cannot set any other type.  */
 500       return EINVAL;
 501     }
 502
 503   /* Record the ownership.  */
 504   mutex->__data.__owner = id;
 505 #ifndef NO_INCR
 506   ++mutex->__data.__nusers;
 507 #endif
 508
 509   LIBC_PROBE (mutex_acquired, 1, mutex);
 510
 511   return 0;
 512 }
 513 #ifndef __pthread_mutex_lock
 514 strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
 515 hidden_def (__pthread_mutex_lock)
 516 #endif
 517
 518
 519 #ifdef NO_INCR
 520 void
 521 __pthread_mutex_cond_lock_adjust (mutex)
 522      pthread_mutex_t *mutex;
 523 {
 524   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
 525   assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
 526   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
 527
 528   /* Record the ownership.  */
 529   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 530   mutex->__data.__owner = id;
 531
 532   if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
 533     ++mutex->__data.__count;
 534 }
 535 #endif