nptl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002-2016 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <sys/param.h>
  24 #include <not-cancel.h>
  25 #include "pthreadP.h"
  26 #include <atomic.h>
  27 #include <lowlevellock.h>
  28 #include <stap-probe.h>
  29
  30 #ifndef lll_lock_elision
  31 #define lll_lock_elision(lock, try_lock, private)       ({ \
  32       lll_lock (lock, private); 0; })
  33 #endif
  34
  35 #ifndef lll_trylock_elision
  36 #define lll_trylock_elision(a,t) lll_trylock(a)
  37 #endif
  38
  39 #ifndef LLL_MUTEX_LOCK
  40 # define LLL_MUTEX_LOCK(mutex) \
  41   lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
  42 # define LLL_MUTEX_TRYLOCK(mutex) \
  43   lll_trylock ((mutex)->__data.__lock)
  44 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
  45   lll_robust_lock ((mutex)->__data.__lock, id, \
  46                    PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
  47 # define LLL_MUTEX_LOCK_ELISION(mutex) \
  48   lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
  49                    PTHREAD_MUTEX_PSHARED (mutex))
  50 # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
  51   lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
  52                    PTHREAD_MUTEX_PSHARED (mutex))
  53 #endif
  54
  55 #ifndef FORCE_ELISION
  56 #define FORCE_ELISION(m, s)
  57 #endif
  58
  59 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
  60      __attribute_noinline__;
  61
  62 int
  63 __pthread_mutex_lock (pthread_mutex_t *mutex)
  64 {
  65   assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
  66
  67   unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  68
  69   LIBC_PROBE (mutex_entry, 1, mutex);
  70
  71   if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
  72                                  | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
  73     return __pthread_mutex_lock_full (mutex);
  74
  75   if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
  76     {
  77       FORCE_ELISION (mutex, goto elision);
  78     simple:
  79       /* Normal mutex.  */
  80       LLL_MUTEX_LOCK (mutex);
  81       assert (mutex->__data.__owner == 0);
  82     }
  83 #ifdef HAVE_ELISION
  84   else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
  85     {
  86   elision: __attribute__((unused))
  87       /* This case can never happen on a system without elision,
  88          as the mutex type initialization functions will not
  89          allow to set the elision flags.  */
  90       /* Don't record owner or users for elision case.  This is a
  91          tail call.  */
  92       return LLL_MUTEX_LOCK_ELISION (mutex);
  93     }
  94 #endif
  95   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  96                              == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  97     {
  98       /* Recursive mutex.  */
  99       pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 100
 101       /* Check whether we already hold the mutex.  */
 102       if (mutex->__data.__owner == id)
 103         {
 104           /* Just bump the counter.  */
 105           if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 106             /* Overflow of the counter.  */
 107             return EAGAIN;
 108
 109           ++mutex->__data.__count;
 110
 111           return 0;
 112         }
 113
 114       /* We have to get the mutex.  */
 115       LLL_MUTEX_LOCK (mutex);
 116
 117       assert (mutex->__data.__owner == 0);
 118       mutex->__data.__count = 1;
 119     }
 120   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
 121                           == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
 122     {
 123       if (! __is_smp)
 124         goto simple;
 125
 126       if (LLL_MUTEX_TRYLOCK (mutex) != 0)
 127         {
 128           int cnt = 0;
 129           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
 130                              mutex->__data.__spins * 2 + 10);
 131           do
 132             {
 133               if (cnt++ >= max_cnt)
 134                 {
 135                   LLL_MUTEX_LOCK (mutex);
 136                   break;
 137                 }
 138               atomic_spin_nop ();
 139             }
 140           while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 141
 142           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 143         }
 144       assert (mutex->__data.__owner == 0);
 145     }
 146   else
 147     {
 148       pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 149       assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
 150       /* Check whether we already hold the mutex.  */
 151       if (__glibc_unlikely (mutex->__data.__owner == id))
 152         return EDEADLK;
 153       goto simple;
 154     }
 155
 156   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 157
 158   /* Record the ownership.  */
 159   mutex->__data.__owner = id;
 160 #ifndef NO_INCR
 161   ++mutex->__data.__nusers;
 162 #endif
 163
 164   LIBC_PROBE (mutex_acquired, 1, mutex);
 165
 166   return 0;
 167 }
 168
 169 static int
 170 __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 171 {
 172   int oldval;
 173   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 174
 175   switch (PTHREAD_MUTEX_TYPE (mutex))
 176     {
 177     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 178     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 179     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 180     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 181       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 182                      &mutex->__data.__list.__next);
 183
 184       oldval = mutex->__data.__lock;
 185       do
 186         {
 187         again:
 188           if ((oldval & FUTEX_OWNER_DIED) != 0)
 189             {
 190               /* The previous owner died.  Try locking the mutex.  */
 191               int newval = id;
 192 #ifdef NO_INCR
 193               newval |= FUTEX_WAITERS;
 194 #else
 195               newval |= (oldval & FUTEX_WAITERS);
 196 #endif
 197
 198               newval
 199                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 200                                                        newval, oldval);
 201
 202               if (newval != oldval)
 203                 {
 204                   oldval = newval;
 205                   goto again;
 206                 }
 207
 208               /* We got the mutex.  */
 209               mutex->__data.__count = 1;
 210               /* But it is inconsistent unless marked otherwise.  */
 211               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 212
 213               ENQUEUE_MUTEX (mutex);
 214               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 215
 216               /* Note that we deliberately exit here.  If we fall
 217                  through to the end of the function __nusers would be
 218                  incremented which is not correct because the old
 219                  owner has to be discounted.  If we are not supposed
 220                  to increment __nusers we actually have to decrement
 221                  it here.  */
 222 #ifdef NO_INCR
 223               --mutex->__data.__nusers;
 224 #endif
 225
 226               return EOWNERDEAD;
 227             }
 228
 229           /* Check whether we already hold the mutex.  */
 230           if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 231             {
 232               int kind = PTHREAD_MUTEX_TYPE (mutex);
 233               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 234                 {
 235                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 236                                  NULL);
 237                   return EDEADLK;
 238                 }
 239
 240               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 241                 {
 242                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 243                                  NULL);
 244
 245                   /* Just bump the counter.  */
 246                   if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 247                     /* Overflow of the counter.  */
 248                     return EAGAIN;
 249
 250                   ++mutex->__data.__count;
 251
 252                   return 0;
 253                 }
 254             }
 255
 256           oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id);
 257
 258           if (__builtin_expect (mutex->__data.__owner
 259                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 260             {
 261               /* This mutex is now not recoverable.  */
 262               mutex->__data.__count = 0;
 263               lll_unlock (mutex->__data.__lock,
 264                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 265               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 266               return ENOTRECOVERABLE;
 267             }
 268         }
 269       while ((oldval & FUTEX_OWNER_DIED) != 0);
 270
 271       mutex->__data.__count = 1;
 272       ENQUEUE_MUTEX (mutex);
 273       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 274       break;
 275
 276     /* The PI support requires the Linux futex system call.  If that's not
 277        available, pthread_mutex_init should never have allowed the type to
 278        be set.  So it will get the default case for an invalid type.  */
 279 #ifdef __NR_futex
 280     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 281     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 282     case PTHREAD_MUTEX_PI_NORMAL_NP:
 283     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 284     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 285     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 286     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 287     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 288       {
 289         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 290         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 291
 292         if (robust)
 293           /* Note: robust PI futexes are signaled by setting bit 0.  */
 294           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 295                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 296                                    | 1));
 297
 298         oldval = mutex->__data.__lock;
 299
 300         /* Check whether we already hold the mutex.  */
 301         if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 302           {
 303             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 304               {
 305                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 306                 return EDEADLK;
 307               }
 308
 309             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 310               {
 311                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 312
 313                 /* Just bump the counter.  */
 314                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 315                   /* Overflow of the counter.  */
 316                   return EAGAIN;
 317
 318                 ++mutex->__data.__count;
 319
 320                 return 0;
 321               }
 322           }
 323
 324         int newval = id;
 325 # ifdef NO_INCR
 326         newval |= FUTEX_WAITERS;
 327 # endif
 328         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 329                                                       newval, 0);
 330
 331         if (oldval != 0)
 332           {
 333             /* The mutex is locked.  The kernel will now take care of
 334                everything.  */
 335             int private = (robust
 336                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 337                            : PTHREAD_MUTEX_PSHARED (mutex));
 338             INTERNAL_SYSCALL_DECL (__err);
 339             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 340                                       __lll_private_flag (FUTEX_LOCK_PI,
 341                                                           private), 1, 0);
 342
 343             if (INTERNAL_SYSCALL_ERROR_P (e, __err)
 344                 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 345                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
 346               {
 347                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 348                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 349                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 350                 /* ESRCH can happen only for non-robust PI mutexes where
 351                    the owner of the lock died.  */
 352                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
 353
 354                 /* Delay the thread indefinitely.  */
 355                 while (1)
 356                   pause_not_cancel ();
 357               }
 358
 359             oldval = mutex->__data.__lock;
 360
 361             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 362           }
 363
 364         if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
 365           {
 366             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 367
 368             /* We got the mutex.  */
 369             mutex->__data.__count = 1;
 370             /* But it is inconsistent unless marked otherwise.  */
 371             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 372
 373             ENQUEUE_MUTEX_PI (mutex);
 374             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 375
 376             /* Note that we deliberately exit here.  If we fall
 377                through to the end of the function __nusers would be
 378                incremented which is not correct because the old owner
 379                has to be discounted.  If we are not supposed to
 380                increment __nusers we actually have to decrement it here.  */
 381 # ifdef NO_INCR
 382             --mutex->__data.__nusers;
 383 # endif
 384
 385             return EOWNERDEAD;
 386           }
 387
 388         if (robust
 389             && __builtin_expect (mutex->__data.__owner
 390                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 391           {
 392             /* This mutex is now not recoverable.  */
 393             mutex->__data.__count = 0;
 394
 395             INTERNAL_SYSCALL_DECL (__err);
 396             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 397                               __lll_private_flag (FUTEX_UNLOCK_PI,
 398                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
 399                               0, 0);
 400
 401             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 402             return ENOTRECOVERABLE;
 403           }
 404
 405         mutex->__data.__count = 1;
 406         if (robust)
 407           {
 408             ENQUEUE_MUTEX_PI (mutex);
 409             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 410           }
 411       }
 412       break;
 413 #endif  /* __NR_futex.  */
 414
 415     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 416     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 417     case PTHREAD_MUTEX_PP_NORMAL_NP:
 418     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 419       {
 420         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 421
 422         oldval = mutex->__data.__lock;
 423
 424         /* Check whether we already hold the mutex.  */
 425         if (mutex->__data.__owner == id)
 426           {
 427             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 428               return EDEADLK;
 429
 430             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 431               {
 432                 /* Just bump the counter.  */
 433                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 434                   /* Overflow of the counter.  */
 435                   return EAGAIN;
 436
 437                 ++mutex->__data.__count;
 438
 439                 return 0;
 440               }
 441           }
 442
 443         int oldprio = -1, ceilval;
 444         do
 445           {
 446             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 447                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 448
 449             if (__pthread_current_priority () > ceiling)
 450               {
 451                 if (oldprio != -1)
 452                   __pthread_tpp_change_priority (oldprio, -1);
 453                 return EINVAL;
 454               }
 455
 456             int retval = __pthread_tpp_change_priority (oldprio, ceiling);
 457             if (retval)
 458               return retval;
 459
 460             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 461             oldprio = ceiling;
 462
 463             oldval
 464               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 465 #ifdef NO_INCR
 466                                                      ceilval | 2,
 467 #else
 468                                                      ceilval | 1,
 469 #endif
 470                                                      ceilval);
 471
 472             if (oldval == ceilval)
 473               break;
 474
 475             do
 476               {
 477                 oldval
 478                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 479                                                          ceilval | 2,
 480                                                          ceilval | 1);
 481
 482                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 483                   break;
 484
 485                 if (oldval != ceilval)
 486                   lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
 487                                   PTHREAD_MUTEX_PSHARED (mutex));
 488               }
 489             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 490                                                         ceilval | 2, ceilval)
 491                    != ceilval);
 492           }
 493         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 494
 495         assert (mutex->__data.__owner == 0);
 496         mutex->__data.__count = 1;
 497       }
 498       break;
 499
 500     default:
 501       /* Correct code cannot set any other type.  */
 502       return EINVAL;
 503     }
 504
 505   /* Record the ownership.  */
 506   mutex->__data.__owner = id;
 507 #ifndef NO_INCR
 508   ++mutex->__data.__nusers;
 509 #endif
 510
 511   LIBC_PROBE (mutex_acquired, 1, mutex);
 512
 513   return 0;
 514 }
 515 #ifndef __pthread_mutex_lock
 516 strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
 517 hidden_def (__pthread_mutex_lock)
 518 #endif
 519
 520
 521 #ifdef NO_INCR
 522 void
 523 internal_function
 524 __pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
 525 {
 526   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
 527   assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
 528   assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
 529
 530   /* Record the ownership.  */
 531   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 532   mutex->__data.__owner = id;
 533
 534   if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
 535     ++mutex->__data.__count;
 536 }
 537 #endif