nptl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002-2007, 2008 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <not-cancel.h>
  25 #include "pthreadP.h"
  26 #include <lowlevellock.h>
  27
  28
  29 #ifndef LLL_MUTEX_LOCK
  30 # define LLL_MUTEX_LOCK(mutex) \
  31   lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
  32 # define LLL_MUTEX_TRYLOCK(mutex) \
  33   lll_trylock ((mutex)->__data.__lock)
  34 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
  35   lll_robust_lock ((mutex)->__data.__lock, id, \
  36                    PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
  37 #endif
  38
  39
  40 int
  41 __pthread_mutex_lock (mutex)
  42      pthread_mutex_t *mutex;
  43 {
  44   assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
  45
  46   int oldval;
  47   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  48
  49   int retval = 0;
  50   switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex),
  51                             PTHREAD_MUTEX_TIMED_NP))
  52     {
  53       /* Recursive mutex.  */
  54     case PTHREAD_MUTEX_RECURSIVE_NP:
  55       /* Check whether we already hold the mutex.  */
  56       if (mutex->__data.__owner == id)
  57         {
  58           /* Just bump the counter.  */
  59           if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
  60             /* Overflow of the counter.  */
  61             return EAGAIN;
  62
  63           ++mutex->__data.__count;
  64
  65           return 0;
  66         }
  67
  68       /* We have to get the mutex.  */
  69       LLL_MUTEX_LOCK (mutex);
  70
  71       assert (mutex->__data.__owner == 0);
  72       mutex->__data.__count = 1;
  73       break;
  74
  75       /* Error checking mutex.  */
  76     case PTHREAD_MUTEX_ERRORCHECK_NP:
  77       /* Check whether we already hold the mutex.  */
  78       if (__builtin_expect (mutex->__data.__owner == id, 0))
  79         return EDEADLK;
  80
  81       /* FALLTHROUGH */
  82
  83     case PTHREAD_MUTEX_TIMED_NP:
  84     simple:
  85       /* Normal mutex.  */
  86       LLL_MUTEX_LOCK (mutex);
  87       assert (mutex->__data.__owner == 0);
  88       break;
  89
  90     case PTHREAD_MUTEX_ADAPTIVE_NP:
  91       if (! __is_smp)
  92         goto simple;
  93
  94       if (LLL_MUTEX_TRYLOCK (mutex) != 0)
  95         {
  96           int cnt = 0;
  97           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
  98                              mutex->__data.__spins * 2 + 10);
  99           do
 100             {
 101               if (cnt++ >= max_cnt)
 102                 {
 103                   LLL_MUTEX_LOCK (mutex);
 104                   break;
 105                 }
 106
 107 #ifdef BUSY_WAIT_NOP
 108               BUSY_WAIT_NOP;
 109 #endif
 110             }
 111           while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 112
 113           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 114         }
 115       assert (mutex->__data.__owner == 0);
 116       break;
 117
 118     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 119     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 120     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 121     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 122       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 123                      &mutex->__data.__list.__next);
 124
 125       oldval = mutex->__data.__lock;
 126       do
 127         {
 128         again:
 129           if ((oldval & FUTEX_OWNER_DIED) != 0)
 130             {
 131               /* The previous owner died.  Try locking the mutex.  */
 132               int newval = id;
 133 #ifdef NO_INCR
 134               newval |= FUTEX_WAITERS;
 135 #else
 136               newval |= (oldval & FUTEX_WAITERS);
 137 #endif
 138
 139               newval
 140                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 141                                                        newval, oldval);
 142
 143               if (newval != oldval)
 144                 {
 145                   oldval = newval;
 146                   goto again;
 147                 }
 148
 149               /* We got the mutex.  */
 150               mutex->__data.__count = 1;
 151               /* But it is inconsistent unless marked otherwise.  */
 152               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 153
 154               ENQUEUE_MUTEX (mutex);
 155               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 156
 157               /* Note that we deliberately exit here.  If we fall
 158                  through to the end of the function __nusers would be
 159                  incremented which is not correct because the old
 160                  owner has to be discounted.  If we are not supposed
 161                  to increment __nusers we actually have to decrement
 162                  it here.  */
 163 #ifdef NO_INCR
 164               --mutex->__data.__nusers;
 165 #endif
 166
 167               return EOWNERDEAD;
 168             }
 169
 170           /* Check whether we already hold the mutex.  */
 171           if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 172             {
 173               int kind = PTHREAD_MUTEX_TYPE (mutex);
 174               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 175                 {
 176                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 177                                  NULL);
 178                   return EDEADLK;
 179                 }
 180
 181               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 182                 {
 183                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 184                                  NULL);
 185
 186                   /* Just bump the counter.  */
 187                   if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 188                     /* Overflow of the counter.  */
 189                     return EAGAIN;
 190
 191                   ++mutex->__data.__count;
 192
 193                   return 0;
 194                 }
 195             }
 196
 197           oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id);
 198
 199           if (__builtin_expect (mutex->__data.__owner
 200                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 201             {
 202               /* This mutex is now not recoverable.  */
 203               mutex->__data.__count = 0;
 204               lll_unlock (mutex->__data.__lock,
 205                           PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 206               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 207               return ENOTRECOVERABLE;
 208             }
 209         }
 210       while ((oldval & FUTEX_OWNER_DIED) != 0);
 211
 212       mutex->__data.__count = 1;
 213       ENQUEUE_MUTEX (mutex);
 214       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 215       break;
 216
 217     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 218     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 219     case PTHREAD_MUTEX_PI_NORMAL_NP:
 220     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 221     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 222     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 223     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 224     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 225       {
 226         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 227         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 228
 229         if (robust)
 230           /* Note: robust PI futexes are signaled by setting bit 0.  */
 231           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 232                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 233                                    | 1));
 234
 235         oldval = mutex->__data.__lock;
 236
 237         /* Check whether we already hold the mutex.  */
 238         if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 239           {
 240             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 241               {
 242                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 243                 return EDEADLK;
 244               }
 245
 246             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 247               {
 248                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 249
 250                 /* Just bump the counter.  */
 251                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 252                   /* Overflow of the counter.  */
 253                   return EAGAIN;
 254
 255                 ++mutex->__data.__count;
 256
 257                 return 0;
 258               }
 259           }
 260
 261         int newval = id;
 262 #ifdef NO_INCR
 263         newval |= FUTEX_WAITERS;
 264 #endif
 265         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 266                                                       newval, 0);
 267
 268         if (oldval != 0)
 269           {
 270             /* The mutex is locked.  The kernel will now take care of
 271                everything.  */
 272             int private = (robust
 273                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 274                            : PTHREAD_MUTEX_PSHARED (mutex));
 275             INTERNAL_SYSCALL_DECL (__err);
 276             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 277                                       __lll_private_flag (FUTEX_LOCK_PI,
 278                                                           private), 1, 0);
 279
 280             if (INTERNAL_SYSCALL_ERROR_P (e, __err)
 281                 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 282                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
 283               {
 284                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 285                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 286                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 287                 /* ESRCH can happen only for non-robust PI mutexes where
 288                    the owner of the lock died.  */
 289                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
 290
 291                 /* Delay the thread indefinitely.  */
 292                 while (1)
 293                   pause_not_cancel ();
 294               }
 295
 296             oldval = mutex->__data.__lock;
 297
 298             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 299           }
 300
 301         if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
 302           {
 303             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 304
 305             /* We got the mutex.  */
 306             mutex->__data.__count = 1;
 307             /* But it is inconsistent unless marked otherwise.  */
 308             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 309
 310             ENQUEUE_MUTEX_PI (mutex);
 311             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 312
 313             /* Note that we deliberately exit here.  If we fall
 314                through to the end of the function __nusers would be
 315                incremented which is not correct because the old owner
 316                has to be discounted.  If we are not supposed to
 317                increment __nusers we actually have to decrement it here.  */
 318 #ifdef NO_INCR
 319             --mutex->__data.__nusers;
 320 #endif
 321
 322             return EOWNERDEAD;
 323           }
 324
 325         if (robust
 326             && __builtin_expect (mutex->__data.__owner
 327                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 328           {
 329             /* This mutex is now not recoverable.  */
 330             mutex->__data.__count = 0;
 331
 332             INTERNAL_SYSCALL_DECL (__err);
 333             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 334                               __lll_private_flag (FUTEX_UNLOCK_PI,
 335                                                   PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 336 ),
 337                               0, 0);
 338
 339             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 340             return ENOTRECOVERABLE;
 341           }
 342
 343         mutex->__data.__count = 1;
 344         if (robust)
 345           {
 346             ENQUEUE_MUTEX_PI (mutex);
 347             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 348           }
 349       }
 350       break;
 351
 352     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 353     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 354     case PTHREAD_MUTEX_PP_NORMAL_NP:
 355     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 356       {
 357         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 358
 359         oldval = mutex->__data.__lock;
 360
 361         /* Check whether we already hold the mutex.  */
 362         if (mutex->__data.__owner == id)
 363           {
 364             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 365               return EDEADLK;
 366
 367             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 368               {
 369                 /* Just bump the counter.  */
 370                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 371                   /* Overflow of the counter.  */
 372                   return EAGAIN;
 373
 374                 ++mutex->__data.__count;
 375
 376                 return 0;
 377               }
 378           }
 379
 380         int oldprio = -1, ceilval;
 381         do
 382           {
 383             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 384                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 385
 386             if (__pthread_current_priority () > ceiling)
 387               {
 388                 if (oldprio != -1)
 389                   __pthread_tpp_change_priority (oldprio, -1);
 390                 return EINVAL;
 391               }
 392
 393             retval = __pthread_tpp_change_priority (oldprio, ceiling);
 394             if (retval)
 395               return retval;
 396
 397             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 398             oldprio = ceiling;
 399
 400             oldval
 401               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 402 #ifdef NO_INCR
 403                                                      ceilval | 2,
 404 #else
 405                                                      ceilval | 1,
 406 #endif
 407                                                      ceilval);
 408
 409             if (oldval == ceilval)
 410               break;
 411
 412             do
 413               {
 414                 oldval
 415                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 416                                                          ceilval | 2,
 417                                                          ceilval | 1);
 418
 419                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 420                   break;
 421
 422                 if (oldval != ceilval)
 423                   lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
 424                                   PTHREAD_MUTEX_PSHARED (mutex));
 425               }
 426             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 427                                                         ceilval | 2, ceilval)
 428                    != ceilval);
 429           }
 430         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 431
 432         assert (mutex->__data.__owner == 0);
 433         mutex->__data.__count = 1;
 434       }
 435       break;
 436
 437     default:
 438       /* Correct code cannot set any other type.  */
 439       return EINVAL;
 440     }
 441
 442   /* Record the ownership.  */
 443   mutex->__data.__owner = id;
 444 #ifndef NO_INCR
 445   ++mutex->__data.__nusers;
 446 #endif
 447
 448   return retval;
 449 }
 450 #ifndef __pthread_mutex_lock
 451 strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
 452 strong_alias (__pthread_mutex_lock, __pthread_mutex_lock_internal)
 453 #endif