nptl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <not-cancel.h>
  25 #include "pthreadP.h"
  26 #include <lowlevellock.h>
  27
  28
  29 #ifndef LLL_MUTEX_LOCK
  30 # define LLL_MUTEX_LOCK(mutex) lll_mutex_lock (mutex)
  31 # define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_trylock (mutex)
  32 # define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_mutex_lock (mutex, id)
  33 #endif
  34
  35
  36 int
  37 __pthread_mutex_lock (mutex)
  38      pthread_mutex_t *mutex;
  39 {
  40   assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
  41
  42   int oldval;
  43   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  44
  45   int retval = 0;
  46   switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex),
  47                             PTHREAD_MUTEX_TIMED_NP))
  48     {
  49       /* Recursive mutex.  */
  50     case PTHREAD_MUTEX_RECURSIVE_NP:
  51       /* Check whether we already hold the mutex.  */
  52       if (mutex->__data.__owner == id)
  53         {
  54           /* Just bump the counter.  */
  55           if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
  56             /* Overflow of the counter.  */
  57             return EAGAIN;
  58
  59           ++mutex->__data.__count;
  60
  61           return 0;
  62         }
  63
  64       /* We have to get the mutex.  */
  65       LLL_MUTEX_LOCK (mutex->__data.__lock);
  66
  67       assert (mutex->__data.__owner == 0);
  68       mutex->__data.__count = 1;
  69       break;
  70
  71       /* Error checking mutex.  */
  72     case PTHREAD_MUTEX_ERRORCHECK_NP:
  73       /* Check whether we already hold the mutex.  */
  74       if (__builtin_expect (mutex->__data.__owner == id, 0))
  75         return EDEADLK;
  76
  77       /* FALLTHROUGH */
  78
  79     case PTHREAD_MUTEX_TIMED_NP:
  80     simple:
  81       /* Normal mutex.  */
  82       LLL_MUTEX_LOCK (mutex->__data.__lock);
  83       assert (mutex->__data.__owner == 0);
  84       break;
  85
  86     case PTHREAD_MUTEX_ADAPTIVE_NP:
  87       if (! __is_smp)
  88         goto simple;
  89
  90       if (LLL_MUTEX_TRYLOCK (mutex->__data.__lock) != 0)
  91         {
  92           int cnt = 0;
  93           int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
  94                              mutex->__data.__spins * 2 + 10);
  95           do
  96             {
  97               if (cnt++ >= max_cnt)
  98                 {
  99                   LLL_MUTEX_LOCK (mutex->__data.__lock);
 100                   break;
 101                 }
 102
 103 #ifdef BUSY_WAIT_NOP
 104               BUSY_WAIT_NOP;
 105 #endif
 106             }
 107           while (LLL_MUTEX_TRYLOCK (mutex->__data.__lock) != 0);
 108
 109           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 110         }
 111       assert (mutex->__data.__owner == 0);
 112       break;
 113
 114     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 115     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 116     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 117     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 118       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 119                      &mutex->__data.__list.__next);
 120
 121       oldval = mutex->__data.__lock;
 122       do
 123         {
 124         again:
 125           if ((oldval & FUTEX_OWNER_DIED) != 0)
 126             {
 127               /* The previous owner died.  Try locking the mutex.  */
 128               int newval = id;
 129 #ifdef NO_INCR
 130               newval |= FUTEX_WAITERS;
 131 #else
 132               newval |= (oldval & FUTEX_WAITERS);
 133 #endif
 134
 135               newval
 136                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 137                                                        newval, oldval);
 138
 139               if (newval != oldval)
 140                 {
 141                   oldval = newval;
 142                   goto again;
 143                 }
 144
 145               /* We got the mutex.  */
 146               mutex->__data.__count = 1;
 147               /* But it is inconsistent unless marked otherwise.  */
 148               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 149
 150               ENQUEUE_MUTEX (mutex);
 151               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 152
 153               /* Note that we deliberately exit here.  If we fall
 154                  through to the end of the function __nusers would be
 155                  incremented which is not correct because the old
 156                  owner has to be discounted.  If we are not supposed
 157                  to increment __nusers we actually have to decrement
 158                  it here.  */
 159 #ifdef NO_INCR
 160               --mutex->__data.__nusers;
 161 #endif
 162
 163               return EOWNERDEAD;
 164             }
 165
 166           /* Check whether we already hold the mutex.  */
 167           if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 168             {
 169               if (mutex->__data.__kind
 170                   == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 171                 {
 172                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 173                                  NULL);
 174                   return EDEADLK;
 175                 }
 176
 177               if (mutex->__data.__kind
 178                   == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 179                 {
 180                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 181                                  NULL);
 182
 183                   /* Just bump the counter.  */
 184                   if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 185                     /* Overflow of the counter.  */
 186                     return EAGAIN;
 187
 188                   ++mutex->__data.__count;
 189
 190                   return 0;
 191                 }
 192             }
 193
 194           oldval = LLL_ROBUST_MUTEX_LOCK (mutex->__data.__lock, id);
 195
 196           if (__builtin_expect (mutex->__data.__owner
 197                                 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 198             {
 199               /* This mutex is now not recoverable.  */
 200               mutex->__data.__count = 0;
 201               lll_mutex_unlock (mutex->__data.__lock);
 202               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 203               return ENOTRECOVERABLE;
 204             }
 205         }
 206       while ((oldval & FUTEX_OWNER_DIED) != 0);
 207
 208       mutex->__data.__count = 1;
 209       ENQUEUE_MUTEX (mutex);
 210       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 211       break;
 212
 213     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 214     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 215     case PTHREAD_MUTEX_PI_NORMAL_NP:
 216     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 217     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 218     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 219     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 220     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 221       {
 222         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 223         int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 224
 225         if (robust)
 226           /* Note: robust PI futexes are signaled by setting bit 0.  */
 227           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 228                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 229                                    | 1));
 230
 231         oldval = mutex->__data.__lock;
 232
 233         /* Check whether we already hold the mutex.  */
 234         if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 235           {
 236             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 237               {
 238                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 239                 return EDEADLK;
 240               }
 241
 242             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 243               {
 244                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 245
 246                 /* Just bump the counter.  */
 247                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 248                   /* Overflow of the counter.  */
 249                   return EAGAIN;
 250
 251                 ++mutex->__data.__count;
 252
 253                 return 0;
 254               }
 255           }
 256
 257         int newval = id;
 258 #ifdef NO_INCR
 259         newval |= FUTEX_WAITERS;
 260 #endif
 261         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 262                                                       newval, 0);
 263
 264         if (oldval != 0)
 265           {
 266             /* The mutex is locked.  The kernel will now take care of
 267                everything.  */
 268             INTERNAL_SYSCALL_DECL (__err);
 269             int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 270                                       FUTEX_LOCK_PI, 1, 0);
 271
 272             if (INTERNAL_SYSCALL_ERROR_P (e, __err)
 273                 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
 274                     || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
 275               {
 276                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
 277                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 278                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 279                 /* ESRCH can happen only for non-robust PI mutexes where
 280                    the owner of the lock died.  */
 281                 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
 282
 283                 /* Delay the thread indefinitely.  */
 284                 while (1)
 285                   pause_not_cancel ();
 286               }
 287
 288             oldval = mutex->__data.__lock;
 289
 290             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 291           }
 292
 293         if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
 294           {
 295             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 296
 297             /* We got the mutex.  */
 298             mutex->__data.__count = 1;
 299             /* But it is inconsistent unless marked otherwise.  */
 300             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 301
 302             ENQUEUE_MUTEX_PI (mutex);
 303             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 304
 305             /* Note that we deliberately exit here.  If we fall
 306                through to the end of the function __nusers would be
 307                incremented which is not correct because the old owner
 308                has to be discounted.  If we are not supposed to
 309                increment __nusers we actually have to decrement it here.  */
 310 #ifdef NO_INCR
 311             --mutex->__data.__nusers;
 312 #endif
 313
 314             return EOWNERDEAD;
 315           }
 316
 317         if (robust
 318             && __builtin_expect (mutex->__data.__owner
 319                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 320           {
 321             /* This mutex is now not recoverable.  */
 322             mutex->__data.__count = 0;
 323
 324             INTERNAL_SYSCALL_DECL (__err);
 325             INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
 326                               FUTEX_UNLOCK_PI, 0, 0);
 327
 328             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 329             return ENOTRECOVERABLE;
 330           }
 331
 332         mutex->__data.__count = 1;
 333         if (robust)
 334           {
 335             ENQUEUE_MUTEX_PI (mutex);
 336             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 337           }
 338       }
 339       break;
 340
 341     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 342     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 343     case PTHREAD_MUTEX_PP_NORMAL_NP:
 344     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 345       {
 346         int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
 347
 348         oldval = mutex->__data.__lock;
 349
 350         /* Check whether we already hold the mutex.  */
 351         if (mutex->__data.__owner == id)
 352           {
 353             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 354               return EDEADLK;
 355
 356             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 357               {
 358                 /* Just bump the counter.  */
 359                 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
 360                   /* Overflow of the counter.  */
 361                   return EAGAIN;
 362
 363                 ++mutex->__data.__count;
 364
 365                 return 0;
 366               }
 367           }
 368
 369         int oldprio = -1, ceilval;
 370         do
 371           {
 372             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 373                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 374
 375             if (__pthread_current_priority () > ceiling)
 376               {
 377                 if (oldprio != -1)
 378                   __pthread_tpp_change_priority (oldprio, -1);
 379                 return EINVAL;
 380               }
 381
 382             retval = __pthread_tpp_change_priority (oldprio, ceiling);
 383             if (retval)
 384               return retval;
 385
 386             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 387             oldprio = ceiling;
 388
 389             oldval
 390               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 391 #ifdef NO_INCR
 392                                                      ceilval | 2,
 393 #else
 394                                                      ceilval | 1,
 395 #endif
 396                                                      ceilval);
 397
 398             if (oldval == ceilval)
 399               break;
 400
 401             do
 402               {
 403                 oldval
 404                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 405                                                          ceilval | 2,
 406                                                          ceilval | 1);
 407
 408                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 409                   break;
 410
 411                 if (oldval != ceilval)
 412                   lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
 413                                   // XYZ check mutex flag
 414                                   LLL_SHARED);
 415               }
 416             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 417                                                         ceilval | 2, ceilval)
 418                    != ceilval);
 419           }
 420         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 421
 422         assert (mutex->__data.__owner == 0);
 423         mutex->__data.__count = 1;
 424       }
 425       break;
 426
 427     default:
 428       /* Correct code cannot set any other type.  */
 429       return EINVAL;
 430     }
 431
 432   /* Record the ownership.  */
 433   mutex->__data.__owner = id;
 434 #ifndef NO_INCR
 435   ++mutex->__data.__nusers;
 436 #endif
 437
 438   return retval;
 439 }
 440 #ifndef __pthread_mutex_lock
 441 strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
 442 strong_alias (__pthread_mutex_lock, __pthread_mutex_lock_internal)
 443 #endif