nptl/pthread_mutex_unlock.c

   1 /* Copyright (C) 2002-2017 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include "pthreadP.h"
  23 #include <lowlevellock.h>
  24 #include <stap-probe.h>
  25
  26 #ifndef lll_unlock_elision
  27 #define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
  28 #endif
  29
  30 static int
  31 internal_function
  32 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
  33      __attribute_noinline__;
  34
  35 int
  36 internal_function attribute_hidden
  37 __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
  38 {
  39   int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  40   if (__builtin_expect (type &
  41                 ~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
  42     return __pthread_mutex_unlock_full (mutex, decr);
  43
  44   if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
  45       == PTHREAD_MUTEX_TIMED_NP)
  46     {
  47       /* Always reset the owner field.  */
  48     normal:
  49       mutex->__data.__owner = 0;
  50       if (decr)
  51         /* One less user.  */
  52         --mutex->__data.__nusers;
  53
  54       /* Unlock.  */
  55       lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));
  56
  57       LIBC_PROBE (mutex_release, 1, mutex);
  58
  59       return 0;
  60     }
  61   else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
  62     {
  63       /* Don't reset the owner/users fields for elision.  */
  64       return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision,
  65                                       PTHREAD_MUTEX_PSHARED (mutex));
  66     }
  67   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  68                               == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  69     {
  70       /* Recursive mutex.  */
  71       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
  72         return EPERM;
  73
  74       if (--mutex->__data.__count != 0)
  75         /* We still hold the mutex.  */
  76         return 0;
  77       goto normal;
  78     }
  79   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  80                               == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
  81     goto normal;
  82   else
  83     {
  84       /* Error checking mutex.  */
  85       assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
  86       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
  87           || ! lll_islocked (mutex->__data.__lock))
  88         return EPERM;
  89       goto normal;
  90     }
  91 }
  92
  93
  94 static int
  95 internal_function
  96 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
  97 {
  98   int newowner = 0;
  99   int private;
 100
 101   switch (PTHREAD_MUTEX_TYPE (mutex))
 102     {
 103     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 104       /* Recursive mutex.  */
 105       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 106           == THREAD_GETMEM (THREAD_SELF, tid)
 107           && __builtin_expect (mutex->__data.__owner
 108                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 109         {
 110           if (--mutex->__data.__count != 0)
 111             /* We still hold the mutex.  */
 112             return ENOTRECOVERABLE;
 113
 114           goto notrecoverable;
 115         }
 116
 117       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 118         return EPERM;
 119
 120       if (--mutex->__data.__count != 0)
 121         /* We still hold the mutex.  */
 122         return 0;
 123
 124       goto robust;
 125
 126     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 127     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 128     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 129       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 130           != THREAD_GETMEM (THREAD_SELF, tid)
 131           || ! lll_islocked (mutex->__data.__lock))
 132         return EPERM;
 133
 134       /* If the previous owner died and the caller did not succeed in
 135          making the state consistent, mark the mutex as unrecoverable
 136          and make all waiters.  */
 137       if (__builtin_expect (mutex->__data.__owner
 138                             == PTHREAD_MUTEX_INCONSISTENT, 0))
 139       notrecoverable:
 140         newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 141
 142     robust:
 143       /* Remove mutex from the list.  */
 144       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 145                      &mutex->__data.__list.__next);
 146       /* We must set op_pending before we dequeue the mutex.  Also see
 147          comments at ENQUEUE_MUTEX.  */
 148       __asm ("" ::: "memory");
 149       DEQUEUE_MUTEX (mutex);
 150
 151       mutex->__data.__owner = newowner;
 152       if (decr)
 153         /* One less user.  */
 154         --mutex->__data.__nusers;
 155
 156       /* Unlock by setting the lock to 0 (not acquired); if the lock had
 157          FUTEX_WAITERS set previously, then wake any waiters.
 158          The unlock operation must be the last access to the mutex to not
 159          violate the mutex destruction requirements (see __lll_unlock).  */
 160       private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
 161       if (__glibc_unlikely ((atomic_exchange_rel (&mutex->__data.__lock, 0)
 162                              & FUTEX_WAITERS) != 0))
 163         lll_futex_wake (&mutex->__data.__lock, 1, private);
 164
 165       /* We must clear op_pending after we release the mutex.
 166          FIXME However, this violates the mutex destruction requirements
 167          because another thread could acquire the mutex, destroy it, and
 168          reuse the memory for something else; then, if this thread crashes,
 169          and the memory happens to have a value equal to the TID, the kernel
 170          will believe it is still related to the mutex (which has been
 171          destroyed already) and will modify some other random object.  */
 172       __asm ("" ::: "memory");
 173       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 174       break;
 175
 176     /* The PI support requires the Linux futex system call.  If that's not
 177        available, pthread_mutex_init should never have allowed the type to
 178        be set.  So it will get the default case for an invalid type.  */
 179 #ifdef __NR_futex
 180     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 181       /* Recursive mutex.  */
 182       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 183         return EPERM;
 184
 185       if (--mutex->__data.__count != 0)
 186         /* We still hold the mutex.  */
 187         return 0;
 188       goto continue_pi_non_robust;
 189
 190     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 191       /* Recursive mutex.  */
 192       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 193           == THREAD_GETMEM (THREAD_SELF, tid)
 194           && __builtin_expect (mutex->__data.__owner
 195                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 196         {
 197           if (--mutex->__data.__count != 0)
 198             /* We still hold the mutex.  */
 199             return ENOTRECOVERABLE;
 200
 201           goto pi_notrecoverable;
 202         }
 203
 204       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 205         return EPERM;
 206
 207       if (--mutex->__data.__count != 0)
 208         /* We still hold the mutex.  */
 209         return 0;
 210
 211       goto continue_pi_robust;
 212
 213     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 214     case PTHREAD_MUTEX_PI_NORMAL_NP:
 215     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 216     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 217     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 218     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 219       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 220           != THREAD_GETMEM (THREAD_SELF, tid)
 221           || ! lll_islocked (mutex->__data.__lock))
 222         return EPERM;
 223
 224       /* If the previous owner died and the caller did not succeed in
 225          making the state consistent, mark the mutex as unrecoverable
 226          and make all waiters.  */
 227       if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
 228           && __builtin_expect (mutex->__data.__owner
 229                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 230       pi_notrecoverable:
 231        newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 232
 233       if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
 234         {
 235         continue_pi_robust:
 236           /* Remove mutex from the list.
 237              Note: robust PI futexes are signaled by setting bit 0.  */
 238           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 239                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 240                                    | 1));
 241           /* We must set op_pending before we dequeue the mutex.  Also see
 242              comments at ENQUEUE_MUTEX.  */
 243           __asm ("" ::: "memory");
 244           DEQUEUE_MUTEX (mutex);
 245         }
 246
 247     continue_pi_non_robust:
 248       mutex->__data.__owner = newowner;
 249       if (decr)
 250         /* One less user.  */
 251         --mutex->__data.__nusers;
 252
 253       /* Unlock.  Load all necessary mutex data before releasing the mutex
 254          to not violate the mutex destruction requirements (see
 255          lll_unlock).  */
 256       int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 257       private = (robust
 258                  ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 259                  : PTHREAD_MUTEX_PSHARED (mutex));
 260       /* Unlock the mutex using a CAS unless there are futex waiters or our
 261          TID is not the value of __lock anymore, in which case we let the
 262          kernel take care of the situation.  Use release MO in the CAS to
 263          synchronize with acquire MO in lock acquisitions.  */
 264       int l = atomic_load_relaxed (&mutex->__data.__lock);
 265       do
 266         {
 267           if (((l & FUTEX_WAITERS) != 0)
 268               || (l != THREAD_GETMEM (THREAD_SELF, tid)))
 269             {
 270               INTERNAL_SYSCALL_DECL (__err);
 271               INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
 272                                 __lll_private_flag (FUTEX_UNLOCK_PI, private));
 273               break;
 274             }
 275         }
 276       while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
 277                                                     &l, 0));
 278
 279       /* This happens after the kernel releases the mutex but violates the
 280          mutex destruction requirements; see comments in the code handling
 281          PTHREAD_MUTEX_ROBUST_NORMAL_NP.  */
 282       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 283       break;
 284 #endif  /* __NR_futex.  */
 285
 286     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 287       /* Recursive mutex.  */
 288       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 289         return EPERM;
 290
 291       if (--mutex->__data.__count != 0)
 292         /* We still hold the mutex.  */
 293         return 0;
 294       goto pp;
 295
 296     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 297       /* Error checking mutex.  */
 298       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
 299           || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
 300         return EPERM;
 301       /* FALLTHROUGH */
 302
 303     case PTHREAD_MUTEX_PP_NORMAL_NP:
 304     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 305       /* Always reset the owner field.  */
 306     pp:
 307       mutex->__data.__owner = 0;
 308
 309       if (decr)
 310         /* One less user.  */
 311         --mutex->__data.__nusers;
 312
 313       /* Unlock.  Use release MO in the CAS to synchronize with acquire MO in
 314          lock acquisitions.  */
 315       int newval;
 316       int oldval = atomic_load_relaxed (&mutex->__data.__lock);
 317       do
 318         {
 319           newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
 320         }
 321       while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
 322                                                     &oldval, newval));
 323
 324       if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
 325         lll_futex_wake (&mutex->__data.__lock, 1,
 326                         PTHREAD_MUTEX_PSHARED (mutex));
 327
 328       int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 329
 330       LIBC_PROBE (mutex_release, 1, mutex);
 331
 332       return __pthread_tpp_change_priority (oldprio, -1);
 333
 334     default:
 335       /* Correct code cannot set any other type.  */
 336       return EINVAL;
 337     }
 338
 339   LIBC_PROBE (mutex_release, 1, mutex);
 340   return 0;
 341 }
 342
 343
 344 int
 345 __pthread_mutex_unlock (pthread_mutex_t *mutex)
 346 {
 347   return __pthread_mutex_unlock_usercnt (mutex, 1);
 348 }
 349 weak_alias (__pthread_mutex_unlock, pthread_mutex_unlock)
 350 hidden_def (__pthread_mutex_unlock)