nptl/pthread_mutex_unlock.c

   1 /* Copyright (C) 2002-2016 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include "pthreadP.h"
  23 #include <lowlevellock.h>
  24 #include <stap-probe.h>
  25
  26 #ifndef lll_unlock_elision
  27 #define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
  28 #endif
  29
  30 static int
  31 internal_function
  32 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
  33      __attribute_noinline__;
  34
  35 int
  36 internal_function attribute_hidden
  37 __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
  38 {
  39   int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  40   if (__builtin_expect (type &
  41                 ~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
  42     return __pthread_mutex_unlock_full (mutex, decr);
  43
  44   if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
  45       == PTHREAD_MUTEX_TIMED_NP)
  46     {
  47       /* Always reset the owner field.  */
  48     normal:
  49       mutex->__data.__owner = 0;
  50       if (decr)
  51         /* One less user.  */
  52         --mutex->__data.__nusers;
  53
  54       /* Unlock.  */
  55       lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));
  56
  57       LIBC_PROBE (mutex_release, 1, mutex);
  58
  59       return 0;
  60     }
  61   else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
  62     {
  63       /* Don't reset the owner/users fields for elision.  */
  64       return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision,
  65                                       PTHREAD_MUTEX_PSHARED (mutex));
  66     }
  67   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  68                               == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  69     {
  70       /* Recursive mutex.  */
  71       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
  72         return EPERM;
  73
  74       if (--mutex->__data.__count != 0)
  75         /* We still hold the mutex.  */
  76         return 0;
  77       goto normal;
  78     }
  79   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  80                               == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
  81     goto normal;
  82   else
  83     {
  84       /* Error checking mutex.  */
  85       assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
  86       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
  87           || ! lll_islocked (mutex->__data.__lock))
  88         return EPERM;
  89       goto normal;
  90     }
  91 }
  92
  93
  94 static int
  95 internal_function
  96 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
  97 {
  98   int newowner = 0;
  99
 100   switch (PTHREAD_MUTEX_TYPE (mutex))
 101     {
 102     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 103       /* Recursive mutex.  */
 104       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 105           == THREAD_GETMEM (THREAD_SELF, tid)
 106           && __builtin_expect (mutex->__data.__owner
 107                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 108         {
 109           if (--mutex->__data.__count != 0)
 110             /* We still hold the mutex.  */
 111             return ENOTRECOVERABLE;
 112
 113           goto notrecoverable;
 114         }
 115
 116       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 117         return EPERM;
 118
 119       if (--mutex->__data.__count != 0)
 120         /* We still hold the mutex.  */
 121         return 0;
 122
 123       goto robust;
 124
 125     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 126     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 127     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 128       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 129           != THREAD_GETMEM (THREAD_SELF, tid)
 130           || ! lll_islocked (mutex->__data.__lock))
 131         return EPERM;
 132
 133       /* If the previous owner died and the caller did not succeed in
 134          making the state consistent, mark the mutex as unrecoverable
 135          and make all waiters.  */
 136       if (__builtin_expect (mutex->__data.__owner
 137                             == PTHREAD_MUTEX_INCONSISTENT, 0))
 138       notrecoverable:
 139         newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 140
 141     robust:
 142       /* Remove mutex from the list.  */
 143       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 144                      &mutex->__data.__list.__next);
 145       DEQUEUE_MUTEX (mutex);
 146
 147       mutex->__data.__owner = newowner;
 148       if (decr)
 149         /* One less user.  */
 150         --mutex->__data.__nusers;
 151
 152       /* Unlock.  */
 153       lll_robust_unlock (mutex->__data.__lock,
 154                          PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 155
 156       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 157       break;
 158
 159     /* The PI support requires the Linux futex system call.  If that's not
 160        available, pthread_mutex_init should never have allowed the type to
 161        be set.  So it will get the default case for an invalid type.  */
 162 #ifdef __NR_futex
 163     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 164       /* Recursive mutex.  */
 165       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 166         return EPERM;
 167
 168       if (--mutex->__data.__count != 0)
 169         /* We still hold the mutex.  */
 170         return 0;
 171       goto continue_pi_non_robust;
 172
 173     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 174       /* Recursive mutex.  */
 175       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 176           == THREAD_GETMEM (THREAD_SELF, tid)
 177           && __builtin_expect (mutex->__data.__owner
 178                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 179         {
 180           if (--mutex->__data.__count != 0)
 181             /* We still hold the mutex.  */
 182             return ENOTRECOVERABLE;
 183
 184           goto pi_notrecoverable;
 185         }
 186
 187       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 188         return EPERM;
 189
 190       if (--mutex->__data.__count != 0)
 191         /* We still hold the mutex.  */
 192         return 0;
 193
 194       goto continue_pi_robust;
 195
 196     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 197     case PTHREAD_MUTEX_PI_NORMAL_NP:
 198     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 199     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 200     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 201     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 202       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 203           != THREAD_GETMEM (THREAD_SELF, tid)
 204           || ! lll_islocked (mutex->__data.__lock))
 205         return EPERM;
 206
 207       /* If the previous owner died and the caller did not succeed in
 208          making the state consistent, mark the mutex as unrecoverable
 209          and make all waiters.  */
 210       if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
 211           && __builtin_expect (mutex->__data.__owner
 212                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 213       pi_notrecoverable:
 214        newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 215
 216       if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
 217         {
 218         continue_pi_robust:
 219           /* Remove mutex from the list.
 220              Note: robust PI futexes are signaled by setting bit 0.  */
 221           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 222                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 223                                    | 1));
 224           DEQUEUE_MUTEX (mutex);
 225         }
 226
 227     continue_pi_non_robust:
 228       mutex->__data.__owner = newowner;
 229       if (decr)
 230         /* One less user.  */
 231         --mutex->__data.__nusers;
 232
 233       /* Unlock.  Load all necessary mutex data before releasing the mutex
 234          to not violate the mutex destruction requirements (see
 235          lll_unlock).  */
 236       int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 237       int private = (robust
 238                      ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 239                      : PTHREAD_MUTEX_PSHARED (mutex));
 240       /* Unlock the mutex using a CAS unless there are futex waiters or our
 241          TID is not the value of __lock anymore, in which case we let the
 242          kernel take care of the situation.  Use release MO in the CAS to
 243          synchronize with acquire MO in lock acquisitions.  */
 244       int l = atomic_load_relaxed (&mutex->__data.__lock);
 245       do
 246         {
 247           if (((l & FUTEX_WAITERS) != 0)
 248               || (l != THREAD_GETMEM (THREAD_SELF, tid)))
 249             {
 250               INTERNAL_SYSCALL_DECL (__err);
 251               INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
 252                                 __lll_private_flag (FUTEX_UNLOCK_PI, private));
 253               break;
 254             }
 255         }
 256       while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
 257                                                     &l, 0));
 258
 259       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 260       break;
 261 #endif  /* __NR_futex.  */
 262
 263     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 264       /* Recursive mutex.  */
 265       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 266         return EPERM;
 267
 268       if (--mutex->__data.__count != 0)
 269         /* We still hold the mutex.  */
 270         return 0;
 271       goto pp;
 272
 273     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 274       /* Error checking mutex.  */
 275       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
 276           || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
 277         return EPERM;
 278       /* FALLTHROUGH */
 279
 280     case PTHREAD_MUTEX_PP_NORMAL_NP:
 281     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 282       /* Always reset the owner field.  */
 283     pp:
 284       mutex->__data.__owner = 0;
 285
 286       if (decr)
 287         /* One less user.  */
 288         --mutex->__data.__nusers;
 289
 290       /* Unlock.  Use release MO in the CAS to synchronize with acquire MO in
 291          lock acquisitions.  */
 292       int newval;
 293       int oldval = atomic_load_relaxed (&mutex->__data.__lock);
 294       do
 295         {
 296           newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
 297         }
 298       while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
 299                                                     &oldval, newval));
 300
 301       if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
 302         lll_futex_wake (&mutex->__data.__lock, 1,
 303                         PTHREAD_MUTEX_PSHARED (mutex));
 304
 305       int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 306
 307       LIBC_PROBE (mutex_release, 1, mutex);
 308
 309       return __pthread_tpp_change_priority (oldprio, -1);
 310
 311     default:
 312       /* Correct code cannot set any other type.  */
 313       return EINVAL;
 314     }
 315
 316   LIBC_PROBE (mutex_release, 1, mutex);
 317   return 0;
 318 }
 319
 320
 321 int
 322 __pthread_mutex_unlock (pthread_mutex_t *mutex)
 323 {
 324   return __pthread_mutex_unlock_usercnt (mutex, 1);
 325 }
 326 strong_alias (__pthread_mutex_unlock, pthread_mutex_unlock)
 327 hidden_def (__pthread_mutex_unlock)