nptl/pthread_mutex_unlock.c

   1 /* Copyright (C) 2002-2015 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include "pthreadP.h"
  23 #include <lowlevellock.h>
  24 #include <stap-probe.h>
  25
  26 #ifndef lll_unlock_elision
  27 #define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
  28 #endif
  29
  30 static int
  31 internal_function
  32 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
  33      __attribute_noinline__;
  34
  35 int
  36 internal_function attribute_hidden
  37 __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
  38 {
  39   int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  40   if (__builtin_expect (type &
  41                 ~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
  42     return __pthread_mutex_unlock_full (mutex, decr);
  43
  44   if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
  45       == PTHREAD_MUTEX_TIMED_NP)
  46     {
  47       /* Always reset the owner field.  */
  48     normal:
  49       mutex->__data.__owner = 0;
  50       if (decr)
  51         /* One less user.  */
  52         --mutex->__data.__nusers;
  53
  54       /* Unlock.  */
  55       lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));
  56
  57       LIBC_PROBE (mutex_release, 1, mutex);
  58
  59       return 0;
  60     }
  61   else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
  62     {
  63       /* Don't reset the owner/users fields for elision.  */
  64       return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision,
  65                                       PTHREAD_MUTEX_PSHARED (mutex));
  66     }
  67   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  68                               == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  69     {
  70       /* Recursive mutex.  */
  71       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
  72         return EPERM;
  73
  74       if (--mutex->__data.__count != 0)
  75         /* We still hold the mutex.  */
  76         return 0;
  77       goto normal;
  78     }
  79   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  80                               == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
  81     goto normal;
  82   else
  83     {
  84       /* Error checking mutex.  */
  85       assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
  86       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
  87           || ! lll_islocked (mutex->__data.__lock))
  88         return EPERM;
  89       goto normal;
  90     }
  91 }
  92
  93
  94 static int
  95 internal_function
  96 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
  97 {
  98   int newowner = 0;
  99
 100   switch (PTHREAD_MUTEX_TYPE (mutex))
 101     {
 102     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 103       /* Recursive mutex.  */
 104       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 105           == THREAD_GETMEM (THREAD_SELF, tid)
 106           && __builtin_expect (mutex->__data.__owner
 107                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 108         {
 109           if (--mutex->__data.__count != 0)
 110             /* We still hold the mutex.  */
 111             return ENOTRECOVERABLE;
 112
 113           goto notrecoverable;
 114         }
 115
 116       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 117         return EPERM;
 118
 119       if (--mutex->__data.__count != 0)
 120         /* We still hold the mutex.  */
 121         return 0;
 122
 123       goto robust;
 124
 125     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 126     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 127     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 128       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 129           != THREAD_GETMEM (THREAD_SELF, tid)
 130           || ! lll_islocked (mutex->__data.__lock))
 131         return EPERM;
 132
 133       /* If the previous owner died and the caller did not succeed in
 134          making the state consistent, mark the mutex as unrecoverable
 135          and make all waiters.  */
 136       if (__builtin_expect (mutex->__data.__owner
 137                             == PTHREAD_MUTEX_INCONSISTENT, 0))
 138       notrecoverable:
 139         newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 140
 141     robust:
 142       /* Remove mutex from the list.  */
 143       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 144                      &mutex->__data.__list.__next);
 145       DEQUEUE_MUTEX (mutex);
 146
 147       mutex->__data.__owner = newowner;
 148       if (decr)
 149         /* One less user.  */
 150         --mutex->__data.__nusers;
 151
 152       /* Unlock.  */
 153       lll_robust_unlock (mutex->__data.__lock,
 154                          PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 155
 156       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 157       break;
 158
 159     /* The PI support requires the Linux futex system call.  If that's not
 160        available, pthread_mutex_init should never have allowed the type to
 161        be set.  So it will get the default case for an invalid type.  */
 162 #ifdef __NR_futex
 163     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 164       /* Recursive mutex.  */
 165       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 166         return EPERM;
 167
 168       if (--mutex->__data.__count != 0)
 169         /* We still hold the mutex.  */
 170         return 0;
 171       goto continue_pi_non_robust;
 172
 173     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 174       /* Recursive mutex.  */
 175       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 176           == THREAD_GETMEM (THREAD_SELF, tid)
 177           && __builtin_expect (mutex->__data.__owner
 178                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 179         {
 180           if (--mutex->__data.__count != 0)
 181             /* We still hold the mutex.  */
 182             return ENOTRECOVERABLE;
 183
 184           goto pi_notrecoverable;
 185         }
 186
 187       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 188         return EPERM;
 189
 190       if (--mutex->__data.__count != 0)
 191         /* We still hold the mutex.  */
 192         return 0;
 193
 194       goto continue_pi_robust;
 195
 196     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 197     case PTHREAD_MUTEX_PI_NORMAL_NP:
 198     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 199     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 200     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 201     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 202       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 203           != THREAD_GETMEM (THREAD_SELF, tid)
 204           || ! lll_islocked (mutex->__data.__lock))
 205         return EPERM;
 206
 207       /* If the previous owner died and the caller did not succeed in
 208          making the state consistent, mark the mutex as unrecoverable
 209          and make all waiters.  */
 210       if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
 211           && __builtin_expect (mutex->__data.__owner
 212                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 213       pi_notrecoverable:
 214        newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 215
 216       if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
 217         {
 218         continue_pi_robust:
 219           /* Remove mutex from the list.
 220              Note: robust PI futexes are signaled by setting bit 0.  */
 221           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 222                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 223                                    | 1));
 224           DEQUEUE_MUTEX (mutex);
 225         }
 226
 227     continue_pi_non_robust:
 228       mutex->__data.__owner = newowner;
 229       if (decr)
 230         /* One less user.  */
 231         --mutex->__data.__nusers;
 232
 233       /* Unlock.  */
 234       if ((mutex->__data.__lock & FUTEX_WAITERS) != 0
 235           || atomic_compare_and_exchange_bool_rel (&mutex->__data.__lock, 0,
 236                                                    THREAD_GETMEM (THREAD_SELF,
 237                                                                   tid)))
 238         {
 239           int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 240           int private = (robust
 241                          ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 242                          : PTHREAD_MUTEX_PSHARED (mutex));
 243           INTERNAL_SYSCALL_DECL (__err);
 244           INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
 245                             __lll_private_flag (FUTEX_UNLOCK_PI, private));
 246         }
 247
 248       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 249       break;
 250 #endif  /* __NR_futex.  */
 251
 252     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 253       /* Recursive mutex.  */
 254       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 255         return EPERM;
 256
 257       if (--mutex->__data.__count != 0)
 258         /* We still hold the mutex.  */
 259         return 0;
 260       goto pp;
 261
 262     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 263       /* Error checking mutex.  */
 264       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
 265           || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
 266         return EPERM;
 267       /* FALLTHROUGH */
 268
 269     case PTHREAD_MUTEX_PP_NORMAL_NP:
 270     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 271       /* Always reset the owner field.  */
 272     pp:
 273       mutex->__data.__owner = 0;
 274
 275       if (decr)
 276         /* One less user.  */
 277         --mutex->__data.__nusers;
 278
 279       /* Unlock.  */
 280       int newval, oldval;
 281       do
 282         {
 283           oldval = mutex->__data.__lock;
 284           newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
 285         }
 286       while (atomic_compare_and_exchange_bool_rel (&mutex->__data.__lock,
 287                                                    newval, oldval));
 288
 289       if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
 290         lll_futex_wake (&mutex->__data.__lock, 1,
 291                         PTHREAD_MUTEX_PSHARED (mutex));
 292
 293       int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 294
 295       LIBC_PROBE (mutex_release, 1, mutex);
 296
 297       return __pthread_tpp_change_priority (oldprio, -1);
 298
 299     default:
 300       /* Correct code cannot set any other type.  */
 301       return EINVAL;
 302     }
 303
 304   LIBC_PROBE (mutex_release, 1, mutex);
 305   return 0;
 306 }
 307
 308
 309 int
 310 __pthread_mutex_unlock (pthread_mutex_t *mutex)
 311 {
 312   return __pthread_mutex_unlock_usercnt (mutex, 1);
 313 }
 314 strong_alias (__pthread_mutex_unlock, pthread_mutex_unlock)
 315 hidden_def (__pthread_mutex_unlock)