nptl/pthread_mutex_unlock.c

   1 /* Copyright (C) 2002-2014 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include "pthreadP.h"
  23 #include <lowlevellock.h>
  24 #include <stap-probe.h>
  25
  26 #ifndef lll_unlock_elision
  27 #define lll_unlock_elision(a,b) ({ lll_unlock (a,b); 0; })
  28 #endif
  29
  30 static int
  31 internal_function
  32 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
  33      __attribute_noinline__;
  34
  35 int
  36 internal_function attribute_hidden
  37 __pthread_mutex_unlock_usercnt (mutex, decr)
  38      pthread_mutex_t *mutex;
  39      int decr;
  40 {
  41   int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  42   if (__builtin_expect (type &
  43                 ~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
  44     return __pthread_mutex_unlock_full (mutex, decr);
  45
  46   if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
  47       == PTHREAD_MUTEX_TIMED_NP)
  48     {
  49       /* Always reset the owner field.  */
  50     normal:
  51       mutex->__data.__owner = 0;
  52       if (decr)
  53         /* One less user.  */
  54         --mutex->__data.__nusers;
  55
  56       /* Unlock.  */
  57       lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));
  58
  59       LIBC_PROBE (mutex_release, 1, mutex);
  60
  61       return 0;
  62     }
  63   else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
  64     {
  65       /* Don't reset the owner/users fields for elision.  */
  66       return lll_unlock_elision (mutex->__data.__lock,
  67                                       PTHREAD_MUTEX_PSHARED (mutex));
  68     }
  69   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  70                               == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  71     {
  72       /* Recursive mutex.  */
  73       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
  74         return EPERM;
  75
  76       if (--mutex->__data.__count != 0)
  77         /* We still hold the mutex.  */
  78         return 0;
  79       goto normal;
  80     }
  81   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  82                               == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
  83     goto normal;
  84   else
  85     {
  86       /* Error checking mutex.  */
  87       assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
  88       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
  89           || ! lll_islocked (mutex->__data.__lock))
  90         return EPERM;
  91       goto normal;
  92     }
  93 }
  94
  95
  96 static int
  97 internal_function
  98 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
  99 {
 100   int newowner = 0;
 101
 102   switch (PTHREAD_MUTEX_TYPE (mutex))
 103     {
 104     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 105       /* Recursive mutex.  */
 106       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 107           == THREAD_GETMEM (THREAD_SELF, tid)
 108           && __builtin_expect (mutex->__data.__owner
 109                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 110         {
 111           if (--mutex->__data.__count != 0)
 112             /* We still hold the mutex.  */
 113             return ENOTRECOVERABLE;
 114
 115           goto notrecoverable;
 116         }
 117
 118       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 119         return EPERM;
 120
 121       if (--mutex->__data.__count != 0)
 122         /* We still hold the mutex.  */
 123         return 0;
 124
 125       goto robust;
 126
 127     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 128     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 129     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 130       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 131           != THREAD_GETMEM (THREAD_SELF, tid)
 132           || ! lll_islocked (mutex->__data.__lock))
 133         return EPERM;
 134
 135       /* If the previous owner died and the caller did not succeed in
 136          making the state consistent, mark the mutex as unrecoverable
 137          and make all waiters.  */
 138       if (__builtin_expect (mutex->__data.__owner
 139                             == PTHREAD_MUTEX_INCONSISTENT, 0))
 140       notrecoverable:
 141         newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 142
 143     robust:
 144       /* Remove mutex from the list.  */
 145       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 146                      &mutex->__data.__list.__next);
 147       DEQUEUE_MUTEX (mutex);
 148
 149       mutex->__data.__owner = newowner;
 150       if (decr)
 151         /* One less user.  */
 152         --mutex->__data.__nusers;
 153
 154       /* Unlock.  */
 155       lll_robust_unlock (mutex->__data.__lock,
 156                          PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 157
 158       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 159       break;
 160
 161     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 162       /* Recursive mutex.  */
 163       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 164         return EPERM;
 165
 166       if (--mutex->__data.__count != 0)
 167         /* We still hold the mutex.  */
 168         return 0;
 169       goto continue_pi_non_robust;
 170
 171     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 172       /* Recursive mutex.  */
 173       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 174           == THREAD_GETMEM (THREAD_SELF, tid)
 175           && __builtin_expect (mutex->__data.__owner
 176                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 177         {
 178           if (--mutex->__data.__count != 0)
 179             /* We still hold the mutex.  */
 180             return ENOTRECOVERABLE;
 181
 182           goto pi_notrecoverable;
 183         }
 184
 185       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 186         return EPERM;
 187
 188       if (--mutex->__data.__count != 0)
 189         /* We still hold the mutex.  */
 190         return 0;
 191
 192       goto continue_pi_robust;
 193
 194     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 195     case PTHREAD_MUTEX_PI_NORMAL_NP:
 196     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 197     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 198     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 199     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 200       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 201           != THREAD_GETMEM (THREAD_SELF, tid)
 202           || ! lll_islocked (mutex->__data.__lock))
 203         return EPERM;
 204
 205       /* If the previous owner died and the caller did not succeed in
 206          making the state consistent, mark the mutex as unrecoverable
 207          and make all waiters.  */
 208       if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
 209           && __builtin_expect (mutex->__data.__owner
 210                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 211       pi_notrecoverable:
 212        newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 213
 214       if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
 215         {
 216         continue_pi_robust:
 217           /* Remove mutex from the list.
 218              Note: robust PI futexes are signaled by setting bit 0.  */
 219           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 220                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 221                                    | 1));
 222           DEQUEUE_MUTEX (mutex);
 223         }
 224
 225     continue_pi_non_robust:
 226       mutex->__data.__owner = newowner;
 227       if (decr)
 228         /* One less user.  */
 229         --mutex->__data.__nusers;
 230
 231       /* Unlock.  */
 232       if ((mutex->__data.__lock & FUTEX_WAITERS) != 0
 233           || atomic_compare_and_exchange_bool_rel (&mutex->__data.__lock, 0,
 234                                                    THREAD_GETMEM (THREAD_SELF,
 235                                                                   tid)))
 236         {
 237           int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 238           int private = (robust
 239                          ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 240                          : PTHREAD_MUTEX_PSHARED (mutex));
 241           INTERNAL_SYSCALL_DECL (__err);
 242           INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
 243                             __lll_private_flag (FUTEX_UNLOCK_PI, private));
 244         }
 245
 246       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 247       break;
 248
 249     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 250       /* Recursive mutex.  */
 251       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 252         return EPERM;
 253
 254       if (--mutex->__data.__count != 0)
 255         /* We still hold the mutex.  */
 256         return 0;
 257       goto pp;
 258
 259     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 260       /* Error checking mutex.  */
 261       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
 262           || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
 263         return EPERM;
 264       /* FALLTHROUGH */
 265
 266     case PTHREAD_MUTEX_PP_NORMAL_NP:
 267     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 268       /* Always reset the owner field.  */
 269     pp:
 270       mutex->__data.__owner = 0;
 271
 272       if (decr)
 273         /* One less user.  */
 274         --mutex->__data.__nusers;
 275
 276       /* Unlock.  */
 277       int newval, oldval;
 278       do
 279         {
 280           oldval = mutex->__data.__lock;
 281           newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
 282         }
 283       while (atomic_compare_and_exchange_bool_rel (&mutex->__data.__lock,
 284                                                    newval, oldval));
 285
 286       if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
 287         lll_futex_wake (&mutex->__data.__lock, 1,
 288                         PTHREAD_MUTEX_PSHARED (mutex));
 289
 290       int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 291
 292       LIBC_PROBE (mutex_release, 1, mutex);
 293
 294       return __pthread_tpp_change_priority (oldprio, -1);
 295
 296     default:
 297       /* Correct code cannot set any other type.  */
 298       return EINVAL;
 299     }
 300
 301   LIBC_PROBE (mutex_release, 1, mutex);
 302   return 0;
 303 }
 304
 305
 306 int
 307 __pthread_mutex_unlock (mutex)
 308      pthread_mutex_t *mutex;
 309 {
 310   return __pthread_mutex_unlock_usercnt (mutex, 1);
 311 }
 312 strong_alias (__pthread_mutex_unlock, pthread_mutex_unlock)
 313 hidden_def (__pthread_mutex_unlock)