nptl/pthread_mutex_unlock.c

   1 /* Copyright (C) 2002-2023 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3
   4    The GNU C Library is free software; you can redistribute it and/or
   5    modify it under the terms of the GNU Lesser General Public
   6    License as published by the Free Software Foundation; either
   7    version 2.1 of the License, or (at your option) any later version.
   8
   9    The GNU C Library is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12    Lesser General Public License for more details.
  13
  14    You should have received a copy of the GNU Lesser General Public
  15    License along with the GNU C Library; if not, see
  16    <https://www.gnu.org/licenses/>.  */
  17
  18 #include <assert.h>
  19 #include <errno.h>
  20 #include <stdlib.h>
  21 #include "pthreadP.h"
  22 #include <lowlevellock.h>
  23 #include <stap-probe.h>
  24 #include <futex-internal.h>
  25 #include <shlib-compat.h>
  26
  27 static int
  28 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
  29      __attribute_noinline__;
  30
  31 /* lll_lock with single-thread optimization.  */
  32 static inline void
  33 lll_mutex_unlock_optimized (pthread_mutex_t *mutex)
  34 {
  35   /* The single-threaded optimization is only valid for private
  36      mutexes.  For process-shared mutexes, the mutex could be in a
  37      shared mapping, so synchronization with another process is needed
  38      even without any threads.  */
  39   int private = PTHREAD_MUTEX_PSHARED (mutex);
  40   if (private == LLL_PRIVATE && SINGLE_THREAD_P)
  41     mutex->__data.__lock = 0;
  42   else
  43     lll_unlock (mutex->__data.__lock, private);
  44 }
  45
  46 int
  47 __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
  48 {
  49   /* See concurrency notes regarding mutex type which is loaded from __kind
  50      in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h.  */
  51   int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  52   if (__builtin_expect (type
  53                         & ~(PTHREAD_MUTEX_KIND_MASK_NP
  54                             |PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
  55     return __pthread_mutex_unlock_full (mutex, decr);
  56
  57   if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
  58       == PTHREAD_MUTEX_TIMED_NP)
  59     {
  60       /* Always reset the owner field.  */
  61     normal:
  62       mutex->__data.__owner = 0;
  63       if (decr)
  64         /* One less user.  */
  65         --mutex->__data.__nusers;
  66
  67       /* Unlock.  */
  68       lll_mutex_unlock_optimized (mutex);
  69
  70       LIBC_PROBE (mutex_release, 1, mutex);
  71
  72       return 0;
  73     }
  74   else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
  75     {
  76       /* Don't reset the owner/users fields for elision.  */
  77       return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision,
  78                                       PTHREAD_MUTEX_PSHARED (mutex));
  79     }
  80   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  81                               == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  82     {
  83       /* Recursive mutex.  */
  84       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
  85         return EPERM;
  86
  87       if (--mutex->__data.__count != 0)
  88         /* We still hold the mutex.  */
  89         return 0;
  90       goto normal;
  91     }
  92   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  93                               == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
  94     goto normal;
  95   else
  96     {
  97       /* Error checking mutex.  */
  98       assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
  99       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
 100           || ! lll_islocked (mutex->__data.__lock))
 101         return EPERM;
 102       goto normal;
 103     }
 104 }
 105 libc_hidden_def (__pthread_mutex_unlock_usercnt)
 106
 107
 108 static int
 109 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
 110 {
 111   int newowner = 0;
 112   int private;
 113
 114   switch (PTHREAD_MUTEX_TYPE (mutex))
 115     {
 116     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 117       /* Recursive mutex.  */
 118       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 119           == THREAD_GETMEM (THREAD_SELF, tid)
 120           && __builtin_expect (mutex->__data.__owner
 121                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 122         {
 123           if (--mutex->__data.__count != 0)
 124             /* We still hold the mutex.  */
 125             return ENOTRECOVERABLE;
 126
 127           goto notrecoverable;
 128         }
 129
 130       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 131         return EPERM;
 132
 133       if (--mutex->__data.__count != 0)
 134         /* We still hold the mutex.  */
 135         return 0;
 136
 137       goto robust;
 138
 139     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 140     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 141     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 142       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 143           != THREAD_GETMEM (THREAD_SELF, tid)
 144           || ! lll_islocked (mutex->__data.__lock))
 145         return EPERM;
 146
 147       /* If the previous owner died and the caller did not succeed in
 148          making the state consistent, mark the mutex as unrecoverable
 149          and make all waiters.  */
 150       if (__builtin_expect (mutex->__data.__owner
 151                             == PTHREAD_MUTEX_INCONSISTENT, 0))
 152       notrecoverable:
 153         newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 154
 155     robust:
 156       /* Remove mutex from the list.  */
 157       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 158                      &mutex->__data.__list.__next);
 159       /* We must set op_pending before we dequeue the mutex.  Also see
 160          comments at ENQUEUE_MUTEX.  */
 161       __asm ("" ::: "memory");
 162       DEQUEUE_MUTEX (mutex);
 163
 164       mutex->__data.__owner = newowner;
 165       if (decr)
 166         /* One less user.  */
 167         --mutex->__data.__nusers;
 168
 169       /* Unlock by setting the lock to 0 (not acquired); if the lock had
 170          FUTEX_WAITERS set previously, then wake any waiters.
 171          The unlock operation must be the last access to the mutex to not
 172          violate the mutex destruction requirements (see __lll_unlock).  */
 173       private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
 174       if (__glibc_unlikely ((atomic_exchange_release (&mutex->__data.__lock, 0)
 175                              & FUTEX_WAITERS) != 0))
 176         futex_wake ((unsigned int *) &mutex->__data.__lock, 1, private);
 177
 178       /* We must clear op_pending after we release the mutex.
 179          FIXME However, this violates the mutex destruction requirements
 180          because another thread could acquire the mutex, destroy it, and
 181          reuse the memory for something else; then, if this thread crashes,
 182          and the memory happens to have a value equal to the TID, the kernel
 183          will believe it is still related to the mutex (which has been
 184          destroyed already) and will modify some other random object.  */
 185       __asm ("" ::: "memory");
 186       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 187       break;
 188
 189     /* The PI support requires the Linux futex system call.  If that's not
 190        available, pthread_mutex_init should never have allowed the type to
 191        be set.  So it will get the default case for an invalid type.  */
 192 #ifdef __NR_futex
 193     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 194       /* Recursive mutex.  */
 195       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 196         return EPERM;
 197
 198       if (--mutex->__data.__count != 0)
 199         /* We still hold the mutex.  */
 200         return 0;
 201       goto continue_pi_non_robust;
 202
 203     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 204       /* Recursive mutex.  */
 205       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 206           == THREAD_GETMEM (THREAD_SELF, tid)
 207           && __builtin_expect (mutex->__data.__owner
 208                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 209         {
 210           if (--mutex->__data.__count != 0)
 211             /* We still hold the mutex.  */
 212             return ENOTRECOVERABLE;
 213
 214           goto pi_notrecoverable;
 215         }
 216
 217       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 218         return EPERM;
 219
 220       if (--mutex->__data.__count != 0)
 221         /* We still hold the mutex.  */
 222         return 0;
 223
 224       goto continue_pi_robust;
 225
 226     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 227     case PTHREAD_MUTEX_PI_NORMAL_NP:
 228     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 229     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 230     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 231     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 232       if ((mutex->__data.__lock & FUTEX_TID_MASK)
 233           != THREAD_GETMEM (THREAD_SELF, tid)
 234           || ! lll_islocked (mutex->__data.__lock))
 235         return EPERM;
 236
 237       /* If the previous owner died and the caller did not succeed in
 238          making the state consistent, mark the mutex as unrecoverable
 239          and make all waiters.  */
 240       /* See concurrency notes regarding __kind in struct __pthread_mutex_s
 241          in sysdeps/nptl/bits/thread-shared-types.h.  */
 242       if ((atomic_load_relaxed (&(mutex->__data.__kind))
 243            & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
 244           && __builtin_expect (mutex->__data.__owner
 245                                == PTHREAD_MUTEX_INCONSISTENT, 0))
 246       pi_notrecoverable:
 247        newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
 248
 249       /* See concurrency notes regarding __kind in struct __pthread_mutex_s
 250          in sysdeps/nptl/bits/thread-shared-types.h.  */
 251       if ((atomic_load_relaxed (&(mutex->__data.__kind))
 252            & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
 253         {
 254         continue_pi_robust:
 255           /* Remove mutex from the list.
 256              Note: robust PI futexes are signaled by setting bit 0.  */
 257           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 258                          (void *) (((uintptr_t) &mutex->__data.__list.__next)
 259                                    | 1));
 260           /* We must set op_pending before we dequeue the mutex.  Also see
 261              comments at ENQUEUE_MUTEX.  */
 262           __asm ("" ::: "memory");
 263           DEQUEUE_MUTEX (mutex);
 264         }
 265
 266     continue_pi_non_robust:
 267       mutex->__data.__owner = newowner;
 268       if (decr)
 269         /* One less user.  */
 270         --mutex->__data.__nusers;
 271
 272       /* Unlock.  Load all necessary mutex data before releasing the mutex
 273          to not violate the mutex destruction requirements (see
 274          lll_unlock).  */
 275       /* See concurrency notes regarding __kind in struct __pthread_mutex_s
 276          in sysdeps/nptl/bits/thread-shared-types.h.  */
 277       int robust = atomic_load_relaxed (&(mutex->__data.__kind))
 278         & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 279       private = (robust
 280                  ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 281                  : PTHREAD_MUTEX_PSHARED (mutex));
 282       /* Unlock the mutex using a CAS unless there are futex waiters or our
 283          TID is not the value of __lock anymore, in which case we let the
 284          kernel take care of the situation.  Use release MO in the CAS to
 285          synchronize with acquire MO in lock acquisitions.  */
 286       int l = atomic_load_relaxed (&mutex->__data.__lock);
 287       do
 288         {
 289           if (((l & FUTEX_WAITERS) != 0)
 290               || (l != THREAD_GETMEM (THREAD_SELF, tid)))
 291             {
 292               futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
 293                                private);
 294               break;
 295             }
 296         }
 297       while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
 298                                                     &l, 0));
 299
 300       /* This happens after the kernel releases the mutex but violates the
 301          mutex destruction requirements; see comments in the code handling
 302          PTHREAD_MUTEX_ROBUST_NORMAL_NP.  */
 303       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 304       break;
 305 #endif  /* __NR_futex.  */
 306
 307     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 308       /* Recursive mutex.  */
 309       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
 310         return EPERM;
 311
 312       if (--mutex->__data.__count != 0)
 313         /* We still hold the mutex.  */
 314         return 0;
 315       goto pp;
 316
 317     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 318       /* Error checking mutex.  */
 319       if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
 320           || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
 321         return EPERM;
 322       /* FALLTHROUGH */
 323
 324     case PTHREAD_MUTEX_PP_NORMAL_NP:
 325     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 326       /* Always reset the owner field.  */
 327     pp:
 328       mutex->__data.__owner = 0;
 329
 330       if (decr)
 331         /* One less user.  */
 332         --mutex->__data.__nusers;
 333
 334       /* Unlock.  Use release MO in the CAS to synchronize with acquire MO in
 335          lock acquisitions.  */
 336       int newval;
 337       int oldval = atomic_load_relaxed (&mutex->__data.__lock);
 338       do
 339         {
 340           newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
 341         }
 342       while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
 343                                                     &oldval, newval));
 344
 345       if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
 346         futex_wake ((unsigned int *)&mutex->__data.__lock, 1,
 347                     PTHREAD_MUTEX_PSHARED (mutex));
 348
 349       int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 350
 351       LIBC_PROBE (mutex_release, 1, mutex);
 352
 353       return __pthread_tpp_change_priority (oldprio, -1);
 354
 355     default:
 356       /* Correct code cannot set any other type.  */
 357       return EINVAL;
 358     }
 359
 360   LIBC_PROBE (mutex_release, 1, mutex);
 361   return 0;
 362 }
 363
 364
 365 int
 366 ___pthread_mutex_unlock (pthread_mutex_t *mutex)
 367 {
 368   return __pthread_mutex_unlock_usercnt (mutex, 1);
 369 }
 370 libc_hidden_ver (___pthread_mutex_unlock, __pthread_mutex_unlock)
 371 #ifndef SHARED
 372 strong_alias (___pthread_mutex_unlock, __pthread_mutex_unlock)
 373 #endif
 374 versioned_symbol (libpthread, ___pthread_mutex_unlock, pthread_mutex_unlock,
 375                   GLIBC_2_0);
 376
 377 #if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_0, GLIBC_2_34)
 378 compat_symbol (libpthread, ___pthread_mutex_unlock, __pthread_mutex_unlock,
 379                GLIBC_2_0);
 380 #endif