nptl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002-2022 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3
   4    The GNU C Library is free software; you can redistribute it and/or
   5    modify it under the terms of the GNU Lesser General Public
   6    License as published by the Free Software Foundation; either
   7    version 2.1 of the License, or (at your option) any later version.
   8
   9    The GNU C Library is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12    Lesser General Public License for more details.
  13
  14    You should have received a copy of the GNU Lesser General Public
  15    License along with the GNU C Library; if not, see
  16    <https://www.gnu.org/licenses/>.  */
  17
  18 #include <assert.h>
  19 #include <errno.h>
  20 #include <stdlib.h>
  21 #include <unistd.h>
  22 #include <sys/param.h>
  23 #include <not-cancel.h>
  24 #include "pthreadP.h"
  25 #include <atomic.h>
  26 #include <futex-internal.h>
  27 #include <stap-probe.h>
  28 #include <shlib-compat.h>
  29
  30 /* Some of the following definitions differ when pthread_mutex_cond_lock.c
  31    includes this file.  */
  32 #ifndef LLL_MUTEX_LOCK
  33 /* lll_lock with single-thread optimization.  */
  34 static inline void
  35 lll_mutex_lock_optimized (pthread_mutex_t *mutex)
  36 {
  37   /* The single-threaded optimization is only valid for private
  38      mutexes.  For process-shared mutexes, the mutex could be in a
  39      shared mapping, so synchronization with another process is needed
  40      even without any threads.  If the lock is already marked as
  41      acquired, POSIX requires that pthread_mutex_lock deadlocks for
  42      normal mutexes, so skip the optimization in that case as
  43      well.  */
  44   int private = PTHREAD_MUTEX_PSHARED (mutex);
  45   if (private == LLL_PRIVATE && SINGLE_THREAD_P && mutex->__data.__lock == 0)
  46     mutex->__data.__lock = 1;
  47   else
  48     lll_lock (mutex->__data.__lock, private);
  49 }
  50
  51 # define LLL_MUTEX_LOCK(mutex)                                          \
  52   lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
  53 # define LLL_MUTEX_LOCK_OPTIMIZED(mutex) lll_mutex_lock_optimized (mutex)
  54 # define LLL_MUTEX_TRYLOCK(mutex) \
  55   lll_trylock ((mutex)->__data.__lock)
  56 # define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0
  57 # define LLL_MUTEX_LOCK_ELISION(mutex) \
  58   lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
  59                    PTHREAD_MUTEX_PSHARED (mutex))
  60 # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
  61   lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
  62                    PTHREAD_MUTEX_PSHARED (mutex))
  63 # define PTHREAD_MUTEX_LOCK ___pthread_mutex_lock
  64 # define PTHREAD_MUTEX_VERSIONS 1
  65 #endif
  66
  67 #ifndef LLL_MUTEX_READ_LOCK
  68 # define LLL_MUTEX_READ_LOCK(mutex) \
  69   atomic_load_relaxed (&(mutex)->__data.__lock)
  70 #endif
  71
  72 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
  73      __attribute_noinline__;
  74
  75 int
  76 PTHREAD_MUTEX_LOCK (pthread_mutex_t *mutex)
  77 {
  78   /* See concurrency notes regarding mutex type which is loaded from __kind
  79      in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h.  */
  80   unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  81
  82   LIBC_PROBE (mutex_entry, 1, mutex);
  83
  84   if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
  85                                  | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
  86     return __pthread_mutex_lock_full (mutex);
  87
  88   if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
  89     {
  90       FORCE_ELISION (mutex, goto elision);
  91     simple:
  92       /* Normal mutex.  */
  93       LLL_MUTEX_LOCK_OPTIMIZED (mutex);
  94       assert (mutex->__data.__owner == 0);
  95     }
  96 #if ENABLE_ELISION_SUPPORT
  97   else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
  98     {
  99   elision: __attribute__((unused))
 100       /* This case can never happen on a system without elision,
 101          as the mutex type initialization functions will not
 102          allow to set the elision flags.  */
 103       /* Don't record owner or users for elision case.  This is a
 104          tail call.  */
 105       return LLL_MUTEX_LOCK_ELISION (mutex);
 106     }
 107 #endif
 108   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
 109                              == PTHREAD_MUTEX_RECURSIVE_NP, 1))
 110     {
 111       /* Recursive mutex.  */
 112       pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 113
 114       /* Check whether we already hold the mutex.  */
 115       if (mutex->__data.__owner == id)
 116         {
 117           /* Just bump the counter.  */
 118           if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 119             /* Overflow of the counter.  */
 120             return EAGAIN;
 121
 122           ++mutex->__data.__count;
 123
 124           return 0;
 125         }
 126
 127       /* We have to get the mutex.  */
 128       LLL_MUTEX_LOCK_OPTIMIZED (mutex);
 129
 130       assert (mutex->__data.__owner == 0);
 131       mutex->__data.__count = 1;
 132     }
 133   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
 134                           == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
 135     {
 136       if (LLL_MUTEX_TRYLOCK (mutex) != 0)
 137         {
 138           int cnt = 0;
 139           int max_cnt = MIN (max_adaptive_count (),
 140                              mutex->__data.__spins * 2 + 10);
 141           do
 142             {
 143               if (cnt++ >= max_cnt)
 144                 {
 145                   LLL_MUTEX_LOCK (mutex);
 146                   break;
 147                 }
 148               atomic_spin_nop ();
 149             }
 150           while (LLL_MUTEX_READ_LOCK (mutex) != 0
 151                  || LLL_MUTEX_TRYLOCK (mutex) != 0);
 152
 153           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 154         }
 155       assert (mutex->__data.__owner == 0);
 156     }
 157   else
 158     {
 159       pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 160       assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
 161       /* Check whether we already hold the mutex.  */
 162       if (__glibc_unlikely (mutex->__data.__owner == id))
 163         return EDEADLK;
 164       goto simple;
 165     }
 166
 167   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 168
 169   /* Record the ownership.  */
 170   mutex->__data.__owner = id;
 171 #ifndef NO_INCR
 172   ++mutex->__data.__nusers;
 173 #endif
 174
 175   LIBC_PROBE (mutex_acquired, 1, mutex);
 176
 177   return 0;
 178 }
 179
 180 static int
 181 __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 182 {
 183   int oldval;
 184   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 185
 186   switch (PTHREAD_MUTEX_TYPE (mutex))
 187     {
 188     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 189     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 190     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 191     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 192       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 193                      &mutex->__data.__list.__next);
 194       /* We need to set op_pending before starting the operation.  Also
 195          see comments at ENQUEUE_MUTEX.  */
 196       __asm ("" ::: "memory");
 197
 198       oldval = mutex->__data.__lock;
 199       /* This is set to FUTEX_WAITERS iff we might have shared the
 200          FUTEX_WAITERS flag with other threads, and therefore need to keep it
 201          set to avoid lost wake-ups.  We have the same requirement in the
 202          simple mutex algorithm.
 203          We start with value zero for a normal mutex, and FUTEX_WAITERS if we
 204          are building the special case mutexes for use from within condition
 205          variables.  */
 206       unsigned int assume_other_futex_waiters = LLL_ROBUST_MUTEX_LOCK_MODIFIER;
 207       while (1)
 208         {
 209           /* Try to acquire the lock through a CAS from 0 (not acquired) to
 210              our TID | assume_other_futex_waiters.  */
 211           if (__glibc_likely (oldval == 0))
 212             {
 213               oldval
 214                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 215                     id | assume_other_futex_waiters, 0);
 216               if (__glibc_likely (oldval == 0))
 217                 break;
 218             }
 219
 220           if ((oldval & FUTEX_OWNER_DIED) != 0)
 221             {
 222               /* The previous owner died.  Try locking the mutex.  */
 223               int newval = id;
 224 #ifdef NO_INCR
 225               /* We are not taking assume_other_futex_waiters into accoount
 226                  here simply because we'll set FUTEX_WAITERS anyway.  */
 227               newval |= FUTEX_WAITERS;
 228 #else
 229               newval |= (oldval & FUTEX_WAITERS) | assume_other_futex_waiters;
 230 #endif
 231
 232               newval
 233                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 234                                                        newval, oldval);
 235
 236               if (newval != oldval)
 237                 {
 238                   oldval = newval;
 239                   continue;
 240                 }
 241
 242               /* We got the mutex.  */
 243               mutex->__data.__count = 1;
 244               /* But it is inconsistent unless marked otherwise.  */
 245               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 246
 247               /* We must not enqueue the mutex before we have acquired it.
 248                  Also see comments at ENQUEUE_MUTEX.  */
 249               __asm ("" ::: "memory");
 250               ENQUEUE_MUTEX (mutex);
 251               /* We need to clear op_pending after we enqueue the mutex.  */
 252               __asm ("" ::: "memory");
 253               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 254
 255               /* Note that we deliberately exit here.  If we fall
 256                  through to the end of the function __nusers would be
 257                  incremented which is not correct because the old
 258                  owner has to be discounted.  If we are not supposed
 259                  to increment __nusers we actually have to decrement
 260                  it here.  */
 261 #ifdef NO_INCR
 262               --mutex->__data.__nusers;
 263 #endif
 264
 265               return EOWNERDEAD;
 266             }
 267
 268           /* Check whether we already hold the mutex.  */
 269           if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 270             {
 271               int kind = PTHREAD_MUTEX_TYPE (mutex);
 272               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 273                 {
 274                   /* We do not need to ensure ordering wrt another memory
 275                      access.  Also see comments at ENQUEUE_MUTEX. */
 276                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 277                                  NULL);
 278                   return EDEADLK;
 279                 }
 280
 281               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 282                 {
 283                   /* We do not need to ensure ordering wrt another memory
 284                      access.  */
 285                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 286                                  NULL);
 287
 288                   /* Just bump the counter.  */
 289                   if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 290                     /* Overflow of the counter.  */
 291                     return EAGAIN;
 292
 293                   ++mutex->__data.__count;
 294
 295                   return 0;
 296                 }
 297             }
 298
 299           /* We cannot acquire the mutex nor has its owner died.  Thus, try
 300              to block using futexes.  Set FUTEX_WAITERS if necessary so that
 301              other threads are aware that there are potentially threads
 302              blocked on the futex.  Restart if oldval changed in the
 303              meantime.  */
 304           if ((oldval & FUTEX_WAITERS) == 0)
 305             {
 306               int val = atomic_compare_and_exchange_val_acq
 307                 (&mutex->__data.__lock, oldval | FUTEX_WAITERS, oldval);
 308               if (val != oldval)
 309                 {
 310                   oldval = val;
 311                   continue;
 312                 }
 313               oldval |= FUTEX_WAITERS;
 314             }
 315
 316           /* It is now possible that we share the FUTEX_WAITERS flag with
 317              another thread; therefore, update assume_other_futex_waiters so
 318              that we do not forget about this when handling other cases
 319              above and thus do not cause lost wake-ups.  */
 320           assume_other_futex_waiters |= FUTEX_WAITERS;
 321
 322           /* Block using the futex and reload current lock value.  */
 323           futex_wait ((unsigned int *) &mutex->__data.__lock, oldval,
 324                       PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 325           oldval = mutex->__data.__lock;
 326         }
 327
 328       /* We have acquired the mutex; check if it is still consistent.  */
 329       if (__builtin_expect (mutex->__data.__owner
 330                             == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 331         {
 332           /* This mutex is now not recoverable.  */
 333           mutex->__data.__count = 0;
 334           int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
 335           lll_unlock (mutex->__data.__lock, private);
 336           /* FIXME This violates the mutex destruction requirements.  See
 337              __pthread_mutex_unlock_full.  */
 338           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 339           return ENOTRECOVERABLE;
 340         }
 341
 342       mutex->__data.__count = 1;
 343       /* We must not enqueue the mutex before we have acquired it.
 344          Also see comments at ENQUEUE_MUTEX.  */
 345       __asm ("" ::: "memory");
 346       ENQUEUE_MUTEX (mutex);
 347       /* We need to clear op_pending after we enqueue the mutex.  */
 348       __asm ("" ::: "memory");
 349       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 350       break;
 351
 352     /* The PI support requires the Linux futex system call.  If that's not
 353        available, pthread_mutex_init should never have allowed the type to
 354        be set.  So it will get the default case for an invalid type.  */
 355 #ifdef __NR_futex
 356     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 357     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 358     case PTHREAD_MUTEX_PI_NORMAL_NP:
 359     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 360     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 361     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 362     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 363     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 364       {
 365         int kind, robust;
 366         {
 367           /* See concurrency notes regarding __kind in struct __pthread_mutex_s
 368              in sysdeps/nptl/bits/thread-shared-types.h.  */
 369           int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
 370           kind = mutex_kind & PTHREAD_MUTEX_KIND_MASK_NP;
 371           robust = mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 372         }
 373
 374         if (robust)
 375           {
 376             /* Note: robust PI futexes are signaled by setting bit 0.  */
 377             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 378                            (void *) (((uintptr_t) &mutex->__data.__list.__next)
 379                                      | 1));
 380             /* We need to set op_pending before starting the operation.  Also
 381                see comments at ENQUEUE_MUTEX.  */
 382             __asm ("" ::: "memory");
 383           }
 384
 385         oldval = mutex->__data.__lock;
 386
 387         /* Check whether we already hold the mutex.  */
 388         if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 389           {
 390             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 391               {
 392                 /* We do not need to ensure ordering wrt another memory
 393                    access.  */
 394                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 395                 return EDEADLK;
 396               }
 397
 398             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 399               {
 400                 /* We do not need to ensure ordering wrt another memory
 401                    access.  */
 402                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 403
 404                 /* Just bump the counter.  */
 405                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 406                   /* Overflow of the counter.  */
 407                   return EAGAIN;
 408
 409                 ++mutex->__data.__count;
 410
 411                 return 0;
 412               }
 413           }
 414
 415         int newval = id;
 416 # ifdef NO_INCR
 417         newval |= FUTEX_WAITERS;
 418 # endif
 419         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 420                                                       newval, 0);
 421
 422         if (oldval != 0)
 423           {
 424             /* The mutex is locked.  The kernel will now take care of
 425                everything.  */
 426             int private = (robust
 427                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 428                            : PTHREAD_MUTEX_PSHARED (mutex));
 429             int e = __futex_lock_pi64 (&mutex->__data.__lock, 0 /* ununsed  */,
 430                                        NULL, private);
 431             if (e == ESRCH || e == EDEADLK)
 432               {
 433                 assert (e != EDEADLK
 434                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 435                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 436                 /* ESRCH can happen only for non-robust PI mutexes where
 437                    the owner of the lock died.  */
 438                 assert (e != ESRCH || !robust);
 439
 440                 /* Delay the thread indefinitely.  */
 441                 while (1)
 442                   __futex_abstimed_wait64 (&(unsigned int){0}, 0,
 443                                            0 /* ignored */, NULL, private);
 444               }
 445
 446             oldval = mutex->__data.__lock;
 447
 448             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 449           }
 450
 451         if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
 452           {
 453             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 454
 455             /* We got the mutex.  */
 456             mutex->__data.__count = 1;
 457             /* But it is inconsistent unless marked otherwise.  */
 458             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 459
 460             /* We must not enqueue the mutex before we have acquired it.
 461                Also see comments at ENQUEUE_MUTEX.  */
 462             __asm ("" ::: "memory");
 463             ENQUEUE_MUTEX_PI (mutex);
 464             /* We need to clear op_pending after we enqueue the mutex.  */
 465             __asm ("" ::: "memory");
 466             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 467
 468             /* Note that we deliberately exit here.  If we fall
 469                through to the end of the function __nusers would be
 470                incremented which is not correct because the old owner
 471                has to be discounted.  If we are not supposed to
 472                increment __nusers we actually have to decrement it here.  */
 473 # ifdef NO_INCR
 474             --mutex->__data.__nusers;
 475 # endif
 476
 477             return EOWNERDEAD;
 478           }
 479
 480         if (robust
 481             && __builtin_expect (mutex->__data.__owner
 482                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 483           {
 484             /* This mutex is now not recoverable.  */
 485             mutex->__data.__count = 0;
 486
 487             futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
 488                              PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 489
 490             /* To the kernel, this will be visible after the kernel has
 491                acquired the mutex in the syscall.  */
 492             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 493             return ENOTRECOVERABLE;
 494           }
 495
 496         mutex->__data.__count = 1;
 497         if (robust)
 498           {
 499             /* We must not enqueue the mutex before we have acquired it.
 500                Also see comments at ENQUEUE_MUTEX.  */
 501             __asm ("" ::: "memory");
 502             ENQUEUE_MUTEX_PI (mutex);
 503             /* We need to clear op_pending after we enqueue the mutex.  */
 504             __asm ("" ::: "memory");
 505             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 506           }
 507       }
 508       break;
 509 #endif  /* __NR_futex.  */
 510
 511     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 512     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 513     case PTHREAD_MUTEX_PP_NORMAL_NP:
 514     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 515       {
 516         /* See concurrency notes regarding __kind in struct __pthread_mutex_s
 517            in sysdeps/nptl/bits/thread-shared-types.h.  */
 518         int kind = atomic_load_relaxed (&(mutex->__data.__kind))
 519           & PTHREAD_MUTEX_KIND_MASK_NP;
 520
 521         oldval = mutex->__data.__lock;
 522
 523         /* Check whether we already hold the mutex.  */
 524         if (mutex->__data.__owner == id)
 525           {
 526             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 527               return EDEADLK;
 528
 529             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 530               {
 531                 /* Just bump the counter.  */
 532                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 533                   /* Overflow of the counter.  */
 534                   return EAGAIN;
 535
 536                 ++mutex->__data.__count;
 537
 538                 return 0;
 539               }
 540           }
 541
 542         int oldprio = -1, ceilval;
 543         do
 544           {
 545             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 546                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 547
 548             if (__pthread_current_priority () > ceiling)
 549               {
 550                 if (oldprio != -1)
 551                   __pthread_tpp_change_priority (oldprio, -1);
 552                 return EINVAL;
 553               }
 554
 555             int retval = __pthread_tpp_change_priority (oldprio, ceiling);
 556             if (retval)
 557               return retval;
 558
 559             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 560             oldprio = ceiling;
 561
 562             oldval
 563               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 564 #ifdef NO_INCR
 565                                                      ceilval | 2,
 566 #else
 567                                                      ceilval | 1,
 568 #endif
 569                                                      ceilval);
 570
 571             if (oldval == ceilval)
 572               break;
 573
 574             do
 575               {
 576                 oldval
 577                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 578                                                          ceilval | 2,
 579                                                          ceilval | 1);
 580
 581                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 582                   break;
 583
 584                 if (oldval != ceilval)
 585                   futex_wait ((unsigned int * ) &mutex->__data.__lock,
 586                               ceilval | 2,
 587                               PTHREAD_MUTEX_PSHARED (mutex));
 588               }
 589             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 590                                                         ceilval | 2, ceilval)
 591                    != ceilval);
 592           }
 593         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 594
 595         assert (mutex->__data.__owner == 0);
 596         mutex->__data.__count = 1;
 597       }
 598       break;
 599
 600     default:
 601       /* Correct code cannot set any other type.  */
 602       return EINVAL;
 603     }
 604
 605   /* Record the ownership.  */
 606   mutex->__data.__owner = id;
 607 #ifndef NO_INCR
 608   ++mutex->__data.__nusers;
 609 #endif
 610
 611   LIBC_PROBE (mutex_acquired, 1, mutex);
 612
 613   return 0;
 614 }
 615
 616 #if PTHREAD_MUTEX_VERSIONS
 617 libc_hidden_ver (___pthread_mutex_lock, __pthread_mutex_lock)
 618 # ifndef SHARED
 619 strong_alias (___pthread_mutex_lock, __pthread_mutex_lock)
 620 # endif
 621 versioned_symbol (libpthread, ___pthread_mutex_lock, pthread_mutex_lock,
 622                   GLIBC_2_0);
 623
 624 # if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_0, GLIBC_2_34)
 625 compat_symbol (libpthread, ___pthread_mutex_lock, __pthread_mutex_lock,
 626                GLIBC_2_0);
 627 # endif
 628 #endif /* PTHREAD_MUTEX_VERSIONS */
 629
 630
 631 #ifdef NO_INCR
 632 void
 633 __pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
 634 {
 635   /* See concurrency notes regarding __kind in struct __pthread_mutex_s
 636      in sysdeps/nptl/bits/thread-shared-types.h.  */
 637   int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
 638   assert ((mutex_kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
 639   assert ((mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
 640   assert ((mutex_kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
 641
 642   /* Record the ownership.  */
 643   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 644   mutex->__data.__owner = id;
 645
 646   if (mutex_kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
 647     ++mutex->__data.__count;
 648 }
 649 #endif