nptl/pthread_mutex_lock.c

   1 /* Copyright (C) 2002-2021 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <sys/param.h>
  24 #include <not-cancel.h>
  25 #include "pthreadP.h"
  26 #include <atomic.h>
  27 #include <futex-internal.h>
  28 #include <stap-probe.h>
  29
  30 /* Some of the following definitions differ when pthread_mutex_cond_lock.c
  31    includes this file.  */
  32 #ifndef LLL_MUTEX_LOCK
  33 # define LLL_MUTEX_LOCK(mutex) \
  34   lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
  35 # define LLL_MUTEX_TRYLOCK(mutex) \
  36   lll_trylock ((mutex)->__data.__lock)
  37 # define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0
  38 # define LLL_MUTEX_LOCK_ELISION(mutex) \
  39   lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
  40                    PTHREAD_MUTEX_PSHARED (mutex))
  41 # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
  42   lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
  43                    PTHREAD_MUTEX_PSHARED (mutex))
  44 #endif
  45
  46 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
  47      __attribute_noinline__;
  48
  49 int
  50 __pthread_mutex_lock (pthread_mutex_t *mutex)
  51 {
  52   /* See concurrency notes regarding mutex type which is loaded from __kind
  53      in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h.  */
  54   unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
  55
  56   LIBC_PROBE (mutex_entry, 1, mutex);
  57
  58   if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
  59                                  | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
  60     return __pthread_mutex_lock_full (mutex);
  61
  62   if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
  63     {
  64       FORCE_ELISION (mutex, goto elision);
  65     simple:
  66       /* Normal mutex.  */
  67       LLL_MUTEX_LOCK (mutex);
  68       assert (mutex->__data.__owner == 0);
  69     }
  70 #if ENABLE_ELISION_SUPPORT
  71   else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
  72     {
  73   elision: __attribute__((unused))
  74       /* This case can never happen on a system without elision,
  75          as the mutex type initialization functions will not
  76          allow to set the elision flags.  */
  77       /* Don't record owner or users for elision case.  This is a
  78          tail call.  */
  79       return LLL_MUTEX_LOCK_ELISION (mutex);
  80     }
  81 #endif
  82   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
  83                              == PTHREAD_MUTEX_RECURSIVE_NP, 1))
  84     {
  85       /* Recursive mutex.  */
  86       pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
  87
  88       /* Check whether we already hold the mutex.  */
  89       if (mutex->__data.__owner == id)
  90         {
  91           /* Just bump the counter.  */
  92           if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
  93             /* Overflow of the counter.  */
  94             return EAGAIN;
  95
  96           ++mutex->__data.__count;
  97
  98           return 0;
  99         }
 100
 101       /* We have to get the mutex.  */
 102       LLL_MUTEX_LOCK (mutex);
 103
 104       assert (mutex->__data.__owner == 0);
 105       mutex->__data.__count = 1;
 106     }
 107   else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
 108                           == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
 109     {
 110       if (LLL_MUTEX_TRYLOCK (mutex) != 0)
 111         {
 112           int cnt = 0;
 113           int max_cnt = MIN (max_adaptive_count (),
 114                              mutex->__data.__spins * 2 + 10);
 115           do
 116             {
 117               if (cnt++ >= max_cnt)
 118                 {
 119                   LLL_MUTEX_LOCK (mutex);
 120                   break;
 121                 }
 122               atomic_spin_nop ();
 123             }
 124           while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 125
 126           mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 127         }
 128       assert (mutex->__data.__owner == 0);
 129     }
 130   else
 131     {
 132       pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 133       assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
 134       /* Check whether we already hold the mutex.  */
 135       if (__glibc_unlikely (mutex->__data.__owner == id))
 136         return EDEADLK;
 137       goto simple;
 138     }
 139
 140   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 141
 142   /* Record the ownership.  */
 143   mutex->__data.__owner = id;
 144 #ifndef NO_INCR
 145   ++mutex->__data.__nusers;
 146 #endif
 147
 148   LIBC_PROBE (mutex_acquired, 1, mutex);
 149
 150   return 0;
 151 }
 152
 153 static int
 154 __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 155 {
 156   int oldval;
 157   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 158
 159   switch (PTHREAD_MUTEX_TYPE (mutex))
 160     {
 161     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
 162     case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
 163     case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
 164     case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
 165       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 166                      &mutex->__data.__list.__next);
 167       /* We need to set op_pending before starting the operation.  Also
 168          see comments at ENQUEUE_MUTEX.  */
 169       __asm ("" ::: "memory");
 170
 171       oldval = mutex->__data.__lock;
 172       /* This is set to FUTEX_WAITERS iff we might have shared the
 173          FUTEX_WAITERS flag with other threads, and therefore need to keep it
 174          set to avoid lost wake-ups.  We have the same requirement in the
 175          simple mutex algorithm.
 176          We start with value zero for a normal mutex, and FUTEX_WAITERS if we
 177          are building the special case mutexes for use from within condition
 178          variables.  */
 179       unsigned int assume_other_futex_waiters = LLL_ROBUST_MUTEX_LOCK_MODIFIER;
 180       while (1)
 181         {
 182           /* Try to acquire the lock through a CAS from 0 (not acquired) to
 183              our TID | assume_other_futex_waiters.  */
 184           if (__glibc_likely (oldval == 0))
 185             {
 186               oldval
 187                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 188                     id | assume_other_futex_waiters, 0);
 189               if (__glibc_likely (oldval == 0))
 190                 break;
 191             }
 192
 193           if ((oldval & FUTEX_OWNER_DIED) != 0)
 194             {
 195               /* The previous owner died.  Try locking the mutex.  */
 196               int newval = id;
 197 #ifdef NO_INCR
 198               /* We are not taking assume_other_futex_waiters into accoount
 199                  here simply because we'll set FUTEX_WAITERS anyway.  */
 200               newval |= FUTEX_WAITERS;
 201 #else
 202               newval |= (oldval & FUTEX_WAITERS) | assume_other_futex_waiters;
 203 #endif
 204
 205               newval
 206                 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 207                                                        newval, oldval);
 208
 209               if (newval != oldval)
 210                 {
 211                   oldval = newval;
 212                   continue;
 213                 }
 214
 215               /* We got the mutex.  */
 216               mutex->__data.__count = 1;
 217               /* But it is inconsistent unless marked otherwise.  */
 218               mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 219
 220               /* We must not enqueue the mutex before we have acquired it.
 221                  Also see comments at ENQUEUE_MUTEX.  */
 222               __asm ("" ::: "memory");
 223               ENQUEUE_MUTEX (mutex);
 224               /* We need to clear op_pending after we enqueue the mutex.  */
 225               __asm ("" ::: "memory");
 226               THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 227
 228               /* Note that we deliberately exit here.  If we fall
 229                  through to the end of the function __nusers would be
 230                  incremented which is not correct because the old
 231                  owner has to be discounted.  If we are not supposed
 232                  to increment __nusers we actually have to decrement
 233                  it here.  */
 234 #ifdef NO_INCR
 235               --mutex->__data.__nusers;
 236 #endif
 237
 238               return EOWNERDEAD;
 239             }
 240
 241           /* Check whether we already hold the mutex.  */
 242           if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 243             {
 244               int kind = PTHREAD_MUTEX_TYPE (mutex);
 245               if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 246                 {
 247                   /* We do not need to ensure ordering wrt another memory
 248                      access.  Also see comments at ENQUEUE_MUTEX. */
 249                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 250                                  NULL);
 251                   return EDEADLK;
 252                 }
 253
 254               if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 255                 {
 256                   /* We do not need to ensure ordering wrt another memory
 257                      access.  */
 258                   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 259                                  NULL);
 260
 261                   /* Just bump the counter.  */
 262                   if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 263                     /* Overflow of the counter.  */
 264                     return EAGAIN;
 265
 266                   ++mutex->__data.__count;
 267
 268                   return 0;
 269                 }
 270             }
 271
 272           /* We cannot acquire the mutex nor has its owner died.  Thus, try
 273              to block using futexes.  Set FUTEX_WAITERS if necessary so that
 274              other threads are aware that there are potentially threads
 275              blocked on the futex.  Restart if oldval changed in the
 276              meantime.  */
 277           if ((oldval & FUTEX_WAITERS) == 0)
 278             {
 279               if (atomic_compare_and_exchange_bool_acq (&mutex->__data.__lock,
 280                                                         oldval | FUTEX_WAITERS,
 281                                                         oldval)
 282                   != 0)
 283                 {
 284                   oldval = mutex->__data.__lock;
 285                   continue;
 286                 }
 287               oldval |= FUTEX_WAITERS;
 288             }
 289
 290           /* It is now possible that we share the FUTEX_WAITERS flag with
 291              another thread; therefore, update assume_other_futex_waiters so
 292              that we do not forget about this when handling other cases
 293              above and thus do not cause lost wake-ups.  */
 294           assume_other_futex_waiters |= FUTEX_WAITERS;
 295
 296           /* Block using the futex and reload current lock value.  */
 297           futex_wait ((unsigned int *) &mutex->__data.__lock, oldval,
 298                       PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 299           oldval = mutex->__data.__lock;
 300         }
 301
 302       /* We have acquired the mutex; check if it is still consistent.  */
 303       if (__builtin_expect (mutex->__data.__owner
 304                             == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 305         {
 306           /* This mutex is now not recoverable.  */
 307           mutex->__data.__count = 0;
 308           int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
 309           lll_unlock (mutex->__data.__lock, private);
 310           /* FIXME This violates the mutex destruction requirements.  See
 311              __pthread_mutex_unlock_full.  */
 312           THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 313           return ENOTRECOVERABLE;
 314         }
 315
 316       mutex->__data.__count = 1;
 317       /* We must not enqueue the mutex before we have acquired it.
 318          Also see comments at ENQUEUE_MUTEX.  */
 319       __asm ("" ::: "memory");
 320       ENQUEUE_MUTEX (mutex);
 321       /* We need to clear op_pending after we enqueue the mutex.  */
 322       __asm ("" ::: "memory");
 323       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 324       break;
 325
 326     /* The PI support requires the Linux futex system call.  If that's not
 327        available, pthread_mutex_init should never have allowed the type to
 328        be set.  So it will get the default case for an invalid type.  */
 329 #ifdef __NR_futex
 330     case PTHREAD_MUTEX_PI_RECURSIVE_NP:
 331     case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
 332     case PTHREAD_MUTEX_PI_NORMAL_NP:
 333     case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
 334     case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
 335     case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
 336     case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
 337     case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
 338       {
 339         int kind, robust;
 340         {
 341           /* See concurrency notes regarding __kind in struct __pthread_mutex_s
 342              in sysdeps/nptl/bits/thread-shared-types.h.  */
 343           int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
 344           kind = mutex_kind & PTHREAD_MUTEX_KIND_MASK_NP;
 345           robust = mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
 346         }
 347
 348         if (robust)
 349           {
 350             /* Note: robust PI futexes are signaled by setting bit 0.  */
 351             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 352                            (void *) (((uintptr_t) &mutex->__data.__list.__next)
 353                                      | 1));
 354             /* We need to set op_pending before starting the operation.  Also
 355                see comments at ENQUEUE_MUTEX.  */
 356             __asm ("" ::: "memory");
 357           }
 358
 359         oldval = mutex->__data.__lock;
 360
 361         /* Check whether we already hold the mutex.  */
 362         if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
 363           {
 364             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 365               {
 366                 /* We do not need to ensure ordering wrt another memory
 367                    access.  */
 368                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 369                 return EDEADLK;
 370               }
 371
 372             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 373               {
 374                 /* We do not need to ensure ordering wrt another memory
 375                    access.  */
 376                 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 377
 378                 /* Just bump the counter.  */
 379                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 380                   /* Overflow of the counter.  */
 381                   return EAGAIN;
 382
 383                 ++mutex->__data.__count;
 384
 385                 return 0;
 386               }
 387           }
 388
 389         int newval = id;
 390 # ifdef NO_INCR
 391         newval |= FUTEX_WAITERS;
 392 # endif
 393         oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 394                                                       newval, 0);
 395
 396         if (oldval != 0)
 397           {
 398             /* The mutex is locked.  The kernel will now take care of
 399                everything.  */
 400             int private = (robust
 401                            ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 402                            : PTHREAD_MUTEX_PSHARED (mutex));
 403             int e = futex_lock_pi64 (&mutex->__data.__lock, NULL, private);
 404             if (e == ESRCH || e == EDEADLK)
 405               {
 406                 assert (e != EDEADLK
 407                         || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 408                             && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 409                 /* ESRCH can happen only for non-robust PI mutexes where
 410                    the owner of the lock died.  */
 411                 assert (e != ESRCH || !robust);
 412
 413                 /* Delay the thread indefinitely.  */
 414                 while (1)
 415                   __futex_abstimed_wait64 (&(unsigned int){0}, 0,
 416                                            0 /* ignored */, NULL, private);
 417               }
 418
 419             oldval = mutex->__data.__lock;
 420
 421             assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
 422           }
 423
 424         if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
 425           {
 426             atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
 427
 428             /* We got the mutex.  */
 429             mutex->__data.__count = 1;
 430             /* But it is inconsistent unless marked otherwise.  */
 431             mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 432
 433             /* We must not enqueue the mutex before we have acquired it.
 434                Also see comments at ENQUEUE_MUTEX.  */
 435             __asm ("" ::: "memory");
 436             ENQUEUE_MUTEX_PI (mutex);
 437             /* We need to clear op_pending after we enqueue the mutex.  */
 438             __asm ("" ::: "memory");
 439             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 440
 441             /* Note that we deliberately exit here.  If we fall
 442                through to the end of the function __nusers would be
 443                incremented which is not correct because the old owner
 444                has to be discounted.  If we are not supposed to
 445                increment __nusers we actually have to decrement it here.  */
 446 # ifdef NO_INCR
 447             --mutex->__data.__nusers;
 448 # endif
 449
 450             return EOWNERDEAD;
 451           }
 452
 453         if (robust
 454             && __builtin_expect (mutex->__data.__owner
 455                                  == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 456           {
 457             /* This mutex is now not recoverable.  */
 458             mutex->__data.__count = 0;
 459
 460             futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
 461                              PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 462
 463             /* To the kernel, this will be visible after the kernel has
 464                acquired the mutex in the syscall.  */
 465             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 466             return ENOTRECOVERABLE;
 467           }
 468
 469         mutex->__data.__count = 1;
 470         if (robust)
 471           {
 472             /* We must not enqueue the mutex before we have acquired it.
 473                Also see comments at ENQUEUE_MUTEX.  */
 474             __asm ("" ::: "memory");
 475             ENQUEUE_MUTEX_PI (mutex);
 476             /* We need to clear op_pending after we enqueue the mutex.  */
 477             __asm ("" ::: "memory");
 478             THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 479           }
 480       }
 481       break;
 482 #endif  /* __NR_futex.  */
 483
 484     case PTHREAD_MUTEX_PP_RECURSIVE_NP:
 485     case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
 486     case PTHREAD_MUTEX_PP_NORMAL_NP:
 487     case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
 488       {
 489         /* See concurrency notes regarding __kind in struct __pthread_mutex_s
 490            in sysdeps/nptl/bits/thread-shared-types.h.  */
 491         int kind = atomic_load_relaxed (&(mutex->__data.__kind))
 492           & PTHREAD_MUTEX_KIND_MASK_NP;
 493
 494         oldval = mutex->__data.__lock;
 495
 496         /* Check whether we already hold the mutex.  */
 497         if (mutex->__data.__owner == id)
 498           {
 499             if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
 500               return EDEADLK;
 501
 502             if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
 503               {
 504                 /* Just bump the counter.  */
 505                 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
 506                   /* Overflow of the counter.  */
 507                   return EAGAIN;
 508
 509                 ++mutex->__data.__count;
 510
 511                 return 0;
 512               }
 513           }
 514
 515         int oldprio = -1, ceilval;
 516         do
 517           {
 518             int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
 519                           >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 520
 521             if (__pthread_current_priority () > ceiling)
 522               {
 523                 if (oldprio != -1)
 524                   __pthread_tpp_change_priority (oldprio, -1);
 525                 return EINVAL;
 526               }
 527
 528             int retval = __pthread_tpp_change_priority (oldprio, ceiling);
 529             if (retval)
 530               return retval;
 531
 532             ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
 533             oldprio = ceiling;
 534
 535             oldval
 536               = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 537 #ifdef NO_INCR
 538                                                      ceilval | 2,
 539 #else
 540                                                      ceilval | 1,
 541 #endif
 542                                                      ceilval);
 543
 544             if (oldval == ceilval)
 545               break;
 546
 547             do
 548               {
 549                 oldval
 550                   = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 551                                                          ceilval | 2,
 552                                                          ceilval | 1);
 553
 554                 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
 555                   break;
 556
 557                 if (oldval != ceilval)
 558                   futex_wait ((unsigned int * ) &mutex->__data.__lock,
 559                               ceilval | 2,
 560                               PTHREAD_MUTEX_PSHARED (mutex));
 561               }
 562             while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 563                                                         ceilval | 2, ceilval)
 564                    != ceilval);
 565           }
 566         while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
 567
 568         assert (mutex->__data.__owner == 0);
 569         mutex->__data.__count = 1;
 570       }
 571       break;
 572
 573     default:
 574       /* Correct code cannot set any other type.  */
 575       return EINVAL;
 576     }
 577
 578   /* Record the ownership.  */
 579   mutex->__data.__owner = id;
 580 #ifndef NO_INCR
 581   ++mutex->__data.__nusers;
 582 #endif
 583
 584   LIBC_PROBE (mutex_acquired, 1, mutex);
 585
 586   return 0;
 587 }
 588 #ifndef __pthread_mutex_lock
 589 weak_alias (__pthread_mutex_lock, pthread_mutex_lock)
 590 hidden_def (__pthread_mutex_lock)
 591 #endif
 592
 593
 594 #ifdef NO_INCR
 595 void
 596 __pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
 597 {
 598   /* See concurrency notes regarding __kind in struct __pthread_mutex_s
 599      in sysdeps/nptl/bits/thread-shared-types.h.  */
 600   int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
 601   assert ((mutex_kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
 602   assert ((mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
 603   assert ((mutex_kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
 604
 605   /* Record the ownership.  */
 606   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 607   mutex->__data.__owner = id;
 608
 609   if (mutex_kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
 610     ++mutex->__data.__count;
 611 }
 612 #endif