nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h

   1 /* Copyright (C) 2003, 2004, 2006-2008, 2009 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #ifndef _LOWLEVELLOCK_H
  20 #define _LOWLEVELLOCK_H 1
  21
  22 #include <time.h>
  23 #include <sys/param.h>
  24 #include <bits/pthreadtypes.h>
  25 #include <atomic.h>
  26 #include <kernel-features.h>
  27
  28 #define SYS_futex               238
  29 #define FUTEX_WAIT              0
  30 #define FUTEX_WAKE              1
  31 #define FUTEX_REQUEUE           3
  32 #define FUTEX_CMP_REQUEUE       4
  33 #define FUTEX_WAKE_OP           5
  34 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  35 #define FUTEX_LOCK_PI           6
  36 #define FUTEX_UNLOCK_PI         7
  37 #define FUTEX_TRYLOCK_PI        8
  38 #define FUTEX_WAIT_BITSET       9
  39 #define FUTEX_WAKE_BITSET       10
  40 #define FUTEX_PRIVATE_FLAG      128
  41 #define FUTEX_CLOCK_REALTIME    256
  42
  43 #define FUTEX_BITSET_MATCH_ANY  0xffffffff
  44
  45 /* Values for 'private' parameter of locking macros.  Yes, the
  46    definition seems to be backwards.  But it is not.  The bit will be
  47    reversed before passing to the system call.  */
  48 #define LLL_PRIVATE     0
  49 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  50
  51
  52 #if !defined NOT_IN_libc || defined IS_IN_rtld
  53 /* In libc.so or ld.so all futexes are private.  */
  54 # ifdef __ASSUME_PRIVATE_FUTEX
  55 #  define __lll_private_flag(fl, private) \
  56   ((fl) | FUTEX_PRIVATE_FLAG)
  57 # else
  58 #  define __lll_private_flag(fl, private) \
  59   ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
  60 # endif
  61 #else
  62 # ifdef __ASSUME_PRIVATE_FUTEX
  63 #  define __lll_private_flag(fl, private) \
  64   (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
  65 # else
  66 #  define __lll_private_flag(fl, private) \
  67   (__builtin_constant_p (private)                                             \
  68    ? ((private) == 0                                                          \
  69       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))            \
  70       : (fl))                                                                 \
  71    : ((fl) | (((private) ^ FUTEX_PRIVATE_FLAG)                                \
  72               & THREAD_GETMEM (THREAD_SELF, header.private_futex))))
  73 # endif
  74 #endif
  75
  76 #define lll_futex_wait(futex, val, private) \
  77   lll_futex_timed_wait (futex, val, NULL, private)
  78
  79 #define lll_futex_timed_wait(futex, val, timespec, private) \
  80   ({                                                                          \
  81     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
  82     register unsigned long int __r3 asm ("3")                                 \
  83       = __lll_private_flag (FUTEX_WAIT, private);                             \
  84     register unsigned long int __r4 asm ("4") = (unsigned long int) (val);    \
  85     register unsigned long int __r5 asm ("5") = (unsigned long int)(timespec);\
  86     register unsigned long int __result asm ("2");                            \
  87                                                                               \
  88     __asm __volatile ("svc %b1"                                               \
  89                       : "=d" (__result)                                       \
  90                       : "i" (SYS_futex), "0" (__r2), "d" (__r3),              \
  91                         "d" (__r4), "d" (__r5)                                \
  92                       : "cc", "memory" );                                     \
  93     __result;                                                                 \
  94   })
  95
  96
  97 #define lll_futex_wake(futex, nr, private) \
  98   ({                                                                          \
  99     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
 100     register unsigned long int __r3 asm ("3")                                 \
 101       = __lll_private_flag (FUTEX_WAKE, private);                             \
 102     register unsigned long int __r4 asm ("4") = (unsigned long int) (nr);     \
 103     register unsigned long int __result asm ("2");                            \
 104                                                                               \
 105     __asm __volatile ("svc %b1"                                               \
 106                       : "=d" (__result)                                       \
 107                       : "i" (SYS_futex), "0" (__r2), "d" (__r3), "d" (__r4)   \
 108                       : "cc", "memory" );                                     \
 109     __result;                                                                 \
 110   })
 111
 112
 113 #define lll_robust_dead(futexv, private) \
 114   do                                                                          \
 115     {                                                                         \
 116       int *__futexp = &(futexv);                                              \
 117                                                                               \
 118       atomic_or (__futexp, FUTEX_OWNER_DIED);                                 \
 119       lll_futex_wake (__futexp, 1, private);                                  \
 120     }                                                                         \
 121   while (0)
 122
 123
 124 /* Returns non-zero if error happened, zero if success.  */
 125 #define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \
 126   ({                                                                          \
 127     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
 128     register unsigned long int __r3 asm ("3")                                 \
 129       = __lll_private_flag (FUTEX_CMP_REQUEUE, private);                      \
 130     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);         \
 131     register unsigned long int __r5 asm ("5") = (long int) (nr_move);         \
 132     register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex);  \
 133     register unsigned long int __r7 asm ("7") = (int) (val);                  \
 134     register unsigned long __result asm ("2");                                \
 135                                                                               \
 136     __asm __volatile ("svc %b1"                                               \
 137                       : "=d" (__result)                                       \
 138                       : "i" (SYS_futex), "0" (__r2), "d" (__r3),              \
 139                         "d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7)        \
 140                       : "cc", "memory" );                                     \
 141     __result > -4096UL;                                                       \
 142   })
 143
 144
 145 /* Returns non-zero if error happened, zero if success.  */
 146 #define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2, private) \
 147   ({                                                                          \
 148     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
 149     register unsigned long int __r3 asm ("3")                                 \
 150       = __lll_private_flag (FUTEX_WAKE_OP, private);                          \
 151     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);         \
 152     register unsigned long int __r5 asm ("5") = (long int) (nr_wake2);        \
 153     register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \
 154     register unsigned long int __r7 asm ("7")                                 \
 155       = (int) FUTEX_OP_CLEAR_WAKE_IF_GT_ONE;                                  \
 156     register unsigned long __result asm ("2");                                \
 157                                                                               \
 158     __asm __volatile ("svc %b1"                                               \
 159                       : "=d" (__result)                                       \
 160                       : "i" (SYS_futex), "0" (__r2), "d" (__r3),              \
 161                         "d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7)        \
 162                       : "cc", "memory" );                                     \
 163     __result > -4096UL;                                                       \
 164   })
 165
 166
 167 #define lll_compare_and_swap(futex, oldval, newval, operation) \
 168   do {                                                                        \
 169     __typeof (futex) __futex = (futex);                                       \
 170     __asm __volatile ("   l   %1,%0\n"                                        \
 171                       "0: " operation "\n"                                    \
 172                       "   cs  %1,%2,%0\n"                                     \
 173                       "   jl  0b\n"                                           \
 174                       "1:"                                                    \
 175                       : "=Q" (*__futex), "=&d" (oldval), "=&d" (newval)       \
 176                       : "m" (*__futex) : "cc", "memory" );                    \
 177   } while (0)
 178
 179
 180 static inline int
 181 __attribute__ ((always_inline))
 182 __lll_trylock (int *futex)
 183 {
 184     unsigned int old;
 185
 186     __asm __volatile ("cs %0,%3,%1"
 187                        : "=d" (old), "=Q" (*futex)
 188                        : "0" (0), "d" (1), "m" (*futex) : "cc", "memory" );
 189     return old != 0;
 190 }
 191 #define lll_trylock(futex) __lll_trylock (&(futex))
 192
 193
 194 static inline int
 195 __attribute__ ((always_inline))
 196 __lll_cond_trylock (int *futex)
 197 {
 198     unsigned int old;
 199
 200     __asm __volatile ("cs %0,%3,%1"
 201                        : "=d" (old), "=Q" (*futex)
 202                        : "0" (0), "d" (2), "m" (*futex) : "cc", "memory" );
 203     return old != 0;
 204 }
 205 #define lll_cond_trylock(futex) __lll_cond_trylock (&(futex))
 206
 207
 208 static inline int
 209 __attribute__ ((always_inline))
 210 __lll_robust_trylock (int *futex, int id)
 211 {
 212     unsigned int old;
 213
 214     __asm __volatile ("cs %0,%3,%1"
 215                        : "=d" (old), "=Q" (*futex)
 216                        : "0" (0), "d" (id), "m" (*futex) : "cc", "memory" );
 217     return old != 0;
 218 }
 219 #define lll_robust_trylock(futex, id) \
 220   __lll_robust_trylock (&(futex), id)
 221
 222
 223 extern void __lll_lock_wait_private (int *futex) attribute_hidden;
 224 extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
 225 extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;
 226
 227 static inline void
 228 __attribute__ ((always_inline))
 229 __lll_lock (int *futex, int private)
 230 {
 231   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
 232     {
 233       if (__builtin_constant_p (private) && private == LLL_PRIVATE)
 234         __lll_lock_wait_private (futex);
 235       else
 236         __lll_lock_wait (futex, private);
 237     }
 238 }
 239 #define lll_lock(futex, private) __lll_lock (&(futex), private)
 240
 241 static inline int
 242 __attribute__ ((always_inline))
 243 __lll_robust_lock (int *futex, int id, int private)
 244 {
 245   int result = 0;
 246   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
 247                         0))
 248     result = __lll_robust_lock_wait (futex, private);
 249   return result;
 250 }
 251 #define lll_robust_lock(futex, id, private) \
 252   __lll_robust_lock (&(futex), id, private)
 253
 254 static inline void
 255 __attribute__ ((always_inline))
 256 __lll_cond_lock (int *futex, int private)
 257 {
 258   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 2, 0), 0))
 259     __lll_lock_wait (futex, private);
 260 }
 261 #define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
 262
 263 #define lll_robust_cond_lock(futex, id, private) \
 264   __lll_robust_lock (&(futex), (id) | FUTEX_WAITERS, private)
 265
 266 extern int __lll_timedlock_wait
 267   (int *futex, const struct timespec *, int private) attribute_hidden;
 268 extern int __lll_robust_timedlock_wait
 269   (int *futex, const struct timespec *, int private) attribute_hidden;
 270
 271 static inline int
 272 __attribute__ ((always_inline))
 273 __lll_timedlock (int *futex, const struct timespec *abstime, int private)
 274 {
 275   int result = 0;
 276   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
 277     result = __lll_timedlock_wait (futex, abstime, private);
 278   return result;
 279 }
 280 #define lll_timedlock(futex, abstime, private) \
 281   __lll_timedlock (&(futex), abstime, private)
 282
 283 static inline int
 284 __attribute__ ((always_inline))
 285 __lll_robust_timedlock (int *futex, const struct timespec *abstime,
 286                         int id, int private)
 287 {
 288   int result = 0;
 289   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
 290                         0))
 291     result = __lll_robust_timedlock_wait (futex, abstime, private);
 292   return result;
 293 }
 294 #define lll_robust_timedlock(futex, abstime, id, private) \
 295   __lll_robust_timedlock (&(futex), abstime, id, private)
 296
 297
 298 #define __lll_unlock(futex, private) \
 299   (void)                                                                      \
 300     ({ int __oldval;                                                          \
 301        int __newval = 0;                                                      \
 302        int *__futexp = (futex);                                               \
 303                                                                               \
 304        lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
 305        if (__builtin_expect (__oldval > 1, 0))                                \
 306          lll_futex_wake (__futexp, 1, private);                               \
 307     })
 308 #define lll_unlock(futex, private) __lll_unlock(&(futex), private)
 309
 310
 311 #define __lll_robust_unlock(futex, private) \
 312   (void)                                                                      \
 313     ({ int __oldval;                                                          \
 314        int __newval = 0;                                                      \
 315        int *__futexp = (futex);                                               \
 316                                                                               \
 317        lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
 318        if (__builtin_expect (__oldval & FUTEX_WAITERS, 0))                    \
 319          lll_futex_wake (__futexp, 1, private);                               \
 320     })
 321 #define lll_robust_unlock(futex, private) \
 322   __lll_robust_unlock(&(futex), private)
 323
 324 #define lll_islocked(futex) \
 325   (futex != 0)
 326
 327
 328 /* Initializers for lock.  */
 329 #define LLL_LOCK_INITIALIZER            (0)
 330 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 331
 332 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
 333    wakeup when the clone terminates.  The memory location contains the
 334    thread ID while the clone is running and is reset to zero
 335    afterwards.  */
 336 #define __lll_wait_tid(ptid) \
 337   do                                                                          \
 338     {                                                                         \
 339       int __tid;                                                              \
 340                                                                               \
 341       while ((__tid = *ptid) != 0)                                            \
 342         lll_futex_wait (ptid, __tid, LLL_SHARED);                             \
 343     }                                                                         \
 344   while (0)
 345 #define lll_wait_tid(tid) __lll_wait_tid(&(tid))
 346
 347 extern int __lll_timedwait_tid (int *, const struct timespec *)
 348      attribute_hidden;
 349
 350 #define lll_timedwait_tid(tid, abstime) \
 351   ({                                                                          \
 352     int __res = 0;                                                            \
 353     if ((tid) != 0)                                                           \
 354       __res = __lll_timedwait_tid (&(tid), (abstime));                        \
 355     __res;                                                                    \
 356   })
 357
 358 #endif  /* lowlevellock.h */