nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h

   1 /* Copyright (C) 2003, 2004, 2006, 2007, 2008 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #ifndef _LOWLEVELLOCK_H
  21 #define _LOWLEVELLOCK_H 1
  22
  23 #include <time.h>
  24 #include <sys/param.h>
  25 #include <bits/pthreadtypes.h>
  26 #include <atomic.h>
  27 #include <kernel-features.h>
  28
  29 #define SYS_futex               238
  30 #define FUTEX_WAIT              0
  31 #define FUTEX_WAKE              1
  32 #define FUTEX_REQUEUE           3
  33 #define FUTEX_CMP_REQUEUE       4
  34 #define FUTEX_WAKE_OP           5
  35 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  36 #define FUTEX_LOCK_PI           6
  37 #define FUTEX_UNLOCK_PI         7
  38 #define FUTEX_TRYLOCK_PI        8
  39 #define FUTEX_WAIT_BITSET       9
  40 #define FUTEX_WAKE_BITSET       10
  41 #define FUTEX_PRIVATE_FLAG      128
  42
  43 /* Values for 'private' parameter of locking macros.  Yes, the
  44    definition seems to be backwards.  But it is not.  The bit will be
  45    reversed before passing to the system call.  */
  46 #define LLL_PRIVATE     0
  47 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  48
  49
  50 #if !defined NOT_IN_libc || defined IS_IN_rtld
  51 /* In libc.so or ld.so all futexes are private.  */
  52 # ifdef __ASSUME_PRIVATE_FUTEX
  53 #  define __lll_private_flag(fl, private) \
  54   ((fl) | FUTEX_PRIVATE_FLAG)
  55 # else
  56 #  define __lll_private_flag(fl, private) \
  57   ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
  58 # endif
  59 #else
  60 # ifdef __ASSUME_PRIVATE_FUTEX
  61 #  define __lll_private_flag(fl, private) \
  62   (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
  63 # else
  64 #  define __lll_private_flag(fl, private) \
  65   (__builtin_constant_p (private)                                             \
  66    ? ((private) == 0                                                          \
  67       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))            \
  68       : (fl))                                                                 \
  69    : ((fl) | (((private) ^ FUTEX_PRIVATE_FLAG)                                \
  70               & THREAD_GETMEM (THREAD_SELF, header.private_futex))))
  71 # endif
  72 #endif
  73
  74 #define lll_futex_wait(futex, val, private) \
  75   lll_futex_timed_wait (futex, val, NULL, private)
  76
  77 #define lll_futex_timed_wait(futex, val, timespec, private) \
  78   ({                                                                          \
  79     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
  80     register unsigned long int __r3 asm ("3")                                 \
  81       = __lll_private_flag (FUTEX_WAIT, private);                             \
  82     register unsigned long int __r4 asm ("4") = (unsigned long int) (val);    \
  83     register unsigned long int __r5 asm ("5") = (unsigned long int)(timespec);\
  84     register unsigned long int __result asm ("2");                            \
  85                                                                               \
  86     __asm __volatile ("svc %b1"                                               \
  87                       : "=d" (__result)                                       \
  88                       : "i" (SYS_futex), "0" (__r2), "d" (__r3),              \
  89                         "d" (__r4), "d" (__r5)                                \
  90                       : "cc", "memory" );                                     \
  91     __result;                                                                 \
  92   })
  93
  94
  95 #define lll_futex_wake(futex, nr, private) \
  96   ({                                                                          \
  97     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
  98     register unsigned long int __r3 asm ("3")                                 \
  99       = __lll_private_flag (FUTEX_WAKE, private);                             \
 100     register unsigned long int __r4 asm ("4") = (unsigned long int) (nr);     \
 101     register unsigned long int __result asm ("2");                            \
 102                                                                               \
 103     __asm __volatile ("svc %b1"                                               \
 104                       : "=d" (__result)                                       \
 105                       : "i" (SYS_futex), "0" (__r2), "d" (__r3), "d" (__r4)   \
 106                       : "cc", "memory" );                                     \
 107     __result;                                                                 \
 108   })
 109
 110
 111 #define lll_robust_dead(futexv, private) \
 112   do                                                                          \
 113     {                                                                         \
 114       int *__futexp = &(futexv);                                              \
 115                                                                               \
 116       atomic_or (__futexp, FUTEX_OWNER_DIED);                                 \
 117       lll_futex_wake (__futexp, 1, private);                                  \
 118     }                                                                         \
 119   while (0)
 120
 121
 122 /* Returns non-zero if error happened, zero if success.  */
 123 #define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \
 124   ({                                                                          \
 125     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
 126     register unsigned long int __r3 asm ("3")                                 \
 127       = __lll_private_flag (FUTEX_CMP_REQUEUE, private);                      \
 128     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);         \
 129     register unsigned long int __r5 asm ("5") = (long int) (nr_move);         \
 130     register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex);  \
 131     register unsigned long int __r7 asm ("7") = (int) (val);                  \
 132     register unsigned long __result asm ("2");                                \
 133                                                                               \
 134     __asm __volatile ("svc %b1"                                               \
 135                       : "=d" (__result)                                       \
 136                       : "i" (SYS_futex), "0" (__r2), "d" (__r3),              \
 137                         "d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7)        \
 138                       : "cc", "memory" );                                     \
 139     __result > -4096UL;                                                       \
 140   })
 141
 142
 143 /* Returns non-zero if error happened, zero if success.  */
 144 #define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2, private) \
 145   ({                                                                          \
 146     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
 147     register unsigned long int __r3 asm ("3")                                 \
 148       = __lll_private_flag (FUTEX_WAKE_OP, private);                          \
 149     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);         \
 150     register unsigned long int __r5 asm ("5") = (long int) (nr_wake2);        \
 151     register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \
 152     register unsigned long int __r7 asm ("7")                                 \
 153       = (int) FUTEX_OP_CLEAR_WAKE_IF_GT_ONE;                                  \
 154     register unsigned long __result asm ("2");                                \
 155                                                                               \
 156     __asm __volatile ("svc %b1"                                               \
 157                       : "=d" (__result)                                       \
 158                       : "i" (SYS_futex), "0" (__r2), "d" (__r3),              \
 159                         "d" (__r4), "d" (__r5), "d" (__r6), "d" (__r7)        \
 160                       : "cc", "memory" );                                     \
 161     __result > -4096UL;                                                       \
 162   })
 163
 164
 165 #define lll_compare_and_swap(futex, oldval, newval, operation) \
 166   do {                                                                        \
 167     __typeof (futex) __futex = (futex);                                       \
 168     __asm __volatile ("   l   %1,%0\n"                                        \
 169                       "0: " operation "\n"                                    \
 170                       "   cs  %1,%2,%0\n"                                     \
 171                       "   jl  0b\n"                                           \
 172                       "1:"                                                    \
 173                       : "=Q" (*__futex), "=&d" (oldval), "=&d" (newval)       \
 174                       : "m" (*__futex) : "cc", "memory" );                    \
 175   } while (0)
 176
 177
 178 static inline int
 179 __attribute__ ((always_inline))
 180 __lll_trylock (int *futex)
 181 {
 182     unsigned int old;
 183
 184     __asm __volatile ("cs %0,%3,%1"
 185                        : "=d" (old), "=Q" (*futex)
 186                        : "0" (0), "d" (1), "m" (*futex) : "cc", "memory" );
 187     return old != 0;
 188 }
 189 #define lll_trylock(futex) __lll_trylock (&(futex))
 190
 191
 192 static inline int
 193 __attribute__ ((always_inline))
 194 __lll_cond_trylock (int *futex)
 195 {
 196     unsigned int old;
 197
 198     __asm __volatile ("cs %0,%3,%1"
 199                        : "=d" (old), "=Q" (*futex)
 200                        : "0" (0), "d" (2), "m" (*futex) : "cc", "memory" );
 201     return old != 0;
 202 }
 203 #define lll_cond_trylock(futex) __lll_cond_trylock (&(futex))
 204
 205
 206 static inline int
 207 __attribute__ ((always_inline))
 208 __lll_robust_trylock (int *futex, int id)
 209 {
 210     unsigned int old;
 211
 212     __asm __volatile ("cs %0,%3,%1"
 213                        : "=d" (old), "=Q" (*futex)
 214                        : "0" (0), "d" (id), "m" (*futex) : "cc", "memory" );
 215     return old != 0;
 216 }
 217 #define lll_robust_trylock(futex, id) \
 218   __lll_robust_trylock (&(futex), id)
 219
 220
 221 extern void __lll_lock_wait_private (int *futex) attribute_hidden;
 222 extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
 223 extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;
 224
 225 static inline void
 226 __attribute__ ((always_inline))
 227 __lll_lock (int *futex, int private)
 228 {
 229   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
 230     {
 231       if (__builtin_constant_p (private) && private == LLL_PRIVATE)
 232         __lll_lock_wait_private (futex);
 233       else
 234         __lll_lock_wait (futex, private);
 235     }
 236 }
 237 #define lll_lock(futex, private) __lll_lock (&(futex), private)
 238
 239 static inline int
 240 __attribute__ ((always_inline))
 241 __lll_robust_lock (int *futex, int id, int private)
 242 {
 243   int result = 0;
 244   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
 245                         0))
 246     result = __lll_robust_lock_wait (futex, private);
 247   return result;
 248 }
 249 #define lll_robust_lock(futex, id, private) \
 250   __lll_robust_lock (&(futex), id, private)
 251
 252 static inline void
 253 __attribute__ ((always_inline))
 254 __lll_cond_lock (int *futex, int private)
 255 {
 256   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 2, 0), 0))
 257     __lll_lock_wait (futex, private);
 258 }
 259 #define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
 260
 261 #define lll_robust_cond_lock(futex, id, private) \
 262   __lll_robust_lock (&(futex), (id) | FUTEX_WAITERS, private)
 263
 264 extern int __lll_timedlock_wait
 265   (int *futex, const struct timespec *, int private) attribute_hidden;
 266 extern int __lll_robust_timedlock_wait
 267   (int *futex, const struct timespec *, int private) attribute_hidden;
 268
 269 static inline int
 270 __attribute__ ((always_inline))
 271 __lll_timedlock (int *futex, const struct timespec *abstime, int private)
 272 {
 273   int result = 0;
 274   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, 1, 0), 0))
 275     result = __lll_timedlock_wait (futex, abstime, private);
 276   return result;
 277 }
 278 #define lll_timedlock(futex, abstime, private) \
 279   __lll_timedlock (&(futex), abstime, private)
 280
 281 static inline int
 282 __attribute__ ((always_inline))
 283 __lll_robust_timedlock (int *futex, const struct timespec *abstime,
 284                         int id, int private)
 285 {
 286   int result = 0;
 287   if (__builtin_expect (atomic_compare_and_exchange_bool_acq (futex, id, 0),
 288                         0))
 289     result = __lll_robust_timedlock_wait (futex, abstime, private);
 290   return result;
 291 }
 292 #define lll_robust_timedlock(futex, abstime, id, private) \
 293   __lll_robust_timedlock (&(futex), abstime, id, private)
 294
 295
 296 #define __lll_unlock(futex, private) \
 297   (void)                                                                      \
 298     ({ int __oldval;                                                          \
 299        int __newval = 0;                                                      \
 300        int *__futexp = (futex);                                               \
 301                                                                               \
 302        lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
 303        if (__builtin_expect (__oldval > 1, 0))                                \
 304          lll_futex_wake (__futexp, 1, private);                               \
 305     })
 306 #define lll_unlock(futex, private) __lll_unlock(&(futex), private)
 307
 308
 309 #define __lll_robust_unlock(futex, private) \
 310   (void)                                                                      \
 311     ({ int __oldval;                                                          \
 312        int __newval = 0;                                                      \
 313        int *__futexp = (futex);                                               \
 314                                                                               \
 315        lll_compare_and_swap (__futexp, __oldval, __newval, "slr %2,%2");      \
 316        if (__builtin_expect (__oldval & FUTEX_WAITERS, 0))                    \
 317          lll_futex_wake (__futexp, 1, private);                               \
 318     })
 319 #define lll_robust_unlock(futex, private) \
 320   __lll_robust_unlock(&(futex), private)
 321
 322 #define lll_islocked(futex) \
 323   (futex != 0)
 324
 325
 326 /* Initializers for lock.  */
 327 #define LLL_LOCK_INITIALIZER            (0)
 328 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 329
 330 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
 331    wakeup when the clone terminates.  The memory location contains the
 332    thread ID while the clone is running and is reset to zero
 333    afterwards.  */
 334 #define __lll_wait_tid(ptid) \
 335   do                                                                          \
 336     {                                                                         \
 337       int __tid;                                                              \
 338                                                                               \
 339       while ((__tid = *ptid) != 0)                                            \
 340         lll_futex_wait (ptid, __tid, LLL_SHARED);                             \
 341     }                                                                         \
 342   while (0)
 343 #define lll_wait_tid(tid) __lll_wait_tid(&(tid))
 344
 345 extern int __lll_timedwait_tid (int *, const struct timespec *)
 346      attribute_hidden;
 347
 348 #define lll_timedwait_tid(tid, abstime) \
 349   ({                                                                          \
 350     int __res = 0;                                                            \
 351     if ((tid) != 0)                                                           \
 352       __res = __lll_timedwait_tid (&(tid), (abstime));                        \
 353     __res;                                                                    \
 354   })
 355
 356 #endif  /* lowlevellock.h */