nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h

   1 /* Copyright (C) 2003, 2004, 2006, 2007, 2008 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Paul Mackerras <paulus@au.ibm.com>, 2003.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #ifndef _LOWLEVELLOCK_H
  21 #define _LOWLEVELLOCK_H 1
  22
  23 #include <time.h>
  24 #include <sys/param.h>
  25 #include <bits/pthreadtypes.h>
  26 #include <atomic.h>
  27 #include <kernel-features.h>
  28
  29 #ifndef __NR_futex
  30 # define __NR_futex             221
  31 #endif
  32 #define FUTEX_WAIT              0
  33 #define FUTEX_WAKE              1
  34 #define FUTEX_REQUEUE           3
  35 #define FUTEX_CMP_REQUEUE       4
  36 #define FUTEX_WAKE_OP           5
  37 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  38 #define FUTEX_LOCK_PI           6
  39 #define FUTEX_UNLOCK_PI         7
  40 #define FUTEX_TRYLOCK_PI        8
  41 #define FUTEX_WAIT_BITSET       9
  42 #define FUTEX_WAKE_BITSET       10
  43 #define FUTEX_PRIVATE_FLAG      128
  44
  45 /* Values for 'private' parameter of locking macros.  Yes, the
  46    definition seems to be backwards.  But it is not.  The bit will be
  47    reversed before passing to the system call.  */
  48 #define LLL_PRIVATE     0
  49 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  50
  51 #if !defined NOT_IN_libc || defined IS_IN_rtld
  52 /* In libc.so or ld.so all futexes are private.  */
  53 # ifdef __ASSUME_PRIVATE_FUTEX
  54 #  define __lll_private_flag(fl, private) \
  55   ((fl) | FUTEX_PRIVATE_FLAG)
  56 # else
  57 #  define __lll_private_flag(fl, private) \
  58   ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
  59 # endif
  60 #else
  61 # ifdef __ASSUME_PRIVATE_FUTEX
  62 #  define __lll_private_flag(fl, private) \
  63   (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
  64 # else
  65 #  define __lll_private_flag(fl, private) \
  66   (__builtin_constant_p (private)                                             \
  67    ? ((private) == 0                                                          \
  68       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))            \
  69       : (fl))                                                                 \
  70    : ((fl) | (((private) ^ FUTEX_PRIVATE_FLAG)                                \
  71               & THREAD_GETMEM (THREAD_SELF, header.private_futex))))
  72 # endif
  73 #endif
  74
  75 #define lll_futex_wait(futexp, val, private) \
  76   lll_futex_timed_wait (futexp, val, NULL, private)
  77
  78 #define lll_futex_timed_wait(futexp, val, timespec, private) \
  79   ({                                                                          \
  80     INTERNAL_SYSCALL_DECL (__err);                                            \
  81     long int __ret;                                                           \
  82                                                                               \
  83     __ret = INTERNAL_SYSCALL (futex, __err, 4, (futexp),                      \
  84                               __lll_private_flag (FUTEX_WAIT, private),       \
  85                               (val), (timespec));                             \
  86     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
  87   })
  88
  89 #define lll_futex_wake(futexp, nr, private) \
  90   ({                                                                          \
  91     INTERNAL_SYSCALL_DECL (__err);                                            \
  92     long int __ret;                                                           \
  93                                                                               \
  94     __ret = INTERNAL_SYSCALL (futex, __err, 4, (futexp),                      \
  95                               __lll_private_flag (FUTEX_WAKE, private),       \
  96                               (nr), 0);                                       \
  97     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
  98   })
  99
 100 #define lll_robust_dead(futexv, private) \
 101   do                                                                          \
 102     {                                                                         \
 103       INTERNAL_SYSCALL_DECL (__err);                                          \
 104       int *__futexp = &(futexv);                                              \
 105                                                                               \
 106       atomic_or (__futexp, FUTEX_OWNER_DIED);                                 \
 107       INTERNAL_SYSCALL (futex, __err, 4, __futexp,                            \
 108                         __lll_private_flag (FUTEX_WAKE, private), 1, 0);      \
 109     }                                                                         \
 110   while (0)
 111
 112 /* Returns non-zero if error happened, zero if success.  */
 113 #define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
 114   ({                                                                          \
 115     INTERNAL_SYSCALL_DECL (__err);                                            \
 116     long int __ret;                                                           \
 117                                                                               \
 118     __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),                      \
 119                               __lll_private_flag (FUTEX_CMP_REQUEUE, private),\
 120                               (nr_wake), (nr_move), (mutex), (val));          \
 121     INTERNAL_SYSCALL_ERROR_P (__ret, __err);                                  \
 122   })
 123
 124 /* Returns non-zero if error happened, zero if success.  */
 125 #define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
 126   ({                                                                          \
 127     INTERNAL_SYSCALL_DECL (__err);                                            \
 128     long int __ret;                                                           \
 129                                                                               \
 130     __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),                      \
 131                               __lll_private_flag (FUTEX_WAKE_OP, private),    \
 132                               (nr_wake), (nr_wake2), (futexp2),               \
 133                               FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);                 \
 134     INTERNAL_SYSCALL_ERROR_P (__ret, __err);                                  \
 135   })
 136
 137
 138 #ifdef UP
 139 # define __lll_acq_instr        ""
 140 # define __lll_rel_instr        ""
 141 #else
 142 # define __lll_acq_instr        "isync"
 143 # ifdef _ARCH_PWR4
 144 /*
 145  * Newer powerpc64 processors support the new "light weight" sync (lwsync)
 146  * So if the build is using -mcpu=[power4,power5,power5+,970] we can
 147  * safely use lwsync.
 148  */
 149 #  define __lll_rel_instr       "lwsync"
 150 # else
 151 /*
 152  * Older powerpc32 processors don't support the new "light weight"
 153  * sync (lwsync).  So the only safe option is to use normal sync
 154  * for all powerpc32 applications.
 155  */
 156 #  define __lll_rel_instr       "sync"
 157 # endif
 158 #endif
 159
 160 /* Set *futex to ID if it is 0, atomically.  Returns the old value */
 161 #define __lll_robust_trylock(futex, id) \
 162   ({ int __val;                                                               \
 163      __asm __volatile ("1:      lwarx   %0,0,%2" MUTEX_HINT_ACQ "\n"          \
 164                        "        cmpwi   0,%0,0\n"                             \
 165                        "        bne     2f\n"                                 \
 166                        "        stwcx.  %3,0,%2\n"                            \
 167                        "        bne-    1b\n"                                 \
 168                        "2:      " __lll_acq_instr                             \
 169                        : "=&r" (__val), "=m" (*futex)                         \
 170                        : "r" (futex), "r" (id), "m" (*futex)                  \
 171                        : "cr0", "memory");                                    \
 172      __val;                                                                   \
 173   })
 174
 175 #define lll_robust_trylock(lock, id) __lll_robust_trylock (&(lock), id)
 176
 177 /* Set *futex to 1 if it is 0, atomically.  Returns the old value */
 178 #define __lll_trylock(futex) __lll_robust_trylock (futex, 1)
 179
 180 #define lll_trylock(lock)       __lll_trylock (&(lock))
 181
 182 /* Set *futex to 2 if it is 0, atomically.  Returns the old value */
 183 #define __lll_cond_trylock(futex) __lll_robust_trylock (futex, 2)
 184
 185 #define lll_cond_trylock(lock)  __lll_cond_trylock (&(lock))
 186
 187
 188 extern void __lll_lock_wait_private (int *futex) attribute_hidden;
 189 extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
 190 extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden;
 191
 192 #define lll_lock(lock, private) \
 193   (void) ({                                                                   \
 194     int *__futex = &(lock);                                                   \
 195     if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, 1, 0),\
 196                           0) != 0)                                            \
 197       {                                                                       \
 198         if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)       \
 199           __lll_lock_wait_private (__futex);                                  \
 200         else                                                                  \
 201           __lll_lock_wait (__futex, private);                                 \
 202       }                                                                       \
 203   })
 204
 205 #define lll_robust_lock(lock, id, private) \
 206   ({                                                                          \
 207     int *__futex = &(lock);                                                   \
 208     int __val = 0;                                                            \
 209     if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, id,  \
 210                                                                 0), 0))       \
 211       __val = __lll_robust_lock_wait (__futex, private);                      \
 212     __val;                                                                    \
 213   })
 214
 215 #define lll_cond_lock(lock, private) \
 216   (void) ({                                                                   \
 217     int *__futex = &(lock);                                                   \
 218     if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, 2, 0),\
 219                           0) != 0)                                            \
 220       __lll_lock_wait (__futex, private);                                     \
 221   })
 222
 223 #define lll_robust_cond_lock(lock, id, private) \
 224   ({                                                                          \
 225     int *__futex = &(lock);                                                   \
 226     int __val = 0;                                                            \
 227     int __id = id | FUTEX_WAITERS;                                            \
 228     if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, __id,\
 229                                                                 0), 0))       \
 230       __val = __lll_robust_lock_wait (__futex, private);                      \
 231     __val;                                                                    \
 232   })
 233
 234
 235 extern int __lll_timedlock_wait
 236   (int *futex, const struct timespec *, int private) attribute_hidden;
 237 extern int __lll_robust_timedlock_wait
 238   (int *futex, const struct timespec *, int private) attribute_hidden;
 239
 240 #define lll_timedlock(lock, abstime, private) \
 241   ({                                                                          \
 242     int *__futex = &(lock);                                                   \
 243     int __val = 0;                                                            \
 244     if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, 1, 0),\
 245                           0) != 0)                                            \
 246       __val = __lll_timedlock_wait (__futex, abstime, private);               \
 247     __val;                                                                    \
 248   })
 249
 250 #define lll_robust_timedlock(lock, abstime, id, private) \
 251   ({                                                                          \
 252     int *__futex = &(lock);                                                   \
 253     int __val = 0;                                                            \
 254     if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, id,  \
 255                                                                 0), 0))       \
 256       __val = __lll_robust_timedlock_wait (__futex, abstime, private);        \
 257     __val;                                                                    \
 258   })
 259
 260 #define lll_unlock(lock, private) \
 261   ((void) ({                                                                  \
 262     int *__futex = &(lock);                                                   \
 263     int __val = atomic_exchange_rel (__futex, 0);                             \
 264     if (__builtin_expect (__val > 1, 0))                                      \
 265       lll_futex_wake (__futex, 1, private);                                   \
 266   }))
 267
 268 #define lll_robust_unlock(lock, private) \
 269   ((void) ({                                                                  \
 270     int *__futex = &(lock);                                                   \
 271     int __val = atomic_exchange_rel (__futex, 0);                             \
 272     if (__builtin_expect (__val & FUTEX_WAITERS, 0))                          \
 273       lll_futex_wake (__futex, 1, private);                                   \
 274   }))
 275
 276 #define lll_islocked(futex) \
 277   (futex != 0)
 278
 279
 280 /* Initializers for lock.  */
 281 #define LLL_LOCK_INITIALIZER            (0)
 282 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 283
 284 /* The states of a lock are:
 285     0  -  untaken
 286     1  -  taken by one user
 287    >1  -  taken by more users */
 288
 289 /* The kernel notifies a process which uses CLONE_CLEARTID via futex
 290    wakeup when the clone terminates.  The memory location contains the
 291    thread ID while the clone is running and is reset to zero
 292    afterwards.  */
 293 #define lll_wait_tid(tid) \
 294   do {                                                                        \
 295     __typeof (tid) __tid;                                                     \
 296     while ((__tid = (tid)) != 0)                                              \
 297       lll_futex_wait (&(tid), __tid, LLL_SHARED);                             \
 298   } while (0)
 299
 300 extern int __lll_timedwait_tid (int *, const struct timespec *)
 301      attribute_hidden;
 302
 303 #define lll_timedwait_tid(tid, abstime) \
 304   ({                                                                          \
 305     int __res = 0;                                                            \
 306     if ((tid) != 0)                                                           \
 307       __res = __lll_timedwait_tid (&(tid), (abstime));                        \
 308     __res;                                                                    \
 309   })
 310
 311 #endif  /* lowlevellock.h */