sysdeps/unix/sysv/linux/i386/lowlevellock.h

   1 /* Copyright (C) 2002-2017 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #ifndef _LOWLEVELLOCK_H
  20 #define _LOWLEVELLOCK_H 1
  21
  22 #include <stap-probe.h>
  23
  24 #ifndef __ASSEMBLER__
  25 # include <time.h>
  26 # include <sys/param.h>
  27 # include <bits/pthreadtypes.h>
  28 # include <kernel-features.h>
  29 # include <tcb-offsets.h>
  30
  31 # ifndef LOCK_INSTR
  32 #  ifdef UP
  33 #   define LOCK_INSTR   /* nothing */
  34 #  else
  35 #   define LOCK_INSTR "lock;"
  36 #  endif
  37 # endif
  38 #else
  39 # ifndef LOCK
  40 #  ifdef UP
  41 #   define LOCK
  42 #  else
  43 #   define LOCK lock
  44 #  endif
  45 # endif
  46 #endif
  47
  48 #include <lowlevellock-futex.h>
  49
  50 /* XXX Remove when no assembler code uses futexes anymore.  */
  51 #define SYS_futex               __NR_futex
  52
  53 #ifndef __ASSEMBLER__
  54
  55 /* Initializer for compatibility lock.  */
  56 #define LLL_LOCK_INITIALIZER            (0)
  57 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
  58 #define LLL_LOCK_INITIALIZER_WAITERS    (2)
  59
  60
  61 /* NB: in the lll_trylock macro we simply return the value in %eax
  62    after the cmpxchg instruction.  In case the operation succeded this
  63    value is zero.  In case the operation failed, the cmpxchg instruction
  64    has loaded the current value of the memory work which is guaranteed
  65    to be nonzero.  */
  66 #if !IS_IN (libc) || defined UP
  67 # define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
  68 #else
  69 # define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
  70                            "je 0f\n\t"                                        \
  71                            "lock\n"                                           \
  72                            "0:\tcmpxchgl %2, %1"
  73 #endif
  74
  75 #define lll_trylock(futex) \
  76   ({ int ret;                                                                 \
  77      __asm __volatile (__lll_trylock_asm                                      \
  78                        : "=a" (ret), "=m" (futex)                             \
  79                        : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
  80                          "0" (LLL_LOCK_INITIALIZER),                          \
  81                          "i" (MULTIPLE_THREADS_OFFSET)                        \
  82                        : "memory");                                           \
  83      ret; })
  84
  85
  86 #define lll_cond_trylock(futex) \
  87   ({ int ret;                                                                 \
  88      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"                           \
  89                        : "=a" (ret), "=m" (futex)                             \
  90                        : "r" (LLL_LOCK_INITIALIZER_WAITERS),                  \
  91                          "m" (futex), "0" (LLL_LOCK_INITIALIZER)              \
  92                        : "memory");                                           \
  93      ret; })
  94
  95 #if !IS_IN (libc) || defined UP
  96 # define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
  97 #else
  98 # define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t"                         \
  99                               "je 0f\n\t"                                     \
 100                               "lock\n"                                        \
 101                               "0:\tcmpxchgl %1, %2\n\t"
 102 #endif
 103
 104 #define lll_lock(futex, private) \
 105   (void)                                                                      \
 106     ({ int ignore1, ignore2;                                                  \
 107        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 108          __asm __volatile (__lll_lock_asm_start                               \
 109                            "jz 18f\n\t"                               \
 110                            "1:\tleal %2, %%ecx\n"                             \
 111                            "2:\tcall __lll_lock_wait_private\n"               \
 112                            "18:"                                              \
 113                            : "=a" (ignore1), "=c" (ignore2), "=m" (futex)     \
 114                            : "0" (0), "1" (1), "m" (futex),                   \
 115                              "i" (MULTIPLE_THREADS_OFFSET)                    \
 116                            : "memory");                                       \
 117        else                                                                   \
 118          {                                                                    \
 119            int ignore3;                                                       \
 120            __asm __volatile (__lll_lock_asm_start                             \
 121                              "jz 18f\n\t"                                     \
 122                              "1:\tleal %2, %%edx\n"                           \
 123                              "0:\tmovl %8, %%ecx\n"                           \
 124                              "2:\tcall __lll_lock_wait\n"                     \
 125                              "18:"                                            \
 126                              : "=a" (ignore1), "=c" (ignore2),                \
 127                                "=m" (futex), "=&d" (ignore3)                  \
 128                              : "1" (1), "m" (futex),                          \
 129                                "i" (MULTIPLE_THREADS_OFFSET), "0" (0),        \
 130                                "g" ((int) (private))                          \
 131                              : "memory");                                     \
 132          }                                                                    \
 133     })
 134
 135
 136 /* Special version of lll_lock which causes the unlock function to
 137    always wakeup waiters.  */
 138 #define lll_cond_lock(futex, private) \
 139   (void)                                                                      \
 140     ({ int ignore1, ignore2, ignore3;                                         \
 141        __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"                     \
 142                          "jz 18f\n\t"                                         \
 143                          "1:\tleal %2, %%edx\n"                               \
 144                          "0:\tmovl %7, %%ecx\n"                               \
 145                          "2:\tcall __lll_lock_wait\n"                         \
 146                          "18:"                                                \
 147                          : "=a" (ignore1), "=c" (ignore2), "=m" (futex),      \
 148                            "=&d" (ignore3)                                    \
 149                          : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
 150                          : "memory");                                         \
 151     })
 152
 153
 154 #define lll_timedlock(futex, timeout, private) \
 155   ({ int result, ignore1, ignore2, ignore3;                                   \
 156      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"                       \
 157                        "jz 18f\n\t"                                           \
 158                        "1:\tleal %3, %%ecx\n"                                 \
 159                        "0:\tmovl %8, %%edx\n"                                 \
 160                        "2:\tcall __lll_timedlock_wait\n"                      \
 161                        "18:"                                                  \
 162                        : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
 163                          "=m" (futex), "=S" (ignore3)                         \
 164                        : "0" (0), "1" (1), "m" (futex), "m" (timeout),        \
 165                          "4" ((int) (private))                                \
 166                        : "memory");                                           \
 167      result; })
 168
 169 extern int __lll_timedlock_elision (int *futex, short *adapt_count,
 170                                          const struct timespec *timeout,
 171                                          int private) attribute_hidden;
 172
 173 #define lll_timedlock_elision(futex, adapt_count, timeout, private)     \
 174   __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
 175
 176 #if !IS_IN (libc) || defined UP
 177 # define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
 178 #else
 179 # define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t"                             \
 180                           "je 0f\n\t"                                         \
 181                           "lock\n"                                            \
 182                           "0:\tsubl $1,%0\n\t"
 183 #endif
 184
 185 #define lll_unlock(futex, private) \
 186   (void)                                                                      \
 187     ({ int ignore;                                                            \
 188        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 189          __asm __volatile (__lll_unlock_asm                                   \
 190                            "je 18f\n\t"                                       \
 191                            "1:\tleal %0, %%eax\n"                             \
 192                            "2:\tcall __lll_unlock_wake_private\n"             \
 193                            "18:"                                              \
 194                            : "=m" (futex), "=&a" (ignore)                     \
 195                            : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET)       \
 196                            : "memory");                                       \
 197        else                                                                   \
 198          {                                                                    \
 199            int ignore2;                                                       \
 200            __asm __volatile (__lll_unlock_asm                                 \
 201                              "je 18f\n\t"                                     \
 202                              "1:\tleal %0, %%eax\n"                           \
 203                              "0:\tmovl %5, %%ecx\n"                           \
 204                              "2:\tcall __lll_unlock_wake\n"                   \
 205                              "18:"                                            \
 206                              : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)  \
 207                              : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex),    \
 208                                "g" ((int) (private))                          \
 209                              : "memory");                                     \
 210          }                                                                    \
 211     })
 212
 213
 214 #define lll_islocked(futex) \
 215   (futex != LLL_LOCK_INITIALIZER)
 216
 217 /* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
 218    wake-up when the clone terminates.  The memory location contains the
 219    thread ID while the clone is running and is reset to zero by the kernel
 220    afterwards.  The kernel up to version 3.16.3 does not use the private futex
 221    operations for futex wake-up when the clone terminates.  */
 222 #define lll_wait_tid(tid) \
 223   do {                                  \
 224     __typeof (tid) __tid;               \
 225     while ((__tid = (tid)) != 0)        \
 226       lll_futex_wait (&(tid), __tid, LLL_SHARED);\
 227   } while (0)
 228
 229 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
 230      __attribute__ ((regparm (2))) attribute_hidden;
 231
 232 /* As lll_wait_tid, but with a timeout.  If the timeout occurs then return
 233    ETIMEDOUT.  If ABSTIME is invalid, return EINVAL.
 234    XXX Note that this differs from the generic version in that we do the
 235    error checking here and not in __lll_timedwait_tid.  */
 236 #define lll_timedwait_tid(tid, abstime) \
 237   ({                                                                          \
 238     int __result = 0;                                                         \
 239     if ((tid) != 0)                                                           \
 240       __result = __lll_timedwait_tid (&(tid), (abstime));                     \
 241     __result; })
 242
 243
 244 extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
 245   attribute_hidden;
 246
 247 extern int __lll_unlock_elision(int *lock, int private)
 248   attribute_hidden;
 249
 250 extern int __lll_trylock_elision(int *lock, short *adapt_count)
 251   attribute_hidden;
 252
 253 #define lll_lock_elision(futex, adapt_count, private) \
 254   __lll_lock_elision (&(futex), &(adapt_count), private)
 255 #define lll_unlock_elision(futex, adapt_count, private) \
 256   __lll_unlock_elision (&(futex), private)
 257 #define lll_trylock_elision(futex, adapt_count) \
 258   __lll_trylock_elision(&(futex), &(adapt_count))
 259
 260 #endif  /* !__ASSEMBLER__ */
 261
 262 #endif  /* lowlevellock.h */