include/asm-x86/bitops.h

   1 #ifndef _ASM_X86_BITOPS_H
   2 #define _ASM_X86_BITOPS_H
   3
   4 /*
   5  * Copyright 1992, Linus Torvalds.
   6  */
   7
   8 #ifndef _LINUX_BITOPS_H
   9 #error only <linux/bitops.h> can be included directly
  10 #endif
  11
  12 #include <linux/compiler.h>
  13 #include <asm/alternative.h>
  14
  15 /*
  16  * These have to be done with inline assembly: that way the bit-setting
  17  * is guaranteed to be atomic. All bit operations return 0 if the bit
  18  * was cleared before the operation and != 0 if it was not.
  19  *
  20  * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
  21  */
  22
  23 #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
  24 /* Technically wrong, but this avoids compilation errors on some gcc
  25    versions. */
  26 #define ADDR "=m" (*(volatile long *) addr)
  27 #define BIT_ADDR "=m" (((volatile int *) addr)[nr >> 5])
  28 #else
  29 #define ADDR "+m" (*(volatile long *) addr)
  30 #define BIT_ADDR "+m" (((volatile int *) addr)[nr >> 5])
  31 #endif
  32 #define BASE_ADDR "m" (*(volatile int *) addr)
  33
  34 /**
  35  * set_bit - Atomically set a bit in memory
  36  * @nr: the bit to set
  37  * @addr: the address to start counting from
  38  *
  39  * This function is atomic and may not be reordered.  See __set_bit()
  40  * if you do not require the atomic guarantees.
  41  *
  42  * Note: there are no guarantees that this function will not be reordered
  43  * on non x86 architectures, so if you are writing portable code,
  44  * make sure not to rely on its reordering guarantees.
  45  *
  46  * Note that @nr may be almost arbitrarily large; this function is not
  47  * restricted to acting on a single-word quantity.
  48  */
  49 static inline void set_bit(int nr, volatile void *addr)
  50 {
  51         asm volatile(LOCK_PREFIX "bts %1,%0"
  52                      : ADDR
  53                      : "Ir" (nr) : "memory");
  54 }
  55
  56 /**
  57  * __set_bit - Set a bit in memory
  58  * @nr: the bit to set
  59  * @addr: the address to start counting from
  60  *
  61  * Unlike set_bit(), this function is non-atomic and may be reordered.
  62  * If it's called on the same region of memory simultaneously, the effect
  63  * may be that only one operation succeeds.
  64  */
  65 static inline void __set_bit(int nr, volatile void *addr)
  66 {
  67         asm volatile("bts %1,%0"
  68                      : ADDR
  69                      : "Ir" (nr) : "memory");
  70 }
  71
  72
  73 /**
  74  * clear_bit - Clears a bit in memory
  75  * @nr: Bit to clear
  76  * @addr: Address to start counting from
  77  *
  78  * clear_bit() is atomic and may not be reordered.  However, it does
  79  * not contain a memory barrier, so if it is used for locking purposes,
  80  * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
  81  * in order to ensure changes are visible on other processors.
  82  */
  83 static inline void clear_bit(int nr, volatile void *addr)
  84 {
  85         asm volatile(LOCK_PREFIX "btr %1,%2"
  86                      : BIT_ADDR : "Ir" (nr), BASE_ADDR);
  87 }
  88
  89 /*
  90  * clear_bit_unlock - Clears a bit in memory
  91  * @nr: Bit to clear
  92  * @addr: Address to start counting from
  93  *
  94  * clear_bit() is atomic and implies release semantics before the memory
  95  * operation. It can be used for an unlock.
  96  */
  97 static inline void clear_bit_unlock(unsigned nr, volatile void *addr)
  98 {
  99         barrier();
 100         clear_bit(nr, addr);
 101 }
 102
 103 static inline void __clear_bit(int nr, volatile void *addr)
 104 {
 105         asm volatile("btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
 106 }
 107
 108 /*
 109  * __clear_bit_unlock - Clears a bit in memory
 110  * @nr: Bit to clear
 111  * @addr: Address to start counting from
 112  *
 113  * __clear_bit() is non-atomic and implies release semantics before the memory
 114  * operation. It can be used for an unlock if no other CPUs can concurrently
 115  * modify other bits in the word.
 116  *
 117  * No memory barrier is required here, because x86 cannot reorder stores past
 118  * older loads. Same principle as spin_unlock.
 119  */
 120 static inline void __clear_bit_unlock(unsigned nr, volatile void *addr)
 121 {
 122         barrier();
 123         __clear_bit(nr, addr);
 124 }
 125
 126 #define smp_mb__before_clear_bit()      barrier()
 127 #define smp_mb__after_clear_bit()       barrier()
 128
 129 /**
 130  * __change_bit - Toggle a bit in memory
 131  * @nr: the bit to change
 132  * @addr: the address to start counting from
 133  *
 134  * Unlike change_bit(), this function is non-atomic and may be reordered.
 135  * If it's called on the same region of memory simultaneously, the effect
 136  * may be that only one operation succeeds.
 137  */
 138 static inline void __change_bit(int nr, volatile void *addr)
 139 {
 140         asm volatile("btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
 141 }
 142
 143 /**
 144  * change_bit - Toggle a bit in memory
 145  * @nr: Bit to change
 146  * @addr: Address to start counting from
 147  *
 148  * change_bit() is atomic and may not be reordered.
 149  * Note that @nr may be almost arbitrarily large; this function is not
 150  * restricted to acting on a single-word quantity.
 151  */
 152 static inline void change_bit(int nr, volatile void *addr)
 153 {
 154         asm volatile(LOCK_PREFIX "btc %1,%2"
 155                      : BIT_ADDR : "Ir" (nr), BASE_ADDR);
 156 }
 157
 158 /**
 159  * test_and_set_bit - Set a bit and return its old value
 160  * @nr: Bit to set
 161  * @addr: Address to count from
 162  *
 163  * This operation is atomic and cannot be reordered.
 164  * It also implies a memory barrier.
 165  */
 166 static inline int test_and_set_bit(int nr, volatile void *addr)
 167 {
 168         int oldbit;
 169
 170         asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
 171                      "sbb %0,%0"
 172                      : "=r" (oldbit), ADDR
 173                      : "Ir" (nr) : "memory");
 174
 175         return oldbit;
 176 }
 177
 178 /**
 179  * test_and_set_bit_lock - Set a bit and return its old value for lock
 180  * @nr: Bit to set
 181  * @addr: Address to count from
 182  *
 183  * This is the same as test_and_set_bit on x86.
 184  */
 185 static inline int test_and_set_bit_lock(int nr, volatile void *addr)
 186 {
 187         return test_and_set_bit(nr, addr);
 188 }
 189
 190 /**
 191  * __test_and_set_bit - Set a bit and return its old value
 192  * @nr: Bit to set
 193  * @addr: Address to count from
 194  *
 195  * This operation is non-atomic and can be reordered.
 196  * If two examples of this operation race, one can appear to succeed
 197  * but actually fail.  You must protect multiple accesses with a lock.
 198  */
 199 static inline int __test_and_set_bit(int nr, volatile void *addr)
 200 {
 201         int oldbit;
 202
 203         asm volatile("bts %2,%3\n\t"
 204                      "sbb %0,%0"
 205                      : "=r" (oldbit), BIT_ADDR
 206                      : "Ir" (nr), BASE_ADDR);
 207         return oldbit;
 208 }
 209
 210 /**
 211  * test_and_clear_bit - Clear a bit and return its old value
 212  * @nr: Bit to clear
 213  * @addr: Address to count from
 214  *
 215  * This operation is atomic and cannot be reordered.
 216  * It also implies a memory barrier.
 217  */
 218 static inline int test_and_clear_bit(int nr, volatile void *addr)
 219 {
 220         int oldbit;
 221
 222         asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
 223                      "sbb %0,%0"
 224                      : "=r" (oldbit), ADDR
 225                      : "Ir" (nr) : "memory");
 226
 227         return oldbit;
 228 }
 229
 230 /**
 231  * __test_and_clear_bit - Clear a bit and return its old value
 232  * @nr: Bit to clear
 233  * @addr: Address to count from
 234  *
 235  * This operation is non-atomic and can be reordered.
 236  * If two examples of this operation race, one can appear to succeed
 237  * but actually fail.  You must protect multiple accesses with a lock.
 238  */
 239 static inline int __test_and_clear_bit(int nr, volatile void *addr)
 240 {
 241         int oldbit;
 242
 243         asm volatile("btr %2,%3\n\t"
 244                      "sbb %0,%0"
 245                      : "=r" (oldbit), BIT_ADDR
 246                      : "Ir" (nr), BASE_ADDR);
 247         return oldbit;
 248 }
 249
 250 /* WARNING: non atomic and it can be reordered! */
 251 static inline int __test_and_change_bit(int nr, volatile void *addr)
 252 {
 253         int oldbit;
 254
 255         asm volatile("btc %2,%3\n\t"
 256                      "sbb %0,%0"
 257                      : "=r" (oldbit), BIT_ADDR
 258                      : "Ir" (nr), BASE_ADDR);
 259
 260         return oldbit;
 261 }
 262
 263 /**
 264  * test_and_change_bit - Change a bit and return its old value
 265  * @nr: Bit to change
 266  * @addr: Address to count from
 267  *
 268  * This operation is atomic and cannot be reordered.
 269  * It also implies a memory barrier.
 270  */
 271 static inline int test_and_change_bit(int nr, volatile void *addr)
 272 {
 273         int oldbit;
 274
 275         asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
 276                      "sbb %0,%0"
 277                      : "=r" (oldbit), ADDR
 278                      : "Ir" (nr) : "memory");
 279
 280         return oldbit;
 281 }
 282
 283 static inline int constant_test_bit(int nr, const volatile void *addr)
 284 {
 285         return ((1UL << (nr % BITS_PER_LONG)) &
 286                 (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
 287 }
 288
 289 static inline int variable_test_bit(int nr, volatile const void *addr)
 290 {
 291         int oldbit;
 292
 293         asm volatile("bt %2,%3\n\t"
 294                      "sbb %0,%0"
 295                      : "=r" (oldbit)
 296                      : "m" (((volatile const int *)addr)[nr >> 5]),
 297                        "Ir" (nr), BASE_ADDR);
 298
 299         return oldbit;
 300 }
 301
 302 #if 0 /* Fool kernel-doc since it doesn't do macros yet */
 303 /**
 304  * test_bit - Determine whether a bit is set
 305  * @nr: bit number to test
 306  * @addr: Address to start counting from
 307  */
 308 static int test_bit(int nr, const volatile unsigned long *addr);
 309 #endif
 310
 311 #define test_bit(nr,addr)                       \
 312         (__builtin_constant_p(nr) ?             \
 313          constant_test_bit((nr),(addr)) :       \
 314          variable_test_bit((nr),(addr)))
 315
 316 #undef BASE_ADDR
 317 #undef BIT_ADDR
 318 #undef ADDR
 319
 320 #ifdef CONFIG_X86_32
 321 # include "bitops_32.h"
 322 #else
 323 # include "bitops_64.h"
 324 #endif
 325
 326 #endif  /* _ASM_X86_BITOPS_H */