include/linux/brlock.h

   1 #ifndef __LINUX_BRLOCK_H
   2 #define __LINUX_BRLOCK_H
   3
   4 /*
   5  * 'Big Reader' read-write spinlocks.
   6  *
   7  * super-fast read/write locks, with write-side penalty. The point
   8  * is to have a per-CPU read/write lock. Readers lock their CPU-local
   9  * readlock, writers must lock all locks to get write access. These
  10  * CPU-read-write locks are semantically identical to normal rwlocks.
  11  * Memory usage is higher as well. (NR_CPUS*L1_CACHE_BYTES bytes)
  12  *
  13  * The most important feature is that these spinlocks do not cause
  14  * cacheline ping-pong in the 'most readonly data' case.
  15  *
  16  * Copyright 2000, Ingo Molnar <mingo@redhat.com>
  17  *
  18  * Registry idea and naming [ crutial! :-) ] by:
  19  *
  20  *                 David S. Miller <davem@redhat.com>
  21  *
  22  * David has an implementation that doesnt use atomic operations in
  23  * the read branch via memory ordering tricks - i guess we need to
  24  * split this up into a per-arch thing? The atomicity issue is a
  25  * secondary item in profiles, at least on x86 platforms.
  26  *
  27  * The atomic op version overhead is indeed a big deal on
  28  * load-locked/store-conditional cpus (ALPHA/MIPS/PPC) and
  29  * compare-and-swap cpus (Sparc64).  So we control which
  30  * implementation to use with a __BRLOCK_USE_ATOMICS define. -DaveM
  31  */
  32
  33 /* Register bigreader lock indices here. */
  34 enum brlock_indices {
  35         BR_GLOBALIRQ_LOCK,
  36         BR_NETPROTO_LOCK,
  37
  38         __BR_END
  39 };
  40
  41 #include <linux/config.h>
  42
  43 #ifdef CONFIG_SMP
  44
  45 #include <linux/cache.h>
  46 #include <linux/spinlock.h>
  47
  48 #if defined(__i386__) || defined(__ia64__)
  49 #define __BRLOCK_USE_ATOMICS
  50 #else
  51 #undef __BRLOCK_USE_ATOMICS
  52 #endif
  53
  54 #ifdef __BRLOCK_USE_ATOMICS
  55 typedef rwlock_t        brlock_read_lock_t;
  56 #else
  57 typedef unsigned int    brlock_read_lock_t;
  58 #endif
  59
  60 /*
  61  * align last allocated index to the next cacheline:
  62  */
  63 #define __BR_IDX_MAX \
  64         (((sizeof(brlock_read_lock_t)*__BR_END + SMP_CACHE_BYTES-1) & ~(SMP_CACHE_BYTES-1)) / sizeof(brlock_read_lock_t))
  65
  66 extern brlock_read_lock_t __brlock_array[NR_CPUS][__BR_IDX_MAX];
  67
  68 #ifndef __BRLOCK_USE_ATOMICS
  69 struct br_wrlock {
  70         spinlock_t lock;
  71 } __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
  72
  73 extern struct br_wrlock __br_write_locks[__BR_IDX_MAX];
  74 #endif
  75
  76 extern void __br_lock_usage_bug (void);
  77
  78 #ifdef __BRLOCK_USE_ATOMICS
  79
  80 static inline void br_read_lock (enum brlock_indices idx)
  81 {
  82         /*
  83          * This causes a link-time bug message if an
  84          * invalid index is used:
  85          */
  86         if (idx >= __BR_END)
  87                 __br_lock_usage_bug();
  88
  89         read_lock(&__brlock_array[smp_processor_id()][idx]);
  90 }
  91
  92 static inline void br_read_unlock (enum brlock_indices idx)
  93 {
  94         if (idx >= __BR_END)
  95                 __br_lock_usage_bug();
  96
  97         read_unlock(&__brlock_array[smp_processor_id()][idx]);
  98 }
  99
 100 #else /* ! __BRLOCK_USE_ATOMICS */
 101 static inline void br_read_lock (enum brlock_indices idx)
 102 {
 103         unsigned int *ctr;
 104         spinlock_t *lock;
 105
 106         /*
 107          * This causes a link-time bug message if an
 108          * invalid index is used:
 109          */
 110         if (idx >= __BR_END)
 111                 __br_lock_usage_bug();
 112
 113         ctr = &__brlock_array[smp_processor_id()][idx];
 114         lock = &__br_write_locks[idx].lock;
 115 again:
 116         (*ctr)++;
 117         mb();
 118         if (spin_is_locked(lock)) {
 119                 (*ctr)--;
 120                 wmb(); /*
 121                         * The release of the ctr must become visible
 122                         * to the other cpus eventually thus wmb(),
 123                         * we don't care if spin_is_locked is reordered
 124                         * before the releasing of the ctr.
 125                         * However IMHO this wmb() is superflous even in theory.
 126                         * It would not be superflous only if on the
 127                         * other CPUs doing a ldl_l instead of an ldl
 128                         * would make a difference and I don't think this is
 129                         * the case.
 130                         * I'd like to clarify this issue further
 131                         * but for now this is a slow path so adding the
 132                         * wmb() will keep us on the safe side.
 133                         */
 134                 while (spin_is_locked(lock))
 135                         barrier();
 136                 goto again;
 137         }
 138 }
 139
 140 static inline void br_read_unlock (enum brlock_indices idx)
 141 {
 142         unsigned int *ctr;
 143
 144         if (idx >= __BR_END)
 145                 __br_lock_usage_bug();
 146
 147         ctr = &__brlock_array[smp_processor_id()][idx];
 148
 149         wmb();
 150         (*ctr)--;
 151 }
 152 #endif /* __BRLOCK_USE_ATOMICS */
 153
 154 /* write path not inlined - it's rare and larger */
 155
 156 extern void FASTCALL(__br_write_lock (enum brlock_indices idx));
 157 extern void FASTCALL(__br_write_unlock (enum brlock_indices idx));
 158
 159 static inline void br_write_lock (enum brlock_indices idx)
 160 {
 161         if (idx >= __BR_END)
 162                 __br_lock_usage_bug();
 163         __br_write_lock(idx);
 164 }
 165
 166 static inline void br_write_unlock (enum brlock_indices idx)
 167 {
 168         if (idx >= __BR_END)
 169                 __br_lock_usage_bug();
 170         __br_write_unlock(idx);
 171 }
 172
 173 #else
 174 # define br_read_lock(idx)      ((void)(idx))
 175 # define br_read_unlock(idx)    ((void)(idx))
 176 # define br_write_lock(idx)     ((void)(idx))
 177 # define br_write_unlock(idx)   ((void)(idx))
 178 #endif
 179
 180 /*
 181  * Now enumerate all of the possible sw/hw IRQ protected
 182  * versions of the interfaces.
 183  */
 184 #define br_read_lock_irqsave(idx, flags) \
 185         do { local_irq_save(flags); br_read_lock(idx); } while (0)
 186
 187 #define br_read_lock_irq(idx) \
 188         do { local_irq_disable(); br_read_lock(idx); } while (0)
 189
 190 #define br_read_lock_bh(idx) \
 191         do { local_bh_disable(); br_read_lock(idx); } while (0)
 192
 193 #define br_write_lock_irqsave(idx, flags) \
 194         do { local_irq_save(flags); br_write_lock(idx); } while (0)
 195
 196 #define br_write_lock_irq(idx) \
 197         do { local_irq_disable(); br_write_lock(idx); } while (0)
 198
 199 #define br_write_lock_bh(idx) \
 200         do { local_bh_disable(); br_write_lock(idx); } while (0)
 201
 202 #define br_read_unlock_irqrestore(idx, flags) \
 203         do { br_read_unlock(irx); local_irq_restore(flags); } while (0)
 204
 205 #define br_read_unlock_irq(idx) \
 206         do { br_read_unlock(idx); local_irq_enable(); } while (0)
 207
 208 #define br_read_unlock_bh(idx) \
 209         do { br_read_unlock(idx); local_bh_enable(); } while (0)
 210
 211 #define br_write_unlock_irqrestore(idx, flags) \
 212         do { br_write_unlock(irx); local_irq_restore(flags); } while (0)
 213
 214 #define br_write_unlock_irq(idx) \
 215         do { br_write_unlock(idx); local_irq_enable(); } while (0)
 216
 217 #define br_write_unlock_bh(idx) \
 218         do { br_write_unlock(idx); local_bh_enable(); } while (0)
 219
 220 #endif /* __LINUX_BRLOCK_H */