kernel/spinlock.c

   1 /*
   2  * Copyright (2004) Linus Torvalds
   3  *
   4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
   5  *
   6  * Copyright (2004, 2005) Ingo Molnar
   7  *
   8  * This file contains the spinlock/rwlock implementations for the
   9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
  10  *
  11  * Note that some architectures have special knowledge about the
  12  * stack frames of these functions in their profile_pc. If you
  13  * change anything significant here that could change the stack
  14  * frame contact the architecture maintainers.
  15  */
  16
  17 #include <linux/linkage.h>
  18 #include <linux/preempt.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/debug_locks.h>
  22 #include <linux/module.h>
  23
  24 int __lockfunc _spin_trylock(spinlock_t *lock)
  25 {
  26         return __spin_trylock(lock);
  27 }
  28 EXPORT_SYMBOL(_spin_trylock);
  29
  30 int __lockfunc _read_trylock(rwlock_t *lock)
  31 {
  32         return __read_trylock(lock);
  33 }
  34 EXPORT_SYMBOL(_read_trylock);
  35
  36 int __lockfunc _write_trylock(rwlock_t *lock)
  37 {
  38         return __write_trylock(lock);
  39 }
  40 EXPORT_SYMBOL(_write_trylock);
  41
  42 /*
  43  * If lockdep is enabled then we use the non-preemption spin-ops
  44  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  45  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  46  */
  47 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
  48
  49 void __lockfunc _read_lock(rwlock_t *lock)
  50 {
  51         __read_lock(lock);
  52 }
  53 EXPORT_SYMBOL(_read_lock);
  54
  55 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
  56 {
  57         return __spin_lock_irqsave(lock);
  58 }
  59 EXPORT_SYMBOL(_spin_lock_irqsave);
  60
  61 void __lockfunc _spin_lock_irq(spinlock_t *lock)
  62 {
  63         __spin_lock_irq(lock);
  64 }
  65 EXPORT_SYMBOL(_spin_lock_irq);
  66
  67 void __lockfunc _spin_lock_bh(spinlock_t *lock)
  68 {
  69         __spin_lock_bh(lock);
  70 }
  71 EXPORT_SYMBOL(_spin_lock_bh);
  72
  73 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
  74 {
  75         return __read_lock_irqsave(lock);
  76 }
  77 EXPORT_SYMBOL(_read_lock_irqsave);
  78
  79 void __lockfunc _read_lock_irq(rwlock_t *lock)
  80 {
  81         __read_lock_irq(lock);
  82 }
  83 EXPORT_SYMBOL(_read_lock_irq);
  84
  85 void __lockfunc _read_lock_bh(rwlock_t *lock)
  86 {
  87         __read_lock_bh(lock);
  88 }
  89 EXPORT_SYMBOL(_read_lock_bh);
  90
  91 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
  92 {
  93         return __write_lock_irqsave(lock);
  94 }
  95 EXPORT_SYMBOL(_write_lock_irqsave);
  96
  97 void __lockfunc _write_lock_irq(rwlock_t *lock)
  98 {
  99         __write_lock_irq(lock);
 100 }
 101 EXPORT_SYMBOL(_write_lock_irq);
 102
 103 void __lockfunc _write_lock_bh(rwlock_t *lock)
 104 {
 105         __write_lock_bh(lock);
 106 }
 107 EXPORT_SYMBOL(_write_lock_bh);
 108
 109 void __lockfunc _spin_lock(spinlock_t *lock)
 110 {
 111         __spin_lock(lock);
 112 }
 113 EXPORT_SYMBOL(_spin_lock);
 114
 115 void __lockfunc _write_lock(rwlock_t *lock)
 116 {
 117         __write_lock(lock);
 118 }
 119 EXPORT_SYMBOL(_write_lock);
 120
 121 #else /* CONFIG_PREEMPT: */
 122
 123 /*
 124  * This could be a long-held lock. We both prepare to spin for a long
 125  * time (making _this_ CPU preemptable if possible), and we also signal
 126  * towards that other CPU that it should break the lock ASAP.
 127  *
 128  * (We do this in a function because inlining it would be excessive.)
 129  */
 130
 131 #define BUILD_LOCK_OPS(op, locktype)                                    \
 132 void __lockfunc _##op##_lock(locktype##_t *lock)                        \
 133 {                                                                       \
 134         for (;;) {                                                      \
 135                 preempt_disable();                                      \
 136                 if (likely(_raw_##op##_trylock(lock)))                  \
 137                         break;                                          \
 138                 preempt_enable();                                       \
 139                                                                         \
 140                 if (!(lock)->break_lock)                                \
 141                         (lock)->break_lock = 1;                         \
 142                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 143                         _raw_##op##_relax(&lock->raw_lock);             \
 144         }                                                               \
 145         (lock)->break_lock = 0;                                         \
 146 }                                                                       \
 147                                                                         \
 148 EXPORT_SYMBOL(_##op##_lock);                                            \
 149                                                                         \
 150 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)       \
 151 {                                                                       \
 152         unsigned long flags;                                            \
 153                                                                         \
 154         for (;;) {                                                      \
 155                 preempt_disable();                                      \
 156                 local_irq_save(flags);                                  \
 157                 if (likely(_raw_##op##_trylock(lock)))                  \
 158                         break;                                          \
 159                 local_irq_restore(flags);                               \
 160                 preempt_enable();                                       \
 161                                                                         \
 162                 if (!(lock)->break_lock)                                \
 163                         (lock)->break_lock = 1;                         \
 164                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 165                         _raw_##op##_relax(&lock->raw_lock);             \
 166         }                                                               \
 167         (lock)->break_lock = 0;                                         \
 168         return flags;                                                   \
 169 }                                                                       \
 170                                                                         \
 171 EXPORT_SYMBOL(_##op##_lock_irqsave);                                    \
 172                                                                         \
 173 void __lockfunc _##op##_lock_irq(locktype##_t *lock)                    \
 174 {                                                                       \
 175         _##op##_lock_irqsave(lock);                                     \
 176 }                                                                       \
 177                                                                         \
 178 EXPORT_SYMBOL(_##op##_lock_irq);                                        \
 179                                                                         \
 180 void __lockfunc _##op##_lock_bh(locktype##_t *lock)                     \
 181 {                                                                       \
 182         unsigned long flags;                                            \
 183                                                                         \
 184         /*                                                      */      \
 185         /* Careful: we must exclude softirqs too, hence the     */      \
 186         /* irq-disabling. We use the generic preemption-aware   */      \
 187         /* function:                                            */      \
 188         /**/                                                            \
 189         flags = _##op##_lock_irqsave(lock);                             \
 190         local_bh_disable();                                             \
 191         local_irq_restore(flags);                                       \
 192 }                                                                       \
 193                                                                         \
 194 EXPORT_SYMBOL(_##op##_lock_bh)
 195
 196 /*
 197  * Build preemption-friendly versions of the following
 198  * lock-spinning functions:
 199  *
 200  *         _[spin|read|write]_lock()
 201  *         _[spin|read|write]_lock_irq()
 202  *         _[spin|read|write]_lock_irqsave()
 203  *         _[spin|read|write]_lock_bh()
 204  */
 205 BUILD_LOCK_OPS(spin, spinlock);
 206 BUILD_LOCK_OPS(read, rwlock);
 207 BUILD_LOCK_OPS(write, rwlock);
 208
 209 #endif /* CONFIG_PREEMPT */
 210
 211 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 212
 213 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
 214 {
 215         preempt_disable();
 216         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 217         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 218 }
 219 EXPORT_SYMBOL(_spin_lock_nested);
 220
 221 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
 222 {
 223         unsigned long flags;
 224
 225         local_irq_save(flags);
 226         preempt_disable();
 227         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 228         LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock,
 229                                 _raw_spin_lock_flags, &flags);
 230         return flags;
 231 }
 232 EXPORT_SYMBOL(_spin_lock_irqsave_nested);
 233
 234 void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
 235                                      struct lockdep_map *nest_lock)
 236 {
 237         preempt_disable();
 238         spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
 239         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 240 }
 241 EXPORT_SYMBOL(_spin_lock_nest_lock);
 242
 243 #endif
 244
 245 void __lockfunc _spin_unlock(spinlock_t *lock)
 246 {
 247         __spin_unlock(lock);
 248 }
 249 EXPORT_SYMBOL(_spin_unlock);
 250
 251 void __lockfunc _write_unlock(rwlock_t *lock)
 252 {
 253         __write_unlock(lock);
 254 }
 255 EXPORT_SYMBOL(_write_unlock);
 256
 257 void __lockfunc _read_unlock(rwlock_t *lock)
 258 {
 259         __read_unlock(lock);
 260 }
 261 EXPORT_SYMBOL(_read_unlock);
 262
 263 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 264 {
 265         __spin_unlock_irqrestore(lock, flags);
 266 }
 267 EXPORT_SYMBOL(_spin_unlock_irqrestore);
 268
 269 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 270 {
 271         __spin_unlock_irq(lock);
 272 }
 273 EXPORT_SYMBOL(_spin_unlock_irq);
 274
 275 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 276 {
 277         __spin_unlock_bh(lock);
 278 }
 279 EXPORT_SYMBOL(_spin_unlock_bh);
 280
 281 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 282 {
 283         __read_unlock_irqrestore(lock, flags);
 284 }
 285 EXPORT_SYMBOL(_read_unlock_irqrestore);
 286
 287 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 288 {
 289         __read_unlock_irq(lock);
 290 }
 291 EXPORT_SYMBOL(_read_unlock_irq);
 292
 293 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 294 {
 295         __read_unlock_bh(lock);
 296 }
 297 EXPORT_SYMBOL(_read_unlock_bh);
 298
 299 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 300 {
 301         __write_unlock_irqrestore(lock, flags);
 302 }
 303 EXPORT_SYMBOL(_write_unlock_irqrestore);
 304
 305 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 306 {
 307         __write_unlock_irq(lock);
 308 }
 309 EXPORT_SYMBOL(_write_unlock_irq);
 310
 311 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 312 {
 313         __write_unlock_bh(lock);
 314 }
 315 EXPORT_SYMBOL(_write_unlock_bh);
 316
 317 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 318 {
 319         return __spin_trylock_bh(lock);
 320 }
 321 EXPORT_SYMBOL(_spin_trylock_bh);
 322
 323 notrace int in_lock_functions(unsigned long addr)
 324 {
 325         /* Linker adds these: start and end of __lockfunc functions */
 326         extern char __lock_text_start[], __lock_text_end[];
 327
 328         return addr >= (unsigned long)__lock_text_start
 329         && addr < (unsigned long)__lock_text_end;
 330 }
 331 EXPORT_SYMBOL(in_lock_functions);