kernel/spinlock.c

   1 /*
   2  * Copyright (2004) Linus Torvalds
   3  *
   4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
   5  *
   6  * Copyright (2004, 2005) Ingo Molnar
   7  *
   8  * This file contains the spinlock/rwlock implementations for the
   9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
  10  *
  11  * Note that some architectures have special knowledge about the
  12  * stack frames of these functions in their profile_pc. If you
  13  * change anything significant here that could change the stack
  14  * frame contact the architecture maintainers.
  15  */
  16
  17 #include <linux/linkage.h>
  18 #include <linux/preempt.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/debug_locks.h>
  22 #include <linux/module.h>
  23
  24 /*
  25  * If lockdep is enabled then we use the non-preemption spin-ops
  26  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  27  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  28  */
  29 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
  30 /*
  31  * The __lock_function inlines are taken from
  32  * include/linux/spinlock_api_smp.h
  33  */
  34 #else
  35 /*
  36  * We build the __lock_function inlines here. They are too large for
  37  * inlining all over the place, but here is only one user per function
  38  * which embedds them into the calling _lock_function below.
  39  *
  40  * This could be a long-held lock. We both prepare to spin for a long
  41  * time (making _this_ CPU preemptable if possible), and we also signal
  42  * towards that other CPU that it should break the lock ASAP.
  43  */
  44 #define BUILD_LOCK_OPS(op, locktype)                                    \
  45 void __lockfunc __##op##_lock(locktype##_t *lock)                       \
  46 {                                                                       \
  47         for (;;) {                                                      \
  48                 preempt_disable();                                      \
  49                 if (likely(_raw_##op##_trylock(lock)))                  \
  50                         break;                                          \
  51                 preempt_enable();                                       \
  52                                                                         \
  53                 if (!(lock)->break_lock)                                \
  54                         (lock)->break_lock = 1;                         \
  55                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
  56                         _raw_##op##_relax(&lock->raw_lock);             \
  57         }                                                               \
  58         (lock)->break_lock = 0;                                         \
  59 }                                                                       \
  60                                                                         \
  61 unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock)      \
  62 {                                                                       \
  63         unsigned long flags;                                            \
  64                                                                         \
  65         for (;;) {                                                      \
  66                 preempt_disable();                                      \
  67                 local_irq_save(flags);                                  \
  68                 if (likely(_raw_##op##_trylock(lock)))                  \
  69                         break;                                          \
  70                 local_irq_restore(flags);                               \
  71                 preempt_enable();                                       \
  72                                                                         \
  73                 if (!(lock)->break_lock)                                \
  74                         (lock)->break_lock = 1;                         \
  75                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
  76                         _raw_##op##_relax(&lock->raw_lock);             \
  77         }                                                               \
  78         (lock)->break_lock = 0;                                         \
  79         return flags;                                                   \
  80 }                                                                       \
  81                                                                         \
  82 void __lockfunc __##op##_lock_irq(locktype##_t *lock)                   \
  83 {                                                                       \
  84         _##op##_lock_irqsave(lock);                                     \
  85 }                                                                       \
  86                                                                         \
  87 void __lockfunc __##op##_lock_bh(locktype##_t *lock)                    \
  88 {                                                                       \
  89         unsigned long flags;                                            \
  90                                                                         \
  91         /*                                                      */      \
  92         /* Careful: we must exclude softirqs too, hence the     */      \
  93         /* irq-disabling. We use the generic preemption-aware   */      \
  94         /* function:                                            */      \
  95         /**/                                                            \
  96         flags = _##op##_lock_irqsave(lock);                             \
  97         local_bh_disable();                                             \
  98         local_irq_restore(flags);                                       \
  99 }                                                                       \
 100
 101 /*
 102  * Build preemption-friendly versions of the following
 103  * lock-spinning functions:
 104  *
 105  *         __[spin|read|write]_lock()
 106  *         __[spin|read|write]_lock_irq()
 107  *         __[spin|read|write]_lock_irqsave()
 108  *         __[spin|read|write]_lock_bh()
 109  */
 110 BUILD_LOCK_OPS(spin, spinlock);
 111 BUILD_LOCK_OPS(read, rwlock);
 112 BUILD_LOCK_OPS(write, rwlock);
 113
 114 #endif
 115
 116 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 117
 118 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
 119 {
 120         preempt_disable();
 121         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 122         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 123 }
 124 EXPORT_SYMBOL(_spin_lock_nested);
 125
 126 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock,
 127                                                    int subclass)
 128 {
 129         unsigned long flags;
 130
 131         local_irq_save(flags);
 132         preempt_disable();
 133         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 134         LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock,
 135                                 _raw_spin_lock_flags, &flags);
 136         return flags;
 137 }
 138 EXPORT_SYMBOL(_spin_lock_irqsave_nested);
 139
 140 void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
 141                                      struct lockdep_map *nest_lock)
 142 {
 143         preempt_disable();
 144         spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
 145         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 146 }
 147 EXPORT_SYMBOL(_spin_lock_nest_lock);
 148
 149 #endif
 150
 151 #ifndef CONFIG_INLINE_SPIN_TRYLOCK
 152 int __lockfunc _spin_trylock(spinlock_t *lock)
 153 {
 154         return __spin_trylock(lock);
 155 }
 156 EXPORT_SYMBOL(_spin_trylock);
 157 #endif
 158
 159 #ifndef CONFIG_INLINE_READ_TRYLOCK
 160 int __lockfunc _read_trylock(rwlock_t *lock)
 161 {
 162         return __read_trylock(lock);
 163 }
 164 EXPORT_SYMBOL(_read_trylock);
 165 #endif
 166
 167 #ifndef CONFIG_INLINE_WRITE_TRYLOCK
 168 int __lockfunc _write_trylock(rwlock_t *lock)
 169 {
 170         return __write_trylock(lock);
 171 }
 172 EXPORT_SYMBOL(_write_trylock);
 173 #endif
 174
 175 #ifndef CONFIG_INLINE_READ_LOCK
 176 void __lockfunc _read_lock(rwlock_t *lock)
 177 {
 178         __read_lock(lock);
 179 }
 180 EXPORT_SYMBOL(_read_lock);
 181 #endif
 182
 183 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
 184 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
 185 {
 186         return __spin_lock_irqsave(lock);
 187 }
 188 EXPORT_SYMBOL(_spin_lock_irqsave);
 189 #endif
 190
 191 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
 192 void __lockfunc _spin_lock_irq(spinlock_t *lock)
 193 {
 194         __spin_lock_irq(lock);
 195 }
 196 EXPORT_SYMBOL(_spin_lock_irq);
 197 #endif
 198
 199 #ifndef CONFIG_INLINE_SPIN_LOCK_BH
 200 void __lockfunc _spin_lock_bh(spinlock_t *lock)
 201 {
 202         __spin_lock_bh(lock);
 203 }
 204 EXPORT_SYMBOL(_spin_lock_bh);
 205 #endif
 206
 207 #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE
 208 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
 209 {
 210         return __read_lock_irqsave(lock);
 211 }
 212 EXPORT_SYMBOL(_read_lock_irqsave);
 213 #endif
 214
 215 #ifndef CONFIG_INLINE_READ_LOCK_IRQ
 216 void __lockfunc _read_lock_irq(rwlock_t *lock)
 217 {
 218         __read_lock_irq(lock);
 219 }
 220 EXPORT_SYMBOL(_read_lock_irq);
 221 #endif
 222
 223 #ifndef CONFIG_INLINE_READ_LOCK_BH
 224 void __lockfunc _read_lock_bh(rwlock_t *lock)
 225 {
 226         __read_lock_bh(lock);
 227 }
 228 EXPORT_SYMBOL(_read_lock_bh);
 229 #endif
 230
 231 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE
 232 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
 233 {
 234         return __write_lock_irqsave(lock);
 235 }
 236 EXPORT_SYMBOL(_write_lock_irqsave);
 237 #endif
 238
 239 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ
 240 void __lockfunc _write_lock_irq(rwlock_t *lock)
 241 {
 242         __write_lock_irq(lock);
 243 }
 244 EXPORT_SYMBOL(_write_lock_irq);
 245 #endif
 246
 247 #ifndef CONFIG_INLINE_WRITE_LOCK_BH
 248 void __lockfunc _write_lock_bh(rwlock_t *lock)
 249 {
 250         __write_lock_bh(lock);
 251 }
 252 EXPORT_SYMBOL(_write_lock_bh);
 253 #endif
 254
 255 #ifndef CONFIG_INLINE_SPIN_LOCK
 256 void __lockfunc _spin_lock(spinlock_t *lock)
 257 {
 258         __spin_lock(lock);
 259 }
 260 EXPORT_SYMBOL(_spin_lock);
 261 #endif
 262
 263 #ifndef CONFIG_INLINE_WRITE_LOCK
 264 void __lockfunc _write_lock(rwlock_t *lock)
 265 {
 266         __write_lock(lock);
 267 }
 268 EXPORT_SYMBOL(_write_lock);
 269 #endif
 270
 271 #ifndef CONFIG_INLINE_SPIN_UNLOCK
 272 void __lockfunc _spin_unlock(spinlock_t *lock)
 273 {
 274         __spin_unlock(lock);
 275 }
 276 EXPORT_SYMBOL(_spin_unlock);
 277 #endif
 278
 279 #ifndef CONFIG_INLINE_WRITE_UNLOCK
 280 void __lockfunc _write_unlock(rwlock_t *lock)
 281 {
 282         __write_unlock(lock);
 283 }
 284 EXPORT_SYMBOL(_write_unlock);
 285 #endif
 286
 287 #ifndef CONFIG_INLINE_READ_UNLOCK
 288 void __lockfunc _read_unlock(rwlock_t *lock)
 289 {
 290         __read_unlock(lock);
 291 }
 292 EXPORT_SYMBOL(_read_unlock);
 293 #endif
 294
 295 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
 296 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 297 {
 298         __spin_unlock_irqrestore(lock, flags);
 299 }
 300 EXPORT_SYMBOL(_spin_unlock_irqrestore);
 301 #endif
 302
 303 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
 304 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 305 {
 306         __spin_unlock_irq(lock);
 307 }
 308 EXPORT_SYMBOL(_spin_unlock_irq);
 309 #endif
 310
 311 #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
 312 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 313 {
 314         __spin_unlock_bh(lock);
 315 }
 316 EXPORT_SYMBOL(_spin_unlock_bh);
 317 #endif
 318
 319 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE
 320 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 321 {
 322         __read_unlock_irqrestore(lock, flags);
 323 }
 324 EXPORT_SYMBOL(_read_unlock_irqrestore);
 325 #endif
 326
 327 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ
 328 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 329 {
 330         __read_unlock_irq(lock);
 331 }
 332 EXPORT_SYMBOL(_read_unlock_irq);
 333 #endif
 334
 335 #ifndef CONFIG_INLINE_READ_UNLOCK_BH
 336 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 337 {
 338         __read_unlock_bh(lock);
 339 }
 340 EXPORT_SYMBOL(_read_unlock_bh);
 341 #endif
 342
 343 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE
 344 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 345 {
 346         __write_unlock_irqrestore(lock, flags);
 347 }
 348 EXPORT_SYMBOL(_write_unlock_irqrestore);
 349 #endif
 350
 351 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ
 352 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 353 {
 354         __write_unlock_irq(lock);
 355 }
 356 EXPORT_SYMBOL(_write_unlock_irq);
 357 #endif
 358
 359 #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH
 360 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 361 {
 362         __write_unlock_bh(lock);
 363 }
 364 EXPORT_SYMBOL(_write_unlock_bh);
 365 #endif
 366
 367 #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH
 368 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 369 {
 370         return __spin_trylock_bh(lock);
 371 }
 372 EXPORT_SYMBOL(_spin_trylock_bh);
 373 #endif
 374
 375 notrace int in_lock_functions(unsigned long addr)
 376 {
 377         /* Linker adds these: start and end of __lockfunc functions */
 378         extern char __lock_text_start[], __lock_text_end[];
 379
 380         return addr >= (unsigned long)__lock_text_start
 381         && addr < (unsigned long)__lock_text_end;
 382 }
 383 EXPORT_SYMBOL(in_lock_functions);