kernel/spinlock.c

   1 /*
   2  * Copyright (2004) Linus Torvalds
   3  *
   4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
   5  *
   6  * Copyright (2004, 2005) Ingo Molnar
   7  *
   8  * This file contains the spinlock/rwlock implementations for the
   9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
  10  */
  11
  12 #include <linux/config.h>
  13 #include <linux/linkage.h>
  14 #include <linux/preempt.h>
  15 #include <linux/spinlock.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/module.h>
  18
  19 /*
  20  * Generic declaration of the raw read_trylock() function,
  21  * architectures are supposed to optimize this:
  22  */
  23 int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock)
  24 {
  25         __raw_read_lock(lock);
  26         return 1;
  27 }
  28 EXPORT_SYMBOL(generic__raw_read_trylock);
  29
  30 int __lockfunc _spin_trylock(spinlock_t *lock)
  31 {
  32         preempt_disable();
  33         if (_raw_spin_trylock(lock))
  34                 return 1;
  35
  36         preempt_enable();
  37         return 0;
  38 }
  39 EXPORT_SYMBOL(_spin_trylock);
  40
  41 int __lockfunc _read_trylock(rwlock_t *lock)
  42 {
  43         preempt_disable();
  44         if (_raw_read_trylock(lock))
  45                 return 1;
  46
  47         preempt_enable();
  48         return 0;
  49 }
  50 EXPORT_SYMBOL(_read_trylock);
  51
  52 int __lockfunc _write_trylock(rwlock_t *lock)
  53 {
  54         preempt_disable();
  55         if (_raw_write_trylock(lock))
  56                 return 1;
  57
  58         preempt_enable();
  59         return 0;
  60 }
  61 EXPORT_SYMBOL(_write_trylock);
  62
  63 #if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP)
  64
  65 void __lockfunc _read_lock(rwlock_t *lock)
  66 {
  67         preempt_disable();
  68         _raw_read_lock(lock);
  69 }
  70 EXPORT_SYMBOL(_read_lock);
  71
  72 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
  73 {
  74         unsigned long flags;
  75
  76         local_irq_save(flags);
  77         preempt_disable();
  78         _raw_spin_lock_flags(lock, &flags);
  79         return flags;
  80 }
  81 EXPORT_SYMBOL(_spin_lock_irqsave);
  82
  83 void __lockfunc _spin_lock_irq(spinlock_t *lock)
  84 {
  85         local_irq_disable();
  86         preempt_disable();
  87         _raw_spin_lock(lock);
  88 }
  89 EXPORT_SYMBOL(_spin_lock_irq);
  90
  91 void __lockfunc _spin_lock_bh(spinlock_t *lock)
  92 {
  93         local_bh_disable();
  94         preempt_disable();
  95         _raw_spin_lock(lock);
  96 }
  97 EXPORT_SYMBOL(_spin_lock_bh);
  98
  99 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
 100 {
 101         unsigned long flags;
 102
 103         local_irq_save(flags);
 104         preempt_disable();
 105         _raw_read_lock(lock);
 106         return flags;
 107 }
 108 EXPORT_SYMBOL(_read_lock_irqsave);
 109
 110 void __lockfunc _read_lock_irq(rwlock_t *lock)
 111 {
 112         local_irq_disable();
 113         preempt_disable();
 114         _raw_read_lock(lock);
 115 }
 116 EXPORT_SYMBOL(_read_lock_irq);
 117
 118 void __lockfunc _read_lock_bh(rwlock_t *lock)
 119 {
 120         local_bh_disable();
 121         preempt_disable();
 122         _raw_read_lock(lock);
 123 }
 124 EXPORT_SYMBOL(_read_lock_bh);
 125
 126 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
 127 {
 128         unsigned long flags;
 129
 130         local_irq_save(flags);
 131         preempt_disable();
 132         _raw_write_lock(lock);
 133         return flags;
 134 }
 135 EXPORT_SYMBOL(_write_lock_irqsave);
 136
 137 void __lockfunc _write_lock_irq(rwlock_t *lock)
 138 {
 139         local_irq_disable();
 140         preempt_disable();
 141         _raw_write_lock(lock);
 142 }
 143 EXPORT_SYMBOL(_write_lock_irq);
 144
 145 void __lockfunc _write_lock_bh(rwlock_t *lock)
 146 {
 147         local_bh_disable();
 148         preempt_disable();
 149         _raw_write_lock(lock);
 150 }
 151 EXPORT_SYMBOL(_write_lock_bh);
 152
 153 void __lockfunc _spin_lock(spinlock_t *lock)
 154 {
 155         preempt_disable();
 156         _raw_spin_lock(lock);
 157 }
 158
 159 EXPORT_SYMBOL(_spin_lock);
 160
 161 void __lockfunc _write_lock(rwlock_t *lock)
 162 {
 163         preempt_disable();
 164         _raw_write_lock(lock);
 165 }
 166
 167 EXPORT_SYMBOL(_write_lock);
 168
 169 #else /* CONFIG_PREEMPT: */
 170
 171 /*
 172  * This could be a long-held lock. We both prepare to spin for a long
 173  * time (making _this_ CPU preemptable if possible), and we also signal
 174  * towards that other CPU that it should break the lock ASAP.
 175  *
 176  * (We do this in a function because inlining it would be excessive.)
 177  */
 178
 179 #define BUILD_LOCK_OPS(op, locktype)                                    \
 180 void __lockfunc _##op##_lock(locktype##_t *lock)                        \
 181 {                                                                       \
 182         preempt_disable();                                              \
 183         for (;;) {                                                      \
 184                 if (likely(_raw_##op##_trylock(lock)))                  \
 185                         break;                                          \
 186                 preempt_enable();                                       \
 187                 if (!(lock)->break_lock)                                \
 188                         (lock)->break_lock = 1;                         \
 189                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 190                         cpu_relax();                                    \
 191                 preempt_disable();                                      \
 192         }                                                               \
 193         (lock)->break_lock = 0;                                         \
 194 }                                                                       \
 195                                                                         \
 196 EXPORT_SYMBOL(_##op##_lock);                                            \
 197                                                                         \
 198 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)       \
 199 {                                                                       \
 200         unsigned long flags;                                            \
 201                                                                         \
 202         preempt_disable();                                              \
 203         for (;;) {                                                      \
 204                 local_irq_save(flags);                                  \
 205                 if (likely(_raw_##op##_trylock(lock)))                  \
 206                         break;                                          \
 207                 local_irq_restore(flags);                               \
 208                                                                         \
 209                 preempt_enable();                                       \
 210                 if (!(lock)->break_lock)                                \
 211                         (lock)->break_lock = 1;                         \
 212                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 213                         cpu_relax();                                    \
 214                 preempt_disable();                                      \
 215         }                                                               \
 216         (lock)->break_lock = 0;                                         \
 217         return flags;                                                   \
 218 }                                                                       \
 219                                                                         \
 220 EXPORT_SYMBOL(_##op##_lock_irqsave);                                    \
 221                                                                         \
 222 void __lockfunc _##op##_lock_irq(locktype##_t *lock)                    \
 223 {                                                                       \
 224         _##op##_lock_irqsave(lock);                                     \
 225 }                                                                       \
 226                                                                         \
 227 EXPORT_SYMBOL(_##op##_lock_irq);                                        \
 228                                                                         \
 229 void __lockfunc _##op##_lock_bh(locktype##_t *lock)                     \
 230 {                                                                       \
 231         unsigned long flags;                                            \
 232                                                                         \
 233         /*                                                      */      \
 234         /* Careful: we must exclude softirqs too, hence the     */      \
 235         /* irq-disabling. We use the generic preemption-aware   */      \
 236         /* function:                                            */      \
 237         /**/                                                            \
 238         flags = _##op##_lock_irqsave(lock);                             \
 239         local_bh_disable();                                             \
 240         local_irq_restore(flags);                                       \
 241 }                                                                       \
 242                                                                         \
 243 EXPORT_SYMBOL(_##op##_lock_bh)
 244
 245 /*
 246  * Build preemption-friendly versions of the following
 247  * lock-spinning functions:
 248  *
 249  *         _[spin|read|write]_lock()
 250  *         _[spin|read|write]_lock_irq()
 251  *         _[spin|read|write]_lock_irqsave()
 252  *         _[spin|read|write]_lock_bh()
 253  */
 254 BUILD_LOCK_OPS(spin, spinlock);
 255 BUILD_LOCK_OPS(read, rwlock);
 256 BUILD_LOCK_OPS(write, rwlock);
 257
 258 #endif /* CONFIG_PREEMPT */
 259
 260 void __lockfunc _spin_unlock(spinlock_t *lock)
 261 {
 262         _raw_spin_unlock(lock);
 263         preempt_enable();
 264 }
 265 EXPORT_SYMBOL(_spin_unlock);
 266
 267 void __lockfunc _write_unlock(rwlock_t *lock)
 268 {
 269         _raw_write_unlock(lock);
 270         preempt_enable();
 271 }
 272 EXPORT_SYMBOL(_write_unlock);
 273
 274 void __lockfunc _read_unlock(rwlock_t *lock)
 275 {
 276         _raw_read_unlock(lock);
 277         preempt_enable();
 278 }
 279 EXPORT_SYMBOL(_read_unlock);
 280
 281 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 282 {
 283         _raw_spin_unlock(lock);
 284         local_irq_restore(flags);
 285         preempt_enable();
 286 }
 287 EXPORT_SYMBOL(_spin_unlock_irqrestore);
 288
 289 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 290 {
 291         _raw_spin_unlock(lock);
 292         local_irq_enable();
 293         preempt_enable();
 294 }
 295 EXPORT_SYMBOL(_spin_unlock_irq);
 296
 297 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 298 {
 299         _raw_spin_unlock(lock);
 300         preempt_enable_no_resched();
 301         local_bh_enable();
 302 }
 303 EXPORT_SYMBOL(_spin_unlock_bh);
 304
 305 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 306 {
 307         _raw_read_unlock(lock);
 308         local_irq_restore(flags);
 309         preempt_enable();
 310 }
 311 EXPORT_SYMBOL(_read_unlock_irqrestore);
 312
 313 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 314 {
 315         _raw_read_unlock(lock);
 316         local_irq_enable();
 317         preempt_enable();
 318 }
 319 EXPORT_SYMBOL(_read_unlock_irq);
 320
 321 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 322 {
 323         _raw_read_unlock(lock);
 324         preempt_enable_no_resched();
 325         local_bh_enable();
 326 }
 327 EXPORT_SYMBOL(_read_unlock_bh);
 328
 329 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 330 {
 331         _raw_write_unlock(lock);
 332         local_irq_restore(flags);
 333         preempt_enable();
 334 }
 335 EXPORT_SYMBOL(_write_unlock_irqrestore);
 336
 337 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 338 {
 339         _raw_write_unlock(lock);
 340         local_irq_enable();
 341         preempt_enable();
 342 }
 343 EXPORT_SYMBOL(_write_unlock_irq);
 344
 345 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 346 {
 347         _raw_write_unlock(lock);
 348         preempt_enable_no_resched();
 349         local_bh_enable();
 350 }
 351 EXPORT_SYMBOL(_write_unlock_bh);
 352
 353 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 354 {
 355         local_bh_disable();
 356         preempt_disable();
 357         if (_raw_spin_trylock(lock))
 358                 return 1;
 359
 360         preempt_enable_no_resched();
 361         local_bh_enable();
 362         return 0;
 363 }
 364 EXPORT_SYMBOL(_spin_trylock_bh);
 365
 366 int in_lock_functions(unsigned long addr)
 367 {
 368         /* Linker adds these: start and end of __lockfunc functions */
 369         extern char __lock_text_start[], __lock_text_end[];
 370
 371         return addr >= (unsigned long)__lock_text_start
 372         && addr < (unsigned long)__lock_text_end;
 373 }
 374 EXPORT_SYMBOL(in_lock_functions);