kernel/2.6.29.6-aldebaran-rt/kernel/spinlock.c

   1 /*
   2  * Copyright (2004) Linus Torvalds
   3  *
   4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
   5  *
   6  * Copyright (2004, 2005) Ingo Molnar
   7  *
   8  * This file contains the spinlock/rwlock implementations for the
   9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
  10  *
  11  * Note that some architectures have special knowledge about the
  12  * stack frames of these functions in their profile_pc. If you
  13  * change anything significant here that could change the stack
  14  * frame contact the architecture maintainers.
  15  */
  16
  17 #include <linux/linkage.h>
  18 #include <linux/preempt.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/debug_locks.h>
  22 #include <linux/module.h>
  23
  24 int __lockfunc __spin_trylock(raw_spinlock_t *lock)
  25 {
  26         preempt_disable();
  27         if (_raw_spin_trylock(lock)) {
  28                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  29                 return 1;
  30         }
  31
  32         preempt_enable();
  33         return 0;
  34 }
  35 EXPORT_SYMBOL(__spin_trylock);
  36
  37 int __lockfunc __spin_trylock_irq(raw_spinlock_t *lock)
  38 {
  39         local_irq_disable();
  40         preempt_disable();
  41
  42         if (_raw_spin_trylock(lock)) {
  43                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  44                 return 1;
  45         }
  46
  47         __preempt_enable_no_resched();
  48         local_irq_enable();
  49         preempt_check_resched();
  50
  51         return 0;
  52 }
  53 EXPORT_SYMBOL(__spin_trylock_irq);
  54
  55 int __lockfunc __spin_trylock_irqsave(raw_spinlock_t *lock,
  56                                          unsigned long *flags)
  57 {
  58         local_irq_save(*flags);
  59         preempt_disable();
  60
  61         if (_raw_spin_trylock(lock)) {
  62                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  63                 return 1;
  64         }
  65
  66         __preempt_enable_no_resched();
  67         local_irq_restore(*flags);
  68         preempt_check_resched();
  69
  70         return 0;
  71 }
  72 EXPORT_SYMBOL(__spin_trylock_irqsave);
  73
  74 int __lockfunc __read_trylock(raw_rwlock_t *lock)
  75 {
  76         preempt_disable();
  77         if (_raw_read_trylock(lock)) {
  78                 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
  79                 return 1;
  80         }
  81
  82         preempt_enable();
  83         return 0;
  84 }
  85 EXPORT_SYMBOL(__read_trylock);
  86
  87 int __lockfunc __write_trylock(raw_rwlock_t *lock)
  88 {
  89         preempt_disable();
  90         if (_raw_write_trylock(lock)) {
  91                 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  92                 return 1;
  93         }
  94
  95         preempt_enable();
  96         return 0;
  97 }
  98 EXPORT_SYMBOL(__write_trylock);
  99
 100 int __lockfunc __write_trylock_irqsave(raw_rwlock_t *lock, unsigned long *flags)
 101 {
 102         int ret;
 103
 104         local_irq_save(*flags);
 105         ret = __write_trylock(lock);
 106         if (ret)
 107                 return ret;
 108
 109         local_irq_restore(*flags);
 110         return 0;
 111 }
 112 EXPORT_SYMBOL(__write_trylock_irqsave);
 113
 114 /*
 115  * If lockdep is enabled then we use the non-preemption spin-ops
 116  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
 117  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
 118  */
 119 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
 120
 121 void __lockfunc __read_lock(raw_rwlock_t *lock)
 122 {
 123         preempt_disable();
 124         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 125         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
 126 }
 127 EXPORT_SYMBOL(__read_lock);
 128
 129 unsigned long __lockfunc __spin_lock_irqsave(raw_spinlock_t *lock)
 130 {
 131         unsigned long flags;
 132
 133         local_irq_save(flags);
 134         preempt_disable();
 135         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 136         /*
 137          * On lockdep we dont want the hand-coded irq-enable of
 138          * _raw_spin_lock_flags() code, because lockdep assumes
 139          * that interrupts are not re-enabled during lock-acquire:
 140          */
 141 #ifdef CONFIG_LOCKDEP
 142         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 143 #else
 144         _raw_spin_lock_flags(lock, &flags);
 145 #endif
 146         return flags;
 147 }
 148 EXPORT_SYMBOL(__spin_lock_irqsave);
 149
 150 void __lockfunc __spin_lock_irq(raw_spinlock_t *lock)
 151 {
 152         local_irq_disable();
 153         preempt_disable();
 154         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 155         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 156 }
 157 EXPORT_SYMBOL(__spin_lock_irq);
 158
 159 void __lockfunc __spin_lock_bh(raw_spinlock_t *lock)
 160 {
 161         local_bh_disable();
 162         preempt_disable();
 163         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 164         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 165 }
 166 EXPORT_SYMBOL(__spin_lock_bh);
 167
 168 unsigned long __lockfunc __read_lock_irqsave(raw_rwlock_t *lock)
 169 {
 170         unsigned long flags;
 171
 172         local_irq_save(flags);
 173         preempt_disable();
 174         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 175         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
 176         return flags;
 177 }
 178 EXPORT_SYMBOL(__read_lock_irqsave);
 179
 180 void __lockfunc __read_lock_irq(raw_rwlock_t *lock)
 181 {
 182         local_irq_disable();
 183         preempt_disable();
 184         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 185         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
 186 }
 187 EXPORT_SYMBOL(__read_lock_irq);
 188
 189 void __lockfunc __read_lock_bh(raw_rwlock_t *lock)
 190 {
 191         local_bh_disable();
 192         preempt_disable();
 193         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 194         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
 195 }
 196 EXPORT_SYMBOL(__read_lock_bh);
 197
 198 unsigned long __lockfunc __write_lock_irqsave(raw_rwlock_t *lock)
 199 {
 200         unsigned long flags;
 201
 202         local_irq_save(flags);
 203         preempt_disable();
 204         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 205         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 206         return flags;
 207 }
 208 EXPORT_SYMBOL(__write_lock_irqsave);
 209
 210 void __lockfunc __write_lock_irq(raw_rwlock_t *lock)
 211 {
 212         local_irq_disable();
 213         preempt_disable();
 214         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 215         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 216 }
 217 EXPORT_SYMBOL(__write_lock_irq);
 218
 219 void __lockfunc __write_lock_bh(raw_rwlock_t *lock)
 220 {
 221         local_bh_disable();
 222         preempt_disable();
 223         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 224         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 225 }
 226 EXPORT_SYMBOL(__write_lock_bh);
 227
 228 void __lockfunc __spin_lock(raw_spinlock_t *lock)
 229 {
 230         preempt_disable();
 231         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 232         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 233 }
 234
 235 EXPORT_SYMBOL(__spin_lock);
 236
 237 void __lockfunc __write_lock(raw_rwlock_t *lock)
 238 {
 239         preempt_disable();
 240         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 241         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
 242 }
 243
 244 EXPORT_SYMBOL(__write_lock);
 245
 246 #else /* CONFIG_PREEMPT: */
 247
 248 /*
 249  * This could be a long-held lock. We both prepare to spin for a long
 250  * time (making _this_ CPU preemptable if possible), and we also signal
 251  * towards that other CPU that it should break the lock ASAP.
 252  *
 253  * (We do this in a function because inlining it would be excessive.)
 254  */
 255
 256 #define BUILD_LOCK_OPS(op, locktype)                                    \
 257 void __lockfunc __##op##_lock(locktype##_t *lock)                       \
 258 {                                                                       \
 259         for (;;) {                                                      \
 260                 preempt_disable();                                      \
 261                 if (likely(_raw_##op##_trylock(lock)))                  \
 262                         break;                                          \
 263                 preempt_enable();                                       \
 264                                                                         \
 265                 if (!(lock)->break_lock)                                \
 266                         (lock)->break_lock = 1;                         \
 267                 while (!__raw_##op##_can_lock(&(lock)->raw_lock) &&     \
 268                                         (lock)->break_lock)             \
 269                         __raw_##op##_relax(&lock->raw_lock);            \
 270         }                                                               \
 271         (lock)->break_lock = 0;                                         \
 272 }                                                                       \
 273                                                                         \
 274 EXPORT_SYMBOL(__##op##_lock);                                           \
 275                                                                         \
 276 unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock)      \
 277 {                                                                       \
 278         unsigned long flags;                                            \
 279                                                                         \
 280         for (;;) {                                                      \
 281                 preempt_disable();                                      \
 282                 local_irq_save(flags);                                  \
 283                 if (likely(_raw_##op##_trylock(lock)))                  \
 284                         break;                                          \
 285                 local_irq_restore(flags);                               \
 286                 preempt_enable();                                       \
 287                                                                         \
 288                 if (!(lock)->break_lock)                                \
 289                         (lock)->break_lock = 1;                         \
 290                 while (!__raw_##op##_can_lock(&(lock)->raw_lock) &&     \
 291                                                  (lock)->break_lock)    \
 292                         __raw_##op##_relax(&lock->raw_lock);            \
 293         }                                                               \
 294         (lock)->break_lock = 0;                                         \
 295         return flags;                                                   \
 296 }                                                                       \
 297                                                                         \
 298 EXPORT_SYMBOL(__##op##_lock_irqsave);                                   \
 299                                                                         \
 300 void __lockfunc __##op##_lock_irq(locktype##_t *lock)                   \
 301 {                                                                       \
 302         __##op##_lock_irqsave(lock);                                    \
 303 }                                                                       \
 304                                                                         \
 305 EXPORT_SYMBOL(__##op##_lock_irq);                                       \
 306                                                                         \
 307 void __lockfunc __##op##_lock_bh(locktype##_t *lock)                    \
 308 {                                                                       \
 309         unsigned long flags;                                            \
 310                                                                         \
 311         /*                                                      */      \
 312         /* Careful: we must exclude softirqs too, hence the     */      \
 313         /* irq-disabling. We use the generic preemption-aware   */      \
 314         /* function:                                            */      \
 315         /**/                                                            \
 316         flags = __##op##_lock_irqsave(lock);                            \
 317         local_bh_disable();                                             \
 318         local_irq_restore(flags);                                       \
 319 }                                                                       \
 320                                                                         \
 321 EXPORT_SYMBOL(__##op##_lock_bh)
 322
 323 /*
 324  * Build preemption-friendly versions of the following
 325  * lock-spinning functions:
 326  *
 327  *         __[spin|read|write]_lock()
 328  *         __[spin|read|write]_lock_irq()
 329  *         __[spin|read|write]_lock_irqsave()
 330  *         __[spin|read|write]_lock_bh()
 331  */
 332 BUILD_LOCK_OPS(spin, raw_spinlock);
 333 BUILD_LOCK_OPS(read, raw_rwlock);
 334 BUILD_LOCK_OPS(write, raw_rwlock);
 335
 336 #endif /* CONFIG_PREEMPT */
 337
 338 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 339
 340 void __lockfunc __spin_lock_nested(raw_spinlock_t *lock, int subclass)
 341 {
 342         preempt_disable();
 343         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 344         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 345 }
 346 EXPORT_SYMBOL(__spin_lock_nested);
 347
 348 void __lockfunc __spin_lock_nest_lock(raw_spinlock_t *lock,
 349                                      struct lockdep_map *nest_lock)
 350 {
 351         preempt_disable();
 352         spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
 353         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 354 }
 355 EXPORT_SYMBOL(__spin_lock_nest_lock);
 356
 357 unsigned long __lockfunc __spin_lock_irqsave_nested(raw_spinlock_t *lock, int subclass)
 358 {
 359         unsigned long flags;
 360
 361         local_irq_save(flags);
 362         preempt_disable();
 363         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 364         /*
 365          * On lockdep we dont want the hand-coded irq-enable of
 366          * _raw_spin_lock_flags() code, because lockdep assumes
 367          * that interrupts are not re-enabled during lock-acquire:
 368          */
 369 #ifdef CONFIG_LOCKDEP
 370         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 371 #else
 372         _raw_spin_lock_flags(lock, &flags);
 373 #endif
 374         return flags;
 375 }
 376 EXPORT_SYMBOL(__spin_lock_irqsave_nested);
 377
 378 #endif
 379
 380 void __lockfunc __spin_unlock(raw_spinlock_t *lock)
 381 {
 382         spin_release(&lock->dep_map, 1, _RET_IP_);
 383         _raw_spin_unlock(lock);
 384         preempt_enable();
 385 }
 386 EXPORT_SYMBOL(__spin_unlock);
 387
 388 void __lockfunc __spin_unlock_no_resched(raw_spinlock_t *lock)
 389 {
 390         spin_release(&lock->dep_map, 1, _RET_IP_);
 391         _raw_spin_unlock(lock);
 392         __preempt_enable_no_resched();
 393 }
 394 /* not exported */
 395
 396 void __lockfunc __write_unlock(raw_rwlock_t *lock)
 397 {
 398         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 399         _raw_write_unlock(lock);
 400         preempt_enable();
 401 }
 402 EXPORT_SYMBOL(__write_unlock);
 403
 404 void __lockfunc __read_unlock(raw_rwlock_t *lock)
 405 {
 406         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 407         _raw_read_unlock(lock);
 408         preempt_enable();
 409 }
 410 EXPORT_SYMBOL(__read_unlock);
 411
 412 void __lockfunc __spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
 413 {
 414         spin_release(&lock->dep_map, 1, _RET_IP_);
 415         _raw_spin_unlock(lock);
 416         __preempt_enable_no_resched();
 417         local_irq_restore(flags);
 418         preempt_check_resched();
 419 }
 420 EXPORT_SYMBOL(__spin_unlock_irqrestore);
 421
 422 void __lockfunc __spin_unlock_irq(raw_spinlock_t *lock)
 423 {
 424         spin_release(&lock->dep_map, 1, _RET_IP_);
 425         _raw_spin_unlock(lock);
 426         __preempt_enable_no_resched();
 427         local_irq_enable();
 428         preempt_check_resched();
 429 }
 430 EXPORT_SYMBOL(__spin_unlock_irq);
 431
 432 void __lockfunc __spin_unlock_bh(raw_spinlock_t *lock)
 433 {
 434         spin_release(&lock->dep_map, 1, _RET_IP_);
 435         _raw_spin_unlock(lock);
 436         __preempt_enable_no_resched();
 437         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 438 }
 439 EXPORT_SYMBOL(__spin_unlock_bh);
 440
 441 void __lockfunc __read_unlock_irqrestore(raw_rwlock_t *lock, unsigned long flags)
 442 {
 443         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 444         _raw_read_unlock(lock);
 445         __preempt_enable_no_resched();
 446         local_irq_restore(flags);
 447         preempt_check_resched();
 448 }
 449 EXPORT_SYMBOL(__read_unlock_irqrestore);
 450
 451 void __lockfunc __read_unlock_irq(raw_rwlock_t *lock)
 452 {
 453         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 454         _raw_read_unlock(lock);
 455         __preempt_enable_no_resched();
 456         local_irq_enable();
 457         preempt_check_resched();
 458 }
 459 EXPORT_SYMBOL(__read_unlock_irq);
 460
 461 void __lockfunc __read_unlock_bh(raw_rwlock_t *lock)
 462 {
 463         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 464         _raw_read_unlock(lock);
 465         __preempt_enable_no_resched();
 466         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 467 }
 468 EXPORT_SYMBOL(__read_unlock_bh);
 469
 470 void __lockfunc __write_unlock_irqrestore(raw_rwlock_t *lock, unsigned long flags)
 471 {
 472         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 473         _raw_write_unlock(lock);
 474         __preempt_enable_no_resched();
 475         local_irq_restore(flags);
 476         preempt_check_resched();
 477 }
 478 EXPORT_SYMBOL(__write_unlock_irqrestore);
 479
 480 void __lockfunc __write_unlock_irq(raw_rwlock_t *lock)
 481 {
 482         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 483         _raw_write_unlock(lock);
 484         __preempt_enable_no_resched();
 485         local_irq_enable();
 486         preempt_check_resched();
 487 }
 488 EXPORT_SYMBOL(__write_unlock_irq);
 489
 490 void __lockfunc __write_unlock_bh(raw_rwlock_t *lock)
 491 {
 492         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 493         _raw_write_unlock(lock);
 494         __preempt_enable_no_resched();
 495         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 496 }
 497 EXPORT_SYMBOL(__write_unlock_bh);
 498
 499 int __lockfunc __spin_trylock_bh(raw_spinlock_t *lock)
 500 {
 501         local_bh_disable();
 502         preempt_disable();
 503         if (_raw_spin_trylock(lock)) {
 504                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
 505                 return 1;
 506         }
 507
 508         __preempt_enable_no_resched();
 509         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 510         return 0;
 511 }
 512 EXPORT_SYMBOL(__spin_trylock_bh);
 513
 514 notrace int in_lock_functions(unsigned long addr)
 515 {
 516         /* Linker adds these: start and end of __lockfunc functions */
 517         extern char __lock_text_start[], __lock_text_end[];
 518
 519         return addr >= (unsigned long)__lock_text_start
 520                 && addr < (unsigned long)__lock_text_end;
 521 }
 522 EXPORT_SYMBOL(in_lock_functions);
 523
 524 void notrace __debug_atomic_dec_and_test(atomic_t *v)
 525 {
 526         static int warn_once = 1;
 527
 528         if (!atomic_read(v) && warn_once) {
 529                 warn_once = 0;
 530                 printk("BUG: atomic counter underflow!\n");
 531                 WARN_ON(1);
 532         }
 533 }