lib/spinlock_debug.c

   1 /*
   2  * Copyright 2005, Red Hat, Inc., Ingo Molnar
   3  * Released under the General Public License (GPL).
   4  *
   5  * This file contains the spinlock/rwlock implementations for
   6  * DEBUG_SPINLOCK.
   7  */
   8
   9 #include <linux/spinlock.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/debug_locks.h>
  12 #include <linux/delay.h>
  13 #include <linux/module.h>
  14
  15 void __spin_lock_init(spinlock_t *lock, const char *name,
  16                       struct lock_class_key *key)
  17 {
  18 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  19         /*
  20          * Make sure we are not reinitializing a held lock:
  21          */
  22         debug_check_no_locks_freed((void *)lock, sizeof(*lock));
  23         lockdep_init_map(&lock->dep_map, name, key);
  24 #endif
  25         lock->raw_lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
  26         lock->magic = SPINLOCK_MAGIC;
  27         lock->owner = SPINLOCK_OWNER_INIT;
  28         lock->owner_cpu = -1;
  29 }
  30
  31 EXPORT_SYMBOL(__spin_lock_init);
  32
  33 void __rwlock_init(rwlock_t *lock, const char *name,
  34                    struct lock_class_key *key)
  35 {
  36 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  37         /*
  38          * Make sure we are not reinitializing a held lock:
  39          */
  40         debug_check_no_locks_freed((void *)lock, sizeof(*lock));
  41         lockdep_init_map(&lock->dep_map, name, key);
  42 #endif
  43         lock->raw_lock = (raw_rwlock_t) __RAW_RW_LOCK_UNLOCKED;
  44         lock->magic = RWLOCK_MAGIC;
  45         lock->owner = SPINLOCK_OWNER_INIT;
  46         lock->owner_cpu = -1;
  47 }
  48
  49 EXPORT_SYMBOL(__rwlock_init);
  50
  51 static void spin_bug(spinlock_t *lock, const char *msg)
  52 {
  53         struct task_struct *owner = NULL;
  54
  55         if (!debug_locks_off())
  56                 return;
  57
  58         if (lock->owner && lock->owner != SPINLOCK_OWNER_INIT)
  59                 owner = lock->owner;
  60         printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n",
  61                 msg, raw_smp_processor_id(),
  62                 current->comm, current->pid);
  63         printk(KERN_EMERG " lock: %p, .magic: %08x, .owner: %s/%d, "
  64                         ".owner_cpu: %d\n",
  65                 lock, lock->magic,
  66                 owner ? owner->comm : "<none>",
  67                 owner ? owner->pid : -1,
  68                 lock->owner_cpu);
  69         dump_stack();
  70 }
  71
  72 #define SPIN_BUG_ON(cond, lock, msg) if (unlikely(cond)) spin_bug(lock, msg)
  73
  74 static inline void
  75 debug_spin_lock_before(spinlock_t *lock)
  76 {
  77         SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
  78         SPIN_BUG_ON(lock->owner == current, lock, "recursion");
  79         SPIN_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
  80                                                         lock, "cpu recursion");
  81 }
  82
  83 static inline void debug_spin_lock_after(spinlock_t *lock)
  84 {
  85         lock->owner_cpu = raw_smp_processor_id();
  86         lock->owner = current;
  87 }
  88
  89 static inline void debug_spin_unlock(spinlock_t *lock)
  90 {
  91         SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
  92         SPIN_BUG_ON(!spin_is_locked(lock), lock, "already unlocked");
  93         SPIN_BUG_ON(lock->owner != current, lock, "wrong owner");
  94         SPIN_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
  95                                                         lock, "wrong CPU");
  96         lock->owner = SPINLOCK_OWNER_INIT;
  97         lock->owner_cpu = -1;
  98 }
  99
 100 static void __spin_lock_debug(spinlock_t *lock)
 101 {
 102         int print_once = 1;
 103         u64 i;
 104
 105         for (;;) {
 106                 for (i = 0; i < loops_per_jiffy * HZ; i++) {
 107                         if (__raw_spin_trylock(&lock->raw_lock))
 108                                 return;
 109                         __delay(1);
 110                 }
 111                 /* lockup suspected: */
 112                 if (print_once) {
 113                         print_once = 0;
 114                         printk(KERN_EMERG "BUG: spinlock lockup on CPU#%d, "
 115                                         "%s/%d, %p\n",
 116                                 raw_smp_processor_id(), current->comm,
 117                                 current->pid, lock);
 118                         dump_stack();
 119                 }
 120         }
 121 }
 122
 123 void _raw_spin_lock(spinlock_t *lock)
 124 {
 125         debug_spin_lock_before(lock);
 126         if (unlikely(!__raw_spin_trylock(&lock->raw_lock)))
 127                 __spin_lock_debug(lock);
 128         debug_spin_lock_after(lock);
 129 }
 130
 131 int _raw_spin_trylock(spinlock_t *lock)
 132 {
 133         int ret = __raw_spin_trylock(&lock->raw_lock);
 134
 135         if (ret)
 136                 debug_spin_lock_after(lock);
 137 #ifndef CONFIG_SMP
 138         /*
 139          * Must not happen on UP:
 140          */
 141         SPIN_BUG_ON(!ret, lock, "trylock failure on UP");
 142 #endif
 143         return ret;
 144 }
 145
 146 void _raw_spin_unlock(spinlock_t *lock)
 147 {
 148         debug_spin_unlock(lock);
 149         __raw_spin_unlock(&lock->raw_lock);
 150 }
 151
 152 static void rwlock_bug(rwlock_t *lock, const char *msg)
 153 {
 154         if (!debug_locks_off())
 155                 return;
 156
 157         printk(KERN_EMERG "BUG: rwlock %s on CPU#%d, %s/%d, %p\n",
 158                 msg, raw_smp_processor_id(), current->comm,
 159                 current->pid, lock);
 160         dump_stack();
 161 }
 162
 163 #define RWLOCK_BUG_ON(cond, lock, msg) if (unlikely(cond)) rwlock_bug(lock, msg)
 164
 165 static void __read_lock_debug(rwlock_t *lock)
 166 {
 167         int print_once = 1;
 168         u64 i;
 169
 170         for (;;) {
 171                 for (i = 0; i < loops_per_jiffy * HZ; i++) {
 172                         if (__raw_read_trylock(&lock->raw_lock))
 173                                 return;
 174                         __delay(1);
 175                 }
 176                 /* lockup suspected: */
 177                 if (print_once) {
 178                         print_once = 0;
 179                         printk(KERN_EMERG "BUG: read-lock lockup on CPU#%d, "
 180                                         "%s/%d, %p\n",
 181                                 raw_smp_processor_id(), current->comm,
 182                                 current->pid, lock);
 183                         dump_stack();
 184                 }
 185         }
 186 }
 187
 188 void _raw_read_lock(rwlock_t *lock)
 189 {
 190         RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
 191         if (unlikely(!__raw_read_trylock(&lock->raw_lock)))
 192                 __read_lock_debug(lock);
 193 }
 194
 195 int _raw_read_trylock(rwlock_t *lock)
 196 {
 197         int ret = __raw_read_trylock(&lock->raw_lock);
 198
 199 #ifndef CONFIG_SMP
 200         /*
 201          * Must not happen on UP:
 202          */
 203         RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
 204 #endif
 205         return ret;
 206 }
 207
 208 void _raw_read_unlock(rwlock_t *lock)
 209 {
 210         RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
 211         __raw_read_unlock(&lock->raw_lock);
 212 }
 213
 214 static inline void debug_write_lock_before(rwlock_t *lock)
 215 {
 216         RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
 217         RWLOCK_BUG_ON(lock->owner == current, lock, "recursion");
 218         RWLOCK_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
 219                                                         lock, "cpu recursion");
 220 }
 221
 222 static inline void debug_write_lock_after(rwlock_t *lock)
 223 {
 224         lock->owner_cpu = raw_smp_processor_id();
 225         lock->owner = current;
 226 }
 227
 228 static inline void debug_write_unlock(rwlock_t *lock)
 229 {
 230         RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
 231         RWLOCK_BUG_ON(lock->owner != current, lock, "wrong owner");
 232         RWLOCK_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
 233                                                         lock, "wrong CPU");
 234         lock->owner = SPINLOCK_OWNER_INIT;
 235         lock->owner_cpu = -1;
 236 }
 237
 238 static void __write_lock_debug(rwlock_t *lock)
 239 {
 240         int print_once = 1;
 241         u64 i;
 242
 243         for (;;) {
 244                 for (i = 0; i < loops_per_jiffy * HZ; i++) {
 245                         if (__raw_write_trylock(&lock->raw_lock))
 246                                 return;
 247                         __delay(1);
 248                 }
 249                 /* lockup suspected: */
 250                 if (print_once) {
 251                         print_once = 0;
 252                         printk(KERN_EMERG "BUG: write-lock lockup on CPU#%d, "
 253                                         "%s/%d, %p\n",
 254                                 raw_smp_processor_id(), current->comm,
 255                                 current->pid, lock);
 256                         dump_stack();
 257                 }
 258         }
 259 }
 260
 261 void _raw_write_lock(rwlock_t *lock)
 262 {
 263         debug_write_lock_before(lock);
 264         if (unlikely(!__raw_write_trylock(&lock->raw_lock)))
 265                 __write_lock_debug(lock);
 266         debug_write_lock_after(lock);
 267 }
 268
 269 int _raw_write_trylock(rwlock_t *lock)
 270 {
 271         int ret = __raw_write_trylock(&lock->raw_lock);
 272
 273         if (ret)
 274                 debug_write_lock_after(lock);
 275 #ifndef CONFIG_SMP
 276         /*
 277          * Must not happen on UP:
 278          */
 279         RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
 280 #endif
 281         return ret;
 282 }
 283
 284 void _raw_write_unlock(rwlock_t *lock)
 285 {
 286         debug_write_unlock(lock);
 287         __raw_write_unlock(&lock->raw_lock);
 288 }