sys/sys/spinlock2.h

   1 /*
   2  * Copyright (c) 2005 Jeffrey M. Hsu.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Jeffrey M. Hsu.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. Neither the name of The DragonFly Project nor the names of its
  16  *    contributors may be used to endorse or promote products derived
  17  *    from this software without specific, prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  22  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  23  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  24  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  25  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  26  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  27  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  28  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  29  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 #ifndef _SYS_SPINLOCK2_H_
  34 #define _SYS_SPINLOCK2_H_
  35
  36 #ifndef _KERNEL
  37
  38 #error "This file should not be included by userland programs."
  39
  40 #else
  41
  42 #ifndef _SYS_SYSTM_H_
  43 #include <sys/systm.h>
  44 #endif
  45 #ifndef _SYS_THREAD2_H_
  46 #include <sys/thread2.h>
  47 #endif
  48 #ifndef _SYS_GLOBALDATA_H_
  49 #include <sys/globaldata.h>
  50 #endif
  51 #include <machine/atomic.h>
  52 #include <machine/cpufunc.h>
  53
  54 extern struct spinlock pmap_spin;
  55
  56 int spin_trylock_contested(struct spinlock *spin);
  57 void _spin_lock_contested(struct spinlock *spin, const char *ident, int count);
  58 void _spin_lock_shared_contested(struct spinlock *spin, const char *ident);
  59
  60 #define spin_lock(spin)                 _spin_lock(spin, __func__)
  61 #define spin_lock_quick(spin)           _spin_lock_quick(spin, __func__)
  62 #define spin_lock_shared(spin)          _spin_lock_shared(spin, __func__)
  63 #define spin_lock_shared_quick(spin)    _spin_lock_shared_quick(spin, __func__)
  64
  65 /*
  66  * Attempt to obtain an exclusive spinlock.  Returns FALSE on failure,
  67  * TRUE on success.
  68  */
  69 static __inline boolean_t
  70 spin_trylock(struct spinlock *spin)
  71 {
  72         globaldata_t gd = mycpu;
  73
  74         crit_enter_raw(gd->gd_curthread);
  75         ++gd->gd_spinlocks;
  76         cpu_ccfence();
  77         if (atomic_cmpset_int(&spin->counta, 0, 1) == 0)
  78                 return (spin_trylock_contested(spin));
  79 #ifdef DEBUG_LOCKS
  80         int i;
  81         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
  82                 if (gd->gd_curthread->td_spinlock_stack_id[i] == 0) {
  83                         gd->gd_curthread->td_spinlock_stack_id[i] = 1;
  84                         gd->gd_curthread->td_spinlock_stack[i] = spin;
  85                         gd->gd_curthread->td_spinlock_caller_pc[i] =
  86                                 __builtin_return_address(0);
  87                         break;
  88                 }
  89         }
  90 #endif
  91         return (TRUE);
  92 }
  93
  94 /*
  95  * Return TRUE if the spinlock is held (we can't tell by whom, though)
  96  */
  97 static __inline int
  98 spin_held(struct spinlock *spin)
  99 {
 100         return((spin->counta & ~SPINLOCK_SHARED) != 0);
 101 }
 102
 103 /*
 104  * Obtain an exclusive spinlock and return.  It is possible for the
 105  * SPINLOCK_SHARED bit to already be set, in which case the contested
 106  * code is called to fix it up.
 107  */
 108 static __inline void
 109 _spin_lock_quick(globaldata_t gd, struct spinlock *spin, const char *ident)
 110 {
 111         int count;
 112
 113         crit_enter_raw(gd->gd_curthread);
 114         ++gd->gd_spinlocks;
 115         cpu_ccfence();
 116
 117         count = atomic_fetchadd_int(&spin->counta, 1);
 118         if (__predict_false(count != 0)) {
 119                 _spin_lock_contested(spin, ident, count);
 120         }
 121 #ifdef DEBUG_LOCKS
 122         int i;
 123         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
 124                 if (gd->gd_curthread->td_spinlock_stack_id[i] == 0) {
 125                         gd->gd_curthread->td_spinlock_stack_id[i] = 1;
 126                         gd->gd_curthread->td_spinlock_stack[i] = spin;
 127                         gd->gd_curthread->td_spinlock_caller_pc[i] =
 128                                 __builtin_return_address(0);
 129                         break;
 130                 }
 131         }
 132 #endif
 133 }
 134
 135 static __inline void
 136 _spin_lock(struct spinlock *spin, const char *ident)
 137 {
 138         _spin_lock_quick(mycpu, spin, ident);
 139 }
 140
 141 /*
 142  * Release an exclusive spinlock.  We can just do this passively, only
 143  * ensuring that our spinlock count is left intact until the mutex is
 144  * cleared.
 145  */
 146 static __inline void
 147 spin_unlock_quick(globaldata_t gd, struct spinlock *spin)
 148 {
 149 #ifdef DEBUG_LOCKS
 150         int i;
 151         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
 152                 if ((gd->gd_curthread->td_spinlock_stack_id[i] == 1) &&
 153                     (gd->gd_curthread->td_spinlock_stack[i] == spin)) {
 154                         gd->gd_curthread->td_spinlock_stack_id[i] = 0;
 155                         gd->gd_curthread->td_spinlock_stack[i] = NULL;
 156                         gd->gd_curthread->td_spinlock_caller_pc[i] = NULL;
 157                         break;
 158                 }
 159         }
 160 #endif
 161         /*
 162          * Don't use a locked instruction here.  To reduce latency we avoid
 163          * reading spin->counta prior to writing to it.
 164          */
 165 #ifdef DEBUG_LOCKS
 166         KKASSERT(spin->counta != 0);
 167 #endif
 168         cpu_sfence();
 169         atomic_add_int(&spin->counta, -1);
 170         cpu_sfence();
 171 #ifdef DEBUG_LOCKS
 172         KKASSERT(gd->gd_spinlocks > 0);
 173 #endif
 174         cpu_ccfence();
 175         --gd->gd_spinlocks;
 176         crit_exit_raw(gd->gd_curthread);
 177 }
 178
 179 static __inline void
 180 spin_unlock(struct spinlock *spin)
 181 {
 182         spin_unlock_quick(mycpu, spin);
 183 }
 184
 185 /*
 186  * Shared spinlock.  Acquire a count, if SPINLOCK_SHARED is not already
 187  * set then try a trivial conversion and drop into the contested code if
 188  * the trivial cocnversion fails.  The SHARED bit is 'cached' when lock
 189  * counts go to 0 so the critical path is typically just the fetchadd.
 190  *
 191  * WARNING!  Due to the way exclusive conflict resolution works, we cannot
 192  *           just unconditionally set the SHARED bit on previous-count == 0.
 193  *           Doing so will interfere with the exclusive contended code.
 194  */
 195 static __inline void
 196 _spin_lock_shared_quick(globaldata_t gd, struct spinlock *spin,
 197                         const char *ident)
 198 {
 199         int counta;
 200
 201         crit_enter_raw(gd->gd_curthread);
 202         ++gd->gd_spinlocks;
 203         cpu_ccfence();
 204
 205         counta = atomic_fetchadd_int(&spin->counta, 1);
 206         if (__predict_false((counta & SPINLOCK_SHARED) == 0)) {
 207                 if (counta != 0 ||
 208                     !atomic_cmpset_int(&spin->counta, 1, SPINLOCK_SHARED | 1)) {
 209                         _spin_lock_shared_contested(spin, ident);
 210                 }
 211         }
 212 #ifdef DEBUG_LOCKS
 213         int i;
 214         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
 215                 if (gd->gd_curthread->td_spinlock_stack_id[i] == 0) {
 216                         gd->gd_curthread->td_spinlock_stack_id[i] = 1;
 217                         gd->gd_curthread->td_spinlock_stack[i] = spin;
 218                         gd->gd_curthread->td_spinlock_caller_pc[i] =
 219                                 __builtin_return_address(0);
 220                         break;
 221                 }
 222         }
 223 #endif
 224 }
 225
 226 /*
 227  * Unlock a shared lock.  For convenience we allow the last transition
 228  * to be to (SPINLOCK_SHARED|0), leaving the SPINLOCK_SHARED bit set
 229  * with a count to 0 which will optimize the next shared lock obtained.
 230  *
 231  * WARNING! In order to implement shared and exclusive spinlocks, an
 232  *          exclusive request will convert a multiply-held shared lock
 233  *          to exclusive and wait for shared holders to unlock.  So keep
 234  *          in mind that as of now the spinlock could actually be in an
 235  *          exclusive state.
 236  */
 237 static __inline void
 238 spin_unlock_shared_quick(globaldata_t gd, struct spinlock *spin)
 239 {
 240 #ifdef DEBUG_LOCKS
 241         int i;
 242         for (i = 0; i < SPINLOCK_DEBUG_ARRAY_SIZE; i++) {
 243                 if ((gd->gd_curthread->td_spinlock_stack_id[i] == 1) &&
 244                     (gd->gd_curthread->td_spinlock_stack[i] == spin)) {
 245                         gd->gd_curthread->td_spinlock_stack_id[i] = 0;
 246                         gd->gd_curthread->td_spinlock_stack[i] = NULL;
 247                         gd->gd_curthread->td_spinlock_caller_pc[i] = NULL;
 248                         break;
 249                 }
 250         }
 251 #endif
 252 #ifdef DEBUG_LOCKS
 253         KKASSERT(spin->counta != 0);
 254 #endif
 255         cpu_sfence();
 256         atomic_add_int(&spin->counta, -1);
 257
 258 #ifdef DEBUG_LOCKS
 259         KKASSERT(gd->gd_spinlocks > 0);
 260 #endif
 261         cpu_ccfence();
 262         --gd->gd_spinlocks;
 263         crit_exit_raw(gd->gd_curthread);
 264 }
 265
 266 static __inline void
 267 _spin_lock_shared(struct spinlock *spin, const char *ident)
 268 {
 269         _spin_lock_shared_quick(mycpu, spin, ident);
 270 }
 271
 272 static __inline void
 273 spin_unlock_shared(struct spinlock *spin)
 274 {
 275         spin_unlock_shared_quick(mycpu, spin);
 276 }
 277
 278 /*
 279  * Attempt to upgrade a shared spinlock to exclusive.  Return non-zero
 280  * on success, 0 on failure.
 281  */
 282 static __inline int
 283 spin_lock_upgrade_try(struct spinlock *spin)
 284 {
 285         if (atomic_cmpset_int(&spin->counta, SPINLOCK_SHARED|1, 1))
 286                 return 1;
 287         else
 288                 return 0;
 289 }
 290
 291 static __inline void
 292 spin_init(struct spinlock *spin, const char *descr __unused)
 293 {
 294         spin->counta = 0;
 295         spin->countb = 0;
 296 #if 0
 297         spin->descr  = descr;
 298 #endif
 299 }
 300
 301 static __inline void
 302 spin_uninit(struct spinlock *spin)
 303 {
 304         /* unused */
 305 }
 306
 307 #endif  /* _KERNEL */
 308 #endif  /* _SYS_SPINLOCK2_H_ */
 309