sys/kern/lwkt_serialize.c

   1 /*
   2  * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.18 2008/10/04 14:22:44 swildner Exp $
  35  */
  36 /*
  37  * This API provides a fast locked-bus-cycle-based serializer.  It's
  38  * basically a low level NON-RECURSIVE exclusive lock that can be held across
  39  * a blocking condition.  It is NOT a mutex.
  40  *
  41  * This serializer is primarily designed for low level situations and
  42  * interrupt/device interaction.  There are two primary facilities.  First,
  43  * the serializer facility itself.  Second, an integrated interrupt handler
  44  * disablement facility.
  45  */
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/proc.h>
  51 #include <sys/rtprio.h>
  52 #include <sys/queue.h>
  53 #include <sys/thread2.h>
  54 #include <sys/serialize.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/ktr.h>
  57 #include <sys/kthread.h>
  58 #include <machine/cpu.h>
  59 #include <machine/cpufunc.h>
  60 #include <machine/specialreg.h>
  61 #include <sys/lock.h>
  62 #include <sys/caps.h>
  63
  64 struct exp_backoff {
  65         int backoff;
  66         int round;
  67         lwkt_serialize_t s;
  68 };
  69
  70 #define SLZ_KTR_STRING          "slz=%p"
  71 #define SLZ_KTR_ARG_SIZE        (sizeof(void *))
  72
  73 #ifndef KTR_SERIALIZER
  74 #define KTR_SERIALIZER  KTR_ALL
  75 #endif
  76
  77 KTR_INFO_MASTER(slz);
  78 KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  79 KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  80 KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  81 KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  82 KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  83 KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  84 KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  85 KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  86 KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  87 #ifdef SMP
  88 KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
  89          "slz=%p bo1=%d bo=%d", (sizeof(void *) + (2 * sizeof(int))));
  90 #endif
  91 KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  92 KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  93
  94 #define logslz(name, slz)               KTR_LOG(slz_ ## name, slz)
  95 #ifdef SMP
  96 #define logslz_spinbo(slz, bo1, bo)     KTR_LOG(slz_spinbo, slz, bo1, bo)
  97 #endif
  98
  99 static void lwkt_serialize_sleep(void *info);
 100 static void lwkt_serialize_wakeup(void *info);
 101
 102 #ifdef SMP
 103 static void lwkt_serialize_adaptive_sleep(void *bo);
 104
 105 static int slz_backoff_limit = 128;
 106 SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
 107     &slz_backoff_limit, 0, "Backoff limit");
 108
 109 static int slz_backoff_shift = 1;
 110 SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
 111     &slz_backoff_shift, 0, "Backoff shift");
 112
 113 static int slz_backoff_round;
 114 TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
 115 SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
 116     &slz_backoff_round, 0,
 117     "Backoff rounding");
 118 #endif  /* SMP */
 119
 120 void
 121 lwkt_serialize_init(lwkt_serialize_t s)
 122 {
 123     atomic_intr_init(&s->interlock);
 124 #ifdef INVARIANTS
 125     s->last_td = (void *)-4;
 126 #endif
 127 }
 128
 129 #ifdef SMP
 130 void
 131 lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
 132 {
 133     struct exp_backoff bo;
 134
 135     bo.backoff = 1;
 136     bo.round = 0;
 137     bo.s = s;
 138
 139     ASSERT_NOT_SERIALIZED(s);
 140
 141     logslz(enter_beg, s);
 142     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
 143     logslz(enter_end, s);
 144 #ifdef INVARIANTS
 145     s->last_td = curthread;
 146 #endif
 147 }
 148 #endif  /* SMP */
 149
 150 void
 151 lwkt_serialize_enter(lwkt_serialize_t s)
 152 {
 153     ASSERT_NOT_SERIALIZED(s);
 154
 155     logslz(enter_beg, s);
 156     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
 157     logslz(enter_end, s);
 158 #ifdef INVARIANTS
 159     s->last_td = curthread;
 160 #endif
 161 }
 162
 163 /*
 164  * Returns non-zero on success
 165  */
 166 int
 167 lwkt_serialize_try(lwkt_serialize_t s)
 168 {
 169     int error;
 170
 171     ASSERT_NOT_SERIALIZED(s);
 172
 173     logslz(try, s);
 174     if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
 175 #ifdef INVARIANTS
 176         s->last_td = curthread;
 177 #endif
 178         logslz(tryok, s);
 179         return(1);
 180     }
 181     logslz(tryfail, s);
 182     return (0);
 183 }
 184
 185 void
 186 lwkt_serialize_exit(lwkt_serialize_t s)
 187 {
 188     ASSERT_SERIALIZED(s);
 189 #ifdef INVARIANTS
 190     s->last_td = (void *)-2;
 191 #endif
 192     logslz(exit_beg, s);
 193     atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 194     logslz(exit_end, s);
 195 }
 196
 197 /*
 198  * Interrupt handler disablement support, used by drivers.  Non-stackable
 199  * (uses bit 30).
 200  */
 201 void
 202 lwkt_serialize_handler_disable(lwkt_serialize_t s)
 203 {
 204     atomic_intr_handler_disable(&s->interlock);
 205 }
 206
 207 void
 208 lwkt_serialize_handler_enable(lwkt_serialize_t s)
 209 {
 210     atomic_intr_handler_enable(&s->interlock);
 211 }
 212
 213 void
 214 lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *),
 215                             void *arg, void *frame)
 216 {
 217     /*
 218      * note: a return value of 0 indicates that the interrupt handler is
 219      * enabled.
 220      */
 221     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 222         logslz(enter_beg, s);
 223         atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
 224         logslz(enter_end, s);
 225 #ifdef INVARIANTS
 226         s->last_td = curthread;
 227 #endif
 228         if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
 229             func(arg, frame);
 230
 231         ASSERT_SERIALIZED(s);
 232 #ifdef INVARIANTS
 233         s->last_td = (void *)-2;
 234 #endif
 235         logslz(exit_beg, s);
 236         atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 237         logslz(exit_end, s);
 238     }
 239 }
 240
 241 /*
 242  * Similar to handler_call but does not block.  Returns 0 on success,
 243  * and 1 on failure.
 244  */
 245 int
 246 lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
 247                            void *arg, void *frame)
 248 {
 249     /*
 250      * note: a return value of 0 indicates that the interrupt handler is
 251      * enabled.
 252      */
 253     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 254         logslz(try, s);
 255         if (atomic_intr_cond_try(&s->interlock) == 0) {
 256 #ifdef INVARIANTS
 257             s->last_td = curthread;
 258 #endif
 259             logslz(tryok, s);
 260
 261             func(arg, frame);
 262
 263             ASSERT_SERIALIZED(s);
 264 #ifdef INVARIANTS
 265             s->last_td = (void *)-2;
 266 #endif
 267             logslz(exit_beg, s);
 268             atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 269             logslz(exit_end, s);
 270             return(0);
 271         }
 272     }
 273     logslz(tryfail, s);
 274     return(1);
 275 }
 276
 277
 278 /*
 279  * Helper functions
 280  *
 281  * It is possible to race an interrupt which acquires and releases the
 282  * bit, then calls wakeup before we actually go to sleep, so we
 283  * need to check that the interlock is still acquired from within
 284  * a critical section prior to sleeping.
 285  */
 286 static void
 287 lwkt_serialize_sleep(void *info)
 288 {
 289     lwkt_serialize_t s = info;
 290
 291     tsleep_interlock(s, 0);
 292     if (atomic_intr_cond_test(&s->interlock) != 0) {
 293         logslz(sleep_beg, s);
 294         tsleep(s, PINTERLOCKED, "slize", 0);
 295         logslz(sleep_end, s);
 296     }
 297 }
 298
 299 #ifdef SMP
 300
 301 static void
 302 lwkt_serialize_adaptive_sleep(void *arg)
 303 {
 304     struct exp_backoff *bo = arg;
 305     lwkt_serialize_t s = bo->s;
 306     int backoff;
 307
 308     /*
 309      * Randomize backoff value
 310      */
 311 #ifdef _RDTSC_SUPPORTED_
 312     if (cpu_feature & CPUID_TSC) {
 313         backoff =
 314         (((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
 315          (bo->backoff - 1)) + 1;
 316     } else
 317 #endif
 318         backoff = bo->backoff;
 319
 320     logslz_spinbo(s, bo->backoff, backoff);
 321
 322     /*
 323      * Quick backoff
 324      */
 325     for (; backoff; --backoff)
 326         cpu_pause();
 327     if (bo->backoff < slz_backoff_limit) {
 328         bo->backoff <<= slz_backoff_shift;
 329         return;
 330     } else {
 331         bo->backoff = 1;
 332         bo->round++;
 333         if (bo->round >= slz_backoff_round)
 334             bo->round = 0;
 335         else
 336             return;
 337     }
 338
 339     tsleep_interlock(s, 0);
 340     if (atomic_intr_cond_test(&s->interlock) != 0) {
 341         logslz(sleep_beg, s);
 342         tsleep(s, PINTERLOCKED, "slize", 0);
 343         logslz(sleep_end, s);
 344     }
 345 }
 346
 347 #endif  /* SMP */
 348
 349 static void
 350 lwkt_serialize_wakeup(void *info)
 351 {
 352     logslz(wakeup_beg, info);
 353     wakeup(info);
 354     logslz(wakeup_end, info);
 355 }
 356
 357 #ifdef SMP
 358 static void
 359 lwkt_serialize_sysinit(void *dummy __unused)
 360 {
 361         if (slz_backoff_round <= 0)
 362                 slz_backoff_round = ncpus * 2;
 363 }
 364 SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
 365         lwkt_serialize_sysinit, NULL);
 366 #endif