sys/kern/lwkt_serialize.c

   1 /*
   2  * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.18 2008/10/04 14:22:44 swildner Exp $
  35  */
  36 /*
  37  * This API provides a fast locked-bus-cycle-based serializer.  It's
  38  * basically a low level NON-RECURSIVE exclusive lock that can be held across
  39  * a blocking condition.  It is NOT a mutex.
  40  *
  41  * This serializer is primarily designed for low level situations and
  42  * interrupt/device interaction.  There are two primary facilities.  First,
  43  * the serializer facility itself.  Second, an integrated interrupt handler
  44  * disablement facility.
  45  */
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/proc.h>
  51 #include <sys/rtprio.h>
  52 #include <sys/queue.h>
  53 #include <sys/thread2.h>
  54 #include <sys/serialize.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/ktr.h>
  57 #include <sys/kthread.h>
  58 #include <machine/cpu.h>
  59 #include <machine/cpufunc.h>
  60 #include <machine/specialreg.h>
  61 #include <sys/lock.h>
  62 #include <sys/caps.h>
  63
  64 struct exp_backoff {
  65         int backoff;
  66         int round;
  67         lwkt_serialize_t s;
  68 };
  69
  70 #define SLZ_KTR_STRING          "slz=%p"
  71 #define SLZ_KTR_ARG_SIZE        (sizeof(void *))
  72
  73 #ifndef KTR_SERIALIZER
  74 #define KTR_SERIALIZER  KTR_ALL
  75 #endif
  76
  77 KTR_INFO_MASTER(slz);
  78 KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  79 KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  80 KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  81 KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  82 KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  83 KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  84 KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  85 KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  86 KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  87 #ifdef SMP
  88 KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
  89          "slz=%p bo1=%d bo=%d", (sizeof(void *) + (2 * sizeof(int))));
  90 #endif
  91 KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  92 KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  93
  94 #define logslz(name, slz)               KTR_LOG(slz_ ## name, slz)
  95 #ifdef SMP
  96 #define logslz_spinbo(slz, bo1, bo)     KTR_LOG(slz_spinbo, slz, bo1, bo)
  97 #endif
  98
  99 static void lwkt_serialize_sleep(void *info);
 100 static void lwkt_serialize_wakeup(void *info);
 101
 102 #ifdef SMP
 103 static void lwkt_serialize_adaptive_sleep(void *bo);
 104
 105 static int slz_backoff_limit = 128;
 106 SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
 107            &slz_backoff_limit, 0, "");
 108
 109 static int slz_backoff_shift = 1;
 110 SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
 111            &slz_backoff_shift, 0, "");
 112
 113 static int slz_backoff_round;
 114 TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
 115 SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
 116            &slz_backoff_round, 0, "");
 117 #endif  /* SMP */
 118
 119 void
 120 lwkt_serialize_init(lwkt_serialize_t s)
 121 {
 122     atomic_intr_init(&s->interlock);
 123 #ifdef INVARIANTS
 124     s->last_td = (void *)-4;
 125 #endif
 126 }
 127
 128 #ifdef SMP
 129 void
 130 lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
 131 {
 132     struct exp_backoff bo;
 133
 134     bo.backoff = 1;
 135     bo.round = 0;
 136     bo.s = s;
 137
 138     ASSERT_NOT_SERIALIZED(s);
 139
 140     logslz(enter_beg, s);
 141     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
 142     logslz(enter_end, s);
 143 #ifdef INVARIANTS
 144     s->last_td = curthread;
 145 #endif
 146 }
 147 #endif  /* SMP */
 148
 149 void
 150 lwkt_serialize_enter(lwkt_serialize_t s)
 151 {
 152     ASSERT_NOT_SERIALIZED(s);
 153
 154     logslz(enter_beg, s);
 155     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
 156     logslz(enter_end, s);
 157 #ifdef INVARIANTS
 158     s->last_td = curthread;
 159 #endif
 160 }
 161
 162 /*
 163  * Returns non-zero on success
 164  */
 165 int
 166 lwkt_serialize_try(lwkt_serialize_t s)
 167 {
 168     int error;
 169
 170     ASSERT_NOT_SERIALIZED(s);
 171
 172     logslz(try, s);
 173     if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
 174 #ifdef INVARIANTS
 175         s->last_td = curthread;
 176 #endif
 177         logslz(tryok, s);
 178         return(1);
 179     }
 180     logslz(tryfail, s);
 181     return (0);
 182 }
 183
 184 void
 185 lwkt_serialize_exit(lwkt_serialize_t s)
 186 {
 187     ASSERT_SERIALIZED(s);
 188 #ifdef INVARIANTS
 189     s->last_td = (void *)-2;
 190 #endif
 191     logslz(exit_beg, s);
 192     atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 193     logslz(exit_end, s);
 194 }
 195
 196 /*
 197  * Interrupt handler disablement support, used by drivers.  Non-stackable
 198  * (uses bit 30).
 199  */
 200 void
 201 lwkt_serialize_handler_disable(lwkt_serialize_t s)
 202 {
 203     atomic_intr_handler_disable(&s->interlock);
 204 }
 205
 206 void
 207 lwkt_serialize_handler_enable(lwkt_serialize_t s)
 208 {
 209     atomic_intr_handler_enable(&s->interlock);
 210 }
 211
 212 void
 213 lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *),
 214                             void *arg, void *frame)
 215 {
 216     /*
 217      * note: a return value of 0 indicates that the interrupt handler is
 218      * enabled.
 219      */
 220     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 221         logslz(enter_beg, s);
 222         atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
 223         logslz(enter_end, s);
 224 #ifdef INVARIANTS
 225         s->last_td = curthread;
 226 #endif
 227         if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
 228             func(arg, frame);
 229
 230         ASSERT_SERIALIZED(s);
 231 #ifdef INVARIANTS
 232         s->last_td = (void *)-2;
 233 #endif
 234         logslz(exit_beg, s);
 235         atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 236         logslz(exit_end, s);
 237     }
 238 }
 239
 240 /*
 241  * Similar to handler_call but does not block.  Returns 0 on success,
 242  * and 1 on failure.
 243  */
 244 int
 245 lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
 246                            void *arg, void *frame)
 247 {
 248     /*
 249      * note: a return value of 0 indicates that the interrupt handler is
 250      * enabled.
 251      */
 252     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 253         logslz(try, s);
 254         if (atomic_intr_cond_try(&s->interlock) == 0) {
 255 #ifdef INVARIANTS
 256             s->last_td = curthread;
 257 #endif
 258             logslz(tryok, s);
 259
 260             func(arg, frame);
 261
 262             ASSERT_SERIALIZED(s);
 263 #ifdef INVARIANTS
 264             s->last_td = (void *)-2;
 265 #endif
 266             logslz(exit_beg, s);
 267             atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 268             logslz(exit_end, s);
 269             return(0);
 270         }
 271     }
 272     logslz(tryfail, s);
 273     return(1);
 274 }
 275
 276
 277 /*
 278  * Helper functions
 279  *
 280  * It is possible to race an interrupt which acquires and releases the
 281  * bit, then calls wakeup before we actually go to sleep, so we
 282  * need to check that the interlock is still acquired from within
 283  * a critical section prior to sleeping.
 284  */
 285 static void
 286 lwkt_serialize_sleep(void *info)
 287 {
 288     lwkt_serialize_t s = info;
 289
 290     tsleep_interlock(s, 0);
 291     if (atomic_intr_cond_test(&s->interlock) != 0) {
 292         logslz(sleep_beg, s);
 293         tsleep(s, PINTERLOCKED, "slize", 0);
 294         logslz(sleep_end, s);
 295     }
 296 }
 297
 298 #ifdef SMP
 299
 300 static void
 301 lwkt_serialize_adaptive_sleep(void *arg)
 302 {
 303     struct exp_backoff *bo = arg;
 304     lwkt_serialize_t s = bo->s;
 305     int backoff;
 306
 307     /*
 308      * Randomize backoff value
 309      */
 310 #ifdef _RDTSC_SUPPORTED_
 311     if (cpu_feature & CPUID_TSC) {
 312         backoff =
 313         (((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
 314          (bo->backoff - 1)) + 1;
 315     } else
 316 #endif
 317         backoff = bo->backoff;
 318
 319     logslz_spinbo(s, bo->backoff, backoff);
 320
 321     /*
 322      * Quick backoff
 323      */
 324     for (; backoff; --backoff)
 325         cpu_pause();
 326     if (bo->backoff < slz_backoff_limit) {
 327         bo->backoff <<= slz_backoff_shift;
 328         return;
 329     } else {
 330         bo->backoff = 1;
 331         bo->round++;
 332         if (bo->round >= slz_backoff_round)
 333             bo->round = 0;
 334         else
 335             return;
 336     }
 337
 338     tsleep_interlock(s, 0);
 339     if (atomic_intr_cond_test(&s->interlock) != 0) {
 340         logslz(sleep_beg, s);
 341         tsleep(s, PINTERLOCKED, "slize", 0);
 342         logslz(sleep_end, s);
 343     }
 344 }
 345
 346 #endif  /* SMP */
 347
 348 static void
 349 lwkt_serialize_wakeup(void *info)
 350 {
 351     logslz(wakeup_beg, info);
 352     wakeup(info);
 353     logslz(wakeup_end, info);
 354 }
 355
 356 #ifdef SMP
 357 static void
 358 lwkt_serialize_sysinit(void *dummy __unused)
 359 {
 360         if (slz_backoff_round <= 0)
 361                 slz_backoff_round = ncpus * 2;
 362 }
 363 SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
 364         lwkt_serialize_sysinit, NULL);
 365 #endif