sys/kern/lwkt_serialize.c

   1 /*
   2  * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.18 2008/10/04 14:22:44 swildner Exp $
  35  */
  36 /*
  37  * This API provides a fast locked-bus-cycle-based serializer.  It's
  38  * basically a low level NON-RECURSIVE exclusive lock that can be held across
  39  * a blocking condition.  It is NOT a mutex.
  40  *
  41  * This serializer is primarily designed for low level situations and
  42  * interrupt/device interaction.  There are two primary facilities.  First,
  43  * the serializer facility itself.  Second, an integrated interrupt handler
  44  * disablement facility.
  45  */
  46
  47 #include "opt_serializer.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/systm.h>
  51 #include <sys/kernel.h>
  52 #include <sys/proc.h>
  53 #include <sys/rtprio.h>
  54 #include <sys/queue.h>
  55 #include <sys/thread2.h>
  56 #include <sys/serialize.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/ktr.h>
  59 #include <sys/kthread.h>
  60 #include <machine/cpu.h>
  61 #include <machine/cpufunc.h>
  62 #include <machine/specialreg.h>
  63 #include <sys/lock.h>
  64 #include <sys/caps.h>
  65
  66 struct exp_backoff {
  67         int backoff;
  68         int round;
  69         lwkt_serialize_t s;
  70 };
  71
  72 #define SLZ_KTR_STRING          "slz=%p"
  73 #define SLZ_KTR_ARG_SIZE        (sizeof(void *))
  74
  75 #ifndef KTR_SERIALIZER
  76 #define KTR_SERIALIZER  KTR_ALL
  77 #endif
  78
  79 KTR_INFO_MASTER(slz);
  80 KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  81 KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  82 KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  83 KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  84 KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  85 KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  86 KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  87 KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  88 KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  89 #ifdef SMP
  90 KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
  91          "slz=%p bo1=%d bo=%d", (sizeof(void *) + (2 * sizeof(int))));
  92 #endif
  93 KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  94 KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  95
  96 #define logslz(name, slz)               KTR_LOG(slz_ ## name, slz)
  97 #ifdef SMP
  98 #define logslz_spinbo(slz, bo1, bo)     KTR_LOG(slz_spinbo, slz, bo1, bo)
  99 #endif
 100
 101 static void lwkt_serialize_sleep(void *info);
 102 static void lwkt_serialize_wakeup(void *info);
 103
 104 #ifdef SMP
 105 static void lwkt_serialize_adaptive_sleep(void *bo);
 106
 107 static int slz_backoff_limit = 128;
 108 SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
 109            &slz_backoff_limit, 0, "");
 110
 111 static int slz_backoff_shift = 1;
 112 SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
 113            &slz_backoff_shift, 0, "");
 114
 115 static int slz_backoff_round;
 116 TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
 117 SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
 118            &slz_backoff_round, 0, "");
 119 #endif  /* SMP */
 120
 121 void
 122 lwkt_serialize_init(lwkt_serialize_t s)
 123 {
 124     atomic_intr_init(&s->interlock);
 125 #ifdef INVARIANTS
 126     s->last_td = (void *)-4;
 127 #endif
 128     s->sleep_cnt = 0;
 129     s->tryfail_cnt = 0;
 130     s->enter_cnt = 0;
 131     s->try_cnt = 0;
 132 }
 133
 134 #ifdef SMP
 135 void
 136 lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
 137 {
 138     struct exp_backoff bo;
 139
 140     bo.backoff = 1;
 141     bo.round = 0;
 142     bo.s = s;
 143
 144     ASSERT_NOT_SERIALIZED(s);
 145
 146     logslz(enter_beg, s);
 147     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
 148     logslz(enter_end, s);
 149 #ifdef INVARIANTS
 150     s->last_td = curthread;
 151 #endif
 152 #ifdef PROFILE_SERIALIZER
 153     s->enter_cnt++;
 154 #endif
 155 }
 156 #endif  /* SMP */
 157
 158 void
 159 lwkt_serialize_enter(lwkt_serialize_t s)
 160 {
 161     ASSERT_NOT_SERIALIZED(s);
 162
 163     logslz(enter_beg, s);
 164     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
 165     logslz(enter_end, s);
 166 #ifdef INVARIANTS
 167     s->last_td = curthread;
 168 #endif
 169 #ifdef PROFILE_SERIALIZER
 170     s->enter_cnt++;
 171 #endif
 172 }
 173
 174 /*
 175  * Returns non-zero on success
 176  */
 177 int
 178 lwkt_serialize_try(lwkt_serialize_t s)
 179 {
 180     int error;
 181
 182     ASSERT_NOT_SERIALIZED(s);
 183
 184 #ifdef PROFILE_SERIALIZER
 185     s->try_cnt++;
 186 #endif
 187     logslz(try, s);
 188     if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
 189 #ifdef INVARIANTS
 190         s->last_td = curthread;
 191 #endif
 192         logslz(tryok, s);
 193         return(1);
 194     }
 195 #ifdef PROFILE_SERIALIZER
 196     s->tryfail_cnt++;
 197 #endif
 198     logslz(tryfail, s);
 199     return (0);
 200 }
 201
 202 void
 203 lwkt_serialize_exit(lwkt_serialize_t s)
 204 {
 205     ASSERT_SERIALIZED(s);
 206 #ifdef INVARIANTS
 207     s->last_td = (void *)-2;
 208 #endif
 209     logslz(exit_beg, s);
 210     atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 211     logslz(exit_end, s);
 212 }
 213
 214 /*
 215  * Interrupt handler disablement support, used by drivers.  Non-stackable
 216  * (uses bit 30).
 217  */
 218 void
 219 lwkt_serialize_handler_disable(lwkt_serialize_t s)
 220 {
 221     atomic_intr_handler_disable(&s->interlock);
 222 }
 223
 224 void
 225 lwkt_serialize_handler_enable(lwkt_serialize_t s)
 226 {
 227     atomic_intr_handler_enable(&s->interlock);
 228 }
 229
 230 void
 231 lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *),
 232                             void *arg, void *frame)
 233 {
 234     /*
 235      * note: a return value of 0 indicates that the interrupt handler is
 236      * enabled.
 237      */
 238     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 239         logslz(enter_beg, s);
 240         atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
 241         logslz(enter_end, s);
 242 #ifdef INVARIANTS
 243         s->last_td = curthread;
 244 #endif
 245 #ifdef PROFILE_SERIALIZER
 246         s->enter_cnt++;
 247 #endif
 248         if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
 249             func(arg, frame);
 250
 251         ASSERT_SERIALIZED(s);
 252 #ifdef INVARIANTS
 253         s->last_td = (void *)-2;
 254 #endif
 255         logslz(exit_beg, s);
 256         atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 257         logslz(exit_end, s);
 258     }
 259 }
 260
 261 /*
 262  * Similar to handler_call but does not block.  Returns 0 on success,
 263  * and 1 on failure.
 264  */
 265 int
 266 lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
 267                            void *arg, void *frame)
 268 {
 269     /*
 270      * note: a return value of 0 indicates that the interrupt handler is
 271      * enabled.
 272      */
 273     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 274 #ifdef PROFILE_SERIALIZER
 275         s->try_cnt++;
 276 #endif
 277         logslz(try, s);
 278         if (atomic_intr_cond_try(&s->interlock) == 0) {
 279 #ifdef INVARIANTS
 280             s->last_td = curthread;
 281 #endif
 282             logslz(tryok, s);
 283
 284             func(arg, frame);
 285
 286             ASSERT_SERIALIZED(s);
 287 #ifdef INVARIANTS
 288             s->last_td = (void *)-2;
 289 #endif
 290             logslz(exit_beg, s);
 291             atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 292             logslz(exit_end, s);
 293             return(0);
 294         }
 295     }
 296 #ifdef PROFILE_SERIALIZER
 297     s->tryfail_cnt++;
 298 #endif
 299     logslz(tryfail, s);
 300     return(1);
 301 }
 302
 303
 304 /*
 305  * Helper functions
 306  *
 307  * It is possible to race an interrupt which acquires and releases the
 308  * bit, then calls wakeup before we actually go to sleep, so we
 309  * need to check that the interlock is still acquired from within
 310  * a critical section prior to sleeping.
 311  */
 312 static void
 313 lwkt_serialize_sleep(void *info)
 314 {
 315     lwkt_serialize_t s = info;
 316     crit_enter();
 317     tsleep_interlock(s);
 318     if (atomic_intr_cond_test(&s->interlock) != 0) {
 319 #ifdef PROFILE_SERIALIZER
 320         s->sleep_cnt++;
 321 #endif
 322         logslz(sleep_beg, s);
 323         tsleep(s, 0, "slize", 0);
 324         logslz(sleep_end, s);
 325     }
 326     crit_exit();
 327 }
 328
 329 #ifdef SMP
 330
 331 static void
 332 lwkt_serialize_adaptive_sleep(void *arg)
 333 {
 334     struct exp_backoff *bo = arg;
 335     lwkt_serialize_t s = bo->s;
 336     int backoff;
 337
 338     /*
 339      * Randomize backoff value
 340      */
 341 #ifdef _RDTSC_SUPPORTED_
 342     if (cpu_feature & CPUID_TSC) {
 343         backoff =
 344         (((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
 345          (bo->backoff - 1)) + 1;
 346     } else
 347 #endif
 348         backoff = bo->backoff;
 349
 350     logslz_spinbo(s, bo->backoff, backoff);
 351
 352     /*
 353      * Quick backoff
 354      */
 355     for (; backoff; --backoff)
 356         cpu_pause();
 357     if (bo->backoff < slz_backoff_limit) {
 358         bo->backoff <<= slz_backoff_shift;
 359         return;
 360     } else {
 361         bo->backoff = 1;
 362         bo->round++;
 363         if (bo->round >= slz_backoff_round)
 364             bo->round = 0;
 365         else
 366             return;
 367     }
 368
 369     crit_enter();
 370     tsleep_interlock(s);
 371     if (atomic_intr_cond_test(&s->interlock) != 0) {
 372 #ifdef PROFILE_SERIALIZER
 373         s->sleep_cnt++;
 374 #endif
 375         logslz(sleep_beg, s);
 376         tsleep(s, 0, "slize", 0);
 377         logslz(sleep_end, s);
 378     }
 379     crit_exit();
 380 }
 381
 382 #endif  /* SMP */
 383
 384 static void
 385 lwkt_serialize_wakeup(void *info)
 386 {
 387     logslz(wakeup_beg, info);
 388     wakeup(info);
 389     logslz(wakeup_end, info);
 390 }
 391
 392 #ifdef SMP
 393 static void
 394 lwkt_serialize_sysinit(void *dummy __unused)
 395 {
 396         if (slz_backoff_round <= 0)
 397                 slz_backoff_round = ncpus * 2;
 398 }
 399 SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
 400         lwkt_serialize_sysinit, NULL);
 401 #endif