sys/kern/lwkt_serialize.c

   1 /*
   2  * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.17 2008/05/24 12:52:49 sephe Exp $
  35  */
  36 /*
  37  * This API provides a fast locked-bus-cycle-based serializer.  It's
  38  * basically a low level NON-RECURSIVE exclusive lock that can be held across
  39  * a blocking condition.  It is NOT a mutex.
  40  *
  41  * This serializer is primarily designed for low level situations and
  42  * interrupt/device interaction.  There are two primary facilities.  First,
  43  * the serializer facility itself.  Second, an integrated interrupt handler
  44  * disablement facility.
  45  */
  46
  47 #include "opt_serializer.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/systm.h>
  51 #include <sys/kernel.h>
  52 #include <sys/proc.h>
  53 #include <sys/rtprio.h>
  54 #include <sys/queue.h>
  55 #include <sys/thread2.h>
  56 #include <sys/serialize.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/ktr.h>
  59 #include <sys/kthread.h>
  60 #include <machine/cpu.h>
  61 #include <machine/cpufunc.h>
  62 #include <machine/specialreg.h>
  63 #include <sys/lock.h>
  64 #include <sys/caps.h>
  65
  66 struct exp_backoff {
  67         int backoff;
  68         int round;
  69         lwkt_serialize_t s;
  70 };
  71
  72 #define SLZ_KTR_STRING          "slz=%p"
  73 #define SLZ_KTR_ARG_SIZE        (sizeof(void *))
  74
  75 #ifndef KTR_SERIALIZER
  76 #define KTR_SERIALIZER  KTR_ALL
  77 #endif
  78
  79 KTR_INFO_MASTER(slz);
  80 KTR_INFO(KTR_SERIALIZER, slz, enter_beg, 0, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  81 KTR_INFO(KTR_SERIALIZER, slz, sleep_beg, 1, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  82 KTR_INFO(KTR_SERIALIZER, slz, sleep_end, 2, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  83 KTR_INFO(KTR_SERIALIZER, slz, exit_end, 3, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  84 KTR_INFO(KTR_SERIALIZER, slz, wakeup_beg, 4, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  85 KTR_INFO(KTR_SERIALIZER, slz, wakeup_end, 5, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  86 KTR_INFO(KTR_SERIALIZER, slz, try, 6, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  87 KTR_INFO(KTR_SERIALIZER, slz, tryfail, 7, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  88 KTR_INFO(KTR_SERIALIZER, slz, tryok, 8, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  89 KTR_INFO(KTR_SERIALIZER, slz, spinbo, 9,
  90          "slz=%p bo1=%d bo=%d", (sizeof(void *) + (2 * sizeof(int))));
  91 KTR_INFO(KTR_SERIALIZER, slz, enter_end, 10, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  92 KTR_INFO(KTR_SERIALIZER, slz, exit_beg, 11, SLZ_KTR_STRING, SLZ_KTR_ARG_SIZE);
  93
  94 #define logslz(name, slz)               KTR_LOG(slz_ ## name, slz)
  95 #define logslz_spinbo(slz, bo1, bo)     KTR_LOG(slz_spinbo, slz, bo1, bo)
  96
  97 static void lwkt_serialize_sleep(void *info);
  98 static void lwkt_serialize_wakeup(void *info);
  99
 100 #ifdef SMP
 101 static void lwkt_serialize_adaptive_sleep(void *bo);
 102
 103 static int slz_backoff_limit = 128;
 104 SYSCTL_INT(_debug, OID_AUTO, serialize_bolimit, CTLFLAG_RW,
 105            &slz_backoff_limit, 0, "");
 106
 107 static int slz_backoff_shift = 1;
 108 SYSCTL_INT(_debug, OID_AUTO, serialize_boshift, CTLFLAG_RW,
 109            &slz_backoff_shift, 0, "");
 110
 111 static int slz_backoff_round;
 112 TUNABLE_INT("debug.serialize_boround", &slz_backoff_round);
 113 SYSCTL_INT(_debug, OID_AUTO, serialize_boround, CTLFLAG_RW,
 114            &slz_backoff_round, 0, "");
 115 #endif  /* SMP */
 116
 117 void
 118 lwkt_serialize_init(lwkt_serialize_t s)
 119 {
 120     atomic_intr_init(&s->interlock);
 121     s->last_td = (void *)-4;
 122
 123     s->sleep_cnt = 0;
 124     s->tryfail_cnt = 0;
 125     s->enter_cnt = 0;
 126     s->try_cnt = 0;
 127 }
 128
 129 #ifdef SMP
 130 void
 131 lwkt_serialize_adaptive_enter(lwkt_serialize_t s)
 132 {
 133     struct exp_backoff bo;
 134
 135     bo.backoff = 1;
 136     bo.round = 0;
 137     bo.s = s;
 138
 139     ASSERT_NOT_SERIALIZED(s);
 140
 141     logslz(enter_beg, s);
 142     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_adaptive_sleep, &bo);
 143     logslz(enter_end, s);
 144
 145     s->last_td = curthread;
 146 #ifdef PROFILE_SERIALIZER
 147     s->enter_cnt++;
 148 #endif
 149 }
 150 #endif  /* SMP */
 151
 152 void
 153 lwkt_serialize_enter(lwkt_serialize_t s)
 154 {
 155     ASSERT_NOT_SERIALIZED(s);
 156
 157     logslz(enter_beg, s);
 158     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
 159     logslz(enter_end, s);
 160
 161     s->last_td = curthread;
 162 #ifdef PROFILE_SERIALIZER
 163     s->enter_cnt++;
 164 #endif
 165 }
 166
 167 /*
 168  * Returns non-zero on success
 169  */
 170 int
 171 lwkt_serialize_try(lwkt_serialize_t s)
 172 {
 173     int error;
 174
 175     ASSERT_NOT_SERIALIZED(s);
 176
 177 #ifdef PROFILE_SERIALIZER
 178     s->try_cnt++;
 179 #endif
 180     logslz(try, s);
 181     if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
 182         s->last_td = curthread;
 183         logslz(tryok, s);
 184         return(1);
 185     }
 186 #ifdef PROFILE_SERIALIZER
 187     s->tryfail_cnt++;
 188 #endif
 189     logslz(tryfail, s);
 190     return (0);
 191 }
 192
 193 void
 194 lwkt_serialize_exit(lwkt_serialize_t s)
 195 {
 196     ASSERT_SERIALIZED(s);
 197     s->last_td = (void *)-2;
 198
 199     logslz(exit_beg, s);
 200     atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 201     logslz(exit_end, s);
 202 }
 203
 204 /*
 205  * Interrupt handler disablement support, used by drivers.  Non-stackable
 206  * (uses bit 30).
 207  */
 208 void
 209 lwkt_serialize_handler_disable(lwkt_serialize_t s)
 210 {
 211     atomic_intr_handler_disable(&s->interlock);
 212 }
 213
 214 void
 215 lwkt_serialize_handler_enable(lwkt_serialize_t s)
 216 {
 217     atomic_intr_handler_enable(&s->interlock);
 218 }
 219
 220 void
 221 lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *),
 222                             void *arg, void *frame)
 223 {
 224     /*
 225      * note: a return value of 0 indicates that the interrupt handler is
 226      * enabled.
 227      */
 228     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 229         logslz(enter_beg, s);
 230         atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
 231         logslz(enter_end, s);
 232
 233         s->last_td = curthread;
 234 #ifdef PROFILE_SERIALIZER
 235         s->enter_cnt++;
 236 #endif
 237         if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
 238             func(arg, frame);
 239
 240         ASSERT_SERIALIZED(s);
 241         s->last_td = (void *)-2;
 242
 243         logslz(exit_beg, s);
 244         atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 245         logslz(exit_end, s);
 246     }
 247 }
 248
 249 /*
 250  * Similar to handler_call but does not block.  Returns 0 on success,
 251  * and 1 on failure.
 252  */
 253 int
 254 lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
 255                            void *arg, void *frame)
 256 {
 257     /*
 258      * note: a return value of 0 indicates that the interrupt handler is
 259      * enabled.
 260      */
 261     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 262 #ifdef PROFILE_SERIALIZER
 263         s->try_cnt++;
 264 #endif
 265         logslz(try, s);
 266         if (atomic_intr_cond_try(&s->interlock) == 0) {
 267             s->last_td = curthread;
 268             logslz(tryok, s);
 269
 270             func(arg, frame);
 271
 272             ASSERT_SERIALIZED(s);
 273             s->last_td = (void *)-2;
 274
 275             logslz(exit_beg, s);
 276             atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 277             logslz(exit_end, s);
 278             return(0);
 279         }
 280     }
 281 #ifdef PROFILE_SERIALIZER
 282     s->tryfail_cnt++;
 283 #endif
 284     logslz(tryfail, s);
 285     return(1);
 286 }
 287
 288
 289 /*
 290  * Helper functions
 291  *
 292  * It is possible to race an interrupt which acquires and releases the
 293  * bit, then calls wakeup before we actually go to sleep, so we
 294  * need to check that the interlock is still acquired from within
 295  * a critical section prior to sleeping.
 296  */
 297 static void
 298 lwkt_serialize_sleep(void *info)
 299 {
 300     lwkt_serialize_t s = info;
 301     crit_enter();
 302     tsleep_interlock(s);
 303     if (atomic_intr_cond_test(&s->interlock) != 0) {
 304 #ifdef PROFILE_SERIALIZER
 305         s->sleep_cnt++;
 306 #endif
 307         logslz(sleep_beg, s);
 308         tsleep(s, 0, "slize", 0);
 309         logslz(sleep_end, s);
 310     }
 311     crit_exit();
 312 }
 313
 314 #ifdef SMP
 315
 316 static void
 317 lwkt_serialize_adaptive_sleep(void *arg)
 318 {
 319     struct exp_backoff *bo = arg;
 320     lwkt_serialize_t s = bo->s;
 321     int backoff;
 322
 323     /*
 324      * Randomize backoff value
 325      */
 326 #ifdef _RDTSC_SUPPORTED_
 327     if (cpu_feature & CPUID_TSC) {
 328         backoff =
 329         (((u_long)rdtsc() ^ (((u_long)curthread) >> 5)) &
 330          (bo->backoff - 1)) + 1;
 331     } else
 332 #endif
 333         backoff = bo->backoff;
 334
 335     logslz_spinbo(s, bo->backoff, backoff);
 336
 337     /*
 338      * Quick backoff
 339      */
 340     for (; backoff; --backoff)
 341         cpu_pause();
 342     if (bo->backoff < slz_backoff_limit) {
 343         bo->backoff <<= slz_backoff_shift;
 344         return;
 345     } else {
 346         bo->backoff = 1;
 347         bo->round++;
 348         if (bo->round >= slz_backoff_round)
 349             bo->round = 0;
 350         else
 351             return;
 352     }
 353
 354     crit_enter();
 355     tsleep_interlock(s);
 356     if (atomic_intr_cond_test(&s->interlock) != 0) {
 357 #ifdef PROFILE_SERIALIZER
 358         s->sleep_cnt++;
 359 #endif
 360         logslz(sleep_beg, s);
 361         tsleep(s, 0, "slize", 0);
 362         logslz(sleep_end, s);
 363     }
 364     crit_exit();
 365 }
 366
 367 #endif  /* SMP */
 368
 369 static void
 370 lwkt_serialize_wakeup(void *info)
 371 {
 372     logslz(wakeup_beg, info);
 373     wakeup(info);
 374     logslz(wakeup_end, info);
 375 }
 376
 377 #ifdef SMP
 378 static void
 379 lwkt_serialize_sysinit(void *dummy __unused)
 380 {
 381         if (slz_backoff_round <= 0)
 382                 slz_backoff_round = ncpus * 2;
 383 }
 384 SYSINIT(lwkt_serialize, SI_SUB_PRE_DRIVERS, SI_ORDER_SECOND,
 385         lwkt_serialize_sysinit, NULL);
 386 #endif