sys/kern/lwkt_serialize.c

   1 /*
   2  * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/kern/lwkt_serialize.c,v 1.11 2008/04/03 12:55:15 sephe Exp $
  35  */
  36 /*
  37  * This API provides a fast locked-bus-cycle-based serializer.  It's
  38  * basically a low level NON-RECURSIVE exclusive lock that can be held across
  39  * a blocking condition.  It is NOT a mutex.
  40  *
  41  * This serializer is primarily designed for low level situations and
  42  * interrupt/device interaction.  There are two primary facilities.  First,
  43  * the serializer facility itself.  Second, an integrated interrupt handler
  44  * disablement facility.
  45  */
  46
  47 #include "opt_serializer.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/systm.h>
  51 #include <sys/kernel.h>
  52 #include <sys/proc.h>
  53 #include <sys/rtprio.h>
  54 #include <sys/queue.h>
  55 #include <sys/thread2.h>
  56 #include <sys/serialize.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/kthread.h>
  59 #include <machine/cpu.h>
  60 #include <sys/lock.h>
  61 #include <sys/caps.h>
  62
  63 static void lwkt_serialize_sleep(void *info);
  64 static void lwkt_serialize_wakeup(void *info);
  65
  66 void
  67 lwkt_serialize_init(lwkt_serialize_t s)
  68 {
  69     atomic_intr_init(&s->interlock);
  70 #ifdef INVARIANTS
  71     s->last_td = (void *)-4;
  72 #endif
  73 #ifdef PROFILE_SERIALIZER
  74     s->sleep_cnt = 0;
  75     s->tryfail_cnt = 0;
  76     s->enter_cnt = 0;
  77     s->try_cnt = 0;
  78 #endif
  79 }
  80
  81 void
  82 lwkt_serialize_enter(lwkt_serialize_t s)
  83 {
  84 #ifdef INVARIANTS
  85     KKASSERT(s->last_td != curthread);
  86 #endif
  87     atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
  88 #ifdef INVARIANTS
  89     s->last_td = curthread;
  90 #endif
  91 #ifdef PROFILE_SERIALIZER
  92     s->enter_cnt++;
  93 #endif
  94 }
  95
  96 /*
  97  * Returns non-zero on success
  98  */
  99 int
 100 lwkt_serialize_try(lwkt_serialize_t s)
 101 {
 102     int error;
 103
 104 #ifdef INVARIANTS
 105     KKASSERT(s->last_td != curthread);
 106 #endif
 107 #ifdef PROFILE_SERIALIZER
 108     s->try_cnt++;
 109 #endif
 110     if ((error = atomic_intr_cond_try(&s->interlock)) == 0) {
 111 #ifdef INVARIANTS
 112         s->last_td = curthread;
 113 #endif
 114         return(1);
 115     }
 116 #ifdef PROFILE_SERIALIZER
 117     s->tryfail_cnt++;
 118 #endif
 119     return (0);
 120 }
 121
 122 void
 123 lwkt_serialize_exit(lwkt_serialize_t s)
 124 {
 125 #ifdef INVARIANTS
 126     KKASSERT(s->last_td == curthread);
 127     s->last_td = (void *)-2;
 128 #endif
 129     atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 130 }
 131
 132 /*
 133  * Interrupt handler disablement support, used by drivers.  Non-stackable
 134  * (uses bit 30).
 135  */
 136 void
 137 lwkt_serialize_handler_disable(lwkt_serialize_t s)
 138 {
 139     atomic_intr_handler_disable(&s->interlock);
 140 }
 141
 142 void
 143 lwkt_serialize_handler_enable(lwkt_serialize_t s)
 144 {
 145     atomic_intr_handler_enable(&s->interlock);
 146 }
 147
 148 void
 149 lwkt_serialize_handler_call(lwkt_serialize_t s, void (*func)(void *, void *),
 150                             void *arg, void *frame)
 151 {
 152     /*
 153      * note: a return value of 0 indicates that the interrupt handler is
 154      * enabled.
 155      */
 156     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 157         atomic_intr_cond_enter(&s->interlock, lwkt_serialize_sleep, s);
 158 #ifdef INVARIANTS
 159         s->last_td = curthread;
 160 #endif
 161 #ifdef PROFILE_SERIALIZER
 162         s->enter_cnt++;
 163 #endif
 164         if (atomic_intr_handler_is_enabled(&s->interlock) == 0)
 165             func(arg, frame);
 166 #ifdef INVARIANTS
 167         KKASSERT(s->last_td == curthread);
 168         s->last_td = (void *)-2;
 169 #endif
 170         atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 171     }
 172 }
 173
 174 /*
 175  * Similar to handler_call but does not block.  Returns 0 on success,
 176  * and 1 on failure.
 177  */
 178 int
 179 lwkt_serialize_handler_try(lwkt_serialize_t s, void (*func)(void *, void *),
 180                            void *arg, void *frame)
 181 {
 182     /*
 183      * note: a return value of 0 indicates that the interrupt handler is
 184      * enabled.
 185      */
 186     if (atomic_intr_handler_is_enabled(&s->interlock) == 0) {
 187 #ifdef PROFILE_SERIALIZER
 188         s->try_cnt++;
 189 #endif
 190         if (atomic_intr_cond_try(&s->interlock) == 0) {
 191 #ifdef INVARIANTS
 192             s->last_td = curthread;
 193 #endif
 194             func(arg, frame);
 195 #ifdef INVARIANTS
 196             KKASSERT(s->last_td == curthread);
 197             s->last_td = (void *)-2;
 198 #endif
 199             atomic_intr_cond_exit(&s->interlock, lwkt_serialize_wakeup, s);
 200             return(0);
 201         }
 202     }
 203 #ifdef PROFILE_SERIALIZER
 204     s->tryfail_cnt++;
 205 #endif
 206     return(1);
 207 }
 208
 209
 210 /*
 211  * Helper functions
 212  *
 213  * It is possible to race an interrupt which acquires and releases the
 214  * bit, then calls wakeup before we actually go to sleep, so we
 215  * need to check that the interlock is still acquired from within
 216  * a critical section prior to sleeping.
 217  */
 218 static void
 219 lwkt_serialize_sleep(void *info)
 220 {
 221     lwkt_serialize_t s = info;
 222     crit_enter();
 223     tsleep_interlock(s);
 224     if (atomic_intr_cond_test(&s->interlock) != 0) {
 225 #ifdef PROFILE_SERIALIZER
 226             s->sleep_cnt++;
 227 #endif
 228             tsleep(s, 0, "slize", 0);
 229     }
 230     crit_exit();
 231 }
 232
 233 static void
 234 lwkt_serialize_wakeup(void *info)
 235 {
 236     wakeup(info);
 237 }