sys/kern/kern_mplock.c

   1 /*
   2  * Copyright (c) 2009 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * Helper functions for MP lock acquisition and release.
  37  */
  38
  39 #include <sys/param.h>
  40 #include <sys/systm.h>
  41 #include <sys/kernel.h>
  42 #include <sys/proc.h>
  43 #include <sys/rtprio.h>
  44 #include <sys/queue.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/kthread.h>
  47 #include <machine/cpu.h>
  48 #include <sys/lock.h>
  49 #include <sys/caps.h>
  50 #include <sys/spinlock.h>
  51 #include <sys/ktr.h>
  52
  53 #include <sys/thread2.h>
  54 #include <sys/mplock2.h>
  55 #include <sys/spinlock2.h>
  56
  57 #ifdef SMP
  58 static int chain_mplock = 0;
  59 static int bgl_yield = 10;
  60 static __int64_t mplock_contention_count = 0;
  61
  62 SYSCTL_INT(_lwkt, OID_AUTO, chain_mplock, CTLFLAG_RW, &chain_mplock, 0, "");
  63 SYSCTL_INT(_lwkt, OID_AUTO, bgl_yield_delay, CTLFLAG_RW, &bgl_yield, 0, "");
  64 SYSCTL_QUAD(_lwkt, OID_AUTO, mplock_contention_count, CTLFLAG_RW,
  65         &mplock_contention_count, 0, "spinning due to MPLOCK contention");
  66
  67 /*
  68  * Kernel Trace
  69  */
  70 #if !defined(KTR_GIANT_CONTENTION)
  71 #define KTR_GIANT_CONTENTION    KTR_ALL
  72 #endif
  73
  74 KTR_INFO_MASTER(giant);
  75 KTR_INFO(KTR_GIANT_CONTENTION, giant, beg, 0,
  76         "thread=%p held %s:%-5d  want %s:%-5d",
  77          sizeof(void *) * 3 + sizeof(int) * 2);
  78 KTR_INFO(KTR_GIANT_CONTENTION, giant, end, 1,
  79         "thread=%p held %s:%-5d  want %s:%-5d",
  80          sizeof(void *) * 3 + sizeof(int) * 2);
  81
  82 #define loggiant(name)                                          \
  83         KTR_LOG(giant_ ## name, curthread,                      \
  84                 mp_lock_holder_file, mp_lock_holder_line,       \
  85                 file, line)
  86
  87 int     mp_lock;
  88 int     cpu_contention_mask;
  89 const char *mp_lock_holder_file;        /* debugging */
  90 int     mp_lock_holder_line;            /* debugging */
  91
  92 /*
  93  * Sets up the initial MP lock state near the start of the kernel boot
  94  */
  95 void
  96 cpu_get_initial_mplock(void)
  97 {
  98         mp_lock = 0;    /* cpu 0 */
  99         curthread->td_mpcount = 1;
 100 }
 101
 102 /*
 103  * Called when the MP lock could not be trvially acquired.  The caller
 104  * has already bumped td_mpcount.
 105  */
 106 void
 107 _get_mplock_contested(const char *file, int line)
 108 {
 109         globaldata_t gd = mycpu;
 110         int ov;
 111         int nv;
 112         const void **stkframe = (const void **)&file;
 113
 114         ++mplock_contention_count;
 115         for (;;) {
 116                 ov = mp_lock;
 117                 nv = gd->gd_cpuid;
 118                 if (ov == gd->gd_cpuid)
 119                         break;
 120                 if (ov == -1) {
 121                         if (atomic_cmpset_int(&mp_lock, ov, gd->gd_cpuid))
 122                                 break;
 123                 } else {
 124                         gd->gd_curthread->td_mplock_stallpc = stkframe[-1];
 125                         loggiant(beg);
 126                         lwkt_switch();
 127                         loggiant(end);
 128                         KKASSERT(gd->gd_cpuid == mp_lock);
 129                         break;
 130                 }
 131         }
 132 }
 133
 134 /*
 135  * Called if td_mpcount went negative or if td_mpcount is 0 and we were
 136  * unable to release the MP lock.  Handles sanity checks and conflicts.
 137  *
 138  * It is possible for the inline release to have raced an interrupt which
 139  * get/rel'd the MP lock, causing the inline's cmpset to fail.  If this
 140  * case occurs mp_lock will either already be in a released state or it
 141  * will have already been acquired by another cpu.
 142  */
 143 void
 144 _rel_mplock_contested(void)
 145 {
 146         globaldata_t gd = mycpu;
 147         int ov;
 148
 149         KKASSERT(gd->gd_curthread->td_mpcount >= 0);
 150         for (;;) {
 151                 ov = mp_lock;
 152                 if (ov != gd->gd_cpuid)
 153                         break;
 154                 if (atomic_cmpset_int(&mp_lock, ov, -1))
 155                         break;
 156         }
 157 }
 158
 159 /*
 160  * Called when try_mplock() fails.
 161  *
 162  * The inline bumped td_mpcount so we have to undo it.
 163  *
 164  * It is possible to race an interrupt which acquired and released the
 165  * MP lock.  When combined with the td_mpcount decrement we do the MP lock
 166  * can wind up in any state and possibly not even owned by us.
 167  *
 168  * It is also possible for this function to be called even if td_mpcount > 1
 169  * if someone bumped it and raced an interrupt which then called try_mpock().
 170  */
 171 void
 172 _try_mplock_contested(const char *file, int line)
 173 {
 174         globaldata_t gd = mycpu;
 175         thread_t td = gd->gd_curthread;
 176         int ov;
 177
 178         --td->td_mpcount;
 179         KKASSERT(td->td_mpcount >= 0);
 180         ++mplock_contention_count;
 181
 182         for (;;) {
 183                 ov = mp_lock;
 184                 if (ov != gd->gd_cpuid)
 185                         break;
 186                 if (atomic_cmpset_int(&mp_lock, ov, -1))
 187                         break;
 188         }
 189 }
 190
 191 /*
 192  * Called when cpu_try_mplock() fails.
 193  *
 194  * The inline did not touch td_mpcount so we do not either.
 195  */
 196 void
 197 _cpu_try_mplock_contested(const char *file, int line)
 198 {
 199         ++mplock_contention_count;
 200 }
 201
 202 /*
 203  * Temporarily yield the MP lock.  This is part of lwkt_user_yield()
 204  * which is kinda hackish.
 205  */
 206 void
 207 yield_mplock(thread_t td)
 208 {
 209         int savecnt;
 210
 211         savecnt = td->td_mpcount;
 212         td->td_mpcount = 1;
 213         rel_mplock();
 214         DELAY(bgl_yield);
 215         get_mplock();
 216         td->td_mpcount = savecnt;
 217 }
 218
 219 #if 0
 220
 221 /*
 222  * The rel_mplock() code will call this function after releasing the
 223  * last reference on the MP lock if cpu_contention_mask is non-zero.
 224  *
 225  * We then chain an IPI to a single other cpu potentially needing the
 226  * lock.  This is a bit heuristical and we can wind up with IPIs flying
 227  * all over the place.
 228  */
 229 static void lwkt_mp_lock_uncontested_remote(void *arg __unused);
 230
 231 void
 232 lwkt_mp_lock_uncontested(void)
 233 {
 234     globaldata_t gd;
 235     globaldata_t dgd;
 236     cpumask_t mask;
 237     cpumask_t tmpmask;
 238     int cpuid;
 239
 240     if (chain_mplock) {
 241         gd = mycpu;
 242         clr_mplock_contention_mask(gd);
 243         mask = cpu_contention_mask;
 244         tmpmask = ~((1 << gd->gd_cpuid) - 1);
 245
 246         if (mask) {
 247             if (mask & tmpmask)
 248                     cpuid = bsfl(mask & tmpmask);
 249             else
 250                     cpuid = bsfl(mask);
 251             atomic_clear_int(&cpu_contention_mask, 1 << cpuid);
 252             dgd = globaldata_find(cpuid);
 253             lwkt_send_ipiq(dgd, lwkt_mp_lock_uncontested_remote, NULL);
 254         }
 255     }
 256 }
 257
 258 /*
 259  * The idea is for this IPI to interrupt a potentially lower priority
 260  * thread, such as a user thread, to allow the scheduler to reschedule
 261  * a higher priority kernel thread that needs the MP lock.
 262  *
 263  * For now we set the LWKT reschedule flag which generates an AST in
 264  * doreti, though theoretically it is also possible to possibly preempt
 265  * here if the underlying thread was operating in user mode.  Nah.
 266  */
 267 static void
 268 lwkt_mp_lock_uncontested_remote(void *arg __unused)
 269 {
 270         need_lwkt_resched();
 271 }
 272
 273 #endif
 274
 275 #endif  /* SMP */