kernel/os/softint.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 #include <sys/types.h>
  27 #include <sys/param.h>
  28 #include <sys/t_lock.h>
  29 #include <sys/systm.h>
  30 #include <sys/spl.h>
  31 #include <sys/cmn_err.h>
  32 #include <sys/debug.h>
  33 #include <sys/kdi_impl.h>
  34 #include <sys/cpuvar.h>
  35 #include <sys/cpuvar.h>
  36 #include <sys/archsystm.h>
  37
  38 /*
  39  * Handle software interrupts through 'softcall' mechanism
  40  *
  41  * At present softcall mechanism uses a global list headed by softhead.
  42  * Entries are added to tail and removed from head so as to preserve FIFO
  43  * nature of entries in the softcall list. softcall() takes care of adding
  44  * entries to the softtail.
  45  *
  46  * softint must take care of executing the entries in the FIFO
  47  * order. It could be called simultaneously from multiple cpus, however only
  48  * one instance of softint should process the softcall list with the exception
  49  * when CPU is stuck due to high interrupt load and can't execute callbacks.
  50  * State diagram is as follows :-
  51  *
  52  *      - Upper half which is same as old state machine
  53  *        (IDLE->PEND->DRAIN->IDLE)
  54  *
  55  *      - Lower half which steals the entries from softcall queue and execute
  56  *        in the context of softint interrupt handler. The interrupt handler
  57  *        is fired on a different CPU by sending a cross-call.
  58  *
  59  * Starting state is IDLE.
  60  *
  61  *                              softint()
  62  *
  63  *
  64  *                              (c)
  65  *      ____________________________________________________
  66  *      |                          ^                         ^
  67  *      v            (a)           |           (b)           |
  68  *      IDLE--------------------->PEND--------------------->DRAIN
  69  *      ^                         |                         |
  70  *      |                         |                         |
  71  *      |                         |                         |
  72  *      |                         |                         |
  73  *      |                         |                         |
  74  *      |                         d                         d
  75  *      |                         |                         |
  76  *      |                         v                         v
  77  *      |                         PEND                      DRAIN
  78  *      |            (e)           &                          &
  79  *      |<-----------------------STEAL                      STEAL
  80  *      ^                                                    |
  81  *      |                                                    |
  82  *      |                         (e)                        v
  83  *      |_________________________<__________________________|
  84  *
  85  *
  86  *
  87  * Edge (a)->(b)->(c) are same as old state machine and these
  88  * are mutually exclusive state.
  89  *
  90  * a - When an entry is being enqueued to softcall queue then the state
  91  *     moves from IDLE to PEND.
  92  *
  93  * b - When interrupt handler has started processing softcall queue.
  94  *
  95  * c - When interrupt handler finished processing softcall queue, the
  96  *     state of machines goes back to IDLE.
  97  *
  98  * d - softcall() generates another softlevel1 iff interrupt handler
  99  *     hasn't run recently.
 100  *
 101  * e - Either PEND|STEAL or DRAIN|STEAL is set. We let softlevel1
 102  *     handler exit because we have processed all the entries.
 103  *
 104  * When CPU is being pinned by higher level interrupts for more than
 105  * softcall_delay clock ticks, SOFT_STEAL is OR'ed so that softlevel1
 106  * handler on the other CPU can drain the queue.
 107  *
 108  * These states are needed for softcall mechanism since Solaris has only
 109  * one interface (ie. siron ) as of now for :
 110  *
 111  * - raising a soft interrupt architecture independently (ie not through
 112  *   setsoftint(..) )
 113  * - to process the softcall queue.
 114  */
 115
 116 #define NSOFTCALLS      200
 117
 118 /*
 119  * Defined states for softcall processing.
 120  */
 121 #define SOFT_IDLE               0x01    /* no processing is needed */
 122 #define SOFT_PEND               0x02    /* softcall list needs processing */
 123 #define SOFT_DRAIN              0x04    /* list is being processed */
 124 #define SOFT_STEAL              0x08    /* list is being stolen for draining */
 125
 126 typedef struct softcall {
 127         void (*sc_func)(void *);        /* function to call */
 128         void *sc_arg;                   /* arg to pass to func */
 129         struct softcall *sc_next;       /* next in list */
 130 } softcall_t;
 131
 132 /*
 133  * softcall list and state variables.
 134  */
 135 static softcall_t *softcalls;
 136 static softcall_t *softhead, *softtail, *softfree;
 137 static uint_t   softcall_state;
 138 static clock_t softcall_tick;
 139 static clock_t softcall_countstart, softcall_lastpoke;
 140 static uint_t softcall_pokecount;
 141
 142 /*
 143  * Max number of pokes per second before increasing softcall_delay
 144  */
 145 uint_t softcall_pokemax = 10;
 146
 147 /*
 148  * This ensures that softcall entries don't get stuck for long. It's expressed
 149  * in 10 milliseconds as 1 unit. When hires_tick is set or other clock frequency
 150  * is used, softcall_init() ensures that it's still expressed as 1 =  10 milli
 151  * seconds.
 152  */
 153 unsigned int softcall_delay = 1;
 154
 155 /*
 156  * The last CPU which will drain softcall queue.
 157  */
 158 static int softcall_latest_cpuid = -1;
 159
 160 /*
 161  * CPUSET to hold the CPU which is processing softcall queue
 162  * currently. There can be more than one CPU having bit set
 163  * but it will happen only when they are stuck.
 164  */
 165 static cpuset_t *softcall_cpuset = NULL;
 166
 167 /*
 168  * protects softcall lists and control variable softcall_state.
 169  */
 170 static kmutex_t softcall_lock;
 171
 172 static void (*kdi_softcall_func)(void);
 173 extern void siron_poke_cpu(cpuset_t);
 174
 175 extern void siron(void);
 176 extern void kdi_siron(void);
 177
 178
 179 void
 180 softcall_init(void)
 181 {
 182         softcall_t *sc;
 183
 184         softcalls = kmem_zalloc(sizeof (softcall_t) * NSOFTCALLS, KM_SLEEP);
 185         softcall_cpuset = kmem_zalloc(sizeof (cpuset_t), KM_SLEEP);
 186         for (sc = softcalls; sc < &softcalls[NSOFTCALLS]; sc++) {
 187                 sc->sc_next = softfree;
 188                 softfree = sc;
 189         }
 190         mutex_init(&softcall_lock, NULL, MUTEX_SPIN,
 191             (void *)ipltospl(SPL8));
 192         softcall_state = SOFT_IDLE;
 193         softcall_tick = ddi_get_lbolt();
 194
 195         /*
 196          * Since softcall_delay is expressed as 1 = 10 milliseconds.
 197          */
 198         softcall_delay = softcall_delay * (hz/100);
 199         CPUSET_ZERO(*softcall_cpuset);
 200 }
 201
 202 /*
 203  * Gets called when softcall queue is not moving forward. We choose
 204  * a CPU and poke except the ones which are already poked.
 205  */
 206 static int
 207 softcall_choose_cpu()
 208 {
 209         cpu_t *cplist = CPU;
 210         cpu_t *cp;
 211         int intr_load = INT_MAX;
 212         int cpuid = -1;
 213         cpuset_t poke;
 214         int s;
 215
 216         ASSERT(getpil() >= DISP_LEVEL);
 217         ASSERT(ncpus > 1);
 218         ASSERT(MUTEX_HELD(&softcall_lock));
 219
 220         CPUSET_ZERO(poke);
 221
 222         /*
 223          * The hint is to start from current CPU.
 224          */
 225         cp = cplist;
 226         do {
 227                 /*
 228                  * Don't select this CPU if :
 229                  *   - in cpuset already
 230                  *   - CPU is not accepting interrupts
 231                  *   - CPU is being offlined
 232                  */
 233                 if (CPU_IN_SET(*softcall_cpuset, cp->cpu_id) ||
 234                     (cp->cpu_flags & CPU_ENABLE) == 0 ||
 235                     (cp == cpu_inmotion))
 236                         continue;
 237 #if defined(__x86)
 238                 /*
 239                  * Don't select this CPU if a hypervisor indicates it
 240                  * isn't currently scheduled onto a physical cpu.  We are
 241                  * looking for a cpu that can respond quickly and the time
 242                  * to get the virtual cpu scheduled and switched to running
 243                  * state is likely to be relatively lengthy.
 244                  */
 245                 if (vcpu_on_pcpu(cp->cpu_id) == VCPU_NOT_ON_PCPU)
 246                         continue;
 247 #endif  /* __x86 */
 248
 249                 /* if CPU is not busy */
 250                 if (cp->cpu_intrload == 0) {
 251                         cpuid = cp->cpu_id;
 252                         break;
 253                 }
 254
 255                 if (cp->cpu_intrload < intr_load) {
 256                         cpuid = cp->cpu_id;
 257                         intr_load = cp->cpu_intrload;
 258                 } else if (cp->cpu_intrload == intr_load) {
 259                         /*
 260                          * We want to poke CPUs having similar
 261                          * load because we don't know which CPU is
 262                          * can acknowledge level1 interrupt. The
 263                          * list of such CPUs should not be large.
 264                          */
 265                         if (cpuid != -1) {
 266                                 /*
 267                                  * Put the last CPU chosen because
 268                                  * it also has same interrupt load.
 269                                  */
 270                                 CPUSET_ADD(poke, cpuid);
 271                                 cpuid = -1;
 272                         }
 273
 274                         CPUSET_ADD(poke, cp->cpu_id);
 275                 }
 276         } while ((cp = cp->cpu_next_onln) != cplist);
 277
 278         /* if we found a CPU which suits best to poke */
 279         if (cpuid != -1) {
 280                 CPUSET_ZERO(poke);
 281                 CPUSET_ADD(poke, cpuid);
 282         }
 283
 284         if (CPUSET_ISNULL(poke)) {
 285                 mutex_exit(&softcall_lock);
 286                 return (0);
 287         }
 288
 289         /*
 290          * We first set the bit in cpuset and then poke.
 291          */
 292         CPUSET_XOR(*softcall_cpuset, poke);
 293         mutex_exit(&softcall_lock);
 294
 295         /*
 296          * If softcall() was called at low pil then we may
 297          * get preempted before we raise PIL. It should be okay
 298          * because we are just going to poke CPUs now or at most
 299          * another thread may start choosing CPUs in this routine.
 300          */
 301         s = splhigh();
 302         siron_poke_cpu(poke);
 303         splx(s);
 304         return (1);
 305 }
 306
 307
 308 /*
 309  * Call function func with argument arg
 310  * at some later time at software interrupt priority
 311  */
 312 void
 313 softcall(void (*func)(void *), void *arg)
 314 {
 315         softcall_t *sc;
 316         clock_t w, now;
 317
 318         /*
 319          * protect against cross-calls
 320          */
 321         mutex_enter(&softcall_lock);
 322         /* coalesce identical softcalls */
 323         for (sc = softhead; sc != 0; sc = sc->sc_next) {
 324                 if (sc->sc_func == func && sc->sc_arg == arg) {
 325                         goto intr;
 326                 }
 327         }
 328
 329         if ((sc = softfree) == 0)
 330                 panic("too many softcalls");
 331
 332         softfree = sc->sc_next;
 333         sc->sc_func = func;
 334         sc->sc_arg = arg;
 335         sc->sc_next = 0;
 336
 337         if (softhead) {
 338                 softtail->sc_next = sc;
 339                 softtail = sc;
 340         } else
 341                 softhead = softtail = sc;
 342
 343 intr:
 344         if (softcall_state & SOFT_IDLE) {
 345                 softcall_state = SOFT_PEND;
 346                 softcall_tick = ddi_get_lbolt();
 347                 mutex_exit(&softcall_lock);
 348                 siron();
 349         } else if (softcall_state & (SOFT_DRAIN|SOFT_PEND)) {
 350                 now = ddi_get_lbolt();
 351                 w = now - softcall_tick;
 352                 if (w <= softcall_delay || ncpus == 1) {
 353                         mutex_exit(&softcall_lock);
 354                         return;
 355                 }
 356                 /*
 357                  * Did we poke less than a second ago?
 358                  */
 359                 if (now - softcall_lastpoke < hz) {
 360                         /*
 361                          * We did, increment the poke count and
 362                          * see if we are poking too often
 363                          */
 364                         if (softcall_pokecount++ == 0)
 365                                 softcall_countstart = now;
 366                         if (softcall_pokecount > softcall_pokemax) {
 367                                 /*
 368                                  * If poking too much increase the delay
 369                                  */
 370                                 if (now - softcall_countstart <= hz)
 371                                         softcall_delay++;
 372                                 softcall_pokecount = 0;
 373                         }
 374                 } else {
 375                         /*
 376                          * poke rate has dropped off, reset the poke monitor
 377                          */
 378                         softcall_pokecount = 0;
 379                 }
 380                 softcall_lastpoke = now;
 381                 if (!(softcall_state & SOFT_STEAL)) {
 382                         softcall_state |= SOFT_STEAL;
 383
 384                         /*
 385                          * We want to give some more chance before
 386                          * fishing around again.
 387                          */
 388                         softcall_tick = now;
 389                 }
 390
 391                 /* softcall_lock will be released by this routine */
 392                 (void) softcall_choose_cpu();
 393         }
 394 }
 395
 396 void
 397 kdi_softcall(void (*func)(void))
 398 {
 399         kdi_softcall_func = func;
 400
 401         if (softhead == NULL)
 402                 kdi_siron();
 403 }
 404
 405 /*
 406  * Called to process software interrupts take one off queue, call it,
 407  * repeat.
 408  *
 409  * Note queue may change during call; softcall_lock, state variables
 410  * softcall_state and softcall_latest_cpuid ensures that -
 411  * - we don't have multiple cpus pulling from the list (thus causing
 412  *   a violation of FIFO order with an exception when we are stuck).
 413  * - we don't miss a new entry having been added to the head.
 414  * - we don't miss a wakeup.
 415  */
 416
 417 void
 418 softint(void)
 419 {
 420         softcall_t *sc = NULL;
 421         void (*func)();
 422         caddr_t arg;
 423         int cpu_id = CPU->cpu_id;
 424
 425         /*
 426          * Don't process softcall queue if current CPU is quiesced or
 427          * offlined. This can happen when a CPU is running pause
 428          * thread but softcall already sent a xcall.
 429          */
 430         if (CPU->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) {
 431                 if (softcall_cpuset != NULL &&
 432                     CPU_IN_SET(*softcall_cpuset, cpu_id)) {
 433                         CPUSET_DEL(*softcall_cpuset, cpu_id);
 434                         goto out;
 435                 }
 436         }
 437
 438         mutex_enter(&softcall_lock);
 439
 440         if (softcall_state & (SOFT_STEAL|SOFT_PEND)) {
 441                 softcall_state = SOFT_DRAIN;
 442         } else  {
 443                 /*
 444                  * The check for softcall_cpuset being
 445                  * NULL is required because it may get
 446                  * called very early during boot.
 447                  */
 448                 if (softcall_cpuset != NULL &&
 449                     CPU_IN_SET(*softcall_cpuset, cpu_id))
 450                         CPUSET_DEL(*softcall_cpuset, cpu_id);
 451                 mutex_exit(&softcall_lock);
 452                 goto out;
 453         }
 454
 455         /*
 456          * Setting softcall_latest_cpuid to current CPU ensures
 457          * that there is only one active softlevel1 handler to
 458          * process softcall queues.
 459          *
 460          * Since softcall_lock lock is dropped before calling
 461          * func (callback), we need softcall_latest_cpuid
 462          * to prevent two softlevel1 hanlders working on the
 463          * queue when the first softlevel1 handler gets
 464          * stuck due to high interrupt load.
 465          */
 466         softcall_latest_cpuid = cpu_id;
 467
 468         /* add ourself to the cpuset */
 469         if (!CPU_IN_SET(*softcall_cpuset, cpu_id))
 470                 CPUSET_ADD(*softcall_cpuset, cpu_id);
 471
 472         for (;;) {
 473                 softcall_tick = ddi_get_lbolt();
 474                 if ((sc = softhead) != NULL) {
 475                         func = sc->sc_func;
 476                         arg = sc->sc_arg;
 477                         softhead = sc->sc_next;
 478                         sc->sc_next = softfree;
 479                         softfree = sc;
 480                 }
 481
 482                 if (sc == NULL) {
 483                         if (CPU_IN_SET(*softcall_cpuset, cpu_id))
 484                                 CPUSET_DEL(*softcall_cpuset, cpu_id);
 485
 486                         softcall_state = SOFT_IDLE;
 487                         ASSERT(softcall_latest_cpuid == cpu_id);
 488                         softcall_latest_cpuid = -1;
 489
 490                         mutex_exit(&softcall_lock);
 491                         break;
 492                 }
 493
 494                 mutex_exit(&softcall_lock);
 495                 func(arg);
 496                 mutex_enter(&softcall_lock);
 497
 498                 /*
 499                  * No longer need softcall processing from current
 500                  * interrupt handler because either
 501                  *  (a) softcall is in SOFT_IDLE state or
 502                  *  (b) There is a CPU already draining softcall
 503                  *      queue and the current softlevel1 is no
 504                  *      longer required.
 505                  */
 506                 if (softcall_latest_cpuid != cpu_id) {
 507                         if (CPU_IN_SET(*softcall_cpuset, cpu_id))
 508                                 CPUSET_DEL(*softcall_cpuset, cpu_id);
 509
 510                         mutex_exit(&softcall_lock);
 511                         break;
 512                 }
 513         }
 514
 515 out:
 516         if ((func = kdi_softcall_func) != NULL) {
 517                 kdi_softcall_func = NULL;
 518                 func();
 519         }
 520 }