4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2018 Joyent, Inc. All rights reserverd.
28 * To understand the present state of interrupt handling on i86pc, we must
29 * first consider the history of interrupt controllers and our way of handling
32 * History of Interrupt Controllers on i86pc
33 * -----------------------------------------
35 * Intel 8259 and 8259A
37 * The first interrupt controller that attained widespread use on i86pc was
38 * the Intel 8259(A) Programmable Interrupt Controller that first saw use with
39 * the 8086. It took up to 8 interrupt sources and combined them into one
40 * output wire. Up to 8 8259s could be slaved together providing up to 64 IRQs.
41 * With the switch to the 8259A, level mode interrupts became possible. For a
42 * long time on i86pc the 8259A was the only way to handle interrupts and it
43 * had its own set of quirks. The 8259A and its corresponding interval timer
44 * the 8254 are programmed using outb and inb instructions.
46 * Intel Advanced Programmable Interrupt Controller (APIC)
48 * Starting around the time of the introduction of the P6 family
49 * microarchitecture (i686) Intel introduced a new interrupt controller.
50 * Instead of having the series of slaved 8259A devices, Intel opted to outfit
51 * each processor with a Local APIC (lapic) and to outfit the system with at
52 * least one, but potentially more, I/O APICs (ioapic). The lapics and ioapics
53 * initially communicated over a dedicated bus, but this has since been
54 * replaced. Each physical core and even hyperthread currently contains its
55 * own local apic, which is not shared. There are a few exceptions for
56 * hyperthreads, but that does not usually concern us.
58 * Instead of talking directly to 8259 for status, sending End Of Interrupt
59 * (EOI), etc. a microprocessor now communicates directly to the lapic. This
60 * also allows for each microprocessor to be able to have independent controls.
61 * The programming method is different from the 8259. Consumers map the lapic
62 * registers into uncacheable memory to read and manipulate the state.
64 * The number of addressable interrupt vectors was increased to 256. However
65 * vectors 0-31 are reserved for the processor exception handling, leaving the
66 * remaining vectors for general use. In addition to hardware generated
67 * interrupts, the lapic provides a way for generating inter-processor
68 * interrupts (IPI) which are the basis for CPU cross calls and CPU pokes.
70 * AMD ended up implementing the Intel APIC architecture in lieu of their work
75 * The x2apic is an extension to the lapic which started showing up around the
76 * same time as the Sandy Bridge chipsets. It provides a new programming mode
77 * as well as new features. The goal of the x2apic is to solve a few problems
78 * with the previous generation of lapic and the x2apic is backwards compatible
79 * with the previous programming and model. The only downsides to using the
80 * backwards compatibility is that you are not able to take advantage of the new
83 * o The APIC ID is increased from an 8-bit value to a 32-bit value. This
84 * increases the maximum number of addressable physical processors beyond
85 * 256. This new ID is assembled in a similar manner as the information that
86 * is obtainable by the extended cpuid topology leaves.
88 * o A new means of generating IPIs was introduced.
90 * o Instead of memory mapping the registers, the x2apic only allows for
91 * programming it through a series of wrmsrs. This has important semantic
92 * side effects. Recall that the registers were previously all mapped to
93 * uncachable memory which meant that all operations to the local apic were
94 * serializing instructions. With the switch to using wrmsrs this has been
95 * relaxed and these operations can no longer be assumed to be serializing
98 * Note for the rest of this we are only going to concern ourselves with the
99 * apic and x2apic which practically all of i86pc has been using now for
102 * Interrupt Priority Levels
103 * -------------------------
105 * On i86pc systems there are a total of fifteen interrupt priority levels
106 * (ipls) which range from 1-15. Level 0 is for normal processing and
107 * non-interrupt processing. To manipulate these values the family of spl
108 * functions (which date back to UNIX on the PDP-11) are used. Specifically,
109 * splr() to raise the priority level and splx() to lower it. One should not
110 * generally call setspl() directly.
112 * Both i86pc and the supported SPARC platforms honor the same conventions for
113 * the meaning behind these IPLs. The most important IPL is the platform's
114 * LOCK_LEVEL (0xa on i86pc). If a thread is above LOCK_LEVEL it _must_ not
115 * sleep on any synchronization object. The only allowed synchronization
116 * primitive is a mutex that has been specifically initialized to be a spin
117 * lock (see mutex_init(9F)). Another important level is DISP_LEVEL (0xb on
118 * i86pc). You must be at DISP_LEVEL if you want to control the dispatcher.
119 * The XC_HI_PIL is the highest level (0xf) and is used during cross-calls.
121 * Each interrupt that is registered in the system fires at a specific IPL.
122 * Generally most interrupts fire below LOCK_LEVEL.
127 * We currently have three sets of PSM (platform specific module) drivers
128 * available. uppc, pcplusmp, and apix. uppc (uni-processor PC) is the original
129 * driver that interacts with the 8259A and 8254. In general, it is not used
130 * anymore given the prevalence of the apic.
132 * The system prefers to use the apix driver over the pcplusmp driver. The apix
133 * driver requires HW support for an x2apic. If there is no x2apic HW, apix
134 * will not be used. In general we prefer using the apix driver over the
135 * pcplusmp driver because it gives us much more flexibility with respect to
136 * interrupts. In the apix driver each local apic has its own independent set
137 * of interrupts, whereas the pcplusmp driver only has a single global set of
138 * interrupts. This is why pcplusmp only supports a finite number of interrupts
139 * per IPL -- generally 16, often less. The apix driver supports using either
140 * the x2apic or the local apic programing modes. The programming mode does not
141 * change the number of interrupts available, just the number of processors
142 * that we can address. For the apix driver, the x2apic mode is enabled if the
143 * system supports interrupt re-mapping, otherwise the module manages the
144 * x2apic in local mode.
146 * When there is no x2apic present, we default back to the pcplusmp PSM driver.
147 * In general, this is not problematic unless you have more than 256
148 * processors in the machine or you do not have enough interrupts available.
150 * Controlling Interrupt Generation on i86pc
151 * -----------------------------------------
153 * There are two different ways to manipulate which interrupts will be
154 * generated on i86pc. Each offers different degrees of control.
156 * The first is through the flags register (eflags and rflags on i386 and amd64
157 * respectively). The IF bit determines whether or not interrupts are enabled
158 * or disabled. This is manipulated in one of several ways. The most common way
159 * is through the cli and sti instructions. These clear the IF flag and set it,
160 * respectively, for the current processor. The other common way is through the
161 * use of the intr_clear and intr_restore functions.
163 * Assuming interrupts are not blocked by the IF flag, then the second form is
164 * through the Processor-Priority Register (PPR). The PPR is used to determine
165 * whether or not a pending interrupt should be delivered. If the ipl of the
166 * new interrupt is higher than the current value in the PPR, then the lapic
167 * will either deliver it immediately (if interrupts are not in progress) or it
168 * will deliver it once the current interrupt processing has issued an EOI. The
169 * highest unmasked interrupt will be the one delivered.
171 * The PPR register is based upon the max of the following two registers in the
172 * lapic, the TPR register (also known as CR8 on amd64) that can be used to
173 * mask interrupt levels, and the current vector. Because the pcplusmp module
174 * always sets TPR appropriately early in the do_interrupt path, we can usually
175 * just think that the PPR is the TPR. The pcplusmp module also issues an EOI
176 * once it has set the TPR, so higher priority interrupts can come in while
177 * we're servicing a lower priority interrupt.
179 * Handling Interrupts
180 * -------------------
182 * Interrupts can be broken down into three categories based on priority and
185 * o High level interrupts
186 * o Low level hardware interrupts
187 * o Low level software interrupts
189 * High Level Interrupts
191 * High level interrupts encompasses both hardware-sourced and software-sourced
192 * interrupts. Examples of high level hardware interrupts include the serial
193 * console. High level software-sourced interrupts are still delivered through
194 * the local apic through IPIs. This is primarily cross calls.
196 * When a high level interrupt comes in, we will raise the SPL and then pin the
197 * current lwp to the processor. We will use its lwp, but our own interrupt
198 * stack and process the high level interrupt in-situ. These handlers are
199 * designed to be very short in nature and cannot go to sleep, only block on a
200 * spin lock. If the interrupt has a lot of work to do, it must generate a
201 * low-priority software interrupt that will be processed later.
203 * Low level hardware interrupts
205 * Low level hardware interrupts start off like their high-level cousins. The
206 * current CPU contains a number of kernel threads (kthread_t) that can be used
207 * to process low level interrupts. These are shared between both low level
208 * hardware and software interrupts. Note that while we run with our
209 * kthread_t, we borrow the pinned threads lwp_t until such a time as we hit a
210 * synchronization object. If we hit one and need to sleep, then the scheduler
211 * will instead create the rest of what we need.
213 * Low level software interrupts
215 * Low level software interrupts are handled in a similar way as hardware
216 * interrupts, but the notification vector is different. Each CPU has a bitmask
217 * of pending software interrupts. We can notify a CPU to process software
218 * interrupts through a specific trap vector as well as through several
219 * checks that are performed throughout the code. These checks will look at
220 * processing software interrupts as we lower our spl.
222 * We attempt to process the highest pending software interrupt that we can
223 * which is greater than our current IPL. If none currently exist, then we move
224 * on. We process a software interrupt in a similar fashion to a hardware
227 * Traditional Interrupt Flow
228 * --------------------------
230 * The following diagram tracks the flow of the traditional uppc and pcplusmp
231 * interrupt handlers. The apix driver has its own version of do_interrupt().
232 * We come into the interrupt handler with all interrupts masked by the IF
233 * flag. This is because we set up the handler using an interrupt-gate, which
234 * is defined architecturally to have cleared the IF flag for us.
236 * +--------------+ +----------------+ +-----------+
237 * | _interrupt() |--->| do_interrupt() |--->| *setlvl() |
238 * +--------------+ +----------------+ +-----------+
241 * low-level| | | softint
242 * HW int | | +---------------------------------------+
243 * +--------------+ | | |
244 * | intr_thread_ |<-----+ | hi-level int |
245 * | prolog() | | +----------+ |
246 * +--------------+ +--->| hilevel_ | Not on intr stack |
247 * | | intr_ |-----------------+ |
249 * +------------+ +----------+ | |
250 * | switch_sp_ | | On intr v |
251 * | and_call() | | Stack +------------+ |
252 * +------------+ | | switch_sp_ | |
253 * | v | and_call() | |
254 * v +-----------+ +------------+ |
255 * +-----------+ | dispatch_ | | |
256 * | dispatch_ | +-------------------| hilevel() |<------------+ |
257 * | hardint() | | +-----------+ |
260 * | +-----+ +----------------------+ +-----+ hi-level |
261 * +---->| sti |->| av_dispatch_autovect |->| cli |---------+ |
262 * +-----+ +----------------------+ +-----+ | |
269 * +--------------+ +----------+ | +----------------+ |
270 * | intr_thread_ | low-level | | hilevel_intr_ | |
271 * | epilog() |<-------------------------------+ | epilog() | |
272 * +--------------+ +----------------+ |
274 * | +----------------------v v---------------------+ |
276 * | +---------------------->| *setlvlx() | |
277 * | | +------------+ |
280 * | | +--------+ +------------------+ +-------------+ |
281 * | | | return |<----| softint pending? |----->| dosoftint() |<-----+
282 * | | +--------+ no +------------------+ yes +-------------+
284 * | | | softint pil too low | |
285 * | | +--------------------------------------+ |
287 * | | +-----------+ +------------+ +-----------+
288 * | | | dispatch_ |<-----| switch_sp_ |<---------| *setspl() |
289 * | | | softint() | | and_call() | +-----------+
290 * | | +-----------+ +------------+
293 * | | +-----+ +----------------------+ +-----+ +------------+
294 * | | | sti |->| av_dispatch_autovect |->| cli |->| dosoftint_ |
295 * | | +-----+ +----------------------+ +-----+ | epilog() |
298 * | +----------------------------------------------------+ |
302 * | thread |<---------------------------------------------------+
307 * +----------------+ +------------+ +-----------+ +-------+ +---------+
308 * | set_base_spl() |->| *setlvlx() |->| splhigh() |->| sti() |->| swtch() |
309 * +----------------+ +------------+ +-----------+ +-------+ +---------+
311 * Calls made on Interrupt Stacks and Epilogue routines
313 * We use the switch_sp_and_call() assembly routine to switch our sp to the
314 * interrupt stacks and then call the appropriate dispatch function. In the
315 * case of interrupts which may block, softints and hardints, we always ensure
316 * that we are still on the interrupt thread when we call the epilog routine.
317 * This is not just important, it's necessary. If the interrupt thread blocked,
318 * we won't return from our switch_sp_and_call() function and instead we'll go
319 * through and set ourselves up to swtch() directly.
324 * The apix module has its own interrupt path. This is done for various
325 * reasons. The first is that rather than having global interrupt vectors, we
326 * now have per-cpu vectors.
328 * The other substantial change is that the apix design does not use the TPR to
329 * mask interrupts below the current level. In fact, except for one special
330 * case, it does not use the TPR at all. Instead, it only uses the IF flag
331 * (cli/sti) to either block all interrupts or allow any interrupts to come in.
332 * The design is such that when interrupts are allowed to come in, if we are
333 * currently servicing a higher priority interupt, the new interrupt is treated
334 * as pending and serviced later. Specifically, in the pcplusmp module's
335 * apic_intr_enter() the code masks interrupts at or below the current
336 * IPL using the TPR before sending EOI, whereas the apix module's
337 * apix_intr_enter() simply sends EOI.
339 * The one special case where the apix code uses the TPR is when it calls
340 * through the apic_reg_ops function pointer apic_write_task_reg in
341 * apix_init_intr() to initially mask all levels and then finally to enable all
344 * Recall that we come into the interrupt handler with all interrupts masked
345 * by the IF flag. This is because we set up the handler using an
346 * interrupt-gate which is defined architecturally to have cleared the IF flag
349 * +--------------+ +---------------------+
350 * | _interrupt() |--->| apix_do_interrupt() |
351 * +--------------+ +---------------------+
353 * hard int? +----+--------+ softint?
354 * | | (but no low-level looping)
357 * +---------+ +-----------+ +----------------------------------+
358 * |apix_add_| check IPL | |
359 * |pending_ |<-------------+------+----------------------+ |
360 * |hardint()| low-level int| hi-level int| |
362 * | check IPL +-----------------+ +---------------+ |
363 * +--+-----+ | apix_intr_ | | apix_hilevel_ | |
364 * | | | thread_prolog() | | intr_prolog() | |
365 * | return +-----------------+ +---------------+ |
367 * | +------------+ | stack? +------------+ |
368 * | | switch_sp_ | +---------| switch_sp_ | |
369 * | | and_call() | | | and_call() | |
370 * | +------------+ | +------------+ |
372 * | +----------------+ +----------------+ |
373 * | | apix_dispatch_ | | apix_dispatch_ | |
374 * | | lowlevel() | | hilevel() | |
375 * | +----------------+ +----------------+ |
378 * | +-------------------------+ |
379 * | |apix_dispatch_by_vector()|----+ |
380 * | +-------------------------+ | |
381 * | !XC_HI_PIL| | | | |
382 * | +---+ +-------+ +---+ | |
383 * | |sti| |*intr()| |cli| | |
384 * | +---+ +-------+ +---+ | hi-level? |
385 * | +---------------------------+----+ |
387 * | +----------------+ +----------------+ |
388 * | | apix_intr_ | | apix_hilevel_ | |
389 * | | thread_epilog()| | intr_epilog() | |
390 * | +----------------+ +----------------+ |
392 * | v-----------------+--------------------------------+ |
394 * | | *setlvlx() | +----------------------------------------------------+
396 * | | | +--------------------------------+ low
397 * v v v------+ v | level
398 * +------------------+ +------------------+ +-----------+ | pending?
399 * | apix_do_pending_ |----->| apix_do_pending_ |----->| apix_do_ |--+
400 * | hilevel() | | hardint() | | softint() | |
401 * +------------------+ +------------------+ +-----------+ return
403 * | while pending | while pending | while pending
404 * | hi-level | low-level | softint
406 * +---------------+ +-----------------+ +-----------------+
407 * | apix_hilevel_ | | apix_intr_ | | apix_do_ |
408 * | intr_prolog() | | thread_prolog() | | softint_prolog()|
409 * +---------------+ +-----------------+ +-----------------+
411 * | stack? +------------+ +------------+ +------------+
412 * +--------| switch_sp_ | | switch_sp_ | | switch_sp_ |
413 * | | and_call() | | and_call() | | and_call() |
414 * | +------------+ +------------+ +------------+
416 * +------------------+ +------------------+ +------------------------+
417 * | apix_dispatch_ | | apix_dispatch_ | | apix_dispatch_softint()|
418 * | pending_hilevel()| | pending_hardint()| +------------------------+
419 * +------------------+ +------------------+ | | | |
421 * | +----------------+ | +----------------+ | | | |
422 * | | apix_hilevel_ | | | apix_intr_ | | | | |
423 * | | intr_epilog() | | | thread_epilog()| | | | |
424 * | +----------------+ | +----------------+ | | | |
426 * | +------------+ | +----------+ +------+ | | |
427 * | | *setlvlx() | | |*setlvlx()| | | | |
428 * | +------------+ | +----------+ | +----------+ | +---------+
429 * | | +---+ |av_ | +---+ |apix_do_ |
430 * +---------------------------------+ |sti| |dispatch_ | |cli| |softint_ |
431 * | apix_dispatch_pending_autovect()| +---+ |softvect()| +---+ |epilog() |
432 * +---------------------------------+ +----------+ +---------+
433 * |!XC_HI_PIL | | | |
434 * +---+ +-------+ +---+ +----------+ +-------+
435 * |sti| |*intr()| |cli| |apix_post_| |*intr()|
436 * +---+ +-------+ +---+ |hardint() | +-------+
440 #include <sys/cpuvar.h>
441 #include <sys/cpu_event.h>
442 #include <sys/regset.h>
444 #include <sys/types.h>
445 #include <sys/thread.h>
446 #include <sys/systm.h>
447 #include <sys/segments.h>
449 #include <sys/trap.h>
450 #include <sys/ftrace.h>
451 #include <sys/traptrace.h>
452 #include <sys/clock.h>
453 #include <sys/panic.h>
454 #include <sys/disp.h>
455 #include <vm/seg_kp.h>
456 #include <sys/stack.h>
457 #include <sys/sysmacros.h>
458 #include <sys/cmn_err.h>
459 #include <sys/kstat.h>
460 #include <sys/smp_impldefs.h>
461 #include <sys/pool_pset.h>
462 #include <sys/zone.h>
463 #include <sys/bitmap.h>
464 #include <sys/archsystm.h>
465 #include <sys/machsystm.h>
466 #include <sys/ontrap.h>
467 #include <sys/x86_archext.h>
468 #include <sys/promif.h>
469 #include <vm/hat_i86.h>
471 #if defined(__amd64) && !defined(__xpv)
472 /* If this fails, then the padding numbers in machcpuvar.h are wrong. */
473 CTASSERT((offsetof(cpu_t
, cpu_m
) + offsetof(struct machcpu
, mcpu_pad
)) <
475 CTASSERT((offsetof(cpu_t
, cpu_m
) + offsetof(struct machcpu
, mcpu_kpti
)) >=
477 CTASSERT((offsetof(cpu_t
, cpu_m
) + offsetof(struct machcpu
, mcpu_kpti_dbg
)) <
479 CTASSERT((offsetof(cpu_t
, cpu_m
) + offsetof(struct machcpu
, mcpu_pad2
)) <
481 CTASSERT(((sizeof (struct kpti_frame
)) & 0xF) == 0);
482 CTASSERT(((offsetof(cpu_t
, cpu_m
) +
483 offsetof(struct machcpu
, mcpu_kpti_dbg
)) & 0xF) == 0);
484 CTASSERT((offsetof(struct kpti_frame
, kf_tr_rsp
) & 0xF) == 0);
487 #if defined(__xpv) && defined(DEBUG)
490 * This panic message is intended as an aid to interrupt debugging.
492 * The associated assertion tests the condition of enabling
493 * events when events are already enabled. The implication
494 * being that whatever code the programmer thought was
495 * protected by having events disabled until the second
496 * enable happened really wasn't protected at all ..
499 int stistipanic
= 1; /* controls the debug panic check */
500 const char *stistimsg
= "stisti";
501 ulong_t laststi
[NCPU
];
504 * This variable tracks the last place events were disabled on each cpu
505 * it assists in debugging when asserts that interrupts are enabled trip.
507 ulong_t lastcli
[NCPU
];
511 void do_interrupt(struct regs
*rp
, trap_trace_rec_t
*ttp
);
513 void (*do_interrupt_common
)(struct regs
*, trap_trace_rec_t
*) = do_interrupt
;
514 uintptr_t (*get_intr_handler
)(int, short) = NULL
;
517 * Set cpu's base SPL level to the highest active interrupt level
522 struct cpu
*cpu
= CPU
;
523 uint16_t active
= (uint16_t)cpu
->cpu_intr_actv
;
525 cpu
->cpu_base_spl
= active
== 0 ? 0 : bsrw_insn(active
);
529 * Do all the work necessary to set up the cpu and thread structures
530 * to dispatch a high-level interrupt.
532 * Returns 0 if we're -not- already on the high-level interrupt stack,
533 * (and *must* switch to it), non-zero if we are already on that stack.
535 * Called with interrupts masked.
536 * The 'pil' is already set to the appropriate level for rp->r_trapno.
539 hilevel_intr_prolog(struct cpu
*cpu
, uint_t pil
, uint_t oldpil
, struct regs
*rp
)
541 struct machcpu
*mcpu
= &cpu
->cpu_m
;
544 hrtime_t now
= tsc_read();
546 ASSERT(pil
> LOCK_LEVEL
);
548 if (pil
== CBE_HIGH_PIL
) {
549 cpu
->cpu_profile_pil
= oldpil
;
550 if (USERMODE(rp
->r_cs
)) {
551 cpu
->cpu_profile_pc
= 0;
552 cpu
->cpu_profile_upc
= rp
->r_pc
;
553 cpu
->cpu_cpcprofile_pc
= 0;
554 cpu
->cpu_cpcprofile_upc
= rp
->r_pc
;
556 cpu
->cpu_profile_pc
= rp
->r_pc
;
557 cpu
->cpu_profile_upc
= 0;
558 cpu
->cpu_cpcprofile_pc
= rp
->r_pc
;
559 cpu
->cpu_cpcprofile_upc
= 0;
563 mask
= cpu
->cpu_intr_actv
& CPU_INTR_ACTV_HIGH_LEVEL_MASK
;
568 * We have interrupted another high-level interrupt.
569 * Load starting timestamp, compute interval, update
570 * cumulative counter.
572 nestpil
= bsrw_insn((uint16_t)mask
);
573 ASSERT(nestpil
< pil
);
575 mcpu
->pil_high_start
[nestpil
- (LOCK_LEVEL
+ 1)];
576 mcpu
->intrstat
[nestpil
][0] += intrtime
;
577 cpu
->cpu_intracct
[cpu
->cpu_mstate
] += intrtime
;
579 * Another high-level interrupt is active below this one, so
580 * there is no need to check for an interrupt thread. That
581 * will be done by the lowest priority high-level interrupt
585 kthread_t
*t
= cpu
->cpu_thread
;
588 * See if we are interrupting a low-level interrupt thread.
589 * If so, account for its time slice only if its time stamp
592 if ((t
->t_flag
& T_INTR_THREAD
) != 0 && t
->t_intr_start
!= 0) {
593 intrtime
= now
- t
->t_intr_start
;
594 mcpu
->intrstat
[t
->t_pil
][0] += intrtime
;
595 cpu
->cpu_intracct
[cpu
->cpu_mstate
] += intrtime
;
601 * Store starting timestamp in CPU structure for this PIL.
603 mcpu
->pil_high_start
[pil
- (LOCK_LEVEL
+ 1)] = now
;
605 ASSERT((cpu
->cpu_intr_actv
& (1 << pil
)) == 0);
609 * To support reentrant level 15 interrupts, we maintain a
610 * recursion count in the top half of cpu_intr_actv. Only
611 * when this count hits zero do we clear the PIL 15 bit from
612 * the lower half of cpu_intr_actv.
614 uint16_t *refcntp
= (uint16_t *)&cpu
->cpu_intr_actv
+ 1;
618 mask
= cpu
->cpu_intr_actv
;
620 cpu
->cpu_intr_actv
|= (1 << pil
);
622 return (mask
& CPU_INTR_ACTV_HIGH_LEVEL_MASK
);
626 * Does most of the work of returning from a high level interrupt.
628 * Returns 0 if there are no more high level interrupts (in which
629 * case we must switch back to the interrupted thread stack) or
630 * non-zero if there are more (in which case we should stay on it).
632 * Called with interrupts masked
635 hilevel_intr_epilog(struct cpu
*cpu
, uint_t pil
, uint_t oldpil
, uint_t vecnum
)
637 struct machcpu
*mcpu
= &cpu
->cpu_m
;
640 hrtime_t now
= tsc_read();
642 ASSERT(mcpu
->mcpu_pri
== pil
);
644 cpu
->cpu_stats
.sys
.intr
[pil
- 1]++;
646 ASSERT(cpu
->cpu_intr_actv
& (1 << pil
));
650 * To support reentrant level 15 interrupts, we maintain a
651 * recursion count in the top half of cpu_intr_actv. Only
652 * when this count hits zero do we clear the PIL 15 bit from
653 * the lower half of cpu_intr_actv.
655 uint16_t *refcntp
= (uint16_t *)&cpu
->cpu_intr_actv
+ 1;
657 ASSERT(*refcntp
> 0);
659 if (--(*refcntp
) == 0)
660 cpu
->cpu_intr_actv
&= ~(1 << pil
);
662 cpu
->cpu_intr_actv
&= ~(1 << pil
);
665 ASSERT(mcpu
->pil_high_start
[pil
- (LOCK_LEVEL
+ 1)] != 0);
667 intrtime
= now
- mcpu
->pil_high_start
[pil
- (LOCK_LEVEL
+ 1)];
668 mcpu
->intrstat
[pil
][0] += intrtime
;
669 cpu
->cpu_intracct
[cpu
->cpu_mstate
] += intrtime
;
672 * Check for lower-pil nested high-level interrupt beneath
673 * current one. If so, place a starting timestamp in its
674 * pil_high_start entry.
676 mask
= cpu
->cpu_intr_actv
& CPU_INTR_ACTV_HIGH_LEVEL_MASK
;
681 * find PIL of nested interrupt
683 nestpil
= bsrw_insn((uint16_t)mask
);
684 ASSERT(nestpil
< pil
);
685 mcpu
->pil_high_start
[nestpil
- (LOCK_LEVEL
+ 1)] = now
;
687 * (Another high-level interrupt is active below this one,
688 * so there is no need to check for an interrupt
689 * thread. That will be done by the lowest priority
690 * high-level interrupt active.)
694 * Check to see if there is a low-level interrupt active.
695 * If so, place a starting timestamp in the thread
698 kthread_t
*t
= cpu
->cpu_thread
;
700 if (t
->t_flag
& T_INTR_THREAD
)
701 t
->t_intr_start
= now
;
704 mcpu
->mcpu_pri
= oldpil
;
705 (void) (*setlvlx
)(oldpil
, vecnum
);
707 return (cpu
->cpu_intr_actv
& CPU_INTR_ACTV_HIGH_LEVEL_MASK
);
711 * Set up the cpu, thread and interrupt thread structures for
712 * executing an interrupt thread. The new stack pointer of the
713 * interrupt thread (which *must* be switched to) is returned.
716 intr_thread_prolog(struct cpu
*cpu
, caddr_t stackptr
, uint_t pil
)
718 struct machcpu
*mcpu
= &cpu
->cpu_m
;
719 kthread_t
*t
, *volatile it
;
720 hrtime_t now
= tsc_read();
723 ASSERT((cpu
->cpu_intr_actv
& (1 << pil
)) == 0);
724 cpu
->cpu_intr_actv
|= (1 << pil
);
727 * Get set to run an interrupt thread.
728 * There should always be an interrupt thread, since we
729 * allocate one for each level on each CPU.
731 * t_intr_start could be zero due to cpu_intr_swtch_enter.
734 if ((t
->t_flag
& T_INTR_THREAD
) && t
->t_intr_start
!= 0) {
735 hrtime_t intrtime
= now
- t
->t_intr_start
;
736 mcpu
->intrstat
[t
->t_pil
][0] += intrtime
;
737 cpu
->cpu_intracct
[cpu
->cpu_mstate
] += intrtime
;
741 ASSERT(SA((uintptr_t)stackptr
) == (uintptr_t)stackptr
);
743 t
->t_sp
= (uintptr_t)stackptr
; /* mark stack in curthread for resume */
746 * unlink the interrupt thread off the cpu
748 * Note that the code in kcpc_overflow_intr -relies- on the
749 * ordering of events here - in particular that t->t_lwp of
750 * the interrupt thread is set to the pinned thread *before*
751 * curthread is changed.
753 it
= cpu
->cpu_intr_thread
;
754 cpu
->cpu_intr_thread
= it
->t_link
;
756 it
->t_lwp
= t
->t_lwp
;
759 * (threads on the interrupt thread free list could have state
760 * preset to TS_ONPROC, but it helps in debugging if
763 it
->t_state
= TS_ONPROC
;
765 cpu
->cpu_thread
= it
; /* new curthread on this cpu */
766 it
->t_pil
= (uchar_t
)pil
;
767 it
->t_pri
= intr_pri
+ (pri_t
)pil
;
768 it
->t_intr_start
= now
;
779 * Called with interrupts disabled
782 intr_thread_epilog(struct cpu
*cpu
, uint_t vec
, uint_t oldpil
)
784 struct machcpu
*mcpu
= &cpu
->cpu_m
;
786 kthread_t
*it
= cpu
->cpu_thread
; /* curthread */
789 hrtime_t now
= tsc_read();
792 cpu
->cpu_stats
.sys
.intr
[pil
- 1]++;
794 ASSERT(it
->t_intr_start
!= 0);
795 intrtime
= now
- it
->t_intr_start
;
796 mcpu
->intrstat
[pil
][0] += intrtime
;
797 cpu
->cpu_intracct
[cpu
->cpu_mstate
] += intrtime
;
799 ASSERT(cpu
->cpu_intr_actv
& (1 << pil
));
800 cpu
->cpu_intr_actv
&= ~(1 << pil
);
803 * If there is still an interrupted thread underneath this one
804 * then the interrupt was never blocked and the return is
805 * fairly simple. Otherwise it isn't.
807 if ((t
= it
->t_intr
) == NULL
) {
809 * The interrupted thread is no longer pinned underneath
810 * the interrupt thread. This means the interrupt must
811 * have blocked, and the interrupted thread has been
812 * unpinned, and has probably been running around the
813 * system for a while.
815 * Since there is no longer a thread under this one, put
816 * this interrupt thread back on the CPU's free list and
817 * resume the idle thread which will dispatch the next
823 cpu
->cpu_stats
.sys
.intrblk
++;
825 * Set CPU's base SPL based on active interrupts bitmask
828 basespl
= cpu
->cpu_base_spl
;
829 mcpu
->mcpu_pri
= basespl
;
830 (*setlvlx
)(basespl
, vec
);
833 it
->t_state
= TS_FREE
;
835 * Return interrupt thread to pool
837 it
->t_link
= cpu
->cpu_intr_thread
;
838 cpu
->cpu_intr_thread
= it
;
840 panic("intr_thread_epilog: swtch returned");
845 * Return interrupt thread to the pool
847 it
->t_link
= cpu
->cpu_intr_thread
;
848 cpu
->cpu_intr_thread
= it
;
849 it
->t_state
= TS_FREE
;
851 basespl
= cpu
->cpu_base_spl
;
852 pil
= MAX(oldpil
, basespl
);
853 mcpu
->mcpu_pri
= pil
;
854 (*setlvlx
)(pil
, vec
);
855 t
->t_intr_start
= now
;
860 * intr_get_time() is a resource for interrupt handlers to determine how
861 * much time has been spent handling the current interrupt. Such a function
862 * is needed because higher level interrupts can arrive during the
863 * processing of an interrupt. intr_get_time() only returns time spent in the
864 * current interrupt handler.
866 * The caller must be calling from an interrupt handler running at a pil
867 * below or at lock level. Timings are not provided for high-level
870 * The first time intr_get_time() is called while handling an interrupt,
871 * it returns the time since the interrupt handler was invoked. Subsequent
872 * calls will return the time since the prior call to intr_get_time(). Time
873 * is returned as ticks. Use scalehrtimef() to convert ticks to nsec.
875 * Theory Of Intrstat[][]:
877 * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
880 * intrstat[pil][0] is a cumulative count of the number of ticks spent
881 * handling all interrupts at the specified pil on this CPU. It is
882 * exported via kstats to the user.
884 * intrstat[pil][1] is always a count of ticks less than or equal to the
885 * value in [0]. The difference between [1] and [0] is the value returned
886 * by a call to intr_get_time(). At the start of interrupt processing,
887 * [0] and [1] will be equal (or nearly so). As the interrupt consumes
888 * time, [0] will increase, but [1] will remain the same. A call to
889 * intr_get_time() will return the difference, then update [1] to be the
890 * same as [0]. Future calls will return the time since the last call.
891 * Finally, when the interrupt completes, [1] is updated to the same as [0].
895 * intr_get_time() works much like a higher level interrupt arriving. It
896 * "checkpoints" the timing information by incrementing intrstat[pil][0]
897 * to include elapsed running time, and by setting t_intr_start to rdtsc.
898 * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
899 * and updates intrstat[pil][1] to be the same as the new value of
902 * In the normal handling of interrupts, after an interrupt handler returns
903 * and the code in intr_thread() updates intrstat[pil][0], it then sets
904 * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
905 * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
908 * Whenever interrupts arrive on a CPU which is handling a lower pil
909 * interrupt, they update the lower pil's [0] to show time spent in the
910 * handler that they've interrupted. This results in a growing discrepancy
911 * between [0] and [1], which is returned the next time intr_get_time() is
912 * called. Time spent in the higher-pil interrupt will not be returned in
913 * the next intr_get_time() call from the original interrupt, because
914 * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
920 struct machcpu
*mcpu
;
922 uint64_t time
, delta
, ret
;
930 ASSERT((cpu
->cpu_intr_actv
& CPU_INTR_ACTV_HIGH_LEVEL_MASK
) == 0);
931 ASSERT(t
->t_flag
& T_INTR_THREAD
);
933 ASSERT(t
->t_intr_start
!= 0);
936 delta
= time
- t
->t_intr_start
;
937 t
->t_intr_start
= time
;
939 time
= mcpu
->intrstat
[pil
][0] + delta
;
940 ret
= time
- mcpu
->intrstat
[pil
][1];
941 mcpu
->intrstat
[pil
][0] = time
;
942 mcpu
->intrstat
[pil
][1] = time
;
943 cpu
->cpu_intracct
[cpu
->cpu_mstate
] += delta
;
956 kthread_t
*t
, *volatile it
;
957 struct machcpu
*mcpu
= &cpu
->cpu_m
;
962 ASSERT(st_pending
== mcpu
->mcpu_softinfo
.st_pending
);
964 pil
= bsrw_insn((uint16_t)st_pending
);
965 if (pil
<= oldpil
|| pil
<= cpu
->cpu_base_spl
)
971 * This is a transliteration of the i386 assembler code for
972 * soft interrupts. One question is "why does this need
973 * to be atomic?" One possible race is -other- processors
974 * posting soft interrupts to us in set_pending() i.e. the
975 * CPU might get preempted just after the address computation,
976 * but just before the atomic transaction, so another CPU would
977 * actually set the original CPU's st_pending bit. However,
978 * it looks like it would be simpler to disable preemption there.
979 * Are there other races for which preemption control doesn't work?
981 * The i386 assembler version -also- checks to see if the bit
982 * being cleared was actually set; if it wasn't, it rechecks
983 * for more. This seems a bit strange, as the only code that
984 * ever clears the bit is -this- code running with interrupts
985 * disabled on -this- CPU. This code would probably be cheaper:
987 * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
990 * and t->t_preempt--/++ around set_pending() even cheaper,
991 * but at this point, correctness is critical, so we slavishly
992 * emulate the i386 port.
994 if (atomic_btr32((uint32_t *)
995 &mcpu
->mcpu_softinfo
.st_pending
, pil
) == 0) {
996 st_pending
= mcpu
->mcpu_softinfo
.st_pending
;
1000 mcpu
->mcpu_pri
= pil
;
1006 * Get set to run interrupt thread.
1007 * There should always be an interrupt thread since we
1008 * allocate one for each level on the CPU.
1010 it
= cpu
->cpu_intr_thread
;
1011 cpu
->cpu_intr_thread
= it
->t_link
;
1013 /* t_intr_start could be zero due to cpu_intr_swtch_enter. */
1014 t
= cpu
->cpu_thread
;
1015 if ((t
->t_flag
& T_INTR_THREAD
) && t
->t_intr_start
!= 0) {
1016 hrtime_t intrtime
= now
- t
->t_intr_start
;
1017 mcpu
->intrstat
[pil
][0] += intrtime
;
1018 cpu
->cpu_intracct
[cpu
->cpu_mstate
] += intrtime
;
1019 t
->t_intr_start
= 0;
1023 * Note that the code in kcpc_overflow_intr -relies- on the
1024 * ordering of events here - in particular that t->t_lwp of
1025 * the interrupt thread is set to the pinned thread *before*
1026 * curthread is changed.
1028 it
->t_lwp
= t
->t_lwp
;
1029 it
->t_state
= TS_ONPROC
;
1032 * Push interrupted thread onto list from new thread.
1033 * Set the new thread as the current one.
1034 * Set interrupted thread's T_SP because if it is the idle thread,
1035 * resume() may use that stack between threads.
1038 ASSERT(SA((uintptr_t)stackptr
) == (uintptr_t)stackptr
);
1039 t
->t_sp
= (uintptr_t)stackptr
;
1042 cpu
->cpu_thread
= it
;
1045 * Set bit for this pil in CPU's interrupt active bitmask.
1047 ASSERT((cpu
->cpu_intr_actv
& (1 << pil
)) == 0);
1048 cpu
->cpu_intr_actv
|= (1 << pil
);
1051 * Initialize thread priority level from intr_pri
1053 it
->t_pil
= (uchar_t
)pil
;
1054 it
->t_pri
= (pri_t
)pil
+ intr_pri
;
1055 it
->t_intr_start
= now
;
1061 dosoftint_epilog(struct cpu
*cpu
, uint_t oldpil
)
1063 struct machcpu
*mcpu
= &cpu
->cpu_m
;
1065 uint_t pil
, basespl
;
1067 hrtime_t now
= tsc_read();
1069 it
= cpu
->cpu_thread
;
1072 cpu
->cpu_stats
.sys
.intr
[pil
- 1]++;
1074 ASSERT(cpu
->cpu_intr_actv
& (1 << pil
));
1075 cpu
->cpu_intr_actv
&= ~(1 << pil
);
1076 intrtime
= now
- it
->t_intr_start
;
1077 mcpu
->intrstat
[pil
][0] += intrtime
;
1078 cpu
->cpu_intracct
[cpu
->cpu_mstate
] += intrtime
;
1081 * If there is still an interrupted thread underneath this one
1082 * then the interrupt was never blocked and the return is
1083 * fairly simple. Otherwise it isn't.
1085 if ((t
= it
->t_intr
) == NULL
) {
1087 * Put thread back on the interrupt thread list.
1088 * This was an interrupt thread, so set CPU's base SPL.
1091 it
->t_state
= TS_FREE
;
1092 it
->t_link
= cpu
->cpu_intr_thread
;
1093 cpu
->cpu_intr_thread
= it
;
1098 panic("dosoftint_epilog: swtch returned");
1100 it
->t_link
= cpu
->cpu_intr_thread
;
1101 cpu
->cpu_intr_thread
= it
;
1102 it
->t_state
= TS_FREE
;
1103 cpu
->cpu_thread
= t
;
1104 if (t
->t_flag
& T_INTR_THREAD
)
1105 t
->t_intr_start
= now
;
1106 basespl
= cpu
->cpu_base_spl
;
1107 pil
= MAX(oldpil
, basespl
);
1108 mcpu
->mcpu_pri
= pil
;
1114 * Make the interrupted thread 'to' be runnable.
1116 * Since t->t_sp has already been saved, t->t_pc is all
1117 * that needs to be set in this function.
1119 * Returns the interrupt level of the interrupt thread.
1123 kthread_t
*it
, /* interrupt thread */
1124 kthread_t
*t
) /* interrupted thread */
1126 extern void _sys_rtt();
1128 ASSERT(it
->t_flag
& T_INTR_THREAD
);
1129 ASSERT(SA(t
->t_sp
) == t
->t_sp
);
1131 t
->t_pc
= (uintptr_t)_sys_rtt
;
1136 * Create interrupt kstats for this CPU.
1139 cpu_create_intrstat(cpu_t
*cp
)
1144 char name
[KSTAT_STRLEN
];
1147 ASSERT(MUTEX_HELD(&cpu_lock
));
1149 if (pool_pset_enabled())
1150 zoneid
= GLOBAL_ZONEID
;
1154 intr_ksp
= kstat_create_zone("cpu", cp
->cpu_id
, "intrstat", "misc",
1155 KSTAT_TYPE_NAMED
, PIL_MAX
* 2, 0, zoneid
);
1158 * Initialize each PIL's named kstat
1160 if (intr_ksp
!= NULL
) {
1161 intr_ksp
->ks_update
= cpu_kstat_intrstat_update
;
1162 knp
= (kstat_named_t
*)intr_ksp
->ks_data
;
1163 intr_ksp
->ks_private
= cp
;
1164 for (i
= 0; i
< PIL_MAX
; i
++) {
1165 (void) snprintf(name
, KSTAT_STRLEN
, "level-%d-time",
1167 kstat_named_init(&knp
[i
* 2], name
, KSTAT_DATA_UINT64
);
1168 (void) snprintf(name
, KSTAT_STRLEN
, "level-%d-count",
1170 kstat_named_init(&knp
[(i
* 2) + 1], name
,
1173 kstat_install(intr_ksp
);
1178 * Delete interrupt kstats for this CPU.
1181 cpu_delete_intrstat(cpu_t
*cp
)
1183 kstat_delete_byname_zone("cpu", cp
->cpu_id
, "intrstat", ALL_ZONES
);
1187 * Convert interrupt statistics from CPU ticks to nanoseconds and
1191 cpu_kstat_intrstat_update(kstat_t
*ksp
, int rw
)
1193 kstat_named_t
*knp
= ksp
->ks_data
;
1194 cpu_t
*cpup
= (cpu_t
*)ksp
->ks_private
;
1198 if (rw
== KSTAT_WRITE
)
1201 for (i
= 0; i
< PIL_MAX
; i
++) {
1202 hrt
= (hrtime_t
)cpup
->cpu_m
.intrstat
[i
+ 1][0];
1204 knp
[i
* 2].value
.ui64
= (uint64_t)hrt
;
1205 knp
[(i
* 2) + 1].value
.ui64
= cpup
->cpu_stats
.sys
.intr
[i
];
1212 * An interrupt thread is ending a time slice, so compute the interval it
1213 * ran for and update the statistic for its PIL.
1216 cpu_intr_swtch_enter(kthread_id_t t
)
1222 ASSERT((t
->t_flag
& T_INTR_THREAD
) != 0);
1223 ASSERT(t
->t_pil
> 0 && t
->t_pil
<= LOCK_LEVEL
);
1226 * We could be here with a zero timestamp. This could happen if:
1227 * an interrupt thread which no longer has a pinned thread underneath
1228 * it (i.e. it blocked at some point in its past) has finished running
1229 * its handler. intr_thread() updated the interrupt statistic for its
1230 * PIL and zeroed its timestamp. Since there was no pinned thread to
1231 * return to, swtch() gets called and we end up here.
1233 * Note that we use atomic ops below (atomic_cas_64 and
1234 * atomic_add_64), which we don't use in the functions above,
1235 * because we're not called with interrupts blocked, but the
1236 * epilog/prolog functions are.
1238 if (t
->t_intr_start
) {
1240 start
= t
->t_intr_start
;
1241 interval
= tsc_read() - start
;
1242 } while (atomic_cas_64(&t
->t_intr_start
, start
, 0) != start
);
1244 cpu
->cpu_m
.intrstat
[t
->t_pil
][0] += interval
;
1246 atomic_add_64((uint64_t *)&cpu
->cpu_intracct
[cpu
->cpu_mstate
],
1249 ASSERT(t
->t_intr
== NULL
);
1253 * An interrupt thread is returning from swtch(). Place a starting timestamp
1254 * in its thread structure.
1257 cpu_intr_swtch_exit(kthread_id_t t
)
1261 ASSERT((t
->t_flag
& T_INTR_THREAD
) != 0);
1262 ASSERT(t
->t_pil
> 0 && t
->t_pil
<= LOCK_LEVEL
);
1265 ts
= t
->t_intr_start
;
1266 } while (atomic_cas_64(&t
->t_intr_start
, ts
, tsc_read()) != ts
);
1270 * Dispatch a hilevel interrupt (one above LOCK_LEVEL)
1274 dispatch_hilevel(uint_t vector
, uint_t arg2
)
1277 av_dispatch_autovect(vector
);
1282 * Dispatch a soft interrupt
1286 dispatch_softint(uint_t oldpil
, uint_t arg2
)
1288 struct cpu
*cpu
= CPU
;
1291 av_dispatch_softvect((int)cpu
->cpu_thread
->t_pil
);
1295 * Must run softint_epilog() on the interrupt thread stack, since
1296 * there may not be a return from it if the interrupt thread blocked.
1298 dosoftint_epilog(cpu
, oldpil
);
1302 * Dispatch a normal interrupt
1305 dispatch_hardint(uint_t vector
, uint_t oldipl
)
1307 struct cpu
*cpu
= CPU
;
1310 av_dispatch_autovect(vector
);
1314 * Must run intr_thread_epilog() on the interrupt thread stack, since
1315 * there may not be a return from it if the interrupt thread blocked.
1317 intr_thread_epilog(cpu
, vector
, oldipl
);
1321 * Deliver any softints the current interrupt priority allows.
1322 * Called with interrupts disabled.
1325 dosoftint(struct regs
*regs
)
1327 struct cpu
*cpu
= CPU
;
1331 while (cpu
->cpu_softinfo
.st_pending
) {
1332 oldipl
= cpu
->cpu_pri
;
1333 newsp
= dosoftint_prolog(cpu
, (caddr_t
)regs
,
1334 cpu
->cpu_softinfo
.st_pending
, oldipl
);
1336 * If returned stack pointer is NULL, priority is too high
1337 * to run any of the pending softints now.
1338 * Break out and they will be run later.
1342 switch_sp_and_call(newsp
, dispatch_softint
, oldipl
, 0);
1347 * Interrupt service routine, called with interrupts disabled.
1351 do_interrupt(struct regs
*rp
, trap_trace_rec_t
*ttp
)
1353 struct cpu
*cpu
= CPU
;
1354 int newipl
, oldipl
= cpu
->cpu_pri
;
1359 ttp
->ttr_marker
= TT_INTERRUPT
;
1360 ttp
->ttr_ipl
= 0xff;
1361 ttp
->ttr_pri
= oldipl
;
1362 ttp
->ttr_spl
= cpu
->cpu_base_spl
;
1363 ttp
->ttr_vector
= 0xff;
1364 #endif /* TRAPTRACE */
1366 cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR
);
1368 ++*(uint16_t *)&cpu
->cpu_m
.mcpu_istamp
;
1371 * If it's a softint go do it now.
1373 if (rp
->r_trapno
== T_SOFTINT
) {
1375 ASSERT(!interrupts_enabled());
1380 * Raise the interrupt priority.
1382 newipl
= (*setlvl
)(oldipl
, (int *)&rp
->r_trapno
);
1384 ttp
->ttr_ipl
= newipl
;
1385 #endif /* TRAPTRACE */
1388 * Bail if it is a spurious interrupt
1392 cpu
->cpu_pri
= newipl
;
1393 vector
= rp
->r_trapno
;
1395 ttp
->ttr_vector
= vector
;
1396 #endif /* TRAPTRACE */
1397 if (newipl
> LOCK_LEVEL
) {
1399 * High priority interrupts run on this cpu's interrupt stack.
1401 if (hilevel_intr_prolog(cpu
, newipl
, oldipl
, rp
) == 0) {
1402 newsp
= cpu
->cpu_intr_stack
;
1403 switch_sp_and_call(newsp
, dispatch_hilevel
, vector
, 0);
1404 } else { /* already on the interrupt stack */
1405 dispatch_hilevel(vector
, 0);
1407 (void) hilevel_intr_epilog(cpu
, newipl
, oldipl
, vector
);
1410 * Run this interrupt in a separate thread.
1412 newsp
= intr_thread_prolog(cpu
, (caddr_t
)rp
, newipl
);
1413 switch_sp_and_call(newsp
, dispatch_hardint
, vector
, oldipl
);
1417 * Deliver any pending soft interrupts.
1419 if (cpu
->cpu_softinfo
.st_pending
)
1425 * Common tasks always done by _sys_rtt, called with interrupts disabled.
1426 * Returns 1 if returning to userland, 0 if returning to system mode.
1429 sys_rtt_common(struct regs
*rp
)
1432 extern void mutex_exit_critical_start();
1433 extern long mutex_exit_critical_size
;
1434 extern void mutex_owner_running_critical_start();
1435 extern long mutex_owner_running_critical_size
;
1440 * Check if returning to user
1442 tp
= CPU
->cpu_thread
;
1443 if (USERMODE(rp
->r_cs
)) {
1447 * Check if AST pending.
1449 if (tp
->t_astflag
) {
1451 * Let trap() handle the AST
1454 rp
->r_trapno
= T_AST
;
1455 trap(rp
, (caddr_t
)0, CPU
->cpu_id
);
1460 pcb
= &tp
->t_lwp
->lwp_pcb
;
1463 * Check to see if we need to initialize the FPU for this
1464 * thread. This should be an uncommon occurrence, but may happen
1465 * in the case where the system creates an lwp through an
1466 * abnormal path such as the agent lwp. Make sure that we still
1467 * happen to have the FPU in a good state.
1469 if ((pcb
->pcb_fpu
.fpu_flags
& FPU_EN
) == 0) {
1473 PCB_SET_UPDATE_FPU(pcb
);
1477 * We are done if segment registers do not need updating.
1479 if (!PCB_NEED_UPDATE(pcb
))
1482 if (PCB_NEED_UPDATE_SEGS(pcb
) && update_sregs(rp
, tp
->t_lwp
)) {
1484 * 1 or more of the selectors is bad.
1485 * Deliver a SIGSEGV.
1487 proc_t
*p
= ttoproc(tp
);
1490 mutex_enter(&p
->p_lock
);
1491 tp
->t_lwp
->lwp_cursig
= SIGSEGV
;
1492 mutex_exit(&p
->p_lock
);
1494 tp
->t_sig_check
= 1;
1497 PCB_CLEAR_UPDATE_SEGS(pcb
);
1499 if (PCB_NEED_UPDATE_FPU(pcb
)) {
1500 fprestore_ctxt(&pcb
->pcb_fpu
);
1502 PCB_CLEAR_UPDATE_FPU(pcb
);
1504 ASSERT0(PCB_NEED_UPDATE(pcb
));
1511 * Assert that we're not trying to return into the syscall return
1512 * trampolines. Things will go baaaaad if we try to do that.
1514 * Note that none of these run with interrupts on, so this should
1515 * never happen (even in the sysexit case the STI doesn't take effect
1516 * until after sysexit finishes).
1518 extern void tr_sysc_ret_start();
1519 extern void tr_sysc_ret_end();
1520 ASSERT(!(rp
->r_pc
>= (uintptr_t)tr_sysc_ret_start
&&
1521 rp
->r_pc
<= (uintptr_t)tr_sysc_ret_end
));
1525 * Here if we are returning to supervisor mode.
1526 * Check for a kernel preemption request.
1528 if (CPU
->cpu_kprunrun
&& (rp
->r_ps
& PS_IE
)) {
1531 * Do nothing if already in kpreempt
1533 if (!tp
->t_preempt_lk
) {
1534 tp
->t_preempt_lk
= 1;
1536 kpreempt(1); /* asynchronous kpreempt call */
1538 tp
->t_preempt_lk
= 0;
1543 * If we interrupted the mutex_exit() critical region we must
1544 * reset the PC back to the beginning to prevent missed wakeups
1545 * See the comments in mutex_exit() for details.
1547 if ((uintptr_t)rp
->r_pc
- (uintptr_t)mutex_exit_critical_start
<
1548 mutex_exit_critical_size
) {
1549 rp
->r_pc
= (greg_t
)mutex_exit_critical_start
;
1553 * If we interrupted the mutex_owner_running() critical region we
1554 * must reset the PC back to the beginning to prevent dereferencing
1555 * of a freed thread pointer. See the comments in mutex_owner_running
1558 if ((uintptr_t)rp
->r_pc
-
1559 (uintptr_t)mutex_owner_running_critical_start
<
1560 mutex_owner_running_critical_size
) {
1561 rp
->r_pc
= (greg_t
)mutex_owner_running_critical_start
;
1568 send_dirint(int cpuid
, int int_level
)
1570 (*send_dirintf
)(cpuid
, int_level
);
1573 #define IS_FAKE_SOFTINT(flag, newpri) \
1574 (((flag) & PS_IE) && \
1575 (((*get_pending_spl)() > (newpri)) || \
1576 bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > (newpri)))
1579 * do_splx routine, takes new ipl to set
1580 * returns the old ipl.
1581 * We are careful not to set priority lower than CPU->cpu_base_pri,
1582 * even though it seems we're raising the priority, it could be set
1583 * higher at any time by an interrupt routine, so we must block interrupts
1584 * and look at CPU->cpu_base_pri
1591 int curpri
, basepri
;
1593 flag
= intr_clear();
1594 cpu
= CPU
; /* ints are disabled, now safe to cache cpu ptr */
1595 curpri
= cpu
->cpu_m
.mcpu_pri
;
1596 basepri
= cpu
->cpu_base_spl
;
1597 if (newpri
< basepri
)
1599 cpu
->cpu_m
.mcpu_pri
= newpri
;
1602 * If we are going to reenable interrupts see if new priority level
1603 * allows pending softint delivery.
1605 if (IS_FAKE_SOFTINT(flag
, newpri
))
1607 ASSERT(!interrupts_enabled());
1613 * Common spl raise routine, takes new ipl to set
1614 * returns the old ipl, will not lower ipl.
1621 int curpri
, basepri
;
1623 flag
= intr_clear();
1624 cpu
= CPU
; /* ints are disabled, now safe to cache cpu ptr */
1625 curpri
= cpu
->cpu_m
.mcpu_pri
;
1627 * Only do something if new priority is larger
1629 if (newpri
> curpri
) {
1630 basepri
= cpu
->cpu_base_spl
;
1631 if (newpri
< basepri
)
1633 cpu
->cpu_m
.mcpu_pri
= newpri
;
1636 * See if new priority level allows pending softint delivery
1638 if (IS_FAKE_SOFTINT(flag
, newpri
))
1648 return (CPU
->cpu_m
.mcpu_pri
);
1654 return (splr(ipltospl(XCALL_PIL
)));
1658 interrupts_enabled(void)
1663 return ((flag
& PS_IE
) == PS_IE
);
1668 assert_ints_enabled(void)
1670 ASSERT(!interrupts_unleashed
|| interrupts_enabled());