2 * Copyright (c) 2008 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * Copyright (c) 1989, 1990 William F. Jolitz.
37 * Copyright (c) 1990 The Regents of the University of California.
38 * All rights reserved.
40 * This code is derived from software contributed to Berkeley by
43 * Redistribution and use in source and binary forms, with or without
44 * modification, are permitted provided that the following conditions
46 * 1. Redistributions of source code must retain the above copyright
47 * notice, this list of conditions and the following disclaimer.
48 * 2. Redistributions in binary form must reproduce the above copyright
49 * notice, this list of conditions and the following disclaimer in the
50 * documentation and/or other materials provided with the distribution.
51 * 3. All advertising materials mentioning features or use of this software
52 * must display the following acknowledgement:
53 * This product includes software developed by the University of
54 * California, Berkeley and its contributors.
55 * 4. Neither the name of the University nor the names of its contributors
56 * may be used to endorse or promote products derived from this software
57 * without specific prior written permission.
59 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
60 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
61 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
62 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
63 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
64 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
65 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
73 * $FreeBSD: src/sys/i386/isa/ipl.s,v 1.32.2.3 2002/05/16 16:03:56 bde Exp $
74 * $DragonFly: src/sys/platform/pc64/amd64/ipl.s,v 1.1 2008/08/29 17:07:10 dillon Exp $
77 #include <machine/asmacros.h>
78 #include <machine/segments.h>
79 #include <machine/ipl.h>
80 #include <machine/lock.h>
81 #include <machine/psl.h>
82 #include <machine/trap.h>
88 * Vector interrupt control section
90 * fpending - Pending interrupts (set when a masked interrupt occurs)
91 * spending - Pending software interrupts
96 .globl fastunpend_count
97 fastunpend_count
: .long 0
105 * - fast interrupts are always called with a critical section
108 * - we release our critical section when scheduling interrupt
109 * or softinterrupt threads in order so they can preempt
110 * (unless we are called manually from a critical section, in
111 * which case there will still be a critical section and
112 * they won't preempt anyway).
114 * - TD_NEST_COUNT prevents splz from nesting too deeply within
115 * itself. It is *not* actually an interrupt nesting count.
116 * PCPU(intr_nesting_level) is an interrupt nesting count.
118 * - We have to be careful in regards to local interrupts
119 * occuring simultaniously with our doreti and splz
126 * Handle return from interrupts, traps and syscalls. This function
127 * checks the cpl for unmasked pending interrupts (fast, normal, or
128 * soft) and schedules them if appropriate, then irets.
130 * If we are in a critical section we cannot run any pending ints
131 * nor can be play with mp_lock.
133 * The stack contains a trapframe at the start of doreti.
137 .type doreti,@function
139 FAKE_MCOUNT
(bintr
) /* init "from" bintr -> doreti */
140 movq $
0,%rax
/* irq mask unavailable due to BGL */
141 movq PCPU
(curthread
),%rbx
142 cli /* interlock with TDPRI_CRIT */
143 cmpl $
0,PCPU
(reqflags
) /* short cut if nothing to do */
145 cmpl $TDPRI_CRIT
,TD_PRI
(%rbx
) /* can't unpend if in critical sec */
147 addl $TDPRI_CRIT
,TD_PRI
(%rbx
) /* force all ints to pending */
149 sti
/* allow new interrupts */
150 movl
%eax
,%ecx
/* irq mask unavailable due to BGL */
152 cli /* disallow YYY remove */
154 testl $RQF_IPIQ
,PCPU
(reqflags
)
156 testl $RQF_TIMER
,PCPU
(reqflags
)
159 testl PCPU
(fpending
),%ecx
/* check for an unmasked fast int */
162 movl PCPU
(spending
),%ecx
/* check for a pending software int */
166 testl $RQF_AST_MASK
,PCPU
(reqflags
) /* any pending ASTs? */
169 /* ASTs are only applicable when returning to userland */
170 testb $SEL_RPL_MASK
,TF_CS
(%rsp
)
174 * Nothing left to do, finish up. Interrupts are still disabled.
175 * %eax contains the mask of IRQ's that are not available due to
176 * BGL requirements. We can only clear RQF_INTPEND if *ALL* pending
177 * interrupts have been processed.
179 subl $TDPRI_CRIT
,TD_PRI
(%rbx
) /* interlocked with cli */
182 andl $~RQF_INTPEND
,PCPU
(reqflags
)
187 * Restore register and iret. iret can fault on %rip (which is
188 * really stupid). If this occurs we re-fault and vector to
189 * doreti_iret_fault().
192 * can be set from user mode, this can result in a kernel mode
193 * exception. The trap code will revector to the *_fault code
194 * which then sets up a T_PROTFLT signal. If the signal is
195 * sent to userland, sendsig() will automatically clean up all
196 * the segment registers to avoid a loop.
199 .globl doreti_syscall_ret
201 POP_FRAME
/* registers and %gs (+cli) */
202 /* special global also used by exception.S */
207 * doreti_iret_fault. Alternative return code for
208 * the case where we get a fault in the doreti_exit code
209 * above. trap() (sys/platform/pc64/amd64/trap.c) catches this specific
210 * case, sends the process a signal and continues in the
211 * corresponding place in the code below.
214 .globl doreti_iret_fault
217 testq $PSL_I
,TF_RFLAGS
(%rsp
)
221 movq $T_PROTFLT
,TF_TRAPNO
(%rsp
)
222 movq $
0,TF_ERR
(%rsp
) /* XXX should be the error code */
223 movq $
0,TF_ADDR
(%rsp
)
224 FAKE_MCOUNT
(TF_RIP
(%rsp
))
228 * FAST interrupt pending. NOTE: stack context holds frame structure
229 * for fast interrupt procedure, do not do random pushes or pops!
233 andl PCPU
(fpending
),%ecx
/* only check fast ints */
234 bsfl
%ecx
, %ecx
/* locate the next dispatchable int */
235 btrl
%ecx
, PCPU
(fpending
) /* is it really still pending? */
237 pushq
%rax
/* save IRQ mask unavailable for BGL */
238 /* NOTE: is also CPL in frame */
241 pushq
%rcx
/* save ecx */
246 /* MP lock successful */
249 call dofastunpend
/* unpend fast intr %ecx */
258 btsl
%ecx
, PCPU
(fpending
) /* oops, couldn't get the MP lock */
259 popq
%rax
/* add to temp. cpl mask to ignore */
260 orl PCPU
(fpending
),%eax
264 * SOFT interrupt pending
266 * Temporarily back-out our critical section to allow an interrupt
267 * preempt us when we schedule it. Bump intr_nesting_level to
268 * prevent the switch code from recursing via splz too deeply.
272 bsfl
%ecx
,%ecx
/* locate the next pending softint */
273 btrl
%ecx
,PCPU
(spending
) /* make sure its still pending */
275 addl $FIRST_SOFTINT
,%ecx
/* actual intr number */
277 movl
%ecx
,%edi
/* argument to C call */
278 incl TD_NEST_COUNT
(%rbx
) /* prevent doreti/splz nesting */
279 subl $TDPRI_CRIT
,TD_PRI
(%rbx
) /* so we can preempt */
280 call sched_ithd
/* YYY must pull in imasks */
281 addl $TDPRI_CRIT
,TD_PRI
(%rbx
)
282 decl TD_NEST_COUNT
(%rbx
)
287 * AST pending. We clear RQF_AST_SIGNAL automatically, the others
288 * are cleared by the trap as they are processed.
290 * Temporarily back-out our critical section because trap() can be
291 * a long-winded call, and we want to be more syscall-like.
293 * YYY theoretically we can call lwkt_switch directly if all we need
294 * to do is a reschedule.
297 andl $~
(RQF_AST_SIGNAL|RQF_AST_UPCALL
),PCPU
(reqflags
)
299 movl
%eax
,%r12d
/* save cpl (can't use stack) */
300 movl $T_ASTFLT
,TF_TRAPNO
(%rsp
)
301 movq
%rsp
,%rdi
/* pass frame by ref (%edi = C arg) */
302 subl $TDPRI_CRIT
,TD_PRI
(%rbx
)
304 addl $TDPRI_CRIT
,TD_PRI
(%rbx
)
305 movl
%r12d
,%eax
/* restore cpl for loop */
310 * IPIQ message pending. We clear RQF_IPIQ automatically.
313 movl
%eax
,%r12d
/* save cpl (can't use stack) */
314 incl PCPU
(intr_nesting_level
)
315 andl $~RQF_IPIQ
,PCPU
(reqflags
)
316 subq $
8,%rsp
/* trapframe->intrframe */
317 movq
%rsp
,%rdi
/* pass frame by ref (C arg) */
318 call lwkt_process_ipiq_frame
319 addq $
8,%rsp
/* intrframe->trapframe */
320 decl PCPU
(intr_nesting_level
)
321 movl
%r12d
,%eax
/* restore cpl for loop */
325 movl
%eax
,%r12d
/* save cpl (can't use stack) */
326 incl PCPU
(intr_nesting_level
)
327 andl $~RQF_TIMER
,PCPU
(reqflags
)
328 subq $
8,%rsp
/* trapframe->intrframe */
329 movq
%rsp
,%rdi
/* pass frame by ref (C arg) */
330 call lapic_timer_process_frame
331 addq $
8,%rsp
/* intrframe->trapframe */
332 decl PCPU
(intr_nesting_level
)
333 movl
%r12d
,%eax
/* restore cpl for loop */
339 * SPLZ() a C callable procedure to dispatch any unmasked pending
340 * interrupts regardless of critical section nesting. ASTs
341 * are not dispatched.
343 * Use %eax to track those IRQs that could not be processed
344 * due to BGL requirements.
351 movq PCPU
(curthread
),%rbx
352 addl $TDPRI_CRIT
,TD_PRI
(%rbx
)
357 movl
%eax
,%ecx
/* ecx = ~CPL */
360 testl $RQF_IPIQ
,PCPU
(reqflags
)
362 testl $RQF_TIMER
,PCPU
(reqflags
)
365 testl PCPU
(fpending
),%ecx
/* check for an unmasked fast int */
368 movl PCPU
(spending
),%ecx
372 subl $TDPRI_CRIT
,TD_PRI
(%rbx
)
375 * Nothing left to do, finish up. Interrupts are still disabled.
376 * If our mask of IRQs we couldn't process due to BGL requirements
377 * is 0 then there are no pending interrupt sources left and we
378 * can clear RQF_INTPEND.
382 andl $~RQF_INTPEND
,PCPU
(reqflags
)
389 * FAST interrupt pending
393 andl PCPU
(fpending
),%ecx
/* only check fast ints */
394 bsfl
%ecx
, %ecx
/* locate the next dispatchable int */
395 btrl
%ecx
, PCPU
(fpending
) /* is it really still pending? */
400 movl
%ecx
,%edi
/* argument to try_mplock */
406 call dofastunpend
/* unpend fast intr %ecx */
415 btsl
%ecx
, PCPU
(fpending
) /* oops, couldn't get the MP lock */
417 orl PCPU
(fpending
),%eax
421 * SOFT interrupt pending
423 * Temporarily back-out our critical section to allow the interrupt
428 bsfl
%ecx
,%ecx
/* locate the next pending softint */
429 btrl
%ecx
,PCPU
(spending
) /* make sure its still pending */
431 addl $FIRST_SOFTINT
,%ecx
/* actual intr number */
434 movl
%ecx
,%edi
/* C argument */
435 subl $TDPRI_CRIT
,TD_PRI
(%rbx
)
436 incl TD_NEST_COUNT
(%rbx
) /* prevent doreti/splz nesting */
437 call sched_ithd
/* YYY must pull in imasks */
438 addl $TDPRI_CRIT
,TD_PRI
(%rbx
)
439 decl TD_NEST_COUNT
(%rbx
) /* prevent doreti/splz nesting */
445 andl $~RQF_IPIQ
,PCPU
(reqflags
)
447 call lwkt_process_ipiq
452 andl $~RQF_TIMER
,PCPU
(reqflags
)
454 call lapic_timer_process
460 * dofastunpend(%ecx:intr)
462 * A FAST interrupt previously made pending can now be run,
463 * execute it by pushing a dummy interrupt frame and
464 * calling ithread_fast_handler to execute or schedule it.
466 * ithread_fast_handler() returns 0 if it wants us to unmask
467 * further interrupts.
470 pushfq ;
/* phys int frame / flags */ \
472 pushq
%rax ;
/* phys int frame / cs */ \
473 pushq
3*8(%rsp
) ;
/* original caller eip */ \
474 subq $TF_RIP
,%rsp ;
/* trap frame */ \
475 movq $
0,TF_XFLAGS
(%rsp
) ;
/* extras */ \
476 movq $
0,TF_TRAPNO
(%rsp
) ;
/* extras */ \
477 movq $
0,TF_ADDR
(%rsp
) ;
/* extras */ \
478 movq $
0,TF_FLAGS
(%rsp
) ;
/* extras */ \
479 movq $
0,TF_ERR
(%rsp
) ;
/* extras */ \
482 addq $TF_RIP+
(3*8),%rsp ; \
485 pushq
%rbp
/* frame for backtrace */
488 pushq
%rcx
/* last part of intrframe = intr */
489 incl fastunpend_count
490 movq
%rsp
,%rdi
/* pass frame by reference C arg */
491 call ithread_fast_handler
/* returns 0 to unmask */
492 popq
%rdi
/* intrframe->trapframe */
493 /* + also rdi C arg to next call */
496 movq MachIntrABI
+ MACHINTR_INTREN
, %rax
497 callq
*%rax
/* MachIntrABI.intren(intr) */