2 * from: vector.s, 386BSD 0.1 unknown origin
3 * $FreeBSD: src/sys/i386/isa/apic_vector.s,v 1.47.2.5 2001/09/01 22:33:38 tegge Exp $
6 #include "opt_auto_eoi.h"
8 #include <machine/asmacros.h>
9 #include <machine/lock.h>
10 #include <machine/psl.h>
11 #include <machine/trap.h>
13 #include <machine_base/icu/icu.h>
14 #include <bus/isa/isa.h>
19 #include <machine_base/apic/ioapic_ipl.h>
20 #include <machine/intr_machdep.h>
22 /* convert an absolute IRQ# into bitmask */
23 #define IRQ_LBIT(irq_num) (1 << ((irq_num) & 0x1f))
25 /* convert an absolute IRQ# into ipending index */
26 #define IRQ_LIDX(irq_num) ((irq_num) >> 5)
29 #define MPLOCKED lock ;
35 * Push an interrupt frame in a format acceptable to doreti, reload
36 * the segment registers for the kernel.
39 pushl $
0 ;
/* dummy error code */ \
40 pushl $
0 ;
/* dummy trap type */ \
41 pushl $
0 ;
/* dummy xflags type */ \
43 pushl
%ds ;
/* save data and extra segments ... */ \
56 * Warning: POP_FRAME can only be used if there is no chance of a
57 * segment register being changed (e.g. by procfs), which is why syscalls
66 addl $
3*4,%esp ;
/* dummy xflags, trap & error codes */ \
68 #define IOAPICADDR(irq_num) \
69 CNAME
(ioapic_irqs
) + IOAPIC_IRQI_SIZE
* (irq_num
) + IOAPIC_IRQI_ADDR
70 #define REDIRIDX(irq_num) \
71 CNAME
(ioapic_irqs
) + IOAPIC_IRQI_SIZE
* (irq_num
) + IOAPIC_IRQI_IDX
72 #define IOAPICFLAGS(irq_num) \
73 CNAME
(ioapic_irqs
) + IOAPIC_IRQI_SIZE
* (irq_num
) + IOAPIC_IRQI_FLAGS
75 #define MASK_IRQ(irq_num) \
76 IOAPIC_IMASK_LOCK ;
/* into critical reg */ \
77 testl $IOAPIC_IRQI_FLAG_MASKED
, IOAPICFLAGS
(irq_num
) ; \
78 jne
7f ;
/* masked, don't mask */ \
79 orl $IOAPIC_IRQI_FLAG_MASKED
, IOAPICFLAGS
(irq_num
) ; \
80 /* set the mask bit */ \
81 movl IOAPICADDR
(irq_num
), %ecx ;
/* ioapic addr */ \
82 movl REDIRIDX
(irq_num
), %eax ;
/* get the index */ \
83 movl
%eax
, (%ecx
) ;
/* write the index */ \
84 orl $IOART_INTMASK
,IOAPIC_WINDOW
(%ecx
) ;
/* set the mask */ \
85 7: ;
/* already masked */ \
86 IOAPIC_IMASK_UNLOCK ; \
89 * Test to see whether we are handling an edge or level triggered INT.
90 * Level-triggered INTs must still be masked as we don't clear the source,
91 * and the EOI cycle would cause redundant INTs to occur.
93 #define MASK_LEVEL_IRQ(irq_num) \
94 testl $IOAPIC_IRQI_FLAG_LEVEL
, IOAPICFLAGS
(irq_num
) ; \
95 jz
9f ;
/* edge, don't mask */ \
100 * Test to see if the source is currntly masked, clear if so.
102 #define UNMASK_IRQ(irq_num) \
105 IOAPIC_IMASK_LOCK ;
/* into critical reg */ \
106 testl $IOAPIC_IRQI_FLAG_MASKED
, IOAPICFLAGS
(irq_num
) ; \
107 je
7f ;
/* bit clear, not masked */ \
108 andl $~IOAPIC_IRQI_FLAG_MASKED
, IOAPICFLAGS
(irq_num
) ; \
109 /* clear mask bit */ \
110 movl IOAPICADDR
(irq_num
),%ecx ;
/* ioapic addr */ \
111 movl REDIRIDX
(irq_num
), %eax ;
/* get the index */ \
112 movl
%eax
,(%ecx
) ;
/* write the index */ \
113 andl $~IOART_INTMASK
,IOAPIC_WINDOW
(%ecx
) ;
/* clear the mask */ \
115 IOAPIC_IMASK_UNLOCK ; \
119 * Interrupt call handlers run in the following sequence:
121 * - Push the trap frame required by doreti
122 * - Mask the interrupt and reenable its source
123 * - If we cannot take the interrupt set its ipending bit and
125 * - If we can take the interrupt clear its ipending bit,
126 * call the handler, then unmask and doreti.
128 * YYY can cache gd base opitner instead of using hidden %fs prefixes.
131 #define INTR_HANDLER(irq_num) \
134 IDTVEC
(ioapic_intr
##irq_num) ; \
136 FAKE_MCOUNT
(15*4(%esp
)) ; \
137 MASK_LEVEL_IRQ
(irq_num
) ; \
139 movl $
0,LA_EOI
(%eax
) ; \
140 movl PCPU
(curthread
),%ebx ; \
141 movl $
0,%eax ;
/* CURRENT CPL IN FRAME (REMOVED) */ \
143 testl $
-1,TD_NEST_COUNT
(%ebx
) ; \
145 testl $
-1,TD_CRITCOUNT
(%ebx
) ; \
148 /* in critical section, make interrupt pending */ \
149 /* set the pending bit and return, leave interrupt masked */ \
150 movl $IRQ_LIDX
(irq_num
),%edx ; \
151 orl $IRQ_LBIT
(irq_num
),PCPU_E4
(ipending
,%edx
) ; \
152 orl $RQF_INTPEND
,PCPU
(reqflags
) ; \
155 /* clear pending bit, run handler */ \
156 movl $IRQ_LIDX
(irq_num
),%edx ; \
157 andl $~IRQ_LBIT
(irq_num
),PCPU_E4
(ipending
,%edx
) ; \
159 pushl
%esp ;
/* pass frame by reference */ \
160 incl TD_CRITCOUNT
(%ebx
) ; \
162 call ithread_fast_handler ;
/* returns 0 to unmask */ \
163 decl TD_CRITCOUNT
(%ebx
) ; \
165 UNMASK_IRQ
(irq_num
) ; \
171 * Handle "spurious INTerrupts".
173 * This is different than the "spurious INTerrupt" generated by an
174 * 8259 PIC for missing INTs. See the APIC documentation for details.
175 * This routine should NOT do an 'EOI' cycle.
182 /* No EOI cycle used here */
189 * Handle TLB shootdowns.
191 * NOTE: Interrupts remain disabled.
199 movl $
0,LA_EOI
(%eax
) /* End Of Interrupt to APIC */
200 FAKE_MCOUNT
(15*4(%esp
))
202 subl $
8,%esp
/* make same as interrupt frame */
203 pushl
%esp
/* pass frame by reference */
204 call smp_invltlb_intr
208 jmp doreti_syscall_ret
211 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
213 * - Signals its receipt.
214 * - Waits for permission to restart.
215 * - Processing pending IPIQ events while waiting.
216 * - Signals its restart.
228 pushl
%ds
/* save current data segment */
232 mov
%ax
, %ds
/* use KERNEL data segment */
237 movl $
0, LA_EOI
(%eax
) /* End Of Interrupt to APIC */
239 movl PCPU
(cpuid
), %eax
240 imull $PCB_SIZE
, %eax
241 leal CNAME
(stoppcbs
)(%eax
), %eax
243 call CNAME
(savectx
) /* Save process context */
247 movl PCPU
(cpuid
), %eax
250 * Indicate that we have stopped and loop waiting for permission
251 * to start again. We must still process IPI events while in a
254 * Interrupts must remain enabled for non-IPI'd per-cpu interrupts
255 * (e.g. Xtimer, Xinvltlb).
258 btsl
%eax
, stopped_cpus
/* stopped_cpus |= (1<<id) */
261 andl $~RQF_IPIQ
,PCPU
(reqflags
)
263 call lwkt_smp_stopped
265 btl %eax
, started_cpus
/* while (!(started_cpus & (1<<id))) */
269 btrl
%eax
, started_cpus
/* started_cpus &= ~(1<<id) */
271 btrl
%eax
, stopped_cpus
/* stopped_cpus &= ~(1<<id) */
276 movl CNAME
(cpustop_restartfunc
), %eax
279 movl $
0, CNAME
(cpustop_restartfunc
) /* One-shot */
284 popl
%ds
/* restore previous data segment */
293 * For now just have one ipiq IPI, but what we really want is
294 * to have one for each source cpu to the APICs don't get stalled
295 * backlogging the requests.
303 movl $
0,LA_EOI
(%eax
) /* End Of Interrupt to APIC */
304 FAKE_MCOUNT
(15*4(%esp
))
306 incl PCPU
(cnt
) + V_IPI
307 movl PCPU
(curthread
),%ebx
308 testl $
-1,TD_CRITCOUNT
(%ebx
)
310 subl $
8,%esp
/* make same as interrupt frame */
311 pushl
%esp
/* pass frame by reference */
312 incl PCPU
(intr_nesting_level
)
313 incl TD_CRITCOUNT
(%ebx
)
315 call lwkt_process_ipiq_frame
316 decl TD_CRITCOUNT
(%ebx
)
317 decl PCPU
(intr_nesting_level
)
319 pushl $
0 /* CPL for frame (REMOVED) */
323 orl $RQF_IPIQ
,PCPU
(reqflags
)
325 jmp doreti_syscall_ret
335 movl $
0,LA_EOI
(%eax
) /* End Of Interrupt to APIC */
336 FAKE_MCOUNT
(15*4(%esp
))
338 incl PCPU
(cnt
) + V_TIMER
339 movl PCPU
(curthread
),%ebx
340 testl $
-1,TD_CRITCOUNT
(%ebx
)
342 testl $
-1,TD_NEST_COUNT
(%ebx
)
344 subl $
8,%esp
/* make same as interrupt frame */
345 pushl
%esp
/* pass frame by reference */
346 incl PCPU
(intr_nesting_level
)
347 incl TD_CRITCOUNT
(%ebx
)
349 call lapic_timer_process_frame
350 decl TD_CRITCOUNT
(%ebx
)
351 decl PCPU
(intr_nesting_level
)
353 pushl $
0 /* CPL for frame (REMOVED) */
357 orl $RQF_TIMER
,PCPU
(reqflags
)
359 jmp doreti_syscall_ret
558 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
559 .globl stopped_cpus, started_cpus
565 .globl CNAME(cpustop_restartfunc)
566 CNAME
(cpustop_restartfunc
):