4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
26 * Copyright (c) 2010, Intel Corporation.
27 * All rights reserved.
31 * To understand how the pcplusmp module interacts with the interrupt subsystem
32 * read the theory statement in uts/i86pc/os/intr.c.
36 * PSMI 1.1 extensions are supported only in 2.6 and later versions.
37 * PSMI 1.2 extensions are supported only in 2.7 and later versions.
38 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
39 * PSMI 1.5 extensions are supported in Solaris Nevada.
40 * PSMI 1.6 extensions are supported in Solaris Nevada.
41 * PSMI 1.7 extensions are supported in Solaris Nevada.
45 #include <sys/processor.h>
48 #include <sys/smp_impldefs.h>
50 #include <sys/acpi/acpi.h>
51 #include <sys/acpica.h>
52 #include <sys/psm_common.h>
56 #include <sys/sunddi.h>
57 #include <sys/ddi_impldefs.h>
59 #include <sys/promif.h>
60 #include <sys/x86_archext.h>
61 #include <sys/cpc_impl.h>
62 #include <sys/uadmin.h>
63 #include <sys/panic.h>
64 #include <sys/debug.h>
65 #include <sys/archsystm.h>
67 #include <sys/machsystm.h>
68 #include <sys/sysmacros.h>
69 #include <sys/cpuvar.h>
70 #include <sys/rm_platter.h>
71 #include <sys/privregs.h>
73 #include <sys/pci_intr_lib.h>
75 #include <sys/clock.h>
76 #include <sys/dditypes.h>
77 #include <sys/sunddi.h>
78 #include <sys/x_call.h>
79 #include <sys/reboot.h>
81 #include <sys/apic_common.h>
82 #include <sys/apic_timer.h>
85 * Local Function Prototypes
87 static void apic_init_intr(void);
92 static int apic_probe(void);
93 static int apic_getclkirq(int ipl
);
94 static void apic_init(void);
95 static void apic_picinit(void);
96 static int apic_post_cpu_start(void);
97 static int apic_intr_enter(int ipl
, int *vect
);
98 static void apic_setspl(int ipl
);
99 static void x2apic_setspl(int ipl
);
100 static int apic_addspl(int ipl
, int vector
, int min_ipl
, int max_ipl
);
101 static int apic_delspl(int ipl
, int vector
, int min_ipl
, int max_ipl
);
102 static int apic_disable_intr(processorid_t cpun
);
103 static void apic_enable_intr(processorid_t cpun
);
104 static int apic_get_ipivect(int ipl
, int type
);
105 static void apic_post_cyclic_setup(void *arg
);
108 * The following vector assignments influence the value of ipltopri and
109 * vectortoipl. Note that vectors 0 - 0x1f are not used. We can program
110 * idle to 0 and IPL 0 to 0xf to differentiate idle in case
111 * we care to do so in future. Note some IPLs which are rarely used
112 * will share the vector ranges and heavily used IPLs (5 and 6) have
115 * This array is used to initialize apic_ipls[] (in apic_init()).
117 * IPL Vector range. as passed to intr_enter
119 * 1,2,3 0x20-0x2f 0x0-0xf
120 * 4 0x30-0x3f 0x10-0x1f
121 * 5 0x40-0x5f 0x20-0x3f
122 * 6 0x60-0x7f 0x40-0x5f
123 * 7,8,9 0x80-0x8f 0x60-0x6f
124 * 10 0x90-0x9f 0x70-0x7f
125 * 11 0xa0-0xaf 0x80-0x8f
127 * 15 0xe0-0xef 0xc0-0xcf
128 * 15 0xf0-0xff 0xd0-0xdf
130 uchar_t apic_vectortoipl
[APIC_AVAIL_VECTOR
/ APIC_VECTOR_PER_IPL
] = {
131 3, 4, 5, 5, 6, 6, 9, 10, 11, 12, 13, 14, 15, 15
134 * The ipl of an ISR at vector X is apic_vectortoipl[X>>4]
135 * NOTE that this is vector as passed into intr_enter which is
136 * programmed vector - 0x20 (APIC_BASE_VECT)
139 uchar_t apic_ipltopri
[MAXIPL
+ 1]; /* unix ipl to apic pri */
140 /* The taskpri to be programmed into apic to mask given ipl */
143 uchar_t apic_cr8pri
[MAXIPL
+ 1]; /* unix ipl to cr8 pri */
147 * Correlation of the hardware vector to the IPL in use, initialized
148 * from apic_vectortoipl[] in apic_init(). The final IPLs may not correlate
149 * to the IPLs in apic_vectortoipl on some systems that share interrupt lines
150 * connected to errata-stricken IOAPICs
152 uchar_t apic_ipls
[APIC_AVAIL_VECTOR
];
155 * Patchable global variables.
157 int apic_enable_hwsoftint
= 0; /* 0 - disable, 1 - enable */
158 int apic_enable_bind_log
= 1; /* 1 - display interrupt binding log */
163 static struct psm_ops apic_ops
= {
175 (int (*)(int))NULL
, /* psm_softlvl_to_irq */
176 (void (*)(int))NULL
, /* psm_set_softintr */
183 (void (*)(void))NULL
, /* psm_hrtimeinit */
186 apic_get_next_processorid
,
193 (int (*)(dev_info_t
*, int))NULL
, /* psm_translate_irq */
194 (void (*)(int, char *))NULL
, /* psm_notify_error */
195 (void (*)(int))NULL
, /* psm_notify_func */
196 apic_timer_reprogram
,
199 apic_post_cyclic_setup
,
201 apic_intr_ops
, /* Advanced DDI Interrupt framework */
202 apic_state
, /* save, restore apic state for S3 */
203 apic_cpu_ops
, /* CPU control interface. */
206 struct psm_ops
*psmops
= &apic_ops
;
208 static struct psm_info apic_psm_info
= {
209 PSM_INFO_VER01_7
, /* version */
210 PSM_OWN_EXCLUSIVE
, /* ownership */
211 (struct psm_ops
*)&apic_ops
, /* operation */
212 APIC_PCPLUSMP_NAME
, /* machine name */
213 "pcplusmp v1.4 compatible",
216 static void *apic_hdlp
;
219 * apic_let_idle_redistribute can have the following values:
220 * 0 - If clock decremented it from 1 to 0, clock has to call redistribute.
221 * apic_redistribute_lock prevents multiple idle cpus from redistributing
223 int apic_num_idle_redistributions
= 0;
224 static int apic_let_idle_redistribute
= 0;
226 /* to gather intr data and redistribute */
227 static void apic_redistribute_compute(void);
230 * This is the loadable module wrapper
236 if (apic_coarse_hrtime
)
237 apic_ops
.psm_gethrtime
= &apic_gettime
;
238 return (psm_mod_init(&apic_hdlp
, &apic_psm_info
));
244 return (psm_mod_fini(&apic_hdlp
, &apic_psm_info
));
248 _info(struct modinfo
*modinfop
)
250 return (psm_mod_info(&apic_hdlp
, &apic_psm_info
, modinfop
));
256 /* check if apix is initialized */
257 if (apix_enable
&& apix_loaded())
258 return (PSM_FAILURE
);
260 apix_enable
= 0; /* continue using pcplusmp PSM */
262 return (apic_probe_common(apic_psm_info
.p_mach_idstring
));
266 apic_xlate_vector_by_irq(uchar_t irq
)
268 if (apic_irq_table
[irq
] == NULL
)
271 return (apic_irq_table
[irq
]->airq_vector
);
280 psm_get_ioapicid
= apic_get_ioapicid
;
281 psm_get_localapicid
= apic_get_localapicid
;
282 psm_xlate_vector_by_irq
= apic_xlate_vector_by_irq
;
284 apic_ipltopri
[0] = APIC_VECTOR_PER_IPL
; /* leave 0 for idle */
285 for (i
= 0; i
< (APIC_AVAIL_VECTOR
/ APIC_VECTOR_PER_IPL
); i
++) {
286 if ((i
< ((APIC_AVAIL_VECTOR
/ APIC_VECTOR_PER_IPL
) - 1)) &&
287 (apic_vectortoipl
[i
+ 1] == apic_vectortoipl
[i
]))
288 /* get to highest vector at the same ipl */
290 for (; j
<= apic_vectortoipl
[i
]; j
++) {
291 apic_ipltopri
[j
] = (i
<< APIC_IPL_SHIFT
) +
295 for (; j
< MAXIPL
+ 1; j
++)
296 /* fill up any empty ipltopri slots */
297 apic_ipltopri
[j
] = (i
<< APIC_IPL_SHIFT
) + APIC_BASE_VECT
;
301 * Make cpu-specific interrupt info point to cr8pri vector
303 for (i
= 0; i
<= MAXIPL
; i
++)
304 apic_cr8pri
[i
] = apic_ipltopri
[i
] >> APIC_IPL_SHIFT
;
305 CPU
->cpu_pri_data
= apic_cr8pri
;
307 if (cpuid_have_cr8access(CPU
))
308 apic_have_32bit_cr8
= 1;
315 processorid_t cpun
= psm_get_cpu_id();
317 uint32_t svr
= AV_UNIT_ENABLE
| APIC_SPUR_INTR
;
319 apic_reg_ops
->apic_write_task_reg(APIC_MASK_ALL
);
321 if (apic_mode
== LOCAL_APIC
) {
323 * We are running APIC in MMIO mode.
325 if (apic_flat_model
) {
326 apic_reg_ops
->apic_write(APIC_FORMAT_REG
,
329 apic_reg_ops
->apic_write(APIC_FORMAT_REG
,
333 apic_reg_ops
->apic_write(APIC_DEST_REG
,
334 AV_HIGH_ORDER
>> cpun
);
337 if (apic_directed_EOI_supported()) {
339 * Setting the 12th bit in the Spurious Interrupt Vector
340 * Register suppresses broadcast EOIs generated by the local
341 * APIC. The suppression of broadcast EOIs happens only when
342 * interrupts are level-triggered.
344 svr
|= APIC_SVR_SUPPRESS_BROADCAST_EOI
;
347 /* need to enable APIC before unmasking NMI */
348 apic_reg_ops
->apic_write(APIC_SPUR_INT_REG
, svr
);
351 * Presence of an invalid vector with delivery mode AV_FIXED can
352 * cause an error interrupt, even if the entry is masked...so
353 * write a valid vector to LVT entries along with the mask bit
356 /* All APICs have timer and LINT0/1 */
357 apic_reg_ops
->apic_write(APIC_LOCAL_TIMER
, AV_MASK
|APIC_RESV_IRQ
);
358 apic_reg_ops
->apic_write(APIC_INT_VECT0
, AV_MASK
|APIC_RESV_IRQ
);
359 apic_reg_ops
->apic_write(APIC_INT_VECT1
, AV_NMI
); /* enable NMI */
362 * On integrated APICs, the number of LVT entries is
363 * 'Max LVT entry' + 1; on 82489DX's (non-integrated
364 * APICs), nlvt is "3" (LINT0, LINT1, and timer)
367 if (apic_cpus
[cpun
].aci_local_ver
< APIC_INTEGRATED_VERS
) {
370 nlvt
= ((apic_reg_ops
->apic_read(APIC_VERS_REG
) >> 16) &
375 /* Enable performance counter overflow interrupt */
377 if (!is_x86_feature(x86_featureset
, X86FSET_MSR
))
378 apic_enable_cpcovf_intr
= 0;
379 if (apic_enable_cpcovf_intr
) {
380 if (apic_cpcovf_vect
== 0) {
381 int ipl
= APIC_PCINT_IPL
;
382 int irq
= apic_get_ipivect(ipl
, -1);
386 apic_irq_table
[irq
]->airq_vector
;
387 ASSERT(apic_cpcovf_vect
);
388 (void) add_avintr(NULL
, ipl
,
389 (avfunc
)kcpc_hw_overflow_intr
,
390 "apic pcint", irq
, NULL
, NULL
, NULL
, NULL
);
391 kcpc_hw_overflow_intr_installed
= 1;
392 kcpc_hw_enable_cpc_intr
=
393 apic_cpcovf_mask_clear
;
395 apic_reg_ops
->apic_write(APIC_PCINT_VECT
,
401 /* Only mask TM intr if the BIOS apparently doesn't use it */
405 lvtval
= apic_reg_ops
->apic_read(APIC_THERM_VECT
);
406 if (((lvtval
& AV_MASK
) == AV_MASK
) ||
407 ((lvtval
& AV_DELIV_MODE
) != AV_SMI
)) {
408 apic_reg_ops
->apic_write(APIC_THERM_VECT
,
409 AV_MASK
|APIC_RESV_IRQ
);
413 /* Enable error interrupt */
415 if (nlvt
>= 4 && apic_enable_error_intr
) {
416 if (apic_errvect
== 0) {
417 int ipl
= 0xf; /* get highest priority intr */
418 int irq
= apic_get_ipivect(ipl
, -1);
421 apic_errvect
= apic_irq_table
[irq
]->airq_vector
;
422 ASSERT(apic_errvect
);
424 * Not PSMI compliant, but we are going to merge
427 (void) add_avintr((void *)NULL
, ipl
,
428 (avfunc
)apic_error_intr
, "apic error intr",
429 irq
, NULL
, NULL
, NULL
, NULL
);
431 apic_reg_ops
->apic_write(APIC_ERR_VECT
, apic_errvect
);
432 apic_reg_ops
->apic_write(APIC_ERROR_STATUS
, 0);
433 apic_reg_ops
->apic_write(APIC_ERROR_STATUS
, 0);
436 /* Enable CMCI interrupt */
437 if (cmi_enable_cmci
) {
439 mutex_enter(&cmci_cpu_setup_lock
);
440 if (cmci_cpu_setup_registered
== 0) {
441 mutex_enter(&cpu_lock
);
442 register_cpu_setup_func(cmci_cpu_setup
, NULL
);
443 mutex_exit(&cpu_lock
);
444 cmci_cpu_setup_registered
= 1;
446 mutex_exit(&cmci_cpu_setup_lock
);
448 if (apic_cmci_vect
== 0) {
450 int irq
= apic_get_ipivect(ipl
, -1);
453 apic_cmci_vect
= apic_irq_table
[irq
]->airq_vector
;
454 ASSERT(apic_cmci_vect
);
456 (void) add_avintr(NULL
, ipl
,
457 (avfunc
)cmi_cmci_trap
,
458 "apic cmci intr", irq
, NULL
, NULL
, NULL
, NULL
);
460 apic_reg_ops
->apic_write(APIC_CMCI_VECT
, apic_cmci_vect
);
471 * Initialize and enable interrupt remapping before apic
472 * hardware initialization
474 apic_intrmap_init(apic_mode
);
477 * On UniSys Model 6520, the BIOS leaves vector 0x20 isr
478 * bit on without clearing it with EOI. Since softint
479 * uses vector 0x20 to interrupt itself, so softint will
480 * not work on this machine. In order to fix this problem
481 * a check is made to verify all the isr bits are clear.
482 * If not, EOIs are issued to clear the bits.
484 for (i
= 7; i
>= 1; i
--) {
485 isr
= apic_reg_ops
->apic_read(APIC_ISR_REG
+ (i
* 4));
487 for (j
= 0; ((j
< 32) && (isr
!= 0)); j
++)
488 if (isr
& (1 << j
)) {
489 apic_reg_ops
->apic_write(
492 apic_error
|= APIC_ERR_BOOT_EOI
;
496 /* set a flag so we know we have run apic_picinit() */
497 apic_picinit_called
= 1;
498 LOCK_INIT_CLEAR(&apic_gethrtime_lock
);
499 LOCK_INIT_CLEAR(&apic_ioapic_lock
);
500 LOCK_INIT_CLEAR(&apic_error_lock
);
501 LOCK_INIT_CLEAR(&apic_mode_switch_lock
);
503 picsetup(); /* initialise the 8259 */
505 /* add nmi handler - least priority nmi handler */
506 LOCK_INIT_CLEAR(&apic_nmi_lock
);
508 if (!psm_add_nmintr(0, (avfunc
) apic_nmi_intr
,
509 "pcplusmp NMI handler", (caddr_t
)NULL
))
510 cmn_err(CE_WARN
, "pcplusmp: Unable to add nmi handler");
513 * Check for directed-EOI capability in the local APIC.
515 if (apic_directed_EOI_supported() == 1) {
516 apic_set_directed_EOI_handler();
521 /* enable apic mode if imcr present */
523 outb(APIC_IMCR_P1
, (uchar_t
)APIC_IMCR_SELECT
);
524 outb(APIC_IMCR_P2
, (uchar_t
)APIC_IMCR_APIC
);
527 ioapic_init_intr(IOAPIC_MASK
);
538 * platform_intr_enter
540 * Called at the beginning of the interrupt service routine to
541 * mask all level equal to and below the interrupt priority
542 * of the interrupting vector. An EOI should be given to
543 * the interrupt controller to enable other HW interrupts.
545 * Return -1 for spurious interrupts
550 apic_intr_enter(int ipl
, int *vectorp
)
556 apic_cpus_info_t
*cpu_infop
;
559 * The real vector delivered is (*vectorp + 0x20), but our caller
560 * subtracts 0x20 from the vector before passing it to us.
561 * (That's why APIC_BASE_VECT is 0x20.)
563 vector
= (uchar_t
)*vectorp
;
565 /* if interrupted by the clock, increment apic_nsec_since_boot */
566 if (vector
== apic_clkvect
) {
568 /* NOTE: this is not MT aware */
570 apic_nsec_since_boot
+= apic_nsec_per_intr
;
572 last_count_read
= apic_hertz_count
;
573 apic_redistribute_compute();
576 /* We will avoid all the book keeping overhead for clock */
577 nipl
= apic_ipls
[vector
];
579 *vectorp
= apic_vector_to_irq
[vector
+ APIC_BASE_VECT
];
580 if (apic_mode
== LOCAL_APIC
) {
582 setcr8((ulong_t
)(apic_ipltopri
[nipl
] >>
585 if (apic_have_32bit_cr8
)
586 setcr8((ulong_t
)(apic_ipltopri
[nipl
] >>
589 LOCAL_APIC_WRITE_REG(APIC_TASK_REG
,
590 (uint32_t)apic_ipltopri
[nipl
]);
592 LOCAL_APIC_WRITE_REG(APIC_EOI_REG
, 0);
594 X2APIC_WRITE(APIC_TASK_REG
, apic_ipltopri
[nipl
]);
595 X2APIC_WRITE(APIC_EOI_REG
, 0);
601 cpu_infop
= &apic_cpus
[psm_get_cpu_id()];
603 if (vector
== (APIC_SPUR_INTR
- APIC_BASE_VECT
)) {
604 cpu_infop
->aci_spur_cnt
++;
605 return (APIC_INT_SPURIOUS
);
608 /* Check if the vector we got is really what we need */
609 if (apic_revector_pending
) {
611 * Disable interrupts for the duration of
612 * the vector translation to prevent a self-race for
613 * the apic_revector_lock. This cannot be done
614 * in apic_xlate_vector because it is recursive and
615 * we want the vector translation to be atomic with
616 * respect to other (higher-priority) interrupts.
618 iflag
= intr_clear();
619 vector
= apic_xlate_vector(vector
+ APIC_BASE_VECT
) -
624 nipl
= apic_ipls
[vector
];
625 *vectorp
= irq
= apic_vector_to_irq
[vector
+ APIC_BASE_VECT
];
627 if (apic_mode
== LOCAL_APIC
) {
629 setcr8((ulong_t
)(apic_ipltopri
[nipl
] >> APIC_IPL_SHIFT
));
631 if (apic_have_32bit_cr8
)
632 setcr8((ulong_t
)(apic_ipltopri
[nipl
] >>
635 LOCAL_APIC_WRITE_REG(APIC_TASK_REG
,
636 (uint32_t)apic_ipltopri
[nipl
]);
639 X2APIC_WRITE(APIC_TASK_REG
, apic_ipltopri
[nipl
]);
642 cpu_infop
->aci_current
[nipl
] = (uchar_t
)irq
;
643 cpu_infop
->aci_curipl
= (uchar_t
)nipl
;
644 cpu_infop
->aci_ISR_in_progress
|= 1 << nipl
;
647 * apic_level_intr could have been assimilated into the irq struct.
648 * but, having it as a character array is more efficient in terms of
649 * cache usage. So, we leave it as is.
651 if (!apic_level_intr
[irq
]) {
652 if (apic_mode
== LOCAL_APIC
) {
653 LOCAL_APIC_WRITE_REG(APIC_EOI_REG
, 0);
655 X2APIC_WRITE(APIC_EOI_REG
, 0);
660 APIC_DEBUG_BUF_PUT(vector
);
661 APIC_DEBUG_BUF_PUT(irq
);
662 APIC_DEBUG_BUF_PUT(nipl
);
663 APIC_DEBUG_BUF_PUT(psm_get_cpu_id());
664 if ((apic_stretch_interrupts
) && (apic_stretch_ISR
& (1 << nipl
)))
665 drv_usecwait(apic_stretch_interrupts
);
667 if (apic_break_on_cpu
== psm_get_cpu_id())
674 * This macro is a common code used by MMIO local apic and X2APIC
677 #define APIC_INTR_EXIT() \
679 cpu_infop = &apic_cpus[psm_get_cpu_id()]; \
680 if (apic_level_intr[irq]) \
681 apic_reg_ops->apic_send_eoi(irq); \
682 cpu_infop->aci_curipl = (uchar_t)prev_ipl; \
683 /* ISR above current pri could not be in progress */ \
684 cpu_infop->aci_ISR_in_progress &= (2 << prev_ipl) - 1; \
688 * Any changes made to this function must also change X2APIC
689 * version of intr_exit.
692 apic_intr_exit(int prev_ipl
, int irq
)
694 apic_cpus_info_t
*cpu_infop
;
697 setcr8((ulong_t
)apic_cr8pri
[prev_ipl
]);
699 if (apic_have_32bit_cr8
)
700 setcr8((ulong_t
)(apic_ipltopri
[prev_ipl
] >> APIC_IPL_SHIFT
));
702 apicadr
[APIC_TASK_REG
] = apic_ipltopri
[prev_ipl
];
709 * Same as apic_intr_exit() except it uses MSR rather than MMIO
710 * to access local apic registers.
713 x2apic_intr_exit(int prev_ipl
, int irq
)
715 apic_cpus_info_t
*cpu_infop
;
717 X2APIC_WRITE(APIC_TASK_REG
, apic_ipltopri
[prev_ipl
]);
722 psm_intr_exit_fn(void)
724 if (apic_mode
== LOCAL_X2APIC
)
725 return (x2apic_intr_exit
);
727 return (apic_intr_exit
);
731 * Mask all interrupts below or equal to the given IPL.
732 * Any changes made to this function must also change X2APIC
739 setcr8((ulong_t
)apic_cr8pri
[ipl
]);
741 if (apic_have_32bit_cr8
)
742 setcr8((ulong_t
)(apic_ipltopri
[ipl
] >> APIC_IPL_SHIFT
));
744 apicadr
[APIC_TASK_REG
] = apic_ipltopri
[ipl
];
747 /* interrupts at ipl above this cannot be in progress */
748 apic_cpus
[psm_get_cpu_id()].aci_ISR_in_progress
&= (2 << ipl
) - 1;
750 * this is a patch fix for the ALR QSMP P5 machine, so that interrupts
751 * have enough time to come in before the priority is raised again
752 * during the idle() loop.
754 if (apic_setspl_delay
)
755 (void) apic_reg_ops
->apic_get_pri();
759 * X2APIC version of setspl.
760 * Mask all interrupts below or equal to the given IPL
763 x2apic_setspl(int ipl
)
765 X2APIC_WRITE(APIC_TASK_REG
, apic_ipltopri
[ipl
]);
767 /* interrupts at ipl above this cannot be in progress */
768 apic_cpus
[psm_get_cpu_id()].aci_ISR_in_progress
&= (2 << ipl
) - 1;
773 apic_addspl(int irqno
, int ipl
, int min_ipl
, int max_ipl
)
775 return (apic_addspl_common(irqno
, ipl
, min_ipl
, max_ipl
));
779 apic_delspl(int irqno
, int ipl
, int min_ipl
, int max_ipl
)
781 return (apic_delspl_common(irqno
, ipl
, min_ipl
, max_ipl
));
785 apic_post_cpu_start(void)
788 static int cpus_started
= 1;
790 /* We know this CPU + BSP started successfully. */
794 * On BSP we would have enabled X2APIC, if supported by processor,
795 * in acpi_probe(), but on AP we do it here.
797 * We enable X2APIC mode only if BSP is running in X2APIC & the
798 * local APIC mode of the current CPU is MMIO (xAPIC).
800 if (apic_mode
== LOCAL_X2APIC
&& apic_detect_x2apic() &&
801 apic_local_mode() == LOCAL_APIC
) {
802 apic_enable_x2apic();
806 * Switch back to x2apic IPI sending method for performance when target
807 * CPU has entered x2apic mode.
809 if (apic_mode
== LOCAL_X2APIC
) {
810 apic_switch_ipi_callback(B_FALSE
);
813 splx(ipltospl(LOCK_LEVEL
));
817 * since some systems don't enable the internal cache on the non-boot
818 * cpus, so we have to enable them here
820 setcr0(getcr0() & ~(CR0_CD
| CR0_NW
));
823 APIC_AV_PENDING_SET();
825 if (apic_mode
== LOCAL_APIC
)
826 APIC_AV_PENDING_SET();
830 * We may be booting, or resuming from suspend; aci_status will
831 * be APIC_CPU_INTR_ENABLE if coming from suspend, so we add the
832 * APIC_CPU_ONLINE flag here rather than setting aci_status completely.
834 cpun
= psm_get_cpu_id();
835 apic_cpus
[cpun
].aci_status
|= APIC_CPU_ONLINE
;
837 apic_reg_ops
->apic_write(APIC_DIVIDE_REG
, apic_divide_reg_init
);
838 return (PSM_SUCCESS
);
842 * type == -1 indicates it is an internal request. Do not change
843 * resv_vector for these requests
846 apic_get_ipivect(int ipl
, int type
)
851 if ((irq
= apic_allocate_irq(APIC_VECTOR(ipl
))) != -1) {
852 if (vector
= apic_allocate_vector(ipl
, irq
, 1)) {
853 apic_irq_table
[irq
]->airq_mps_intr_index
=
855 apic_irq_table
[irq
]->airq_vector
= vector
;
857 apic_resv_vector
[ipl
] = vector
;
862 apic_error
|= APIC_ERR_GET_IPIVECT_FAIL
;
863 return (-1); /* shouldn't happen */
867 apic_getclkirq(int ipl
)
871 if ((irq
= apic_get_ipivect(ipl
, -1)) == -1)
874 * Note the vector in apic_clkvect for per clock handling.
876 apic_clkvect
= apic_irq_table
[irq
]->airq_vector
- APIC_BASE_VECT
;
877 APIC_VERBOSE_IOAPIC((CE_NOTE
, "get_clkirq: vector = %x\n",
883 * Try and disable all interrupts. We just assign interrupts to other
884 * processors based on policy. If any were bound by user request, we
885 * let them continue and return failure. We do not bother to check
886 * for cache affinity while rebinding.
890 apic_disable_intr(processorid_t cpun
)
892 int bind_cpu
= 0, i
, hardbound
= 0;
896 iflag
= intr_clear();
897 lock_set(&apic_ioapic_lock
);
899 for (i
= 0; i
<= APIC_MAX_VECTOR
; i
++) {
900 if (apic_reprogram_info
[i
].done
== B_FALSE
) {
901 if (apic_reprogram_info
[i
].bindcpu
== cpun
) {
903 * CPU is busy -- it's the target of
904 * a pending reprogramming attempt
906 lock_clear(&apic_ioapic_lock
);
908 return (PSM_FAILURE
);
913 apic_cpus
[cpun
].aci_status
&= ~APIC_CPU_INTR_ENABLE
;
915 apic_cpus
[cpun
].aci_curipl
= 0;
917 i
= apic_min_device_irq
;
918 for (; i
<= apic_max_device_irq
; i
++) {
920 * If there are bound interrupts on this cpu, then
921 * rebind them to other processors.
923 if ((irq_ptr
= apic_irq_table
[i
]) != NULL
) {
924 ASSERT((irq_ptr
->airq_temp_cpu
== IRQ_UNBOUND
) ||
925 (irq_ptr
->airq_temp_cpu
== IRQ_UNINIT
) ||
926 (apic_cpu_in_range(irq_ptr
->airq_temp_cpu
)));
928 if (irq_ptr
->airq_temp_cpu
== (cpun
| IRQ_USER_BOUND
)) {
933 if (irq_ptr
->airq_temp_cpu
== cpun
) {
936 apic_find_cpu(APIC_CPU_INTR_ENABLE
);
937 } while (apic_rebind_all(irq_ptr
, bind_cpu
));
942 lock_clear(&apic_ioapic_lock
);
946 cmn_err(CE_WARN
, "Could not disable interrupts on %d"
947 "due to user bound interrupts", cpun
);
948 return (PSM_FAILURE
);
951 return (PSM_SUCCESS
);
955 * Bind interrupts to the CPU's local APIC.
956 * Interrupts should not be bound to a CPU's local APIC until the CPU
957 * is ready to receive interrupts.
960 apic_enable_intr(processorid_t cpun
)
966 iflag
= intr_clear();
967 lock_set(&apic_ioapic_lock
);
969 apic_cpus
[cpun
].aci_status
|= APIC_CPU_INTR_ENABLE
;
971 i
= apic_min_device_irq
;
972 for (i
= apic_min_device_irq
; i
<= apic_max_device_irq
; i
++) {
973 if ((irq_ptr
= apic_irq_table
[i
]) != NULL
) {
974 if ((irq_ptr
->airq_cpu
& ~IRQ_USER_BOUND
) == cpun
) {
975 (void) apic_rebind_all(irq_ptr
,
981 if (apic_cpus
[cpun
].aci_status
& APIC_CPU_SUSPEND
)
982 apic_cpus
[cpun
].aci_status
&= ~APIC_CPU_SUSPEND
;
984 lock_clear(&apic_ioapic_lock
);
989 * If this module needs a periodic handler for the interrupt distribution, it
990 * can be added here. The argument to the periodic handler is not currently
991 * used, but is reserved for future.
994 apic_post_cyclic_setup(void *arg
)
996 _NOTE(ARGUNUSED(arg
))
997 /* cpu_lock is held */
998 /* set up a periodic handler for intr redistribution */
1001 * In peridoc mode intr redistribution processing is done in
1002 * apic_intr_enter during clk intr processing
1007 * Register a periodical handler for the redistribution processing.
1008 * On X86, CY_LOW_LEVEL is mapped to the level 2 interrupt, so
1009 * DDI_IPL_2 should be passed to ddi_periodic_add() here.
1011 apic_periodic_id
= ddi_periodic_add(
1012 (void (*)(void *))apic_redistribute_compute
, NULL
,
1013 apic_redistribute_sample_interval
, DDI_IPL_2
);
1017 apic_redistribute_compute(void)
1021 if (apic_enable_dynamic_migration
) {
1022 if (++apic_nticks
== apic_sample_factor_redistribution
) {
1024 * Time to call apic_intr_redistribute().
1025 * reset apic_nticks. This will cause max_busy
1026 * to be calculated below and if it is more than
1027 * apic_int_busy, we will do the whole thing
1032 for (i
= 0; i
< apic_nproc
; i
++) {
1033 if (!apic_cpu_in_range(i
))
1037 * Check if curipl is non zero & if ISR is in
1040 if (((j
= apic_cpus
[i
].aci_curipl
) != 0) &&
1041 (apic_cpus
[i
].aci_ISR_in_progress
& (1 << j
))) {
1044 apic_cpus
[i
].aci_busy
++;
1045 irq
= apic_cpus
[i
].aci_current
[j
];
1046 apic_irq_table
[irq
]->airq_busy
++;
1050 (apic_cpus
[i
].aci_busy
> max_busy
))
1051 max_busy
= apic_cpus
[i
].aci_busy
;
1054 if (max_busy
> apic_int_busy_mark
) {
1056 * We could make the following check be
1057 * skipped > 1 in which case, we get a
1058 * redistribution at half the busy mark (due to
1059 * double interval). Need to be able to collect
1060 * more empirical data to decide if that is a
1061 * good strategy. Punt for now.
1063 if (apic_skipped_redistribute
) {
1064 apic_cleanup_busy();
1065 apic_skipped_redistribute
= 0;
1067 apic_intr_redistribute();
1070 apic_skipped_redistribute
++;
1077 * The following functions are in the platform specific file so that they
1078 * can be different functions depending on whether we are running on
1079 * bare metal or a hypervisor.
1083 * Check to make sure there are enough irq slots
1086 apic_check_free_irqs(int count
)
1091 for (i
= APIC_FIRST_FREE_IRQ
; i
< APIC_RESV_IRQ
; i
++) {
1092 if ((apic_irq_table
[i
] == NULL
) ||
1093 apic_irq_table
[i
]->airq_mps_intr_index
== FREE_INDEX
) {
1094 if (++avail
>= count
)
1095 return (PSM_SUCCESS
);
1098 return (PSM_FAILURE
);
1102 * This function allocates "count" MSI vector(s) for the given "dip/pri/type"
1105 apic_alloc_msi_vectors(dev_info_t
*dip
, int inum
, int count
, int pri
,
1109 uchar_t start
, irqno
;
1114 DDI_INTR_IMPLDBG((CE_CONT
, "apic_alloc_msi_vectors: dip=0x%p "
1115 "inum=0x%x pri=0x%x count=0x%x behavior=%d\n",
1116 (void *)dip
, inum
, pri
, count
, behavior
));
1119 if (behavior
== DDI_INTR_ALLOC_STRICT
&&
1120 apic_multi_msi_enable
== 0)
1122 if (apic_multi_msi_enable
== 0)
1126 if ((rcount
= apic_navail_vector(dip
, pri
)) > count
)
1128 else if (rcount
== 0 || (rcount
< count
&&
1129 behavior
== DDI_INTR_ALLOC_STRICT
))
1132 /* if not ISP2, then round it down */
1134 rcount
= 1 << (highbit(rcount
) - 1);
1136 mutex_enter(&airq_mutex
);
1138 for (start
= 0; rcount
> 0; rcount
>>= 1) {
1139 if ((start
= apic_find_multi_vectors(pri
, rcount
)) != 0 ||
1140 behavior
== DDI_INTR_ALLOC_STRICT
)
1145 /* no vector available */
1146 mutex_exit(&airq_mutex
);
1150 if (apic_check_free_irqs(rcount
) == PSM_FAILURE
) {
1151 /* not enough free irq slots available */
1152 mutex_exit(&airq_mutex
);
1156 major
= (dip
!= NULL
) ? ddi_driver_major(dip
) : 0;
1157 for (i
= 0; i
< rcount
; i
++) {
1158 if ((irqno
= apic_allocate_irq(apic_first_avail_irq
)) ==
1161 * shouldn't happen because of the
1162 * apic_check_free_irqs() check earlier
1164 mutex_exit(&airq_mutex
);
1165 DDI_INTR_IMPLDBG((CE_CONT
, "apic_alloc_msi_vectors: "
1166 "apic_allocate_irq failed\n"));
1169 apic_max_device_irq
= max(irqno
, apic_max_device_irq
);
1170 apic_min_device_irq
= min(irqno
, apic_min_device_irq
);
1171 irqptr
= apic_irq_table
[irqno
];
1173 if (apic_vector_to_irq
[start
+ i
] != APIC_RESV_IRQ
)
1174 DDI_INTR_IMPLDBG((CE_CONT
, "apic_alloc_msi_vectors: "
1175 "apic_vector_to_irq is not APIC_RESV_IRQ\n"));
1177 apic_vector_to_irq
[start
+ i
] = (uchar_t
)irqno
;
1179 irqptr
->airq_vector
= (uchar_t
)(start
+ i
);
1180 irqptr
->airq_ioapicindex
= (uchar_t
)inum
; /* start */
1181 irqptr
->airq_intin_no
= (uchar_t
)rcount
;
1182 irqptr
->airq_ipl
= pri
;
1183 irqptr
->airq_vector
= start
+ i
;
1184 irqptr
->airq_origirq
= (uchar_t
)(inum
+ i
);
1185 irqptr
->airq_share_id
= 0;
1186 irqptr
->airq_mps_intr_index
= MSI_INDEX
;
1187 irqptr
->airq_dip
= dip
;
1188 irqptr
->airq_major
= major
;
1189 if (i
== 0) /* they all bound to the same cpu */
1190 cpu
= irqptr
->airq_cpu
= apic_bind_intr(dip
, irqno
,
1193 irqptr
->airq_cpu
= cpu
;
1194 DDI_INTR_IMPLDBG((CE_CONT
, "apic_alloc_msi_vectors: irq=0x%x "
1195 "dip=0x%p vector=0x%x origirq=0x%x pri=0x%x\n", irqno
,
1196 (void *)irqptr
->airq_dip
, irqptr
->airq_vector
,
1197 irqptr
->airq_origirq
, pri
));
1199 mutex_exit(&airq_mutex
);
1204 * This function allocates "count" MSI-X vector(s) for the given "dip/pri/type"
1207 apic_alloc_msix_vectors(dev_info_t
*dip
, int inum
, int count
, int pri
,
1213 mutex_enter(&airq_mutex
);
1215 if ((rcount
= apic_navail_vector(dip
, pri
)) > count
)
1217 else if (rcount
== 0 || (rcount
< count
&&
1218 behavior
== DDI_INTR_ALLOC_STRICT
)) {
1223 if (apic_check_free_irqs(rcount
) == PSM_FAILURE
) {
1224 /* not enough free irq slots available */
1229 major
= (dip
!= NULL
) ? ddi_driver_major(dip
) : 0;
1230 for (i
= 0; i
< rcount
; i
++) {
1231 uchar_t vector
, irqno
;
1234 if ((irqno
= apic_allocate_irq(apic_first_avail_irq
)) ==
1237 * shouldn't happen because of the
1238 * apic_check_free_irqs() check earlier
1240 DDI_INTR_IMPLDBG((CE_CONT
, "apic_alloc_msix_vectors: "
1241 "apic_allocate_irq failed\n"));
1245 if ((vector
= apic_allocate_vector(pri
, irqno
, 1)) == 0) {
1247 * shouldn't happen because of the
1248 * apic_navail_vector() call earlier
1250 DDI_INTR_IMPLDBG((CE_CONT
, "apic_alloc_msix_vectors: "
1251 "apic_allocate_vector failed\n"));
1255 apic_max_device_irq
= max(irqno
, apic_max_device_irq
);
1256 apic_min_device_irq
= min(irqno
, apic_min_device_irq
);
1257 irqptr
= apic_irq_table
[irqno
];
1258 irqptr
->airq_vector
= (uchar_t
)vector
;
1259 irqptr
->airq_ipl
= pri
;
1260 irqptr
->airq_origirq
= (uchar_t
)(inum
+ i
);
1261 irqptr
->airq_share_id
= 0;
1262 irqptr
->airq_mps_intr_index
= MSIX_INDEX
;
1263 irqptr
->airq_dip
= dip
;
1264 irqptr
->airq_major
= major
;
1265 irqptr
->airq_cpu
= apic_bind_intr(dip
, irqno
, 0xff, 0xff);
1268 mutex_exit(&airq_mutex
);
1273 * Allocate a free vector for irq at ipl. Takes care of merging of multiple
1274 * IPLs into a single APIC level as well as stretching some IPLs onto multiple
1275 * levels. APIC_HI_PRI_VECTS interrupts are reserved for high priority
1276 * requests and allocated only when pri is set.
1279 apic_allocate_vector(int ipl
, int irq
, int pri
)
1281 int lowest
, highest
, i
;
1283 highest
= apic_ipltopri
[ipl
] + APIC_VECTOR_MASK
;
1284 lowest
= apic_ipltopri
[ipl
- 1] + APIC_VECTOR_PER_IPL
;
1286 if (highest
< lowest
) /* Both ipl and ipl - 1 map to same pri */
1287 lowest
-= APIC_VECTOR_PER_IPL
;
1290 if (apic_restrict_vector
) /* for testing shared interrupt logic */
1291 highest
= lowest
+ apic_restrict_vector
+ APIC_HI_PRI_VECTS
;
1294 highest
-= APIC_HI_PRI_VECTS
;
1296 for (i
= lowest
; i
<= highest
; i
++) {
1297 if (APIC_CHECK_RESERVE_VECTORS(i
))
1299 if (apic_vector_to_irq
[i
] == APIC_RESV_IRQ
) {
1300 apic_vector_to_irq
[i
] = (uchar_t
)irq
;
1308 /* Mark vector as not being used by any irq */
1310 apic_free_vector(uchar_t vector
)
1312 apic_vector_to_irq
[vector
] = APIC_RESV_IRQ
;
1316 * Call rebind to do the actual programming.
1317 * Must be called with interrupts disabled and apic_ioapic_lock held
1318 * 'p' is polymorphic -- if this function is called to process a deferred
1319 * reprogramming, p is of type 'struct ioapic_reprogram_data *', from which
1320 * the irq pointer is retrieved. If not doing deferred reprogramming,
1321 * p is of the type 'apic_irq_t *'.
1323 * apic_ioapic_lock must be held across this call, as it protects apic_rebind
1324 * and it protects apic_get_next_bind_cpu() from a race in which a CPU can be
1325 * taken offline after a cpu is selected, but before apic_rebind is called to
1326 * bind interrupts to it.
1329 apic_setup_io_intr(void *p
, int irq
, boolean_t deferred
)
1332 struct ioapic_reprogram_data
*drep
= NULL
;
1336 drep
= (struct ioapic_reprogram_data
*)p
;
1337 ASSERT(drep
!= NULL
);
1338 irqptr
= drep
->irqp
;
1340 irqptr
= (apic_irq_t
*)p
;
1342 ASSERT(irqptr
!= NULL
);
1344 rv
= apic_rebind(irqptr
, apic_irq_table
[irq
]->airq_cpu
, drep
);
1347 * CPU is not up or interrupts are disabled. Fall back to
1348 * the first available CPU
1350 rv
= apic_rebind(irqptr
, apic_find_cpu(APIC_CPU_INTR_ENABLE
),
1359 apic_modify_vector(uchar_t vector
, int irq
)
1361 apic_vector_to_irq
[vector
] = (uchar_t
)irq
;
1366 apic_get_apic_type(void)
1368 return (apic_psm_info
.p_mach_idstring
);
1372 x2apic_update_psm(void)
1374 struct psm_ops
*pops
= &apic_ops
;
1376 ASSERT(pops
!= NULL
);
1378 pops
->psm_intr_exit
= x2apic_intr_exit
;
1379 pops
->psm_setspl
= x2apic_setspl
;
1381 pops
->psm_send_ipi
= x2apic_send_ipi
;
1382 send_dirintf
= pops
->psm_send_ipi
;
1384 apic_mode
= LOCAL_X2APIC
;