4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2010, Intel Corporation.
26 * All rights reserved.
30 * PSMI 1.1 extensions are supported only in 2.6 and later versions.
31 * PSMI 1.2 extensions are supported only in 2.7 and later versions.
32 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
33 * PSMI 1.5 extensions are supported in Solaris Nevada.
34 * PSMI 1.6 extensions are supported in Solaris Nevada.
35 * PSMI 1.7 extensions are supported in Solaris Nevada.
39 #include <sys/processor.h>
42 #include <sys/smp_impldefs.h>
43 #include <sys/inttypes.h>
45 #include <sys/acpi/acpi.h>
46 #include <sys/acpica.h>
47 #include <sys/psm_common.h>
49 #include <sys/apic_common.h>
52 #include <sys/sunddi.h>
53 #include <sys/ddi_impldefs.h>
55 #include <sys/promif.h>
56 #include <sys/x86_archext.h>
57 #include <sys/cpc_impl.h>
58 #include <sys/uadmin.h>
59 #include <sys/panic.h>
60 #include <sys/debug.h>
61 #include <sys/archsystm.h>
63 #include <sys/machsystm.h>
64 #include <sys/cpuvar.h>
65 #include <sys/rm_platter.h>
66 #include <sys/privregs.h>
67 #include <sys/cyclic.h>
69 #include <sys/pci_intr_lib.h>
70 #include <sys/sunndi.h>
72 #include <sys/clock.h>
75 * Part of mp_platfrom_common.c that's used only by pcplusmp & xpv_psm
77 * These functions may be moved to xpv_psm later when apix and pcplusmp
82 * Local Function Prototypes
84 static void apic_mark_vector(uchar_t oldvector
, uchar_t newvector
);
85 static void apic_xlate_vector_free_timeout_handler(void *arg
);
86 static int apic_check_stuck_interrupt(apic_irq_t
*irq_ptr
, int old_bind_cpu
,
87 int new_bind_cpu
, int apicindex
, int intin_no
, int which_irq
,
88 struct ioapic_reprogram_data
*drep
);
89 static int apic_setup_irq_table(dev_info_t
*dip
, int irqno
,
90 struct apic_io_intr
*intrp
, struct intrspec
*ispec
, iflag_t
*intr_flagp
,
92 static void apic_try_deferred_reprogram(int ipl
, int vect
);
93 static void delete_defer_repro_ent(int which_irq
);
94 static void apic_ioapic_wait_pending_clear(int ioapicindex
,
97 extern int apic_acpi_translate_pci_irq(dev_info_t
*dip
, int busid
, int devid
,
98 int ipin
, int *pci_irqp
, iflag_t
*intr_flagp
);
99 extern int apic_handle_pci_pci_bridge(dev_info_t
*idip
, int child_devno
,
100 int child_ipin
, struct apic_io_intr
**intrp
);
101 extern uchar_t
acpi_find_ioapic(int irq
);
102 extern struct apic_io_intr
*apic_find_io_intr_w_busid(int irqno
, int busid
);
103 extern int apic_find_bus_id(int bustype
);
104 extern int apic_find_intin(uchar_t ioapic
, uchar_t intin
);
105 extern void apic_record_rdt_entry(apic_irq_t
*irqptr
, int irq
);
107 extern int apic_sci_vect
;
108 extern iflag_t apic_sci_flags
;
109 /* ACPI HPET interrupt configuration; -1 if HPET not used */
110 extern int apic_hpet_vect
;
111 extern iflag_t apic_hpet_flags
;
112 extern int apic_intr_policy
;
113 extern char *psm_name
;
116 * number of bits per byte, from <sys/param.h>
118 #define UCHAR_MAX UINT8_MAX
120 /* Max wait time (in repetitions) for flags to clear in an RDT entry. */
121 extern int apic_max_reps_clear_pending
;
123 /* The irq # is implicit in the array index: */
124 struct ioapic_reprogram_data apic_reprogram_info
[APIC_MAX_VECTOR
+1];
126 * APIC_MAX_VECTOR + 1 is the maximum # of IRQs as well. ioapic_reprogram_info
127 * is indexed by IRQ number, NOT by vector number.
130 extern int apic_int_busy_mark
;
131 extern int apic_int_free_mark
;
132 extern int apic_diff_for_redistribution
;
133 extern int apic_sample_factor_redistribution
;
134 extern int apic_redist_cpu_skip
;
135 extern int apic_num_imbalance
;
136 extern int apic_num_rebind
;
138 /* timeout for xlate_vector, mark_vector */
139 int apic_revector_timeout
= 16 * 10000; /* 160 millisec */
141 extern int apic_defconf
;
142 extern int apic_irq_translate
;
144 extern int apic_use_acpi_madt_only
; /* 1=ONLY use MADT from ACPI */
146 extern uchar_t apic_io_vectbase
[MAX_IO_APIC
];
148 extern boolean_t ioapic_mask_workaround
[MAX_IO_APIC
];
151 * First available slot to be used as IRQ index into the apic_irq_table
152 * for those interrupts (like MSI/X) that don't have a physical IRQ.
154 extern int apic_first_avail_irq
;
157 * apic_defer_reprogram_lock ensures that only one processor is handling
158 * deferred interrupt programming at *_intr_exit time.
160 static lock_t apic_defer_reprogram_lock
;
163 * The current number of deferred reprogrammings outstanding
165 uint_t apic_reprogram_outstanding
= 0;
169 * Counters that keep track of deferred reprogramming stats
171 uint_t apic_intr_deferrals
= 0;
172 uint_t apic_intr_deliver_timeouts
= 0;
173 uint_t apic_last_ditch_reprogram_failures
= 0;
174 uint_t apic_deferred_setup_failures
= 0;
175 uint_t apic_defer_repro_total_retries
= 0;
176 uint_t apic_defer_repro_successes
= 0;
177 uint_t apic_deferred_spurious_enters
= 0;
180 extern int apic_io_max
;
181 extern struct apic_io_intr
*apic_io_intrp
;
183 uchar_t apic_vector_to_irq
[APIC_MAX_VECTOR
+1];
185 extern uint32_t eisa_level_intr_mask
;
186 /* At least MSB will be set if EISA bus */
188 extern int apic_pci_bus_total
;
189 extern uchar_t apic_single_pci_busid
;
192 * Following declarations are for revectoring; used when ISRs at different
195 static lock_t apic_revector_lock
;
196 int apic_revector_pending
= 0;
197 static uchar_t
*apic_oldvec_to_newvec
;
198 static uchar_t
*apic_newvec_to_oldvec
;
200 /* ACPI Interrupt Source Override Structure ptr */
201 extern ACPI_MADT_INTERRUPT_OVERRIDE
*acpi_isop
;
202 extern int acpi_iso_cnt
;
205 * Auto-configuration routines
209 * Initialise vector->ipl and ipl->pri arrays. level_intr and irqtable
210 * are also set to NULL. vector->irq is set to a value which cannot map
211 * to a real irq to show that it is free.
214 apic_init_common(void)
220 * Initialize apic_ipls from apic_vectortoipl. This array is
221 * used in apic_intr_enter to determine the IPL to use for the
222 * corresponding vector. On some systems, due to hardware errata
223 * and interrupt sharing, the IPL may not correspond to the IPL listed
224 * in apic_vectortoipl (see apic_addspl and apic_delspl).
226 for (i
= 0; i
< (APIC_AVAIL_VECTOR
/ APIC_VECTOR_PER_IPL
); i
++) {
227 indx
= i
* APIC_VECTOR_PER_IPL
;
229 for (j
= 0; j
< APIC_VECTOR_PER_IPL
; j
++, indx
++)
230 apic_ipls
[indx
] = apic_vectortoipl
[i
];
233 /* cpu 0 is always up (for now) */
234 apic_cpus
[0].aci_status
= APIC_CPU_ONLINE
| APIC_CPU_INTR_ENABLE
;
236 iptr
= (int *)&apic_irq_table
[0];
237 for (i
= 0; i
<= APIC_MAX_VECTOR
; i
++) {
238 apic_level_intr
[i
] = 0;
240 apic_vector_to_irq
[i
] = APIC_RESV_IRQ
;
242 /* These *must* be initted to B_TRUE! */
243 apic_reprogram_info
[i
].done
= B_TRUE
;
244 apic_reprogram_info
[i
].irqp
= NULL
;
245 apic_reprogram_info
[i
].tries
= 0;
246 apic_reprogram_info
[i
].bindcpu
= 0;
250 * Allocate a dummy irq table entry for the reserved entry.
251 * This takes care of the race between removing an irq and
252 * clock detecting a CPU in that irq during interrupt load
255 apic_irq_table
[APIC_RESV_IRQ
] =
256 kmem_zalloc(sizeof (apic_irq_t
), KM_SLEEP
);
258 mutex_init(&airq_mutex
, NULL
, MUTEX_DEFAULT
, NULL
);
262 ioapic_init_intr(int mask_apic
)
265 struct intrspec ispec
;
270 LOCK_INIT_CLEAR(&apic_revector_lock
);
271 LOCK_INIT_CLEAR(&apic_defer_reprogram_lock
);
273 /* mask interrupt vectors */
274 for (j
= 0; j
< apic_io_max
&& mask_apic
; j
++) {
278 /* Bits 23-16 define the maximum redirection entries */
279 intin_max
= (ioapic_read(ioapic_ix
, APIC_VERS_CMD
) >> 16)
281 for (i
= 0; i
<= intin_max
; i
++)
282 ioapic_write(ioapic_ix
, APIC_RDT_CMD
+ 2 * i
, AV_MASK
);
286 * Hack alert: deal with ACPI SCI interrupt chicken/egg here
288 if (apic_sci_vect
> 0) {
290 * acpica has already done add_avintr(); we just
291 * to finish the job by mimicing translate_irq()
293 * Fake up an intrspec and setup the tables
295 ispec
.intrspec_vec
= apic_sci_vect
;
296 ispec
.intrspec_pri
= SCI_IPL
;
298 if (apic_setup_irq_table(NULL
, apic_sci_vect
, NULL
,
299 &ispec
, &apic_sci_flags
, DDI_INTR_TYPE_FIXED
) < 0) {
300 cmn_err(CE_WARN
, "!apic: SCI setup failed");
303 irqptr
= apic_irq_table
[apic_sci_vect
];
305 iflag
= intr_clear();
306 lock_set(&apic_ioapic_lock
);
308 /* Program I/O APIC */
309 (void) apic_setup_io_intr(irqptr
, apic_sci_vect
, B_FALSE
);
311 lock_clear(&apic_ioapic_lock
);
314 irqptr
->airq_share
++;
318 * Hack alert: deal with ACPI HPET interrupt chicken/egg here.
320 if (apic_hpet_vect
> 0) {
322 * hpet has already done add_avintr(); we just need
323 * to finish the job by mimicing translate_irq()
325 * Fake up an intrspec and setup the tables
327 ispec
.intrspec_vec
= apic_hpet_vect
;
328 ispec
.intrspec_pri
= CBE_HIGH_PIL
;
330 if (apic_setup_irq_table(NULL
, apic_hpet_vect
, NULL
,
331 &ispec
, &apic_hpet_flags
, DDI_INTR_TYPE_FIXED
) < 0) {
332 cmn_err(CE_WARN
, "!apic: HPET setup failed");
335 irqptr
= apic_irq_table
[apic_hpet_vect
];
337 iflag
= intr_clear();
338 lock_set(&apic_ioapic_lock
);
340 /* Program I/O APIC */
341 (void) apic_setup_io_intr(irqptr
, apic_hpet_vect
, B_FALSE
);
343 lock_clear(&apic_ioapic_lock
);
346 irqptr
->airq_share
++;
351 * Add mask bits to disable interrupt vector from happening
352 * at or above IPL. In addition, it should remove mask bits
353 * to enable interrupt vectors below the given IPL.
355 * Both add and delspl are complicated by the fact that different interrupts
356 * may share IRQs. This can happen in two ways.
357 * 1. The same H/W line is shared by more than 1 device
358 * 1a. with interrupts at different IPLs
359 * 1b. with interrupts at same IPL
360 * 2. We ran out of vectors at a given IPL and started sharing vectors.
361 * 1b and 2 should be handled gracefully, except for the fact some ISRs
362 * will get called often when no interrupt is pending for the device.
363 * For 1a, we handle it at the higher IPL.
367 apic_addspl_common(int irqno
, int ipl
, int min_ipl
, int max_ipl
)
371 apic_irq_t
*irqptr
, *irqheadptr
;
374 ASSERT(max_ipl
<= UCHAR_MAX
);
375 irqindex
= IRQINDEX(irqno
);
377 if ((irqindex
== -1) || (!apic_irq_table
[irqindex
]))
378 return (PSM_FAILURE
);
380 mutex_enter(&airq_mutex
);
381 irqptr
= irqheadptr
= apic_irq_table
[irqindex
];
383 DDI_INTR_IMPLDBG((CE_CONT
, "apic_addspl: dip=0x%p type=%d irqno=0x%x "
384 "vector=0x%x\n", (void *)irqptr
->airq_dip
,
385 irqptr
->airq_mps_intr_index
, irqno
, irqptr
->airq_vector
));
388 if (VIRTIRQ(irqindex
, irqptr
->airq_share_id
) == irqno
)
390 irqptr
= irqptr
->airq_next
;
392 irqptr
->airq_share
++;
394 mutex_exit(&airq_mutex
);
396 /* return if it is not hardware interrupt */
397 if (irqptr
->airq_mps_intr_index
== RESERVE_INDEX
)
398 return (PSM_SUCCESS
);
400 /* Or if there are more interupts at a higher IPL */
402 return (PSM_SUCCESS
);
405 * if apic_picinit() has not been called yet, just return.
406 * At the end of apic_picinit(), we will call setup_io_intr().
409 if (!apic_picinit_called
)
410 return (PSM_SUCCESS
);
413 * Upgrade vector if max_ipl is not earlier ipl. If we cannot allocate,
416 if (irqptr
->airq_ipl
!= max_ipl
&&
417 !ioapic_mask_workaround
[irqptr
->airq_ioapicindex
]) {
419 vector
= apic_allocate_vector(max_ipl
, irqindex
, 1);
421 irqptr
->airq_share
--;
422 return (PSM_FAILURE
);
425 apic_mark_vector(irqptr
->airq_vector
, vector
);
427 irqptr
->airq_vector
= vector
;
428 irqptr
->airq_ipl
= (uchar_t
)max_ipl
;
430 * reprogram irq being added and every one else
431 * who is not in the UNINIT state
433 if ((VIRTIRQ(irqindex
, irqptr
->airq_share_id
) ==
434 irqno
) || (irqptr
->airq_temp_cpu
!= IRQ_UNINIT
)) {
435 apic_record_rdt_entry(irqptr
, irqindex
);
437 iflag
= intr_clear();
438 lock_set(&apic_ioapic_lock
);
440 (void) apic_setup_io_intr(irqptr
, irqindex
,
443 lock_clear(&apic_ioapic_lock
);
446 irqptr
= irqptr
->airq_next
;
448 return (PSM_SUCCESS
);
450 } else if (irqptr
->airq_ipl
!= max_ipl
&&
451 ioapic_mask_workaround
[irqptr
->airq_ioapicindex
]) {
453 * We cannot upgrade the vector, but we can change
454 * the IPL that this vector induces.
456 * Note that we subtract APIC_BASE_VECT from the vector
457 * here because this array is used in apic_intr_enter
458 * (no need to add APIC_BASE_VECT in that hot code
459 * path since we can do it in the rarely-executed path
462 apic_ipls
[irqptr
->airq_vector
- APIC_BASE_VECT
] =
467 irqptr
->airq_ipl
= (uchar_t
)max_ipl
;
468 irqptr
= irqptr
->airq_next
;
471 return (PSM_SUCCESS
);
476 iflag
= intr_clear();
477 lock_set(&apic_ioapic_lock
);
479 (void) apic_setup_io_intr(irqptr
, irqindex
, B_FALSE
);
481 lock_clear(&apic_ioapic_lock
);
484 return (PSM_SUCCESS
);
488 * Recompute mask bits for the given interrupt vector.
489 * If there is no interrupt servicing routine for this
490 * vector, this function should disable interrupt vector
491 * from happening at all IPLs. If there are still
492 * handlers using the given vector, this function should
493 * disable the given vector from happening below the lowest
494 * IPL of the remaining hadlers.
498 apic_delspl_common(int irqno
, int ipl
, int min_ipl
, int max_ipl
)
504 apic_irq_t
*irqptr
, *preirqptr
, *irqheadptr
, *irqp
;
507 mutex_enter(&airq_mutex
);
508 irqindex
= IRQINDEX(irqno
);
509 irqptr
= preirqptr
= irqheadptr
= apic_irq_table
[irqindex
];
511 DDI_INTR_IMPLDBG((CE_CONT
, "apic_delspl: dip=0x%p type=%d irqno=0x%x "
512 "vector=0x%x\n", (void *)irqptr
->airq_dip
,
513 irqptr
->airq_mps_intr_index
, irqno
, irqptr
->airq_vector
));
516 if (VIRTIRQ(irqindex
, irqptr
->airq_share_id
) == irqno
)
519 irqptr
= irqptr
->airq_next
;
523 irqptr
->airq_share
--;
525 mutex_exit(&airq_mutex
);
528 * If there are more interrupts at a higher IPL, we don't need
529 * to disable anything.
532 return (PSM_SUCCESS
);
534 /* return if it is not hardware interrupt */
535 if (irqptr
->airq_mps_intr_index
== RESERVE_INDEX
)
536 return (PSM_SUCCESS
);
538 if (!apic_picinit_called
) {
540 * Clear irq_struct. If two devices shared an intpt
541 * line & 1 unloaded before picinit, we are hosed. But, then
542 * we hope the machine survive.
544 irqptr
->airq_mps_intr_index
= FREE_INDEX
;
545 irqptr
->airq_temp_cpu
= IRQ_UNINIT
;
546 apic_free_vector(irqptr
->airq_vector
);
547 return (PSM_SUCCESS
);
550 * Downgrade vector to new max_ipl if needed. If we cannot allocate,
551 * use old IPL. Not very elegant, but it should work.
553 if ((irqptr
->airq_ipl
!= max_ipl
) && (max_ipl
!= PSM_INVALID_IPL
) &&
554 !ioapic_mask_workaround
[irqptr
->airq_ioapicindex
]) {
556 if ((vector
= apic_allocate_vector(max_ipl
, irqno
, 1))) {
557 apic_mark_vector(irqheadptr
->airq_vector
, vector
);
560 irqp
->airq_vector
= vector
;
561 irqp
->airq_ipl
= (uchar_t
)max_ipl
;
562 if (irqp
->airq_temp_cpu
!= IRQ_UNINIT
) {
563 apic_record_rdt_entry(irqp
, irqindex
);
565 iflag
= intr_clear();
566 lock_set(&apic_ioapic_lock
);
568 (void) apic_setup_io_intr(irqp
,
571 lock_clear(&apic_ioapic_lock
);
574 irqp
= irqp
->airq_next
;
578 } else if (irqptr
->airq_ipl
!= max_ipl
&&
579 max_ipl
!= PSM_INVALID_IPL
&&
580 ioapic_mask_workaround
[irqptr
->airq_ioapicindex
]) {
583 * We cannot downgrade the IPL of the vector below the vector's
584 * hardware priority. If we did, it would be possible for a
585 * higher-priority hardware vector to interrupt a CPU running at an IPL
586 * lower than the hardware priority of the interrupting vector (but
587 * higher than the soft IPL of this IRQ). When this happens, we would
588 * then try to drop the IPL BELOW what it was (effectively dropping
589 * below base_spl) which would be potentially catastrophic.
591 * (e.g. Suppose the hardware vector associated with this IRQ is 0x40
592 * (hardware IPL of 4). Further assume that the old IPL of this IRQ
593 * was 4, but the new IPL is 1. If we forced vector 0x40 to result in
594 * an IPL of 1, it would be possible for the processor to be executing
595 * at IPL 3 and for an interrupt to come in on vector 0x40, interrupting
596 * the currently-executing ISR. When apic_intr_enter consults
597 * apic_irqs[], it will return 1, bringing the IPL of the CPU down to 1
598 * so even though the processor was running at IPL 4, an IPL 1
599 * interrupt will have interrupted it, which must not happen)).
601 * Effectively, this means that the hardware priority corresponding to
602 * the IRQ's IPL (in apic_ipls[]) cannot be lower than the vector's
605 * (In the above example, then, after removal of the IPL 4 device's
606 * interrupt handler, the new IPL will continue to be 4 because the
607 * hardware priority that IPL 1 implies is lower than the hardware
608 * priority of the vector used.)
610 /* apic_ipls is indexed by vector, starting at APIC_BASE_VECT */
611 const int apic_ipls_index
= irqptr
->airq_vector
-
613 const int vect_inherent_hwpri
= irqptr
->airq_vector
>>
617 * If there are still devices using this IRQ, determine the
620 if (irqptr
->airq_share
) {
621 int vect_desired_hwpri
, hwpri
;
623 ASSERT(max_ipl
< MAXIPL
);
624 vect_desired_hwpri
= apic_ipltopri
[max_ipl
] >>
628 * If the desired IPL's hardware priority is lower
629 * than that of the vector, use the hardware priority
630 * of the vector to determine the new IPL.
632 hwpri
= (vect_desired_hwpri
< vect_inherent_hwpri
) ?
633 vect_inherent_hwpri
: vect_desired_hwpri
;
636 * Now, to get the right index for apic_vectortoipl,
637 * we need to subtract APIC_BASE_VECT from the
638 * hardware-vector-equivalent (in hwpri). Since hwpri
639 * is already shifted, we shift APIC_BASE_VECT before
640 * doing the subtraction.
642 hwpri
-= (APIC_BASE_VECT
>> APIC_IPL_SHIFT
);
645 ASSERT(hwpri
< MAXIPL
);
646 max_ipl
= apic_vectortoipl
[hwpri
];
647 apic_ipls
[apic_ipls_index
] = max_ipl
;
651 irqp
->airq_ipl
= (uchar_t
)max_ipl
;
652 irqp
= irqp
->airq_next
;
656 * No more devices on this IRQ, so reset this vector's
657 * element in apic_ipls to the original IPL for this
660 apic_ipls
[apic_ipls_index
] =
661 apic_vectortoipl
[vect_inherent_hwpri
];
666 * If there are still active interrupts, we are done.
668 if (irqptr
->airq_share
)
669 return (PSM_SUCCESS
);
671 iflag
= intr_clear();
672 lock_set(&apic_ioapic_lock
);
674 if (irqptr
->airq_mps_intr_index
== MSI_INDEX
) {
676 * Disable the MSI vector
677 * Make sure we only disable on the last
678 * of the multi-MSI support
680 if (i_ddi_intr_get_current_nenables(irqptr
->airq_dip
) == 1) {
681 apic_pci_msi_disable_mode(irqptr
->airq_dip
,
684 } else if (irqptr
->airq_mps_intr_index
== MSIX_INDEX
) {
686 * Disable the MSI-X vector
687 * needs to clear its mask and addr/data for each MSI-X
689 apic_pci_msi_unconfigure(irqptr
->airq_dip
, DDI_INTR_TYPE_MSIX
,
690 irqptr
->airq_origirq
);
692 * Make sure we only disable on the last MSI-X
694 if (i_ddi_intr_get_current_nenables(irqptr
->airq_dip
) == 1) {
695 apic_pci_msi_disable_mode(irqptr
->airq_dip
,
700 * The assumption here is that this is safe, even for
701 * systems with IOAPICs that suffer from the hardware
702 * erratum because all devices have been quiesced before
703 * they unregister their interrupt handlers. If that
704 * assumption turns out to be false, this mask operation
705 * can induce the same erratum result we're trying to
708 ioapic_ix
= irqptr
->airq_ioapicindex
;
709 intin
= irqptr
->airq_intin_no
;
710 ioapic_write(ioapic_ix
, APIC_RDT_CMD
+ 2 * intin
, AV_MASK
);
713 apic_vt_ops
->apic_intrmap_free_entry(&irqptr
->airq_intrmap_private
);
716 * This irq entry is the only one in the chain.
718 if (irqheadptr
->airq_next
== NULL
) {
719 ASSERT(irqheadptr
== irqptr
);
720 bind_cpu
= irqptr
->airq_temp_cpu
;
721 if (((uint32_t)bind_cpu
!= IRQ_UNBOUND
) &&
722 ((uint32_t)bind_cpu
!= IRQ_UNINIT
)) {
723 ASSERT(apic_cpu_in_range(bind_cpu
));
724 if (bind_cpu
& IRQ_USER_BOUND
) {
725 /* If hardbound, temp_cpu == cpu */
726 bind_cpu
&= ~IRQ_USER_BOUND
;
727 apic_cpus
[bind_cpu
].aci_bound
--;
729 apic_cpus
[bind_cpu
].aci_temp_bound
--;
731 irqptr
->airq_temp_cpu
= IRQ_UNINIT
;
732 irqptr
->airq_mps_intr_index
= FREE_INDEX
;
733 lock_clear(&apic_ioapic_lock
);
735 apic_free_vector(irqptr
->airq_vector
);
736 return (PSM_SUCCESS
);
740 * If we get here, we are sharing the vector and there are more than
741 * one active irq entries in the chain.
743 lock_clear(&apic_ioapic_lock
);
746 mutex_enter(&airq_mutex
);
747 /* Remove the irq entry from the chain */
748 if (irqptr
== irqheadptr
) { /* The irq entry is at the head */
749 apic_irq_table
[irqindex
] = irqptr
->airq_next
;
751 preirqptr
->airq_next
= irqptr
->airq_next
;
753 /* Free the irq entry */
754 kmem_free(irqptr
, sizeof (apic_irq_t
));
755 mutex_exit(&airq_mutex
);
757 return (PSM_SUCCESS
);
761 * apic_introp_xlate() replaces apic_translate_irq() and is
762 * called only from apic_intr_ops(). With the new ADII framework,
763 * the priority can no longer be retrieved through i_ddi_get_intrspec().
764 * It has to be passed in from the caller.
767 * Success: irqno for the given device
771 apic_introp_xlate(dev_info_t
*dip
, struct intrspec
*ispec
, int type
)
774 int dev_len
, pci_irq
, newirq
, bustype
, devid
, busid
, i
;
775 int irqno
= ispec
->intrspec_vec
;
776 ddi_acc_handle_t cfg_handle
;
778 struct apic_io_intr
*intrp
;
780 ACPI_SUBTABLE_HEADER
*hp
;
781 ACPI_MADT_INTERRUPT_OVERRIDE
*isop
;
783 int parent_is_pci_or_pciex
= 0;
784 int child_is_pciex
= 0;
786 DDI_INTR_IMPLDBG((CE_CONT
, "apic_introp_xlate: dip=0x%p name=%s "
787 "type=%d irqno=0x%x\n", (void *)dip
, ddi_get_name(dip
), type
,
790 dev_len
= sizeof (dev_type
);
791 if (ddi_getlongprop_buf(DDI_DEV_T_ANY
, ddi_get_parent(dip
),
792 DDI_PROP_DONTPASS
, "device_type", (caddr_t
)dev_type
,
793 &dev_len
) == DDI_PROP_SUCCESS
) {
794 if ((strcmp(dev_type
, "pci") == 0) ||
795 (strcmp(dev_type
, "pciex") == 0))
796 parent_is_pci_or_pciex
= 1;
799 if (ddi_getlongprop_buf(DDI_DEV_T_ANY
, dip
,
800 DDI_PROP_DONTPASS
, "compatible", (caddr_t
)dev_type
,
801 &dev_len
) == DDI_PROP_SUCCESS
) {
802 if (strstr(dev_type
, "pciex"))
806 if (DDI_INTR_IS_MSI_OR_MSIX(type
)) {
807 if ((airqp
= apic_find_irq(dip
, ispec
, type
)) != NULL
) {
808 airqp
->airq_iflag
.bustype
=
809 child_is_pciex
? BUS_PCIE
: BUS_PCI
;
810 return (apic_vector_to_irq
[airqp
->airq_vector
]);
812 return (apic_setup_irq_table(dip
, irqno
, NULL
, ispec
,
818 /* check if we have already translated this irq */
819 mutex_enter(&airq_mutex
);
820 newirq
= apic_min_device_irq
;
821 for (; newirq
<= apic_max_device_irq
; newirq
++) {
822 airqp
= apic_irq_table
[newirq
];
824 if ((airqp
->airq_dip
== dip
) &&
825 (airqp
->airq_origirq
== irqno
) &&
826 (airqp
->airq_mps_intr_index
!= FREE_INDEX
)) {
828 mutex_exit(&airq_mutex
);
829 return (VIRTIRQ(newirq
, airqp
->airq_share_id
));
831 airqp
= airqp
->airq_next
;
834 mutex_exit(&airq_mutex
);
839 if ((dip
== NULL
) || (!apic_irq_translate
&& !apic_enable_acpi
))
842 if (parent_is_pci_or_pciex
) {
844 if (acpica_get_bdf(dip
, &busid
, &devid
, NULL
) != 0)
846 if (busid
== 0 && apic_pci_bus_total
== 1)
847 busid
= (int)apic_single_pci_busid
;
849 if (pci_config_setup(dip
, &cfg_handle
) != DDI_SUCCESS
)
851 ipin
= pci_config_get8(cfg_handle
, PCI_CONF_IPIN
) - PCI_INTA
;
852 pci_config_teardown(&cfg_handle
);
853 if (apic_enable_acpi
&& !apic_use_acpi_madt_only
) {
854 if (apic_acpi_translate_pci_irq(dip
, busid
, devid
,
855 ipin
, &pci_irq
, &intr_flag
) != ACPI_PSM_SUCCESS
)
858 intr_flag
.bustype
= child_is_pciex
? BUS_PCIE
: BUS_PCI
;
859 return (apic_setup_irq_table(dip
, pci_irq
, NULL
, ispec
,
862 pci_irq
= ((devid
& 0x1f) << 2) | (ipin
& 0x3);
863 if ((intrp
= apic_find_io_intr_w_busid(pci_irq
, busid
))
865 if ((pci_irq
= apic_handle_pci_pci_bridge(dip
,
866 devid
, ipin
, &intrp
)) == -1)
869 return (apic_setup_irq_table(dip
, pci_irq
, intrp
, ispec
,
872 } else if (strcmp(dev_type
, "isa") == 0)
874 else if (strcmp(dev_type
, "eisa") == 0)
878 if (apic_enable_acpi
&& !apic_use_acpi_madt_only
) {
879 /* search iso entries first */
880 if (acpi_iso_cnt
!= 0) {
881 hp
= (ACPI_SUBTABLE_HEADER
*)acpi_isop
;
883 while (i
< acpi_iso_cnt
) {
885 ACPI_MADT_TYPE_INTERRUPT_OVERRIDE
) {
887 (ACPI_MADT_INTERRUPT_OVERRIDE
*) hp
;
888 if (isop
->Bus
== 0 &&
889 isop
->SourceIrq
== irqno
) {
890 newirq
= isop
->GlobalIrq
;
893 ACPI_MADT_POLARITY_MASK
;
896 ACPI_MADT_TRIGGER_MASK
)
898 intr_flag
.bustype
= BUS_ISA
;
900 return (apic_setup_irq_table(
901 dip
, newirq
, NULL
, ispec
,
907 hp
= (ACPI_SUBTABLE_HEADER
*)(((char *)hp
) +
911 intr_flag
.intr_po
= INTR_PO_ACTIVE_HIGH
;
912 intr_flag
.intr_el
= INTR_EL_EDGE
;
913 intr_flag
.bustype
= BUS_ISA
;
914 return (apic_setup_irq_table(dip
, irqno
, NULL
, ispec
,
917 if (bustype
== 0) /* not initialized */
918 bustype
= eisa_level_intr_mask
? BUS_EISA
: BUS_ISA
;
919 for (i
= 0; i
< 2; i
++) {
920 if (((busid
= apic_find_bus_id(bustype
)) != -1) &&
921 ((intrp
= apic_find_io_intr_w_busid(irqno
, busid
))
923 if ((newirq
= apic_setup_irq_table(dip
, irqno
,
924 intrp
, ispec
, NULL
, type
)) != -1) {
929 bustype
= (bustype
== BUS_EISA
) ? BUS_ISA
: BUS_EISA
;
933 /* MPS default configuration */
935 newirq
= apic_setup_irq_table(dip
, irqno
, NULL
, ispec
, NULL
, type
);
938 ASSERT(IRQINDEX(newirq
) == irqno
);
939 ASSERT(apic_irq_table
[irqno
]);
944 * Attempt to share vector with someone else
947 apic_share_vector(int irqno
, iflag_t
*intr_flagp
, short intr_index
, int ipl
,
948 uchar_t ioapicindex
, uchar_t ipin
, apic_irq_t
**irqptrp
)
951 apic_irq_t
*tmpirqp
= NULL
;
953 apic_irq_t
*irqptr
, dummyirq
;
954 int newirq
, chosen_irq
= -1, share
= 127;
955 int lowest
, highest
, i
;
958 DDI_INTR_IMPLDBG((CE_CONT
, "apic_share_vector: irqno=0x%x "
959 "intr_index=0x%x ipl=0x%x\n", irqno
, intr_index
, ipl
));
961 highest
= apic_ipltopri
[ipl
] + APIC_VECTOR_MASK
;
962 lowest
= apic_ipltopri
[ipl
-1] + APIC_VECTOR_PER_IPL
;
964 if (highest
< lowest
) /* Both ipl and ipl-1 map to same pri */
965 lowest
-= APIC_VECTOR_PER_IPL
;
966 dummyirq
.airq_mps_intr_index
= intr_index
;
967 dummyirq
.airq_ioapicindex
= ioapicindex
;
968 dummyirq
.airq_intin_no
= ipin
;
970 dummyirq
.airq_iflag
= *intr_flagp
;
971 apic_record_rdt_entry(&dummyirq
, irqno
);
972 for (i
= lowest
; i
<= highest
; i
++) {
973 newirq
= apic_vector_to_irq
[i
];
974 if (newirq
== APIC_RESV_IRQ
)
976 irqptr
= apic_irq_table
[newirq
];
978 if ((dummyirq
.airq_rdt_entry
& 0xFF00) !=
979 (irqptr
->airq_rdt_entry
& 0xFF00))
983 if (irqptr
->airq_share
< share
) {
984 share
= irqptr
->airq_share
;
988 if (chosen_irq
!= -1) {
990 * Assign a share id which is free or which is larger
991 * than the largest one.
994 mutex_enter(&airq_mutex
);
995 irqptr
= apic_irq_table
[chosen_irq
];
997 if (irqptr
->airq_mps_intr_index
== FREE_INDEX
) {
998 share_id
= irqptr
->airq_share_id
;
1001 if (share_id
<= irqptr
->airq_share_id
)
1002 share_id
= irqptr
->airq_share_id
+ 1;
1006 irqptr
= irqptr
->airq_next
;
1009 irqptr
= kmem_zalloc(sizeof (apic_irq_t
), KM_SLEEP
);
1010 irqptr
->airq_temp_cpu
= IRQ_UNINIT
;
1012 apic_irq_table
[chosen_irq
]->airq_next
;
1013 apic_irq_table
[chosen_irq
]->airq_next
= irqptr
;
1015 tmpirqp
= apic_irq_table
[chosen_irq
];
1018 irqptr
->airq_mps_intr_index
= intr_index
;
1019 irqptr
->airq_ioapicindex
= ioapicindex
;
1020 irqptr
->airq_intin_no
= ipin
;
1022 irqptr
->airq_iflag
= *intr_flagp
;
1023 irqptr
->airq_vector
= apic_irq_table
[chosen_irq
]->airq_vector
;
1024 irqptr
->airq_share_id
= share_id
;
1025 apic_record_rdt_entry(irqptr
, irqno
);
1028 /* shuffle the pointers to test apic_delspl path */
1030 tmpirqp
->airq_next
= irqptr
->airq_next
;
1031 irqptr
->airq_next
= apic_irq_table
[chosen_irq
];
1032 apic_irq_table
[chosen_irq
] = irqptr
;
1035 mutex_exit(&airq_mutex
);
1036 return (VIRTIRQ(chosen_irq
, share_id
));
1042 * Allocate/Initialize the apic_irq_table[] entry for given irqno. If the entry
1043 * is used already, we will try to allocate a new irqno.
1050 apic_setup_irq_table(dev_info_t
*dip
, int irqno
, struct apic_io_intr
*intrp
,
1051 struct intrspec
*ispec
, iflag_t
*intr_flagp
, int type
)
1053 int origirq
= ispec
->intrspec_vec
;
1054 uchar_t ipl
= ispec
->intrspec_pri
;
1055 int newirq
, intr_index
;
1056 uchar_t ipin
, ioapic
, ioapicindex
, vector
;
1061 DDI_INTR_IMPLDBG((CE_CONT
, "apic_setup_irq_table: dip=0x%p type=%d "
1062 "irqno=0x%x origirq=0x%x\n", (void *)dip
, type
, irqno
, origirq
));
1064 ASSERT(ispec
!= NULL
);
1066 major
= (dip
!= NULL
) ? ddi_driver_major(dip
) : 0;
1068 if (DDI_INTR_IS_MSI_OR_MSIX(type
)) {
1069 /* MSI/X doesn't need to setup ioapic stuffs */
1072 ipin
= (uchar_t
)0xff;
1073 intr_index
= (type
== DDI_INTR_TYPE_MSI
) ? MSI_INDEX
:
1075 mutex_enter(&airq_mutex
);
1076 if ((irqno
= apic_allocate_irq(apic_first_avail_irq
)) == -1) {
1077 mutex_exit(&airq_mutex
);
1078 /* need an irq for MSI/X to index into autovect[] */
1079 cmn_err(CE_WARN
, "No interrupt irq: %s instance %d",
1080 ddi_get_name(dip
), ddi_get_instance(dip
));
1083 mutex_exit(&airq_mutex
);
1085 } else if (intrp
!= NULL
) {
1086 intr_index
= (int)(intrp
- apic_io_intrp
);
1087 ioapic
= intrp
->intr_destid
;
1088 ipin
= intrp
->intr_destintin
;
1089 /* Find ioapicindex. If destid was ALL, we will exit with 0. */
1090 for (ioapicindex
= apic_io_max
- 1; ioapicindex
; ioapicindex
--)
1091 if (apic_io_id
[ioapicindex
] == ioapic
)
1093 ASSERT((ioapic
== apic_io_id
[ioapicindex
]) ||
1094 (ioapic
== INTR_ALL_APIC
));
1096 /* check whether this intin# has been used by another irqno */
1097 if ((newirq
= apic_find_intin(ioapicindex
, ipin
)) != -1) {
1101 } else if (intr_flagp
!= NULL
) {
1103 intr_index
= ACPI_INDEX
;
1104 ioapicindex
= acpi_find_ioapic(irqno
);
1105 ASSERT(ioapicindex
!= 0xFF);
1106 ioapic
= apic_io_id
[ioapicindex
];
1107 ipin
= irqno
- apic_io_vectbase
[ioapicindex
];
1108 if (apic_irq_table
[irqno
] &&
1109 apic_irq_table
[irqno
]->airq_mps_intr_index
== ACPI_INDEX
) {
1110 ASSERT(apic_irq_table
[irqno
]->airq_intin_no
== ipin
&&
1111 apic_irq_table
[irqno
]->airq_ioapicindex
==
1117 /* default configuration */
1119 ioapic
= apic_io_id
[ioapicindex
];
1120 ipin
= (uchar_t
)irqno
;
1121 intr_index
= DEFAULT_INDEX
;
1124 if (ispec
== NULL
) {
1125 APIC_VERBOSE_IOAPIC((CE_WARN
, "No intrspec for irqno = %x\n",
1127 } else if ((vector
= apic_allocate_vector(ipl
, irqno
, 0)) == 0) {
1128 if ((newirq
= apic_share_vector(irqno
, intr_flagp
, intr_index
,
1129 ipl
, ioapicindex
, ipin
, &irqptr
)) != -1) {
1130 irqptr
->airq_ipl
= ipl
;
1131 irqptr
->airq_origirq
= (uchar_t
)origirq
;
1132 irqptr
->airq_dip
= dip
;
1133 irqptr
->airq_major
= major
;
1134 sdip
= apic_irq_table
[IRQINDEX(newirq
)]->airq_dip
;
1135 /* This is OK to do really */
1137 cmn_err(CE_WARN
, "Sharing vectors: %s"
1138 " instance %d and SCI",
1139 ddi_get_name(dip
), ddi_get_instance(dip
));
1141 cmn_err(CE_WARN
, "Sharing vectors: %s"
1142 " instance %d and %s instance %d",
1143 ddi_get_name(sdip
), ddi_get_instance(sdip
),
1144 ddi_get_name(dip
), ddi_get_instance(dip
));
1148 /* try high priority allocation now that share has failed */
1149 if ((vector
= apic_allocate_vector(ipl
, irqno
, 1)) == 0) {
1150 cmn_err(CE_WARN
, "No interrupt vector: %s instance %d",
1151 ddi_get_name(dip
), ddi_get_instance(dip
));
1156 mutex_enter(&airq_mutex
);
1157 if (apic_irq_table
[irqno
] == NULL
) {
1158 irqptr
= kmem_zalloc(sizeof (apic_irq_t
), KM_SLEEP
);
1159 irqptr
->airq_temp_cpu
= IRQ_UNINIT
;
1160 apic_irq_table
[irqno
] = irqptr
;
1162 irqptr
= apic_irq_table
[irqno
];
1163 if (irqptr
->airq_mps_intr_index
!= FREE_INDEX
) {
1165 * The slot is used by another irqno, so allocate
1166 * a free irqno for this interrupt
1168 newirq
= apic_allocate_irq(apic_first_avail_irq
);
1170 mutex_exit(&airq_mutex
);
1174 irqptr
= apic_irq_table
[irqno
];
1175 if (irqptr
== NULL
) {
1176 irqptr
= kmem_zalloc(sizeof (apic_irq_t
),
1178 irqptr
->airq_temp_cpu
= IRQ_UNINIT
;
1179 apic_irq_table
[irqno
] = irqptr
;
1181 vector
= apic_modify_vector(vector
, newirq
);
1184 apic_max_device_irq
= max(irqno
, apic_max_device_irq
);
1185 apic_min_device_irq
= min(irqno
, apic_min_device_irq
);
1186 mutex_exit(&airq_mutex
);
1187 irqptr
->airq_ioapicindex
= ioapicindex
;
1188 irqptr
->airq_intin_no
= ipin
;
1189 irqptr
->airq_ipl
= ipl
;
1190 irqptr
->airq_vector
= vector
;
1191 irqptr
->airq_origirq
= (uchar_t
)origirq
;
1192 irqptr
->airq_share_id
= 0;
1193 irqptr
->airq_mps_intr_index
= (short)intr_index
;
1194 irqptr
->airq_dip
= dip
;
1195 irqptr
->airq_major
= major
;
1196 irqptr
->airq_cpu
= apic_bind_intr(dip
, irqno
, ioapic
, ipin
);
1198 irqptr
->airq_iflag
= *intr_flagp
;
1200 if (!DDI_INTR_IS_MSI_OR_MSIX(type
)) {
1201 /* setup I/O APIC entry for non-MSI/X interrupts */
1202 apic_record_rdt_entry(irqptr
, irqno
);
1208 * return the cpu to which this intr should be bound.
1209 * Check properties or any other mechanism to see if user wants it
1210 * bound to a specific CPU. If so, return the cpu id with high bit set.
1211 * If not, use the policy to choose a cpu and return the id.
1214 apic_bind_intr(dev_info_t
*dip
, int irq
, uchar_t ioapicid
, uchar_t intin
)
1216 int instance
, instno
, prop_len
, bind_cpu
, count
;
1220 char *name
, *drv_name
, *prop_val
, *cptr
;
1225 if (apic_intr_policy
== INTR_LOWEST_PRIORITY
)
1226 return (IRQ_UNBOUND
);
1228 if (apic_nproc
== 1)
1232 rc
= DDI_PROP_NOT_FOUND
;
1233 major
= (major_t
)-1;
1235 name
= ddi_get_name(dip
);
1236 major
= ddi_name_to_major(name
);
1237 drv_name
= ddi_major_to_name(major
);
1238 instance
= ddi_get_instance(dip
);
1239 if (apic_intr_policy
== INTR_ROUND_ROBIN_WITH_AFFINITY
) {
1240 i
= apic_min_device_irq
;
1241 for (; i
<= apic_max_device_irq
; i
++) {
1243 if ((i
== irq
) || (apic_irq_table
[i
] == NULL
) ||
1244 (apic_irq_table
[i
]->airq_mps_intr_index
1248 if ((apic_irq_table
[i
]->airq_major
== major
) &&
1249 (!(apic_irq_table
[i
]->airq_cpu
&
1252 cpu
= apic_irq_table
[i
]->airq_cpu
;
1255 "!%s: %s (%s) instance #%d "
1256 "irq 0x%x vector 0x%x ioapic 0x%x "
1257 "intin 0x%x is bound to cpu %d\n",
1259 name
, drv_name
, instance
, irq
,
1260 apic_irq_table
[irq
]->airq_vector
,
1261 ioapicid
, intin
, cpu
);
1267 * search for "drvname"_intpt_bind_cpus property first, the
1268 * syntax of the property should be "a[,b,c,...]" where
1269 * instance 0 binds to cpu a, instance 1 binds to cpu b,
1270 * instance 3 binds to cpu c...
1271 * ddi_getlongprop() will search /option first, then /
1272 * if "drvname"_intpt_bind_cpus doesn't exist, then find
1273 * intpt_bind_cpus property. The syntax is the same, and
1274 * it applies to all the devices if its "drvname" specific
1275 * property doesn't exist
1277 (void) strcpy(prop_name
, drv_name
);
1278 (void) strcat(prop_name
, "_intpt_bind_cpus");
1279 rc
= ddi_getlongprop(DDI_DEV_T_ANY
, dip
, 0, prop_name
,
1280 (caddr_t
)&prop_val
, &prop_len
);
1281 if (rc
!= DDI_PROP_SUCCESS
) {
1282 rc
= ddi_getlongprop(DDI_DEV_T_ANY
, dip
, 0,
1283 "intpt_bind_cpus", (caddr_t
)&prop_val
, &prop_len
);
1286 if (rc
== DDI_PROP_SUCCESS
) {
1287 for (i
= count
= 0; i
< (prop_len
- 1); i
++)
1288 if (prop_val
[i
] == ',')
1290 if (prop_val
[i
-1] != ',')
1293 * if somehow the binding instances defined in the
1294 * property are not enough for this instno., then
1295 * reuse the pattern for the next instance until
1296 * it reaches the requested instno
1298 instno
= instance
% count
;
1304 bind_cpu
= stoi(&cptr
);
1305 kmem_free(prop_val
, prop_len
);
1306 /* if specific CPU is bogus, then default to next cpu */
1307 if (!apic_cpu_in_range(bind_cpu
)) {
1308 cmn_err(CE_WARN
, "%s: %s=%s: CPU %d not present",
1309 psm_name
, prop_name
, prop_val
, bind_cpu
);
1310 rc
= DDI_PROP_NOT_FOUND
;
1312 /* indicate that we are bound at user request */
1313 bind_cpu
|= IRQ_USER_BOUND
;
1316 * no need to check apic_cpus[].aci_status, if specific CPU is
1317 * not up, then post_cpu_start will handle it.
1320 if (rc
!= DDI_PROP_SUCCESS
) {
1321 iflag
= intr_clear();
1322 lock_set(&apic_ioapic_lock
);
1323 bind_cpu
= apic_get_next_bind_cpu();
1324 lock_clear(&apic_ioapic_lock
);
1325 intr_restore(iflag
);
1328 if (drv_name
!= NULL
)
1329 cmn_err(CE_CONT
, "!%s: %s (%s) instance %d irq 0x%x "
1330 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1331 psm_name
, name
, drv_name
, instance
, irq
,
1332 apic_irq_table
[irq
]->airq_vector
, ioapicid
, intin
,
1333 bind_cpu
& ~IRQ_USER_BOUND
);
1335 cmn_err(CE_CONT
, "!%s: irq 0x%x "
1336 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1337 psm_name
, irq
, apic_irq_table
[irq
]->airq_vector
, ioapicid
,
1338 intin
, bind_cpu
& ~IRQ_USER_BOUND
);
1340 return ((uint32_t)bind_cpu
);
1344 * Mark vector as being in the process of being deleted. Interrupts
1345 * may still come in on some CPU. The moment an interrupt comes with
1346 * the new vector, we know we can free the old one. Called only from
1347 * addspl and delspl with interrupts disabled. Because an interrupt
1348 * can be shared, but no interrupt from either device may come in,
1349 * we also use a timeout mechanism, which we arbitrarily set to
1350 * apic_revector_timeout microseconds.
1353 apic_mark_vector(uchar_t oldvector
, uchar_t newvector
)
1357 iflag
= intr_clear();
1358 lock_set(&apic_revector_lock
);
1359 if (!apic_oldvec_to_newvec
) {
1360 apic_oldvec_to_newvec
=
1361 kmem_zalloc(sizeof (newvector
) * APIC_MAX_VECTOR
* 2,
1364 if (!apic_oldvec_to_newvec
) {
1366 * This failure is not catastrophic.
1367 * But, the oldvec will never be freed.
1369 apic_error
|= APIC_ERR_MARK_VECTOR_FAIL
;
1370 lock_clear(&apic_revector_lock
);
1371 intr_restore(iflag
);
1374 apic_newvec_to_oldvec
= &apic_oldvec_to_newvec
[APIC_MAX_VECTOR
];
1377 /* See if we already did this for drivers which do double addintrs */
1378 if (apic_oldvec_to_newvec
[oldvector
] != newvector
) {
1379 apic_oldvec_to_newvec
[oldvector
] = newvector
;
1380 apic_newvec_to_oldvec
[newvector
] = oldvector
;
1381 apic_revector_pending
++;
1383 lock_clear(&apic_revector_lock
);
1384 intr_restore(iflag
);
1385 (void) timeout(apic_xlate_vector_free_timeout_handler
,
1386 (void *)(uintptr_t)oldvector
, drv_usectohz(apic_revector_timeout
));
1390 * xlate_vector is called from intr_enter if revector_pending is set.
1391 * It will xlate it if needed and mark the old vector as free.
1394 apic_xlate_vector(uchar_t vector
)
1396 uchar_t newvector
, oldvector
= 0;
1398 lock_set(&apic_revector_lock
);
1399 /* Do we really need to do this ? */
1400 if (!apic_revector_pending
) {
1401 lock_clear(&apic_revector_lock
);
1404 if ((newvector
= apic_oldvec_to_newvec
[vector
]) != 0)
1408 * The incoming vector is new . See if a stale entry is
1411 if ((oldvector
= apic_newvec_to_oldvec
[vector
]) != 0)
1416 apic_revector_pending
--;
1417 apic_oldvec_to_newvec
[oldvector
] = 0;
1418 apic_newvec_to_oldvec
[newvector
] = 0;
1419 apic_free_vector(oldvector
);
1420 lock_clear(&apic_revector_lock
);
1421 /* There could have been more than one reprogramming! */
1422 return (apic_xlate_vector(newvector
));
1424 lock_clear(&apic_revector_lock
);
1429 apic_xlate_vector_free_timeout_handler(void *arg
)
1432 uchar_t oldvector
, newvector
;
1434 oldvector
= (uchar_t
)(uintptr_t)arg
;
1435 iflag
= intr_clear();
1436 lock_set(&apic_revector_lock
);
1437 if ((newvector
= apic_oldvec_to_newvec
[oldvector
]) != 0) {
1438 apic_free_vector(oldvector
);
1439 apic_oldvec_to_newvec
[oldvector
] = 0;
1440 apic_newvec_to_oldvec
[newvector
] = 0;
1441 apic_revector_pending
--;
1444 lock_clear(&apic_revector_lock
);
1445 intr_restore(iflag
);
1449 * Bind interrupt corresponding to irq_ptr to bind_cpu.
1450 * Must be called with interrupts disabled and apic_ioapic_lock held
1453 apic_rebind(apic_irq_t
*irq_ptr
, int bind_cpu
,
1454 struct ioapic_reprogram_data
*drep
)
1456 int ioapicindex
, intin_no
;
1457 uint32_t airq_temp_cpu
;
1458 apic_cpus_info_t
*cpu_infop
;
1463 which_irq
= apic_vector_to_irq
[irq_ptr
->airq_vector
];
1465 intin_no
= irq_ptr
->airq_intin_no
;
1466 ioapicindex
= irq_ptr
->airq_ioapicindex
;
1467 airq_temp_cpu
= irq_ptr
->airq_temp_cpu
;
1468 if (airq_temp_cpu
!= IRQ_UNINIT
&& airq_temp_cpu
!= IRQ_UNBOUND
) {
1469 if (airq_temp_cpu
& IRQ_USER_BOUND
)
1470 /* Mask off high bit so it can be used as array index */
1471 airq_temp_cpu
&= ~IRQ_USER_BOUND
;
1473 ASSERT(apic_cpu_in_range(airq_temp_cpu
));
1477 * Can't bind to a CPU that's not accepting interrupts:
1479 cpu_infop
= &apic_cpus
[bind_cpu
& ~IRQ_USER_BOUND
];
1480 if (!(cpu_infop
->aci_status
& APIC_CPU_INTR_ENABLE
))
1484 * If we are about to change the interrupt vector for this interrupt,
1485 * and this interrupt is level-triggered, attached to an IOAPIC,
1486 * has been delivered to a CPU and that CPU has not handled it
1487 * yet, we cannot reprogram the IOAPIC now.
1489 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr
->airq_mps_intr_index
)) {
1491 rdt_entry
= READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex
,
1494 if ((irq_ptr
->airq_vector
!= RDT_VECTOR(rdt_entry
)) &&
1495 apic_check_stuck_interrupt(irq_ptr
, airq_temp_cpu
,
1496 bind_cpu
, ioapicindex
, intin_no
, which_irq
, drep
) != 0) {
1502 * NOTE: We do not unmask the RDT here, as an interrupt MAY
1503 * still come in before we have a chance to reprogram it below.
1504 * The reprogramming below will simultaneously change and
1505 * unmask the RDT entry.
1508 if ((uint32_t)bind_cpu
== IRQ_UNBOUND
) {
1509 irdt
.ir_lo
= AV_LDEST
| AV_LOPRI
|
1510 irq_ptr
->airq_rdt_entry
;
1512 irdt
.ir_hi
= AV_TOALL
>> APIC_ID_BIT_OFFSET
;
1514 apic_vt_ops
->apic_intrmap_alloc_entry(
1515 &irq_ptr
->airq_intrmap_private
, NULL
,
1516 DDI_INTR_TYPE_FIXED
, 1, ioapicindex
);
1517 apic_vt_ops
->apic_intrmap_map_entry(
1518 irq_ptr
->airq_intrmap_private
, (void *)&irdt
,
1519 DDI_INTR_TYPE_FIXED
, 1);
1520 apic_vt_ops
->apic_intrmap_record_rdt(
1521 irq_ptr
->airq_intrmap_private
, &irdt
);
1523 /* Write the RDT entry -- no specific CPU binding */
1524 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex
, intin_no
,
1525 irdt
.ir_hi
| AV_TOALL
);
1527 if (airq_temp_cpu
!= IRQ_UNINIT
&& airq_temp_cpu
!=
1529 apic_cpus
[airq_temp_cpu
].aci_temp_bound
--;
1532 * Write the vector, trigger, and polarity portion of
1535 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex
, intin_no
,
1538 irq_ptr
->airq_temp_cpu
= IRQ_UNBOUND
;
1543 if (bind_cpu
& IRQ_USER_BOUND
) {
1544 cpu_infop
->aci_bound
++;
1546 cpu_infop
->aci_temp_bound
++;
1548 ASSERT(apic_cpu_in_range(bind_cpu
));
1550 if ((airq_temp_cpu
!= IRQ_UNBOUND
) && (airq_temp_cpu
!= IRQ_UNINIT
)) {
1551 apic_cpus
[airq_temp_cpu
].aci_temp_bound
--;
1553 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr
->airq_mps_intr_index
)) {
1555 irdt
.ir_lo
= AV_PDEST
| AV_FIXED
| irq_ptr
->airq_rdt_entry
;
1556 irdt
.ir_hi
= cpu_infop
->aci_local_id
;
1558 apic_vt_ops
->apic_intrmap_alloc_entry(
1559 &irq_ptr
->airq_intrmap_private
, NULL
, DDI_INTR_TYPE_FIXED
,
1561 apic_vt_ops
->apic_intrmap_map_entry(
1562 irq_ptr
->airq_intrmap_private
,
1563 (void *)&irdt
, DDI_INTR_TYPE_FIXED
, 1);
1564 apic_vt_ops
->apic_intrmap_record_rdt(
1565 irq_ptr
->airq_intrmap_private
, &irdt
);
1567 /* Write the RDT entry -- bind to a specific CPU: */
1568 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex
, intin_no
,
1571 /* Write the vector, trigger, and polarity portion of the RDT */
1572 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex
, intin_no
,
1576 int type
= (irq_ptr
->airq_mps_intr_index
== MSI_INDEX
) ?
1577 DDI_INTR_TYPE_MSI
: DDI_INTR_TYPE_MSIX
;
1578 if (type
== DDI_INTR_TYPE_MSI
) {
1579 if (irq_ptr
->airq_ioapicindex
==
1580 irq_ptr
->airq_origirq
) {
1582 DDI_INTR_IMPLDBG((CE_CONT
, "apic_rebind: call "
1583 "apic_pci_msi_enable_vector\n"));
1584 apic_pci_msi_enable_vector(irq_ptr
,
1585 type
, which_irq
, irq_ptr
->airq_vector
,
1586 irq_ptr
->airq_intin_no
,
1587 cpu_infop
->aci_local_id
);
1589 if ((irq_ptr
->airq_ioapicindex
+
1590 irq_ptr
->airq_intin_no
- 1) ==
1591 irq_ptr
->airq_origirq
) { /* last one */
1592 DDI_INTR_IMPLDBG((CE_CONT
, "apic_rebind: call "
1593 "apic_pci_msi_enable_mode\n"));
1594 apic_pci_msi_enable_mode(irq_ptr
->airq_dip
,
1597 } else { /* MSI-X */
1598 apic_pci_msi_enable_vector(irq_ptr
, type
,
1599 irq_ptr
->airq_origirq
, irq_ptr
->airq_vector
, 1,
1600 cpu_infop
->aci_local_id
);
1601 apic_pci_msi_enable_mode(irq_ptr
->airq_dip
, type
,
1602 irq_ptr
->airq_origirq
);
1605 irq_ptr
->airq_temp_cpu
= (uint32_t)bind_cpu
;
1606 apic_redist_cpu_skip
&= ~(1 << (bind_cpu
& ~IRQ_USER_BOUND
));
1611 apic_last_ditch_clear_remote_irr(int ioapic_ix
, int intin_no
)
1613 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
, intin_no
)
1614 & AV_REMOTE_IRR
) != 0) {
1616 * Trying to clear the bit through normal
1617 * channels has failed. So as a last-ditch
1618 * effort, try to set the trigger mode to
1619 * edge, then to level. This has been
1620 * observed to work on many systems.
1622 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1624 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1625 intin_no
) & ~AV_LEVEL
);
1627 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1629 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1630 intin_no
) | AV_LEVEL
);
1633 * If the bit's STILL set, this interrupt may
1636 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1637 intin_no
) & AV_REMOTE_IRR
) != 0) {
1639 prom_printf("%s: Remote IRR still "
1640 "not clear for IOAPIC %d intin %d.\n"
1641 "\tInterrupts to this pin may cease "
1642 "functioning.\n", psm_name
, ioapic_ix
,
1645 apic_last_ditch_reprogram_failures
++;
1652 * This function is protected by apic_ioapic_lock coupled with the
1653 * fact that interrupts are disabled.
1656 delete_defer_repro_ent(int which_irq
)
1658 ASSERT(which_irq
>= 0);
1659 ASSERT(which_irq
<= 255);
1660 ASSERT(LOCK_HELD(&apic_ioapic_lock
));
1662 if (apic_reprogram_info
[which_irq
].done
)
1665 apic_reprogram_info
[which_irq
].done
= B_TRUE
;
1668 apic_defer_repro_total_retries
+=
1669 apic_reprogram_info
[which_irq
].tries
;
1671 apic_defer_repro_successes
++;
1674 if (--apic_reprogram_outstanding
== 0) {
1676 setlvlx
= psm_intr_exit_fn();
1682 * Interrupts must be disabled during this function to prevent
1683 * self-deadlock. Interrupts are disabled because this function
1684 * is called from apic_check_stuck_interrupt(), which is called
1685 * from apic_rebind(), which requires its caller to disable interrupts.
1688 add_defer_repro_ent(apic_irq_t
*irq_ptr
, int which_irq
, int new_bind_cpu
)
1690 ASSERT(which_irq
>= 0);
1691 ASSERT(which_irq
<= 255);
1692 ASSERT(!interrupts_enabled());
1695 * On the off-chance that there's already a deferred
1696 * reprogramming on this irq, check, and if so, just update the
1697 * CPU and irq pointer to which the interrupt is targeted, then return.
1699 if (!apic_reprogram_info
[which_irq
].done
) {
1700 apic_reprogram_info
[which_irq
].bindcpu
= new_bind_cpu
;
1701 apic_reprogram_info
[which_irq
].irqp
= irq_ptr
;
1705 apic_reprogram_info
[which_irq
].irqp
= irq_ptr
;
1706 apic_reprogram_info
[which_irq
].bindcpu
= new_bind_cpu
;
1707 apic_reprogram_info
[which_irq
].tries
= 0;
1709 * This must be the last thing set, since we're not
1710 * grabbing any locks, apic_try_deferred_reprogram() will
1711 * make its decision about using this entry iff done
1714 apic_reprogram_info
[which_irq
].done
= B_FALSE
;
1717 * If there were previously no deferred reprogrammings, change
1718 * setlvlx to call apic_try_deferred_reprogram()
1720 if (++apic_reprogram_outstanding
== 1) {
1722 setlvlx
= apic_try_deferred_reprogram
;
1727 apic_try_deferred_reprogram(int prev_ipl
, int irq
)
1731 struct ioapic_reprogram_data
*drep
;
1733 (*psm_intr_exit_fn())(prev_ipl
, irq
);
1735 if (!lock_try(&apic_defer_reprogram_lock
)) {
1740 * Acquire the apic_ioapic_lock so that any other operations that
1741 * may affect the apic_reprogram_info state are serialized.
1742 * It's still possible for the last deferred reprogramming to clear
1743 * between the time we entered this function and the time we get to
1744 * the for loop below. In that case, *setlvlx will have been set
1745 * back to *_intr_exit and drep will be NULL. (There's no way to
1746 * stop that from happening -- we would need to grab a lock before
1747 * calling *setlvlx, which is neither realistic nor prudent).
1749 iflag
= intr_clear();
1750 lock_set(&apic_ioapic_lock
);
1753 * For each deferred RDT entry, try to reprogram it now. Note that
1754 * there is no lock acquisition to read apic_reprogram_info because
1755 * '.done' is set only after the other fields in the structure are set.
1759 for (reproirq
= 0; reproirq
<= APIC_MAX_VECTOR
; reproirq
++) {
1760 if (apic_reprogram_info
[reproirq
].done
== B_FALSE
) {
1761 drep
= &apic_reprogram_info
[reproirq
];
1767 * Either we found a deferred action to perform, or
1768 * we entered this function spuriously, after *setlvlx
1769 * was restored to point to *_intr_exit. Any other
1770 * permutation is invalid.
1772 ASSERT(drep
!= NULL
|| *setlvlx
== psm_intr_exit_fn());
1775 * Though we can't really do anything about errors
1776 * at this point, keep track of them for reporting.
1777 * Note that it is very possible for apic_setup_io_intr
1778 * to re-register this very timeout if the Remote IRR bit
1779 * has not yet cleared.
1784 if (apic_setup_io_intr(drep
, reproirq
, B_TRUE
) != 0) {
1785 apic_deferred_setup_failures
++;
1788 apic_deferred_spurious_enters
++;
1792 (void) apic_setup_io_intr(drep
, reproirq
, B_TRUE
);
1795 lock_clear(&apic_ioapic_lock
);
1796 intr_restore(iflag
);
1798 lock_clear(&apic_defer_reprogram_lock
);
1802 apic_ioapic_wait_pending_clear(int ioapic_ix
, int intin_no
)
1807 * Wait for the delivery pending bit to clear.
1809 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
, intin_no
) &
1810 (AV_LEVEL
|AV_PENDING
)) == (AV_LEVEL
|AV_PENDING
)) {
1813 * If we're still waiting on the delivery of this interrupt,
1814 * continue to wait here until it is delivered (this should be
1815 * a very small amount of time, but include a timeout just in
1818 for (waited
= 0; waited
< apic_max_reps_clear_pending
;
1820 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1821 intin_no
) & AV_PENDING
) == 0) {
1830 * Checks to see if the IOAPIC interrupt entry specified has its Remote IRR
1831 * bit set. Calls functions that modify the function that setlvlx points to,
1832 * so that the reprogramming can be retried very shortly.
1834 * This function will mask the RDT entry if the interrupt is level-triggered.
1835 * (The caller is responsible for unmasking the RDT entry.)
1837 * Returns non-zero if the caller should defer IOAPIC reprogramming.
1840 apic_check_stuck_interrupt(apic_irq_t
*irq_ptr
, int old_bind_cpu
,
1841 int new_bind_cpu
, int ioapic_ix
, int intin_no
, int which_irq
,
1842 struct ioapic_reprogram_data
*drep
)
1849 * Wait for the delivery pending bit to clear.
1854 apic_ioapic_wait_pending_clear(ioapic_ix
, intin_no
);
1857 * Mask the RDT entry, but only if it's a level-triggered
1860 rdt_entry
= READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1862 if ((rdt_entry
& (AV_LEVEL
|AV_MASK
)) == AV_LEVEL
) {
1865 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
, intin_no
,
1866 AV_MASK
| rdt_entry
);
1869 if ((rdt_entry
& AV_LEVEL
) == AV_LEVEL
) {
1871 * If there was a race and an interrupt was injected
1872 * just before we masked, check for that case here.
1873 * Then, unmask the RDT entry and try again. If we're
1874 * on our last try, don't unmask (because we want the
1875 * RDT entry to remain masked for the rest of the
1878 rdt_entry
= READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1880 if ((rdt_entry
& AV_PENDING
) &&
1881 (reps
< apic_max_reps_clear_pending
)) {
1883 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1884 intin_no
, rdt_entry
& ~AV_MASK
);
1888 } while ((rdt_entry
& AV_PENDING
) &&
1889 (reps
< apic_max_reps_clear_pending
));
1892 if (rdt_entry
& AV_PENDING
)
1893 apic_intr_deliver_timeouts
++;
1897 * If the remote IRR bit is set, then the interrupt has been sent
1898 * to a CPU for processing. We have no choice but to wait for
1899 * that CPU to process the interrupt, at which point the remote IRR
1900 * bit will be cleared.
1902 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
, intin_no
) &
1903 (AV_LEVEL
|AV_REMOTE_IRR
)) == (AV_LEVEL
|AV_REMOTE_IRR
)) {
1906 * If the CPU that this RDT is bound to is NOT the current
1907 * CPU, wait until that CPU handles the interrupt and ACKs
1908 * it. If this interrupt is not bound to any CPU (that is,
1909 * if it's bound to the logical destination of "anyone"), it
1910 * may have been delivered to the current CPU so handle that
1911 * case by deferring the reprogramming (below).
1913 if ((old_bind_cpu
!= IRQ_UNBOUND
) &&
1914 (old_bind_cpu
!= IRQ_UNINIT
) &&
1915 (old_bind_cpu
!= psm_get_cpu_id())) {
1916 for (waited
= 0; waited
< apic_max_reps_clear_pending
;
1918 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix
,
1919 intin_no
) & AV_REMOTE_IRR
) == 0) {
1921 delete_defer_repro_ent(which_irq
);
1923 /* Remote IRR has cleared! */
1930 * If we waited and the Remote IRR bit is still not cleared,
1931 * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
1932 * times for this interrupt, try the last-ditch workaround:
1934 if (drep
&& drep
->tries
>= APIC_REPROGRAM_MAX_TRIES
) {
1936 apic_last_ditch_clear_remote_irr(ioapic_ix
, intin_no
);
1938 /* Mark this one as reprogrammed: */
1939 delete_defer_repro_ent(which_irq
);
1944 apic_intr_deferrals
++;
1948 * If waiting for the Remote IRR bit (above) didn't
1949 * allow it to clear, defer the reprogramming.
1950 * Add a new deferred-programming entry if the
1951 * caller passed a NULL one (and update the existing one
1952 * in case anything changed).
1954 add_defer_repro_ent(irq_ptr
, which_irq
, new_bind_cpu
);
1958 /* Inform caller to defer IOAPIC programming: */
1964 /* Remote IRR is clear */
1965 delete_defer_repro_ent(which_irq
);
1971 * Called to migrate all interrupts at an irq to another cpu.
1972 * Must be called with interrupts disabled and apic_ioapic_lock held
1975 apic_rebind_all(apic_irq_t
*irq_ptr
, int bind_cpu
)
1977 apic_irq_t
*irqptr
= irq_ptr
;
1981 if (irqptr
->airq_temp_cpu
!= IRQ_UNINIT
)
1982 retval
|= apic_rebind(irqptr
, bind_cpu
, NULL
);
1983 irqptr
= irqptr
->airq_next
;
1990 * apic_intr_redistribute does all the messy computations for identifying
1991 * which interrupt to move to which CPU. Currently we do just one interrupt
1992 * at a time. This reduces the time we spent doing all this within clock
1993 * interrupt. When it is done in idle, we could do more than 1.
1994 * First we find the most busy and the most free CPU (time in ISR only)
1995 * skipping those CPUs that has been identified as being ineligible (cpu_skip)
1996 * Then we look for IRQs which are closest to the difference between the
1997 * most busy CPU and the average ISR load. We try to find one whose load
1998 * is less than difference.If none exists, then we chose one larger than the
1999 * difference, provided it does not make the most idle CPU worse than the
2000 * most busy one. In the end, we clear all the busy fields for CPUs. For
2001 * IRQs, they are cleared as they are scanned.
2004 apic_intr_redistribute(void)
2006 int busiest_cpu
, most_free_cpu
;
2007 int cpu_free
, cpu_busy
, max_busy
, min_busy
;
2009 int average_busy
, cpus_online
;
2012 apic_cpus_info_t
*cpu_infop
;
2013 apic_irq_t
*min_busy_irq
= NULL
;
2014 apic_irq_t
*max_busy_irq
= NULL
;
2016 busiest_cpu
= most_free_cpu
= -1;
2017 cpu_free
= cpu_busy
= max_busy
= average_busy
= 0;
2018 min_free
= apic_sample_factor_redistribution
;
2021 * Below we will check for CPU_INTR_ENABLE, bound, temp_bound, temp_cpu
2022 * without ioapic_lock. That is OK as we are just doing statistical
2023 * sampling anyway and any inaccuracy now will get corrected next time
2024 * The call to rebind which actually changes things will make sure
2025 * we are consistent.
2027 for (i
= 0; i
< apic_nproc
; i
++) {
2028 if (apic_cpu_in_range(i
) &&
2029 !(apic_redist_cpu_skip
& (1 << i
)) &&
2030 (apic_cpus
[i
].aci_status
& APIC_CPU_INTR_ENABLE
)) {
2032 cpu_infop
= &apic_cpus
[i
];
2034 * If no unbound interrupts or only 1 total on this
2037 if (!cpu_infop
->aci_temp_bound
||
2038 (cpu_infop
->aci_bound
+ cpu_infop
->aci_temp_bound
)
2040 apic_redist_cpu_skip
|= 1 << i
;
2044 busy
= cpu_infop
->aci_busy
;
2045 average_busy
+= busy
;
2047 if (max_busy
< busy
) {
2051 if (min_free
> busy
) {
2055 if (busy
> apic_int_busy_mark
) {
2058 if (busy
< apic_int_free_mark
)
2063 if ((cpu_busy
&& cpu_free
) ||
2064 (max_busy
>= (min_free
+ apic_diff_for_redistribution
))) {
2066 apic_num_imbalance
++;
2068 if (apic_verbose
& APIC_VERBOSE_IOAPIC_FLAG
) {
2070 "redistribute busy=%x free=%x max=%x min=%x",
2071 cpu_busy
, cpu_free
, max_busy
, min_free
);
2076 average_busy
/= cpus_online
;
2078 diff
= max_busy
- average_busy
;
2079 min_busy
= max_busy
; /* start with the max possible value */
2081 min_busy_irq
= max_busy_irq
= NULL
;
2082 i
= apic_min_device_irq
;
2083 for (; i
<= apic_max_device_irq
; i
++) {
2084 apic_irq_t
*irq_ptr
;
2085 /* Change to linked list per CPU ? */
2086 if ((irq_ptr
= apic_irq_table
[i
]) == NULL
)
2088 /* Check for irq_busy & decide which one to move */
2089 /* Also zero them for next round */
2090 if ((irq_ptr
->airq_temp_cpu
== busiest_cpu
) &&
2091 irq_ptr
->airq_busy
) {
2092 if (irq_ptr
->airq_busy
< diff
) {
2094 * Check for least busy CPU,
2095 * best fit or what ?
2097 if (max_busy
< irq_ptr
->airq_busy
) {
2099 * Most busy within the
2100 * required differential
2102 max_busy
= irq_ptr
->airq_busy
;
2103 max_busy_irq
= irq_ptr
;
2106 if (min_busy
> irq_ptr
->airq_busy
) {
2108 * least busy, but more than
2112 (diff
+ average_busy
-
2115 * Making sure new cpu
2122 min_busy_irq
= irq_ptr
;
2127 irq_ptr
->airq_busy
= 0;
2130 if (max_busy_irq
!= NULL
) {
2132 if (apic_verbose
& APIC_VERBOSE_IOAPIC_FLAG
) {
2133 prom_printf("rebinding %x to %x",
2134 max_busy_irq
->airq_vector
, most_free_cpu
);
2137 iflag
= intr_clear();
2138 if (lock_try(&apic_ioapic_lock
)) {
2139 if (apic_rebind_all(max_busy_irq
,
2140 most_free_cpu
) == 0) {
2141 /* Make change permenant */
2142 max_busy_irq
->airq_cpu
=
2143 (uint32_t)most_free_cpu
;
2145 lock_clear(&apic_ioapic_lock
);
2147 intr_restore(iflag
);
2149 } else if (min_busy_irq
!= NULL
) {
2151 if (apic_verbose
& APIC_VERBOSE_IOAPIC_FLAG
) {
2152 prom_printf("rebinding %x to %x",
2153 min_busy_irq
->airq_vector
, most_free_cpu
);
2157 iflag
= intr_clear();
2158 if (lock_try(&apic_ioapic_lock
)) {
2159 if (apic_rebind_all(min_busy_irq
,
2160 most_free_cpu
) == 0) {
2161 /* Make change permenant */
2162 min_busy_irq
->airq_cpu
=
2163 (uint32_t)most_free_cpu
;
2165 lock_clear(&apic_ioapic_lock
);
2167 intr_restore(iflag
);
2170 if (cpu_busy
!= (1 << busiest_cpu
)) {
2171 apic_redist_cpu_skip
|= 1 << busiest_cpu
;
2173 * We leave cpu_skip set so that next time we
2174 * can choose another cpu
2181 * found nothing. Could be that we skipped over valid CPUs
2182 * or we have balanced everything. If we had a variable
2183 * ticks_for_redistribution, it could be increased here.
2184 * apic_int_busy, int_free etc would also need to be
2187 if (apic_redist_cpu_skip
)
2188 apic_redist_cpu_skip
= 0;
2190 for (i
= 0; i
< apic_nproc
; i
++) {
2191 if (apic_cpu_in_range(i
)) {
2192 apic_cpus
[i
].aci_busy
= 0;
2198 apic_cleanup_busy(void)
2201 apic_irq_t
*irq_ptr
;
2203 for (i
= 0; i
< apic_nproc
; i
++) {
2204 if (apic_cpu_in_range(i
)) {
2205 apic_cpus
[i
].aci_busy
= 0;
2209 for (i
= apic_min_device_irq
; i
<= apic_max_device_irq
; i
++) {
2210 if ((irq_ptr
= apic_irq_table
[i
]) != NULL
)
2211 irq_ptr
->airq_busy
= 0;