2 * Copyright (c) 1996, by Steve Passe
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 * derived from this software without specific prior written permission.
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * $FreeBSD: src/sys/i386/i386/mp_machdep.c,v 1.115.2.15 2003/03/14 21:22:35 jhb Exp $
26 * $DragonFly: src/sys/platform/pc32/i386/mp_machdep.c,v 1.60 2008/06/07 12:03:52 mneumann Exp $
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/sysctl.h>
35 #include <sys/malloc.h>
36 #include <sys/memrange.h>
37 #include <sys/cons.h> /* cngetc() */
38 #include <sys/machintr.h>
40 #include <sys/mplock2.h>
43 #include <vm/vm_param.h>
45 #include <vm/vm_kern.h>
46 #include <vm/vm_extern.h>
48 #include <vm/vm_map.h>
54 #include <machine/smp.h>
55 #include <machine_base/apic/apicreg.h>
56 #include <machine/atomic.h>
57 #include <machine/cpufunc.h>
58 #include <machine_base/apic/mpapic.h>
59 #include <machine/psl.h>
60 #include <machine/segments.h>
61 #include <machine/tss.h>
62 #include <machine/specialreg.h>
63 #include <machine/globaldata.h>
64 #include <machine/pmap_inval.h>
66 #include <machine/md_var.h> /* setidt() */
67 #include <machine_base/icu/icu.h> /* IPIs */
68 #include <machine_base/isa/intr_machdep.h> /* IPIs */
70 #define FIXUP_EXTRA_APIC_INTS 8 /* additional entries we may create */
72 #define WARMBOOT_TARGET 0
73 #define WARMBOOT_OFF (KERNBASE + 0x0467)
74 #define WARMBOOT_SEG (KERNBASE + 0x0469)
76 #define BIOS_BASE (0xf0000)
77 #define BIOS_SIZE (0x10000)
78 #define BIOS_COUNT (BIOS_SIZE/4)
80 #define CMOS_REG (0x70)
81 #define CMOS_DATA (0x71)
82 #define BIOS_RESET (0x0f)
83 #define BIOS_WARM (0x0a)
85 #define PROCENTRY_FLAG_EN 0x01
86 #define PROCENTRY_FLAG_BP 0x02
87 #define IOAPICENTRY_FLAG_EN 0x01
90 /* MP Floating Pointer Structure */
91 typedef struct MPFPS
{
104 /* MP Configuration Table Header */
105 typedef struct MPCTH
{
107 u_short base_table_length
;
111 u_char product_id
[12];
112 u_int32_t oem_table_pointer
;
113 u_short oem_table_size
;
115 u_int32_t apic_address
;
116 u_short extended_table_length
;
117 u_char extended_table_checksum
;
122 typedef struct PROCENTRY
{
127 u_int32_t cpu_signature
;
128 u_int32_t feature_flags
;
133 typedef struct BUSENTRY
{
139 typedef struct IOAPICENTRY
{
144 u_int32_t apic_address
;
145 } *io_apic_entry_ptr
;
147 typedef struct INTENTRY
{
157 /* descriptions of MP basetable entries */
158 typedef struct BASETABLE_ENTRY
{
167 vm_size_t mp_cth_mapsz
;
170 typedef int (*mptable_iter_func
)(void *, const void *, int);
173 * this code MUST be enabled here and in mpboot.s.
174 * it follows the very early stages of AP boot by placing values in CMOS ram.
175 * it NORMALLY will never be needed and thus the primitive method for enabling.
178 #if defined(CHECK_POINTS)
179 #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA))
180 #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
182 #define CHECK_INIT(D); \
183 CHECK_WRITE(0x34, (D)); \
184 CHECK_WRITE(0x35, (D)); \
185 CHECK_WRITE(0x36, (D)); \
186 CHECK_WRITE(0x37, (D)); \
187 CHECK_WRITE(0x38, (D)); \
188 CHECK_WRITE(0x39, (D));
190 #define CHECK_PRINT(S); \
191 kprintf("%s: %d, %d, %d, %d, %d, %d\n", \
200 #else /* CHECK_POINTS */
202 #define CHECK_INIT(D)
203 #define CHECK_PRINT(S)
205 #endif /* CHECK_POINTS */
208 * Values to send to the POST hardware.
210 #define MP_BOOTADDRESS_POST 0x10
211 #define MP_PROBE_POST 0x11
212 #define MPTABLE_PASS1_POST 0x12
214 #define MP_START_POST 0x13
215 #define MP_ENABLE_POST 0x14
216 #define MPTABLE_PASS2_POST 0x15
218 #define START_ALL_APS_POST 0x16
219 #define INSTALL_AP_TRAMP_POST 0x17
220 #define START_AP_POST 0x18
222 #define MP_ANNOUNCE_POST 0x19
224 static int need_hyperthreading_fixup
;
225 static u_int logical_cpus
;
226 u_int logical_cpus_mask
;
228 static int madt_probe_test
;
229 TUNABLE_INT("hw.madt_probe_test", &madt_probe_test
);
231 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
232 int current_postcode
;
234 /** XXX FIXME: what system files declare these??? */
235 extern struct region_descriptor r_gdt
, r_idt
;
237 int mp_naps
; /* # of Applications processors */
239 static int mp_nbusses
; /* # of busses */
240 int mp_napics
; /* # of IO APICs */
242 vm_offset_t cpu_apic_address
;
244 vm_offset_t io_apic_address
[NAPICID
]; /* NAPICID is more than enough */
245 u_int32_t
*io_apic_versions
;
249 u_int32_t cpu_apic_versions
[MAXCPU
];
251 extern int64_t tsc_offsets
[];
253 extern u_long ebda_addr
;
256 struct apic_intmapinfo int_to_apicintpin
[APIC_INTMAPSIZE
];
260 * APIC ID logical/physical mapping structures.
261 * We oversize these to simplify boot-time config.
263 int cpu_num_to_apic_id
[NAPICID
];
265 int io_num_to_apic_id
[NAPICID
];
267 int apic_id_to_logical
[NAPICID
];
269 /* AP uses this during bootstrap. Do not staticize. */
274 * SMP page table page. Setup by locore to point to a page table
275 * page from which we allocate per-cpu privatespace areas io_apics,
279 #define IO_MAPPING_START_INDEX \
280 (SMP_MAXCPU * sizeof(struct privatespace) / PAGE_SIZE)
282 extern pt_entry_t
*SMPpt
;
284 struct pcb stoppcbs
[MAXCPU
];
286 extern inthand_t
IDTVEC(fast_syscall
), IDTVEC(fast_syscall32
);
288 static basetable_entry basetable_entry_types
[] =
290 {0, 20, "Processor"},
298 * Local data and functions.
301 static u_int boot_address
;
302 static u_int base_memory
;
303 static int mp_finish
;
305 static void mp_enable(u_int boot_addr
);
307 static int mptable_iterate_entries(const mpcth_t
,
308 mptable_iter_func
, void *);
309 static int mptable_probe(void);
310 static int mptable_check(vm_paddr_t
);
311 static long mptable_search_sig(u_int32_t target
, int count
);
312 static void mptable_hyperthread_fixup(u_int id_mask
);
313 static void mptable_pass1(struct mptable_pos
*);
314 static int mptable_pass2(struct mptable_pos
*);
315 static void mptable_default(int type
);
316 static void mptable_fix(void);
317 static int mptable_map(struct mptable_pos
*, vm_paddr_t
);
318 static void mptable_unmap(struct mptable_pos
*);
321 static void setup_apic_irq_mapping(void);
322 static int apic_int_is_bus_type(int intr
, int bus_type
);
324 static int start_all_aps(u_int boot_addr
);
326 static void install_ap_tramp(u_int boot_addr
);
328 static int start_ap(struct mdglobaldata
*gd
, u_int boot_addr
, int smibest
);
329 static int smitest(void);
331 static cpumask_t smp_startup_mask
= 1; /* which cpus have been started */
332 cpumask_t smp_active_mask
= 1; /* which cpus are ready for IPIs etc? */
333 SYSCTL_INT(_machdep
, OID_AUTO
, smp_active
, CTLFLAG_RD
, &smp_active_mask
, 0, "");
334 static u_int bootMP_size
;
337 * Calculate usable address in base memory for AP trampoline code.
340 mp_bootaddress(u_int basemem
)
342 POSTCODE(MP_BOOTADDRESS_POST
);
344 base_memory
= basemem
;
346 bootMP_size
= mptramp_end
- mptramp_start
;
347 boot_address
= trunc_page(basemem
* 1024); /* round down to 4k boundary */
348 if (((basemem
* 1024) - boot_address
) < bootMP_size
)
349 boot_address
-= PAGE_SIZE
; /* not enough, lower by 4k */
350 /* 3 levels of page table pages */
351 mptramp_pagetables
= boot_address
- (PAGE_SIZE
* 3);
353 return mptramp_pagetables
;
358 * Look for an Intel MP spec table (ie, SMP capable hardware).
367 * Make sure our SMPpt[] page table is big enough to hold all the
370 KKASSERT(IO_MAPPING_START_INDEX
< NPTEPG
- 2);
372 POSTCODE(MP_PROBE_POST
);
374 /* see if EBDA exists */
375 if (ebda_addr
!= 0) {
376 /* search first 1K of EBDA */
377 target
= (u_int32_t
)ebda_addr
;
378 if ((x
= mptable_search_sig(target
, 1024 / 4)) > 0)
381 /* last 1K of base memory, effective 'top of base' passed in */
382 target
= (u_int32_t
)(base_memory
- 0x400);
383 if ((x
= mptable_search_sig(target
, 1024 / 4)) > 0)
387 /* search the BIOS */
388 target
= (u_int32_t
)BIOS_BASE
;
389 if ((x
= mptable_search_sig(target
, BIOS_COUNT
)) > 0)
396 struct mptable_check_cbarg
{
402 mptable_check_callback(void *xarg
, const void *pos
, int type
)
404 const struct PROCENTRY
*ent
;
405 struct mptable_check_cbarg
*arg
= xarg
;
411 if ((ent
->cpu_flags
& PROCENTRY_FLAG_EN
) == 0)
415 if (ent
->cpu_flags
& PROCENTRY_FLAG_BP
) {
416 if (arg
->found_bsp
) {
417 kprintf("more than one BSP in base MP table\n");
426 mptable_check(vm_paddr_t mpfps_paddr
)
428 struct mptable_pos mpt
;
429 struct mptable_check_cbarg arg
;
433 if (mpfps_paddr
== 0)
436 error
= mptable_map(&mpt
, mpfps_paddr
);
440 if (mpt
.mp_fps
->mpfb1
!= 0)
448 if (cth
->apic_address
== 0)
451 bzero(&arg
, sizeof(arg
));
452 error
= mptable_iterate_entries(cth
, mptable_check_callback
, &arg
);
454 if (arg
.cpu_count
== 0) {
455 kprintf("MP table contains no processor entries\n");
457 } else if (!arg
.found_bsp
) {
458 kprintf("MP table does not contains BSP entry\n");
468 mptable_iterate_entries(const mpcth_t cth
, mptable_iter_func func
, void *arg
)
470 int count
, total_size
;
471 const void *position
;
473 KKASSERT(cth
->base_table_length
>= sizeof(struct MPCTH
));
474 total_size
= cth
->base_table_length
- sizeof(struct MPCTH
);
475 position
= (const uint8_t *)cth
+ sizeof(struct MPCTH
);
476 count
= cth
->entry_count
;
481 KKASSERT(total_size
>= 0);
482 if (total_size
== 0) {
483 kprintf("invalid base MP table, "
484 "entry count and length mismatch\n");
488 type
= *(const uint8_t *)position
;
490 case 0: /* processor_entry */
491 case 1: /* bus_entry */
492 case 2: /* io_apic_entry */
493 case 3: /* int_entry */
494 case 4: /* int_entry */
497 kprintf("unknown base MP table entry type %d\n", type
);
501 if (total_size
< basetable_entry_types
[type
].length
) {
502 kprintf("invalid base MP table length, "
503 "does not contain all entries\n");
506 total_size
-= basetable_entry_types
[type
].length
;
508 error
= func(arg
, position
, type
);
512 position
= (const uint8_t *)position
+
513 basetable_entry_types
[type
].length
;
520 * Startup the SMP processors.
525 POSTCODE(MP_START_POST
);
526 mp_enable(boot_address
);
531 * Print various information about the SMP system hardware and setup.
538 POSTCODE(MP_ANNOUNCE_POST
);
540 kprintf("DragonFly/MP: Multiprocessor motherboard\n");
541 kprintf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
542 kprintf(", version: 0x%08x", cpu_apic_versions
[0]);
543 kprintf(", at 0x%08jx\n", (intmax_t)cpu_apic_address
);
544 for (x
= 1; x
<= mp_naps
; ++x
) {
545 kprintf(" cpu%d (AP): apic id: %2d", x
, CPU_TO_ID(x
));
546 kprintf(", version: 0x%08x", cpu_apic_versions
[x
]);
547 kprintf(", at 0x%08jx\n", (intmax_t)cpu_apic_address
);
551 for (x
= 0; x
< mp_napics
; ++x
) {
552 kprintf(" io%d (APIC): apic id: %2d", x
, IO_TO_ID(x
));
553 kprintf(", version: 0x%08x", io_apic_versions
[x
]);
554 kprintf(", at 0x%08lx\n", io_apic_address
[x
]);
557 kprintf(" Warning: APIC I/O disabled\n");
562 * AP cpu's call this to sync up protected mode.
564 * WARNING! %gs is not set up on entry. This routine sets up %gs.
570 int x
, myid
= bootAP
;
572 struct mdglobaldata
*md
;
573 struct privatespace
*ps
;
575 ps
= &CPU_prvspace
[myid
];
577 gdt_segs
[GPROC0_SEL
].ssd_base
=
578 (long) &ps
->mdglobaldata
.gd_common_tss
;
579 ps
->mdglobaldata
.mi
.gd_prvspace
= ps
;
581 /* We fill the 32-bit segment descriptors */
582 for (x
= 0; x
< NGDT
; x
++) {
583 if (x
!= GPROC0_SEL
&& x
!= (GPROC0_SEL
+ 1))
584 ssdtosd(&gdt_segs
[x
], &gdt
[myid
* NGDT
+ x
]);
586 /* And now a 64-bit one */
587 ssdtosyssd(&gdt_segs
[GPROC0_SEL
],
588 (struct system_segment_descriptor
*)&gdt
[myid
* NGDT
+ GPROC0_SEL
]);
590 r_gdt
.rd_limit
= NGDT
* sizeof(gdt
[0]) - 1;
591 r_gdt
.rd_base
= (long) &gdt
[myid
* NGDT
];
592 lgdt(&r_gdt
); /* does magic intra-segment return */
594 /* lgdt() destroys the GSBASE value, so we load GSBASE after lgdt() */
595 wrmsr(MSR_FSBASE
, 0); /* User value */
596 wrmsr(MSR_GSBASE
, (u_int64_t
)ps
);
597 wrmsr(MSR_KGSBASE
, 0); /* XXX User value while we're in the kernel */
603 mdcpu
->gd_currentldt
= _default_ldt
;
606 gsel_tss
= GSEL(GPROC0_SEL
, SEL_KPL
);
607 gdt
[myid
* NGDT
+ GPROC0_SEL
].sd_type
= SDT_SYSTSS
;
609 md
= mdcpu
; /* loaded through %gs:0 (mdglobaldata.mi.gd_prvspace)*/
611 md
->gd_common_tss
.tss_rsp0
= 0; /* not used until after switch */
613 md
->gd_common_tss
.tss_ioopt
= (sizeof md
->gd_common_tss
) << 16;
615 md
->gd_tss_gdt
= &gdt
[myid
* NGDT
+ GPROC0_SEL
];
616 md
->gd_common_tssd
= *md
->gd_tss_gdt
;
618 /* double fault stack */
619 md
->gd_common_tss
.tss_ist1
=
620 (long)&md
->mi
.gd_prvspace
->idlestack
[
621 sizeof(md
->mi
.gd_prvspace
->idlestack
)];
626 * Set to a known state:
627 * Set by mpboot.s: CR0_PG, CR0_PE
628 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM
631 cr0
&= ~(CR0_CD
| CR0_NW
| CR0_EM
);
634 /* Set up the fast syscall stuff */
635 msr
= rdmsr(MSR_EFER
) | EFER_SCE
;
636 wrmsr(MSR_EFER
, msr
);
637 wrmsr(MSR_LSTAR
, (u_int64_t
)IDTVEC(fast_syscall
));
638 wrmsr(MSR_CSTAR
, (u_int64_t
)IDTVEC(fast_syscall32
));
639 msr
= ((u_int64_t
)GSEL(GCODE_SEL
, SEL_KPL
) << 32) |
640 ((u_int64_t
)GSEL(GUCODE32_SEL
, SEL_UPL
) << 48);
641 wrmsr(MSR_STAR
, msr
);
642 wrmsr(MSR_SF_MASK
, PSL_NT
|PSL_T
|PSL_I
|PSL_C
|PSL_D
);
644 pmap_set_opt(); /* PSE/4MB pages, etc */
646 /* Initialize the PAT MSR. */
650 /* set up CPU registers and state */
653 /* set up SSE/NX registers */
656 /* set up FPU state on the AP */
657 npxinit(__INITIAL_NPXCW__
);
659 /* disable the APIC, just to be SURE */
660 lapic
->svr
&= ~APIC_SVR_ENABLE
;
662 /* data returned to BSP */
663 cpu_apic_versions
[0] = lapic
->version
;
666 /*******************************************************************
667 * local functions and data
671 * start the SMP system
674 mp_enable(u_int boot_addr
)
681 vm_paddr_t mpfps_paddr
;
683 POSTCODE(MP_ENABLE_POST
);
685 if (madt_probe_test
) {
688 mpfps_paddr
= mptable_probe();
689 if (mptable_check(mpfps_paddr
))
694 struct mptable_pos mpt
;
696 mptable_map(&mpt
, mpfps_paddr
);
699 * We can safely map physical memory into SMPpt after
700 * mptable_pass1() completes.
704 if (cpu_apic_address
== 0)
705 panic("mp_enable: no local apic!\n");
707 /* examine the MP table for needed info */
708 x
= mptable_pass2(&mpt
);
713 * can't process default configs till the
714 * CPU APIC is pmapped
719 /* post scan cleanup */
723 * lapic not mapped yet (pmap_init is called too late)
725 lapic
= pmap_mapdev_uncacheable(cpu_apic_address
,
726 sizeof(struct LAPIC
));
728 vm_paddr_t madt_paddr
;
731 madt_paddr
= madt_probe();
733 panic("mp_enable: madt_probe failed\n");
735 cpu_apic_address
= madt_pass1(madt_paddr
);
736 if (cpu_apic_address
== 0)
737 panic("mp_enable: no local apic (madt)!\n");
740 * lapic not mapped yet (pmap_init is called too late)
742 * XXX: where is the best place to set lapic?
744 lapic
= pmap_mapdev_uncacheable(cpu_apic_address
,
745 sizeof(struct LAPIC
));
747 bsp_apic_id
= (lapic
->id
& 0xff000000) >> 24;
748 if (madt_pass2(madt_paddr
, bsp_apic_id
))
749 panic("mp_enable: madt_pass2 failed\n");
754 setup_apic_irq_mapping();
756 /* fill the LOGICAL io_apic_versions table */
757 for (apic
= 0; apic
< mp_napics
; ++apic
) {
758 ux
= io_apic_read(apic
, IOAPIC_VER
);
759 io_apic_versions
[apic
] = ux
;
760 io_apic_set_id(apic
, IO_TO_ID(apic
));
763 /* program each IO APIC in the system */
764 for (apic
= 0; apic
< mp_napics
; ++apic
)
765 if (io_apic_setup(apic
) < 0)
766 panic("IO APIC setup failure");
771 * These are required for SMP operation
774 /* install a 'Spurious INTerrupt' vector */
775 setidt(XSPURIOUSINT_OFFSET
, Xspuriousint
,
776 SDT_SYSIGT
, SEL_KPL
, 0);
778 /* install an inter-CPU IPI for TLB invalidation */
779 setidt(XINVLTLB_OFFSET
, Xinvltlb
,
780 SDT_SYSIGT
, SEL_KPL
, 0);
782 /* install an inter-CPU IPI for IPIQ messaging */
783 setidt(XIPIQ_OFFSET
, Xipiq
,
784 SDT_SYSIGT
, SEL_KPL
, 0);
786 /* install a timer vector */
787 setidt(XTIMER_OFFSET
, Xtimer
,
788 SDT_SYSIGT
, SEL_KPL
, 0);
790 /* install an inter-CPU IPI for CPU stop/restart */
791 setidt(XCPUSTOP_OFFSET
, Xcpustop
,
792 SDT_SYSIGT
, SEL_KPL
, 0);
794 /* start each Application Processor */
795 start_all_aps(boot_addr
);
800 * look for the MP spec signature
803 /* string defined by the Intel MP Spec as identifying the MP table */
804 #define MP_SIG 0x5f504d5f /* _MP_ */
805 #define NEXT(X) ((X) += 4)
807 mptable_search_sig(u_int32_t target
, int count
)
813 KKASSERT(target
!= 0);
815 map_size
= count
* sizeof(u_int32_t
);
816 addr
= pmap_mapdev((vm_paddr_t
)target
, map_size
);
819 for (x
= 0; x
< count
; NEXT(x
)) {
820 if (addr
[x
] == MP_SIG
) {
821 /* make array index a byte index */
822 ret
= target
+ (x
* sizeof(u_int32_t
));
827 pmap_unmapdev((vm_offset_t
)addr
, map_size
);
832 typedef struct BUSDATA
{
834 enum busTypes bus_type
;
837 typedef struct INTDATA
{
847 typedef struct BUSTYPENAME
{
854 static bus_type_name bus_type_table
[] =
860 {UNKNOWN_BUSTYPE
, "---"},
863 {UNKNOWN_BUSTYPE
, "---"},
864 {UNKNOWN_BUSTYPE
, "---"},
865 {UNKNOWN_BUSTYPE
, "---"},
866 {UNKNOWN_BUSTYPE
, "---"},
867 {UNKNOWN_BUSTYPE
, "---"},
869 {UNKNOWN_BUSTYPE
, "---"},
870 {UNKNOWN_BUSTYPE
, "---"},
871 {UNKNOWN_BUSTYPE
, "---"},
872 {UNKNOWN_BUSTYPE
, "---"},
874 {UNKNOWN_BUSTYPE
, "---"}
877 /* from MP spec v1.4, table 5-1 */
878 static int default_data
[7][5] =
880 /* nbus, id0, type0, id1, type1 */
881 {1, 0, ISA
, 255, 255},
882 {1, 0, EISA
, 255, 255},
883 {1, 0, EISA
, 255, 255},
884 {1, 0, MCA
, 255, 255},
886 {2, 0, EISA
, 1, PCI
},
891 static bus_datum
*bus_data
;
893 /* the IO INT data, one entry per possible APIC INTerrupt */
894 static io_int
*io_apic_ints
;
899 static int processor_entry (proc_entry_ptr entry
, int cpu
);
901 static int bus_entry (bus_entry_ptr entry
, int bus
);
902 static int io_apic_entry (io_apic_entry_ptr entry
, int apic
);
903 static int int_entry (int_entry_ptr entry
, int intr
);
904 static int lookup_bus_type (char *name
);
909 * 1st pass on motherboard's Intel MP specification table.
912 * cpu_apic_address (common to all CPUs)
918 * need_hyperthreading_fixup
922 mptable_pass1(struct mptable_pos
*mpt
)
935 POSTCODE(MPTABLE_PASS1_POST
);
938 KKASSERT(fps
!= NULL
);
941 /* clear various tables */
942 for (x
= 0; x
< NAPICID
; ++x
) {
943 io_apic_address
[x
] = ~0; /* IO APIC address table */
947 /* init everything to empty */
956 /* check for use of 'default' configuration */
957 if (fps
->mpfb1
!= 0) {
958 /* use default addresses */
959 cpu_apic_address
= DEFAULT_APIC_BASE
;
961 io_apic_address
[0] = DEFAULT_IO_APIC_BASE
;
964 /* fill in with defaults */
965 mp_naps
= 2; /* includes BSP */
967 mp_nbusses
= default_data
[fps
->mpfb1
- 1][0];
975 panic("MP Configuration Table Header MISSING!");
977 cpu_apic_address
= (vm_offset_t
) cth
->apic_address
;
979 /* walk the table, recording info of interest */
980 totalSize
= cth
->base_table_length
- sizeof(struct MPCTH
);
981 position
= (u_char
*) cth
+ sizeof(struct MPCTH
);
982 count
= cth
->entry_count
;
985 switch (type
= *(u_char
*) position
) {
986 case 0: /* processor_entry */
987 if (((proc_entry_ptr
)position
)->cpu_flags
988 & PROCENTRY_FLAG_EN
) {
991 ((proc_entry_ptr
)position
)->apic_id
;
994 case 1: /* bus_entry */
999 case 2: /* io_apic_entry */
1001 if (((io_apic_entry_ptr
)position
)->apic_flags
1002 & IOAPICENTRY_FLAG_EN
)
1003 io_apic_address
[mp_napics
++] =
1004 (vm_offset_t
)((io_apic_entry_ptr
)
1005 position
)->apic_address
;
1008 case 3: /* int_entry */
1013 case 4: /* int_entry */
1016 panic("mpfps Base Table HOSED!");
1020 totalSize
-= basetable_entry_types
[type
].length
;
1021 position
= (uint8_t *)position
+
1022 basetable_entry_types
[type
].length
;
1026 /* qualify the numbers */
1027 if (mp_naps
> MAXCPU
) {
1028 kprintf("Warning: only using %d of %d available CPUs!\n",
1033 /* See if we need to fixup HT logical CPUs. */
1034 mptable_hyperthread_fixup(id_mask
);
1036 --mp_naps
; /* subtract the BSP */
1041 * 2nd pass on motherboard's Intel MP specification table.
1045 * ID_TO_IO(N), phy APIC ID to log CPU/IO table
1046 * CPU_TO_ID(N), logical CPU to APIC ID table
1047 * IO_TO_ID(N), logical IO to APIC ID table
1052 mptable_pass2(struct mptable_pos
*mpt
)
1054 struct PROCENTRY proc
;
1062 int apic
, bus
, cpu
, intr
;
1065 POSTCODE(MPTABLE_PASS2_POST
);
1068 KKASSERT(fps
!= NULL
);
1070 /* Initialize fake proc entry for use with HT fixup. */
1071 bzero(&proc
, sizeof(proc
));
1073 proc
.cpu_flags
= PROCENTRY_FLAG_EN
;
1076 MALLOC(io_apic_versions
, u_int32_t
*, sizeof(u_int32_t
) * mp_napics
,
1077 M_DEVBUF
, M_WAITOK
);
1078 MALLOC(ioapic
, volatile ioapic_t
**, sizeof(ioapic_t
*) * mp_napics
,
1079 M_DEVBUF
, M_WAITOK
| M_ZERO
);
1080 MALLOC(io_apic_ints
, io_int
*, sizeof(io_int
) * (nintrs
+ FIXUP_EXTRA_APIC_INTS
),
1081 M_DEVBUF
, M_WAITOK
);
1082 MALLOC(bus_data
, bus_datum
*, sizeof(bus_datum
) * mp_nbusses
,
1083 M_DEVBUF
, M_WAITOK
);
1087 for (i
= 0; i
< mp_napics
; i
++) {
1088 ioapic
[i
] = permanent_io_mapping(io_apic_address
[i
]);
1092 /* clear various tables */
1093 for (x
= 0; x
< NAPICID
; ++x
) {
1094 CPU_TO_ID(x
) = -1; /* logical CPU to APIC ID table */
1096 ID_TO_IO(x
) = -1; /* phy APIC ID to log CPU/IO table */
1097 IO_TO_ID(x
) = -1; /* logical IO to APIC ID table */
1102 /* clear bus data table */
1103 for (x
= 0; x
< mp_nbusses
; ++x
)
1104 bus_data
[x
].bus_id
= 0xff;
1106 /* clear IO APIC INT table */
1107 for (x
= 0; x
< (nintrs
+ 1); ++x
) {
1108 io_apic_ints
[x
].int_type
= 0xff;
1109 io_apic_ints
[x
].int_vector
= 0xff;
1113 /* record whether PIC or virtual-wire mode */
1114 machintr_setvar_simple(MACHINTR_VAR_IMCR_PRESENT
, fps
->mpfb2
& 0x80);
1116 /* check for use of 'default' configuration */
1117 if (fps
->mpfb1
!= 0)
1118 return fps
->mpfb1
; /* return default configuration type */
1122 panic("MP Configuration Table Header MISSING!");
1124 /* walk the table, recording info of interest */
1125 totalSize
= cth
->base_table_length
- sizeof(struct MPCTH
);
1126 position
= (u_char
*) cth
+ sizeof(struct MPCTH
);
1127 count
= cth
->entry_count
;
1128 apic
= bus
= intr
= 0;
1129 cpu
= 1; /* pre-count the BSP */
1132 switch (type
= *(u_char
*) position
) {
1134 if (processor_entry(position
, cpu
))
1137 if (need_hyperthreading_fixup
) {
1139 * Create fake mptable processor entries
1140 * and feed them to processor_entry() to
1141 * enumerate the logical CPUs.
1143 proc
.apic_id
= ((proc_entry_ptr
)position
)->apic_id
;
1144 for (i
= 1; i
< logical_cpus
; i
++) {
1146 processor_entry(&proc
, cpu
);
1147 logical_cpus_mask
|= (1 << cpu
);
1154 if (bus_entry(position
, bus
))
1160 if (io_apic_entry(position
, apic
))
1166 if (int_entry(position
, intr
))
1171 /* int_entry(position); */
1174 panic("mpfps Base Table HOSED!");
1178 totalSize
-= basetable_entry_types
[type
].length
;
1179 position
= (uint8_t *)position
+ basetable_entry_types
[type
].length
;
1182 if (CPU_TO_ID(0) < 0)
1183 panic("NO BSP found!");
1185 /* report fact that its NOT a default configuration */
1191 * Check if we should perform a hyperthreading "fix-up" to
1192 * enumerate any logical CPU's that aren't already listed
1195 * XXX: We assume that all of the physical CPUs in the
1196 * system have the same number of logical CPUs.
1198 * XXX: We assume that APIC ID's are allocated such that
1199 * the APIC ID's for a physical processor are aligned
1200 * with the number of logical CPU's in the processor.
1203 mptable_hyperthread_fixup(u_int id_mask
)
1205 int i
, id
, lcpus_max
;
1207 if ((cpu_feature
& CPUID_HTT
) == 0)
1210 lcpus_max
= (cpu_procinfo
& CPUID_HTT_CORES
) >> 16;
1214 if (strcmp(cpu_vendor
, "GenuineIntel") == 0) {
1216 * INSTRUCTION SET REFERENCE, A-M (#253666)
1217 * Page 3-181, Table 3-20
1218 * "The nearest power-of-2 integer that is not smaller
1219 * than EBX[23:16] is the number of unique initial APIC
1220 * IDs reserved for addressing different logical
1221 * processors in a physical package."
1223 for (i
= 0; ; ++i
) {
1224 if ((1 << i
) >= lcpus_max
) {
1231 if (mp_naps
== lcpus_max
) {
1232 /* We have nothing to fix */
1234 } else if (mp_naps
== 1) {
1235 /* XXX this may be incorrect */
1236 logical_cpus
= lcpus_max
;
1238 int cur
, prev
, dist
;
1241 * Calculate the distances between two nearest
1242 * APIC IDs. If all such distances are same,
1243 * then it is the number of missing cpus that
1244 * we are going to fill later.
1246 dist
= cur
= prev
= -1;
1247 for (id
= 0; id
< MAXCPU
; ++id
) {
1248 if ((id_mask
& 1 << id
) == 0)
1253 int new_dist
= cur
- prev
;
1259 * Make sure that all distances
1260 * between two nearest APIC IDs
1263 if (dist
!= new_dist
)
1271 /* Must be power of 2 */
1272 if (dist
& (dist
- 1))
1275 /* Can't exceed CPU package capacity */
1276 if (dist
> lcpus_max
)
1277 logical_cpus
= lcpus_max
;
1279 logical_cpus
= dist
;
1283 * For each APIC ID of a CPU that is set in the mask,
1284 * scan the other candidate APIC ID's for this
1285 * physical processor. If any of those ID's are
1286 * already in the table, then kill the fixup.
1288 for (id
= 0; id
< MAXCPU
; id
++) {
1289 if ((id_mask
& 1 << id
) == 0)
1291 /* First, make sure we are on a logical_cpus boundary. */
1292 if (id
% logical_cpus
!= 0)
1294 for (i
= id
+ 1; i
< id
+ logical_cpus
; i
++)
1295 if ((id_mask
& 1 << i
) != 0)
1300 * Ok, the ID's checked out, so enable the fixup. We have to fixup
1301 * mp_naps right now.
1303 need_hyperthreading_fixup
= 1;
1304 mp_naps
*= logical_cpus
;
1308 mptable_map(struct mptable_pos
*mpt
, vm_paddr_t mpfps_paddr
)
1312 vm_size_t cth_mapsz
= 0;
1314 bzero(mpt
, sizeof(*mpt
));
1316 fps
= pmap_mapdev(mpfps_paddr
, sizeof(*fps
));
1317 if (fps
->pap
!= 0) {
1319 * Map configuration table header to get
1320 * the base table size
1322 cth
= pmap_mapdev(fps
->pap
, sizeof(*cth
));
1323 cth_mapsz
= cth
->base_table_length
;
1324 pmap_unmapdev((vm_offset_t
)cth
, sizeof(*cth
));
1326 if (cth_mapsz
< sizeof(*cth
)) {
1327 kprintf("invalid base MP table length %d\n",
1329 pmap_unmapdev((vm_offset_t
)fps
, sizeof(*fps
));
1334 * Map the base table
1336 cth
= pmap_mapdev(fps
->pap
, cth_mapsz
);
1341 mpt
->mp_cth_mapsz
= cth_mapsz
;
1347 mptable_unmap(struct mptable_pos
*mpt
)
1349 if (mpt
->mp_cth
!= NULL
) {
1350 pmap_unmapdev((vm_offset_t
)mpt
->mp_cth
, mpt
->mp_cth_mapsz
);
1352 mpt
->mp_cth_mapsz
= 0;
1354 if (mpt
->mp_fps
!= NULL
) {
1355 pmap_unmapdev((vm_offset_t
)mpt
->mp_fps
, sizeof(*mpt
->mp_fps
));
1363 assign_apic_irq(int apic
, int intpin
, int irq
)
1367 if (int_to_apicintpin
[irq
].ioapic
!= -1)
1368 panic("assign_apic_irq: inconsistent table");
1370 int_to_apicintpin
[irq
].ioapic
= apic
;
1371 int_to_apicintpin
[irq
].int_pin
= intpin
;
1372 int_to_apicintpin
[irq
].apic_address
= ioapic
[apic
];
1373 int_to_apicintpin
[irq
].redirindex
= IOAPIC_REDTBL
+ 2 * intpin
;
1375 for (x
= 0; x
< nintrs
; x
++) {
1376 if ((io_apic_ints
[x
].int_type
== 0 ||
1377 io_apic_ints
[x
].int_type
== 3) &&
1378 io_apic_ints
[x
].int_vector
== 0xff &&
1379 io_apic_ints
[x
].dst_apic_id
== IO_TO_ID(apic
) &&
1380 io_apic_ints
[x
].dst_apic_int
== intpin
)
1381 io_apic_ints
[x
].int_vector
= irq
;
1386 revoke_apic_irq(int irq
)
1392 if (int_to_apicintpin
[irq
].ioapic
== -1)
1393 panic("revoke_apic_irq: inconsistent table");
1395 oldapic
= int_to_apicintpin
[irq
].ioapic
;
1396 oldintpin
= int_to_apicintpin
[irq
].int_pin
;
1398 int_to_apicintpin
[irq
].ioapic
= -1;
1399 int_to_apicintpin
[irq
].int_pin
= 0;
1400 int_to_apicintpin
[irq
].apic_address
= NULL
;
1401 int_to_apicintpin
[irq
].redirindex
= 0;
1403 for (x
= 0; x
< nintrs
; x
++) {
1404 if ((io_apic_ints
[x
].int_type
== 0 ||
1405 io_apic_ints
[x
].int_type
== 3) &&
1406 io_apic_ints
[x
].int_vector
!= 0xff &&
1407 io_apic_ints
[x
].dst_apic_id
== IO_TO_ID(oldapic
) &&
1408 io_apic_ints
[x
].dst_apic_int
== oldintpin
)
1409 io_apic_ints
[x
].int_vector
= 0xff;
1417 allocate_apic_irq(int intr
)
1423 if (io_apic_ints
[intr
].int_vector
!= 0xff)
1424 return; /* Interrupt handler already assigned */
1426 if (io_apic_ints
[intr
].int_type
!= 0 &&
1427 (io_apic_ints
[intr
].int_type
!= 3 ||
1428 (io_apic_ints
[intr
].dst_apic_id
== IO_TO_ID(0) &&
1429 io_apic_ints
[intr
].dst_apic_int
== 0)))
1430 return; /* Not INT or ExtInt on != (0, 0) */
1433 while (irq
< APIC_INTMAPSIZE
&&
1434 int_to_apicintpin
[irq
].ioapic
!= -1)
1437 if (irq
>= APIC_INTMAPSIZE
)
1438 return; /* No free interrupt handlers */
1440 apic
= ID_TO_IO(io_apic_ints
[intr
].dst_apic_id
);
1441 intpin
= io_apic_ints
[intr
].dst_apic_int
;
1443 assign_apic_irq(apic
, intpin
, irq
);
1448 swap_apic_id(int apic
, int oldid
, int newid
)
1455 return; /* Nothing to do */
1457 kprintf("Changing APIC ID for IO APIC #%d from %d to %d in MP table\n",
1458 apic
, oldid
, newid
);
1460 /* Swap physical APIC IDs in interrupt entries */
1461 for (x
= 0; x
< nintrs
; x
++) {
1462 if (io_apic_ints
[x
].dst_apic_id
== oldid
)
1463 io_apic_ints
[x
].dst_apic_id
= newid
;
1464 else if (io_apic_ints
[x
].dst_apic_id
== newid
)
1465 io_apic_ints
[x
].dst_apic_id
= oldid
;
1468 /* Swap physical APIC IDs in IO_TO_ID mappings */
1469 for (oapic
= 0; oapic
< mp_napics
; oapic
++)
1470 if (IO_TO_ID(oapic
) == newid
)
1473 if (oapic
< mp_napics
) {
1474 kprintf("Changing APIC ID for IO APIC #%d from "
1475 "%d to %d in MP table\n",
1476 oapic
, newid
, oldid
);
1477 IO_TO_ID(oapic
) = oldid
;
1479 IO_TO_ID(apic
) = newid
;
1484 fix_id_to_io_mapping(void)
1488 for (x
= 0; x
< NAPICID
; x
++)
1491 for (x
= 0; x
<= mp_naps
; x
++)
1492 if (CPU_TO_ID(x
) < NAPICID
)
1493 ID_TO_IO(CPU_TO_ID(x
)) = x
;
1495 for (x
= 0; x
< mp_napics
; x
++)
1496 if (IO_TO_ID(x
) < NAPICID
)
1497 ID_TO_IO(IO_TO_ID(x
)) = x
;
1502 first_free_apic_id(void)
1506 for (freeid
= 0; freeid
< NAPICID
; freeid
++) {
1507 for (x
= 0; x
<= mp_naps
; x
++)
1508 if (CPU_TO_ID(x
) == freeid
)
1512 for (x
= 0; x
< mp_napics
; x
++)
1513 if (IO_TO_ID(x
) == freeid
)
1524 io_apic_id_acceptable(int apic
, int id
)
1526 int cpu
; /* Logical CPU number */
1527 int oapic
; /* Logical IO APIC number for other IO APIC */
1530 return 0; /* Out of range */
1532 for (cpu
= 0; cpu
<= mp_naps
; cpu
++)
1533 if (CPU_TO_ID(cpu
) == id
)
1534 return 0; /* Conflict with CPU */
1536 for (oapic
= 0; oapic
< mp_napics
&& oapic
< apic
; oapic
++)
1537 if (IO_TO_ID(oapic
) == id
)
1538 return 0; /* Conflict with other APIC */
1540 return 1; /* ID is acceptable for IO APIC */
1545 io_apic_find_int_entry(int apic
, int pin
)
1549 /* search each of the possible INTerrupt sources */
1550 for (x
= 0; x
< nintrs
; ++x
) {
1551 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
1552 (pin
== io_apic_ints
[x
].dst_apic_int
))
1553 return (&io_apic_ints
[x
]);
1561 * parse an Intel MP specification table
1569 int apic
; /* IO APIC unit number */
1570 int freeid
; /* Free physical APIC ID */
1571 int physid
; /* Current physical IO APIC ID */
1573 int bus_0
= 0; /* Stop GCC warning */
1574 int bus_pci
= 0; /* Stop GCC warning */
1578 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
1579 * did it wrong. The MP spec says that when more than 1 PCI bus
1580 * exists the BIOS must begin with bus entries for the PCI bus and use
1581 * actual PCI bus numbering. This implies that when only 1 PCI bus
1582 * exists the BIOS can choose to ignore this ordering, and indeed many
1583 * MP motherboards do ignore it. This causes a problem when the PCI
1584 * sub-system makes requests of the MP sub-system based on PCI bus
1585 * numbers. So here we look for the situation and renumber the
1586 * busses and associated INTs in an effort to "make it right".
1589 /* find bus 0, PCI bus, count the number of PCI busses */
1590 for (num_pci_bus
= 0, x
= 0; x
< mp_nbusses
; ++x
) {
1591 if (bus_data
[x
].bus_id
== 0) {
1594 if (bus_data
[x
].bus_type
== PCI
) {
1600 * bus_0 == slot of bus with ID of 0
1601 * bus_pci == slot of last PCI bus encountered
1604 /* check the 1 PCI bus case for sanity */
1605 /* if it is number 0 all is well */
1606 if (num_pci_bus
== 1 &&
1607 bus_data
[bus_pci
].bus_id
!= 0) {
1609 /* mis-numbered, swap with whichever bus uses slot 0 */
1611 /* swap the bus entry types */
1612 bus_data
[bus_pci
].bus_type
= bus_data
[bus_0
].bus_type
;
1613 bus_data
[bus_0
].bus_type
= PCI
;
1615 /* swap each relavant INTerrupt entry */
1616 id
= bus_data
[bus_pci
].bus_id
;
1617 for (x
= 0; x
< nintrs
; ++x
) {
1618 if (io_apic_ints
[x
].src_bus_id
== id
) {
1619 io_apic_ints
[x
].src_bus_id
= 0;
1621 else if (io_apic_ints
[x
].src_bus_id
== 0) {
1622 io_apic_ints
[x
].src_bus_id
= id
;
1627 /* Assign IO APIC IDs.
1629 * First try the existing ID. If a conflict is detected, try
1630 * the ID in the MP table. If a conflict is still detected, find
1633 * We cannot use the ID_TO_IO table before all conflicts has been
1634 * resolved and the table has been corrected.
1636 for (apic
= 0; apic
< mp_napics
; ++apic
) { /* For all IO APICs */
1638 /* First try to use the value set by the BIOS */
1639 physid
= io_apic_get_id(apic
);
1640 if (io_apic_id_acceptable(apic
, physid
)) {
1641 if (IO_TO_ID(apic
) != physid
)
1642 swap_apic_id(apic
, IO_TO_ID(apic
), physid
);
1646 /* Then check if the value in the MP table is acceptable */
1647 if (io_apic_id_acceptable(apic
, IO_TO_ID(apic
)))
1650 /* Last resort, find a free APIC ID and use it */
1651 freeid
= first_free_apic_id();
1652 if (freeid
>= NAPICID
)
1653 panic("No free physical APIC IDs found");
1655 if (io_apic_id_acceptable(apic
, freeid
)) {
1656 swap_apic_id(apic
, IO_TO_ID(apic
), freeid
);
1659 panic("Free physical APIC ID not usable");
1661 fix_id_to_io_mapping();
1663 /* detect and fix broken Compaq MP table */
1664 if (apic_int_type(0, 0) == -1) {
1665 kprintf("APIC_IO: MP table broken: 8259->APIC entry missing!\n");
1666 io_apic_ints
[nintrs
].int_type
= 3; /* ExtInt */
1667 io_apic_ints
[nintrs
].int_vector
= 0xff; /* Unassigned */
1668 /* XXX fixme, set src bus id etc, but it doesn't seem to hurt */
1669 io_apic_ints
[nintrs
].dst_apic_id
= IO_TO_ID(0);
1670 io_apic_ints
[nintrs
].dst_apic_int
= 0; /* Pin 0 */
1672 } else if (apic_int_type(0, 0) == 0) {
1673 kprintf("APIC_IO: MP table broken: ExtINT entry corrupt!\n");
1674 for (x
= 0; x
< nintrs
; ++x
)
1675 if ((0 == ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
1676 (0 == io_apic_ints
[x
].dst_apic_int
)) {
1677 io_apic_ints
[x
].int_type
= 3;
1678 io_apic_ints
[x
].int_vector
= 0xff;
1684 * Fix missing IRQ 15 when IRQ 14 is an ISA interrupt. IDE
1685 * controllers universally come in pairs. If IRQ 14 is specified
1686 * as an ISA interrupt, then IRQ 15 had better be too.
1688 * [ Shuttle XPC / AMD Athlon X2 ]
1689 * The MPTable is missing an entry for IRQ 15. Note that the
1690 * ACPI table has an entry for both 14 and 15.
1692 if (apic_int_type(0, 14) == 0 && apic_int_type(0, 15) == -1) {
1693 kprintf("APIC_IO: MP table broken: IRQ 15 not ISA when IRQ 14 is!\n");
1694 io14
= io_apic_find_int_entry(0, 14);
1695 io_apic_ints
[nintrs
] = *io14
;
1696 io_apic_ints
[nintrs
].src_bus_irq
= 15;
1697 io_apic_ints
[nintrs
].dst_apic_int
= 15;
1705 /* Assign low level interrupt handlers */
1707 setup_apic_irq_mapping(void)
1713 for (x
= 0; x
< APIC_INTMAPSIZE
; x
++) {
1714 int_to_apicintpin
[x
].ioapic
= -1;
1715 int_to_apicintpin
[x
].int_pin
= 0;
1716 int_to_apicintpin
[x
].apic_address
= NULL
;
1717 int_to_apicintpin
[x
].redirindex
= 0;
1720 /* First assign ISA/EISA interrupts */
1721 for (x
= 0; x
< nintrs
; x
++) {
1722 int_vector
= io_apic_ints
[x
].src_bus_irq
;
1723 if (int_vector
< APIC_INTMAPSIZE
&&
1724 io_apic_ints
[x
].int_vector
== 0xff &&
1725 int_to_apicintpin
[int_vector
].ioapic
== -1 &&
1726 (apic_int_is_bus_type(x
, ISA
) ||
1727 apic_int_is_bus_type(x
, EISA
)) &&
1728 io_apic_ints
[x
].int_type
== 0) {
1729 assign_apic_irq(ID_TO_IO(io_apic_ints
[x
].dst_apic_id
),
1730 io_apic_ints
[x
].dst_apic_int
,
1735 /* Assign ExtInt entry if no ISA/EISA interrupt 0 entry */
1736 for (x
= 0; x
< nintrs
; x
++) {
1737 if (io_apic_ints
[x
].dst_apic_int
== 0 &&
1738 io_apic_ints
[x
].dst_apic_id
== IO_TO_ID(0) &&
1739 io_apic_ints
[x
].int_vector
== 0xff &&
1740 int_to_apicintpin
[0].ioapic
== -1 &&
1741 io_apic_ints
[x
].int_type
== 3) {
1742 assign_apic_irq(0, 0, 0);
1747 /* Assign PCI interrupts */
1748 for (x
= 0; x
< nintrs
; ++x
) {
1749 if (io_apic_ints
[x
].int_type
== 0 &&
1750 io_apic_ints
[x
].int_vector
== 0xff &&
1751 apic_int_is_bus_type(x
, PCI
))
1752 allocate_apic_irq(x
);
1759 mp_set_cpuids(int cpu_id
, int apic_id
)
1761 CPU_TO_ID(cpu_id
) = apic_id
;
1762 ID_TO_CPU(apic_id
) = cpu_id
;
1766 processor_entry(proc_entry_ptr entry
, int cpu
)
1770 /* check for usability */
1771 if (!(entry
->cpu_flags
& PROCENTRY_FLAG_EN
))
1774 if(entry
->apic_id
>= NAPICID
)
1775 panic("CPU APIC ID out of range (0..%d)", NAPICID
- 1);
1776 /* check for BSP flag */
1777 if (entry
->cpu_flags
& PROCENTRY_FLAG_BP
) {
1778 mp_set_cpuids(0, entry
->apic_id
);
1779 return 0; /* its already been counted */
1782 /* add another AP to list, if less than max number of CPUs */
1783 else if (cpu
< MAXCPU
) {
1784 mp_set_cpuids(cpu
, entry
->apic_id
);
1794 bus_entry(bus_entry_ptr entry
, int bus
)
1799 /* encode the name into an index */
1800 for (x
= 0; x
< 6; ++x
) {
1801 if ((c
= entry
->bus_type
[x
]) == ' ')
1807 if ((x
= lookup_bus_type(name
)) == UNKNOWN_BUSTYPE
)
1808 panic("unknown bus type: '%s'", name
);
1810 bus_data
[bus
].bus_id
= entry
->bus_id
;
1811 bus_data
[bus
].bus_type
= x
;
1817 io_apic_entry(io_apic_entry_ptr entry
, int apic
)
1819 if (!(entry
->apic_flags
& IOAPICENTRY_FLAG_EN
))
1822 IO_TO_ID(apic
) = entry
->apic_id
;
1823 if (entry
->apic_id
< NAPICID
)
1824 ID_TO_IO(entry
->apic_id
) = apic
;
1830 lookup_bus_type(char *name
)
1834 for (x
= 0; x
< MAX_BUSTYPE
; ++x
)
1835 if (strcmp(bus_type_table
[x
].name
, name
) == 0)
1836 return bus_type_table
[x
].type
;
1838 return UNKNOWN_BUSTYPE
;
1842 int_entry(int_entry_ptr entry
, int intr
)
1846 io_apic_ints
[intr
].int_type
= entry
->int_type
;
1847 io_apic_ints
[intr
].int_flags
= entry
->int_flags
;
1848 io_apic_ints
[intr
].src_bus_id
= entry
->src_bus_id
;
1849 io_apic_ints
[intr
].src_bus_irq
= entry
->src_bus_irq
;
1850 if (entry
->dst_apic_id
== 255) {
1851 /* This signal goes to all IO APICS. Select an IO APIC
1852 with sufficient number of interrupt pins */
1853 for (apic
= 0; apic
< mp_napics
; apic
++)
1854 if (((io_apic_read(apic
, IOAPIC_VER
) &
1855 IOART_VER_MAXREDIR
) >> MAXREDIRSHIFT
) >=
1856 entry
->dst_apic_int
)
1858 if (apic
< mp_napics
)
1859 io_apic_ints
[intr
].dst_apic_id
= IO_TO_ID(apic
);
1861 io_apic_ints
[intr
].dst_apic_id
= entry
->dst_apic_id
;
1863 io_apic_ints
[intr
].dst_apic_id
= entry
->dst_apic_id
;
1864 io_apic_ints
[intr
].dst_apic_int
= entry
->dst_apic_int
;
1870 apic_int_is_bus_type(int intr
, int bus_type
)
1874 for (bus
= 0; bus
< mp_nbusses
; ++bus
)
1875 if ((bus_data
[bus
].bus_id
== io_apic_ints
[intr
].src_bus_id
)
1876 && ((int) bus_data
[bus
].bus_type
== bus_type
))
1883 * Given a traditional ISA INT mask, return an APIC mask.
1886 isa_apic_mask(u_int isa_mask
)
1891 #if defined(SKIP_IRQ15_REDIRECT)
1892 if (isa_mask
== (1 << 15)) {
1893 kprintf("skipping ISA IRQ15 redirect\n");
1896 #endif /* SKIP_IRQ15_REDIRECT */
1898 isa_irq
= ffs(isa_mask
); /* find its bit position */
1899 if (isa_irq
== 0) /* doesn't exist */
1901 --isa_irq
; /* make it zero based */
1903 apic_pin
= isa_apic_irq(isa_irq
); /* look for APIC connection */
1907 return (1 << apic_pin
); /* convert pin# to a mask */
1911 * Determine which APIC pin an ISA/EISA INT is attached to.
1913 #define INTTYPE(I) (io_apic_ints[(I)].int_type)
1914 #define INTPIN(I) (io_apic_ints[(I)].dst_apic_int)
1915 #define INTIRQ(I) (io_apic_ints[(I)].int_vector)
1916 #define INTAPIC(I) (ID_TO_IO(io_apic_ints[(I)].dst_apic_id))
1918 #define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq)
1920 isa_apic_irq(int isa_irq
)
1924 for (intr
= 0; intr
< nintrs
; ++intr
) { /* check each record */
1925 if (INTTYPE(intr
) == 0) { /* standard INT */
1926 if (SRCBUSIRQ(intr
) == isa_irq
) {
1927 if (apic_int_is_bus_type(intr
, ISA
) ||
1928 apic_int_is_bus_type(intr
, EISA
)) {
1929 if (INTIRQ(intr
) == 0xff)
1930 return -1; /* unassigned */
1931 return INTIRQ(intr
); /* found */
1936 return -1; /* NOT found */
1941 * Determine which APIC pin a PCI INT is attached to.
1943 #define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id)
1944 #define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1945 #define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03)
1947 pci_apic_irq(int pciBus
, int pciDevice
, int pciInt
)
1951 --pciInt
; /* zero based */
1953 for (intr
= 0; intr
< nintrs
; ++intr
) { /* check each record */
1954 if ((INTTYPE(intr
) == 0) /* standard INT */
1955 && (SRCBUSID(intr
) == pciBus
)
1956 && (SRCBUSDEVICE(intr
) == pciDevice
)
1957 && (SRCBUSLINE(intr
) == pciInt
)) { /* a candidate IRQ */
1958 if (apic_int_is_bus_type(intr
, PCI
)) {
1959 if (INTIRQ(intr
) == 0xff) {
1960 kprintf("IOAPIC: pci_apic_irq() "
1962 return -1; /* unassigned */
1964 return INTIRQ(intr
); /* exact match */
1969 return -1; /* NOT found */
1973 next_apic_irq(int irq
)
1980 for (intr
= 0; intr
< nintrs
; intr
++) {
1981 if (INTIRQ(intr
) != irq
|| INTTYPE(intr
) != 0)
1983 bus
= SRCBUSID(intr
);
1984 bustype
= apic_bus_type(bus
);
1985 if (bustype
!= ISA
&&
1991 if (intr
>= nintrs
) {
1994 for (ointr
= intr
+ 1; ointr
< nintrs
; ointr
++) {
1995 if (INTTYPE(ointr
) != 0)
1997 if (bus
!= SRCBUSID(ointr
))
1999 if (bustype
== PCI
) {
2000 if (SRCBUSDEVICE(intr
) != SRCBUSDEVICE(ointr
))
2002 if (SRCBUSLINE(intr
) != SRCBUSLINE(ointr
))
2005 if (bustype
== ISA
|| bustype
== EISA
) {
2006 if (SRCBUSIRQ(intr
) != SRCBUSIRQ(ointr
))
2009 if (INTPIN(intr
) == INTPIN(ointr
))
2013 if (ointr
>= nintrs
) {
2016 return INTIRQ(ointr
);
2031 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
2034 * Exactly what this means is unclear at this point. It is a solution
2035 * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard
2036 * could route any of the ISA INTs to upper (>15) IRQ values. But most would
2037 * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
2041 undirect_isa_irq(int rirq
)
2045 kprintf("Freeing redirected ISA irq %d.\n", rirq
);
2046 /** FIXME: tickle the MB redirector chip */
2050 kprintf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq
);
2057 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
2060 undirect_pci_irq(int rirq
)
2064 kprintf("Freeing redirected PCI irq %d.\n", rirq
);
2066 /** FIXME: tickle the MB redirector chip */
2070 kprintf("Freeing (NOT implemented) redirected PCI irq %d.\n",
2080 * given a bus ID, return:
2081 * the bus type if found
2085 apic_bus_type(int id
)
2089 for (x
= 0; x
< mp_nbusses
; ++x
)
2090 if (bus_data
[x
].bus_id
== id
)
2091 return bus_data
[x
].bus_type
;
2097 * given a LOGICAL APIC# and pin#, return:
2098 * the associated src bus ID if found
2102 apic_src_bus_id(int apic
, int pin
)
2106 /* search each of the possible INTerrupt sources */
2107 for (x
= 0; x
< nintrs
; ++x
)
2108 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2109 (pin
== io_apic_ints
[x
].dst_apic_int
))
2110 return (io_apic_ints
[x
].src_bus_id
);
2112 return -1; /* NOT found */
2116 * given a LOGICAL APIC# and pin#, return:
2117 * the associated src bus IRQ if found
2121 apic_src_bus_irq(int apic
, int pin
)
2125 for (x
= 0; x
< nintrs
; x
++)
2126 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2127 (pin
== io_apic_ints
[x
].dst_apic_int
))
2128 return (io_apic_ints
[x
].src_bus_irq
);
2130 return -1; /* NOT found */
2135 * given a LOGICAL APIC# and pin#, return:
2136 * the associated INTerrupt type if found
2140 apic_int_type(int apic
, int pin
)
2144 /* search each of the possible INTerrupt sources */
2145 for (x
= 0; x
< nintrs
; ++x
) {
2146 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2147 (pin
== io_apic_ints
[x
].dst_apic_int
))
2148 return (io_apic_ints
[x
].int_type
);
2150 return -1; /* NOT found */
2154 * Return the IRQ associated with an APIC pin
2157 apic_irq(int apic
, int pin
)
2162 for (x
= 0; x
< nintrs
; ++x
) {
2163 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2164 (pin
== io_apic_ints
[x
].dst_apic_int
)) {
2165 res
= io_apic_ints
[x
].int_vector
;
2168 if (apic
!= int_to_apicintpin
[res
].ioapic
)
2169 panic("apic_irq: inconsistent table %d/%d", apic
, int_to_apicintpin
[res
].ioapic
);
2170 if (pin
!= int_to_apicintpin
[res
].int_pin
)
2171 panic("apic_irq inconsistent table (2)");
2180 * given a LOGICAL APIC# and pin#, return:
2181 * the associated trigger mode if found
2185 apic_trigger(int apic
, int pin
)
2189 /* search each of the possible INTerrupt sources */
2190 for (x
= 0; x
< nintrs
; ++x
)
2191 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2192 (pin
== io_apic_ints
[x
].dst_apic_int
))
2193 return ((io_apic_ints
[x
].int_flags
>> 2) & 0x03);
2195 return -1; /* NOT found */
2200 * given a LOGICAL APIC# and pin#, return:
2201 * the associated 'active' level if found
2205 apic_polarity(int apic
, int pin
)
2209 /* search each of the possible INTerrupt sources */
2210 for (x
= 0; x
< nintrs
; ++x
)
2211 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2212 (pin
== io_apic_ints
[x
].dst_apic_int
))
2213 return (io_apic_ints
[x
].int_flags
& 0x03);
2215 return -1; /* NOT found */
2221 * set data according to MP defaults
2222 * FIXME: probably not complete yet...
2225 mptable_default(int type
)
2227 int ap_cpu_id
, boot_cpu_id
;
2228 #if defined(APIC_IO)
2231 #endif /* APIC_IO */
2234 kprintf(" MP default config type: %d\n", type
);
2237 kprintf(" bus: ISA, APIC: 82489DX\n");
2240 kprintf(" bus: EISA, APIC: 82489DX\n");
2243 kprintf(" bus: EISA, APIC: 82489DX\n");
2246 kprintf(" bus: MCA, APIC: 82489DX\n");
2249 kprintf(" bus: ISA+PCI, APIC: Integrated\n");
2252 kprintf(" bus: EISA+PCI, APIC: Integrated\n");
2255 kprintf(" bus: MCA+PCI, APIC: Integrated\n");
2258 kprintf(" future type\n");
2264 boot_cpu_id
= (lapic
->id
& APIC_ID_MASK
) >> 24;
2265 ap_cpu_id
= (boot_cpu_id
== 0) ? 1 : 0;
2268 CPU_TO_ID(0) = boot_cpu_id
;
2269 ID_TO_CPU(boot_cpu_id
) = 0;
2271 /* one and only AP */
2272 CPU_TO_ID(1) = ap_cpu_id
;
2273 ID_TO_CPU(ap_cpu_id
) = 1;
2275 #if defined(APIC_IO)
2276 /* one and only IO APIC */
2277 io_apic_id
= (io_apic_read(0, IOAPIC_ID
) & APIC_ID_MASK
) >> 24;
2280 * sanity check, refer to MP spec section 3.6.6, last paragraph
2281 * necessary as some hardware isn't properly setting up the IO APIC
2283 #if defined(REALLY_ANAL_IOAPICID_VALUE)
2284 if (io_apic_id
!= 2) {
2286 if ((io_apic_id
== 0) || (io_apic_id
== 1) || (io_apic_id
== 15)) {
2287 #endif /* REALLY_ANAL_IOAPICID_VALUE */
2288 io_apic_set_id(0, 2);
2291 IO_TO_ID(0) = io_apic_id
;
2292 ID_TO_IO(io_apic_id
) = 0;
2293 #endif /* APIC_IO */
2295 /* fill out bus entries */
2305 bus_data
[0].bus_id
= default_data
[type
- 1][1];
2306 bus_data
[0].bus_type
= default_data
[type
- 1][2];
2307 bus_data
[1].bus_id
= default_data
[type
- 1][3];
2308 bus_data
[1].bus_type
= default_data
[type
- 1][4];
2312 /* case 4: case 7: MCA NOT supported */
2313 default: /* illegal/reserved */
2314 panic("BAD default MP config: %d", type
);
2318 #if defined(APIC_IO)
2319 /* general cases from MP v1.4, table 5-2 */
2320 for (pin
= 0; pin
< 16; ++pin
) {
2321 io_apic_ints
[pin
].int_type
= 0;
2322 io_apic_ints
[pin
].int_flags
= 0x05; /* edge/active-hi */
2323 io_apic_ints
[pin
].src_bus_id
= 0;
2324 io_apic_ints
[pin
].src_bus_irq
= pin
; /* IRQ2 caught below */
2325 io_apic_ints
[pin
].dst_apic_id
= io_apic_id
;
2326 io_apic_ints
[pin
].dst_apic_int
= pin
; /* 1-to-1 */
2329 /* special cases from MP v1.4, table 5-2 */
2331 io_apic_ints
[2].int_type
= 0xff; /* N/C */
2332 io_apic_ints
[13].int_type
= 0xff; /* N/C */
2333 #if !defined(APIC_MIXED_MODE)
2335 panic("sorry, can't support type 2 default yet");
2336 #endif /* APIC_MIXED_MODE */
2339 io_apic_ints
[2].src_bus_irq
= 0; /* ISA IRQ0 is on APIC INT 2 */
2342 io_apic_ints
[0].int_type
= 0xff; /* N/C */
2344 io_apic_ints
[0].int_type
= 3; /* vectored 8259 */
2345 #endif /* APIC_IO */
2349 * Map a physical memory address representing I/O into KVA. The I/O
2350 * block is assumed not to cross a page boundary.
2353 permanent_io_mapping(vm_paddr_t pa
)
2355 KKASSERT(pa
< 0x100000000LL
);
2357 return pmap_mapdev_uncacheable(pa
, PAGE_SIZE
);
2361 * start each AP in our list
2364 start_all_aps(u_int boot_addr
)
2366 vm_offset_t va
= boot_address
+ KERNBASE
;
2367 u_int64_t
*pt4
, *pt3
, *pt2
;
2373 u_char mpbiosreason
;
2374 u_long mpbioswarmvec
;
2375 struct mdglobaldata
*gd
;
2376 struct privatespace
*ps
;
2378 POSTCODE(START_ALL_APS_POST
);
2380 /* Initialize BSP's local APIC */
2381 apic_initialize(TRUE
);
2383 /* install the AP 1st level boot code */
2384 pmap_kenter(va
, boot_address
);
2385 cpu_invlpg((void *)va
); /* JG XXX */
2386 bcopy(mptramp_start
, (void *)va
, bootMP_size
);
2388 /* Locate the page tables, they'll be below the trampoline */
2389 pt4
= (u_int64_t
*)(uintptr_t)(mptramp_pagetables
+ KERNBASE
);
2390 pt3
= pt4
+ (PAGE_SIZE
) / sizeof(u_int64_t
);
2391 pt2
= pt3
+ (PAGE_SIZE
) / sizeof(u_int64_t
);
2393 /* Create the initial 1GB replicated page tables */
2394 for (i
= 0; i
< 512; i
++) {
2395 /* Each slot of the level 4 pages points to the same level 3 page */
2396 pt4
[i
] = (u_int64_t
)(uintptr_t)(mptramp_pagetables
+ PAGE_SIZE
);
2397 pt4
[i
] |= PG_V
| PG_RW
| PG_U
;
2399 /* Each slot of the level 3 pages points to the same level 2 page */
2400 pt3
[i
] = (u_int64_t
)(uintptr_t)(mptramp_pagetables
+ (2 * PAGE_SIZE
));
2401 pt3
[i
] |= PG_V
| PG_RW
| PG_U
;
2403 /* The level 2 page slots are mapped with 2MB pages for 1GB. */
2404 pt2
[i
] = i
* (2 * 1024 * 1024);
2405 pt2
[i
] |= PG_V
| PG_RW
| PG_PS
| PG_U
;
2408 /* save the current value of the warm-start vector */
2409 mpbioswarmvec
= *((u_int32_t
*) WARMBOOT_OFF
);
2410 outb(CMOS_REG
, BIOS_RESET
);
2411 mpbiosreason
= inb(CMOS_DATA
);
2413 /* setup a vector to our boot code */
2414 *((volatile u_short
*) WARMBOOT_OFF
) = WARMBOOT_TARGET
;
2415 *((volatile u_short
*) WARMBOOT_SEG
) = (boot_address
>> 4);
2416 outb(CMOS_REG
, BIOS_RESET
);
2417 outb(CMOS_DATA
, BIOS_WARM
); /* 'warm-start' */
2420 * If we have a TSC we can figure out the SMI interrupt rate.
2421 * The SMI does not necessarily use a constant rate. Spend
2422 * up to 250ms trying to figure it out.
2425 if (cpu_feature
& CPUID_TSC
) {
2426 set_apic_timer(275000);
2427 smilast
= read_apic_timer();
2428 for (x
= 0; x
< 20 && read_apic_timer(); ++x
) {
2429 smicount
= smitest();
2430 if (smibest
== 0 || smilast
- smicount
< smibest
)
2431 smibest
= smilast
- smicount
;
2434 if (smibest
> 250000)
2437 smibest
= smibest
* (int64_t)1000000 /
2438 get_apic_timer_frequency();
2442 kprintf("SMI Frequency (worst case): %d Hz (%d us)\n",
2443 1000000 / smibest
, smibest
);
2446 for (x
= 1; x
<= mp_naps
; ++x
) {
2448 /* This is a bit verbose, it will go away soon. */
2450 /* first page of AP's private space */
2451 pg
= x
* x86_64_btop(sizeof(struct privatespace
));
2453 /* allocate new private data page(s) */
2454 gd
= (struct mdglobaldata
*)kmem_alloc(&kernel_map
,
2455 MDGLOBALDATA_BASEALLOC_SIZE
);
2457 gd
= &CPU_prvspace
[x
].mdglobaldata
; /* official location */
2458 bzero(gd
, sizeof(*gd
));
2459 gd
->mi
.gd_prvspace
= ps
= &CPU_prvspace
[x
];
2461 /* prime data page for it to use */
2462 mi_gdinit(&gd
->mi
, x
);
2464 gd
->gd_CMAP1
= &SMPpt
[pg
+ 0];
2465 gd
->gd_CMAP2
= &SMPpt
[pg
+ 1];
2466 gd
->gd_CMAP3
= &SMPpt
[pg
+ 2];
2467 gd
->gd_PMAP1
= &SMPpt
[pg
+ 3];
2468 gd
->gd_CADDR1
= ps
->CPAGE1
;
2469 gd
->gd_CADDR2
= ps
->CPAGE2
;
2470 gd
->gd_CADDR3
= ps
->CPAGE3
;
2471 gd
->gd_PADDR1
= (pt_entry_t
*)ps
->PPAGE1
;
2472 gd
->mi
.gd_ipiq
= (void *)kmem_alloc(&kernel_map
, sizeof(lwkt_ipiq
) * (mp_naps
+ 1));
2473 bzero(gd
->mi
.gd_ipiq
, sizeof(lwkt_ipiq
) * (mp_naps
+ 1));
2475 /* setup a vector to our boot code */
2476 *((volatile u_short
*) WARMBOOT_OFF
) = WARMBOOT_TARGET
;
2477 *((volatile u_short
*) WARMBOOT_SEG
) = (boot_addr
>> 4);
2478 outb(CMOS_REG
, BIOS_RESET
);
2479 outb(CMOS_DATA
, BIOS_WARM
); /* 'warm-start' */
2482 * Setup the AP boot stack
2484 bootSTK
= &ps
->idlestack
[UPAGES
*PAGE_SIZE
/2];
2487 /* attempt to start the Application Processor */
2488 CHECK_INIT(99); /* setup checkpoints */
2489 if (!start_ap(gd
, boot_addr
, smibest
)) {
2490 kprintf("AP #%d (PHY# %d) failed!\n", x
, CPU_TO_ID(x
));
2491 CHECK_PRINT("trace"); /* show checkpoints */
2492 /* better panic as the AP may be running loose */
2493 kprintf("panic y/n? [y] ");
2494 if (cngetc() != 'n')
2497 CHECK_PRINT("trace"); /* show checkpoints */
2499 /* record its version info */
2500 cpu_apic_versions
[x
] = cpu_apic_versions
[0];
2503 /* set ncpus to 1 + highest logical cpu. Not all may have come up */
2506 /* ncpus2 -- ncpus rounded down to the nearest power of 2 */
2507 for (shift
= 0; (1 << shift
) <= ncpus
; ++shift
)
2510 ncpus2_shift
= shift
;
2511 ncpus2
= 1 << shift
;
2512 ncpus2_mask
= ncpus2
- 1;
2514 /* ncpus_fit -- ncpus rounded up to the nearest power of 2 */
2515 if ((1 << shift
) < ncpus
)
2517 ncpus_fit
= 1 << shift
;
2518 ncpus_fit_mask
= ncpus_fit
- 1;
2520 /* build our map of 'other' CPUs */
2521 mycpu
->gd_other_cpus
= smp_startup_mask
& ~(1 << mycpu
->gd_cpuid
);
2522 mycpu
->gd_ipiq
= (void *)kmem_alloc(&kernel_map
, sizeof(lwkt_ipiq
) * ncpus
);
2523 bzero(mycpu
->gd_ipiq
, sizeof(lwkt_ipiq
) * ncpus
);
2525 /* fill in our (BSP) APIC version */
2526 cpu_apic_versions
[0] = lapic
->version
;
2528 /* restore the warmstart vector */
2529 *(u_long
*) WARMBOOT_OFF
= mpbioswarmvec
;
2530 outb(CMOS_REG
, BIOS_RESET
);
2531 outb(CMOS_DATA
, mpbiosreason
);
2534 * NOTE! The idlestack for the BSP was setup by locore. Finish
2535 * up, clean out the P==V mapping we did earlier.
2539 /* number of APs actually started */
2545 * load the 1st level AP boot code into base memory.
2548 /* targets for relocation */
2549 extern void bigJump(void);
2550 extern void bootCodeSeg(void);
2551 extern void bootDataSeg(void);
2552 extern void MPentry(void);
2553 extern u_int MP_GDT
;
2554 extern u_int mp_gdtbase
;
2559 install_ap_tramp(u_int boot_addr
)
2562 int size
= *(int *) ((u_long
) & bootMP_size
);
2563 u_char
*src
= (u_char
*) ((u_long
) bootMP
);
2564 u_char
*dst
= (u_char
*) boot_addr
+ KERNBASE
;
2565 u_int boot_base
= (u_int
) bootMP
;
2570 POSTCODE(INSTALL_AP_TRAMP_POST
);
2572 for (x
= 0; x
< size
; ++x
)
2576 * modify addresses in code we just moved to basemem. unfortunately we
2577 * need fairly detailed info about mpboot.s for this to work. changes
2578 * to mpboot.s might require changes here.
2581 /* boot code is located in KERNEL space */
2582 dst
= (u_char
*) boot_addr
+ KERNBASE
;
2584 /* modify the lgdt arg */
2585 dst32
= (u_int32_t
*) (dst
+ ((u_int
) & mp_gdtbase
- boot_base
));
2586 *dst32
= boot_addr
+ ((u_int
) & MP_GDT
- boot_base
);
2588 /* modify the ljmp target for MPentry() */
2589 dst32
= (u_int32_t
*) (dst
+ ((u_int
) bigJump
- boot_base
) + 1);
2590 *dst32
= ((u_int
) MPentry
- KERNBASE
);
2592 /* modify the target for boot code segment */
2593 dst16
= (u_int16_t
*) (dst
+ ((u_int
) bootCodeSeg
- boot_base
));
2594 dst8
= (u_int8_t
*) (dst16
+ 1);
2595 *dst16
= (u_int
) boot_addr
& 0xffff;
2596 *dst8
= ((u_int
) boot_addr
>> 16) & 0xff;
2598 /* modify the target for boot data segment */
2599 dst16
= (u_int16_t
*) (dst
+ ((u_int
) bootDataSeg
- boot_base
));
2600 dst8
= (u_int8_t
*) (dst16
+ 1);
2601 *dst16
= (u_int
) boot_addr
& 0xffff;
2602 *dst8
= ((u_int
) boot_addr
>> 16) & 0xff;
2608 * This function starts the AP (application processor) identified
2609 * by the APIC ID 'physicalCpu'. It does quite a "song and dance"
2610 * to accomplish this. This is necessary because of the nuances
2611 * of the different hardware we might encounter. It ain't pretty,
2612 * but it seems to work.
2614 * NOTE: eventually an AP gets to ap_init(), which is called just
2615 * before the AP goes into the LWKT scheduler's idle loop.
2618 start_ap(struct mdglobaldata
*gd
, u_int boot_addr
, int smibest
)
2622 u_long icr_lo
, icr_hi
;
2624 POSTCODE(START_AP_POST
);
2626 /* get the PHYSICAL APIC ID# */
2627 physical_cpu
= CPU_TO_ID(gd
->mi
.gd_cpuid
);
2629 /* calculate the vector */
2630 vector
= (boot_addr
>> 12) & 0xff;
2632 /* We don't want anything interfering */
2635 /* Make sure the target cpu sees everything */
2639 * Try to detect when a SMI has occurred, wait up to 200ms.
2641 * If a SMI occurs during an AP reset but before we issue
2642 * the STARTUP command, the AP may brick. To work around
2643 * this problem we hold off doing the AP startup until
2644 * after we have detected the SMI. Hopefully another SMI
2645 * will not occur before we finish the AP startup.
2647 * Retries don't seem to help. SMIs have a window of opportunity
2648 * and if USB->legacy keyboard emulation is enabled in the BIOS
2649 * the interrupt rate can be quite high.
2651 * NOTE: Don't worry about the L1 cache load, it might bloat
2652 * ldelta a little but ndelta will be so huge when the SMI
2653 * occurs the detection logic will still work fine.
2656 set_apic_timer(200000);
2661 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
2662 * and running the target CPU. OR this INIT IPI might be latched (P5
2663 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
2666 * see apic/apicreg.h for icr bit definitions.
2668 * TIME CRITICAL CODE, DO NOT DO ANY KPRINTFS IN THE HOT PATH.
2672 * Setup the address for the target AP. We can setup
2673 * icr_hi once and then just trigger operations with
2676 icr_hi
= lapic
->icr_hi
& ~APIC_ID_MASK
;
2677 icr_hi
|= (physical_cpu
<< 24);
2678 icr_lo
= lapic
->icr_lo
& 0xfff00000;
2679 lapic
->icr_hi
= icr_hi
;
2682 * Do an INIT IPI: assert RESET
2684 * Use edge triggered mode to assert INIT
2686 lapic
->icr_lo
= icr_lo
| 0x00004500;
2687 while (lapic
->icr_lo
& APIC_DELSTAT_MASK
)
2691 * The spec calls for a 10ms delay but we may have to use a
2692 * MUCH lower delay to avoid bricking an AP due to a fast SMI
2693 * interrupt. We have other loops here too and dividing by 2
2694 * doesn't seem to be enough even after subtracting 350us,
2695 * so we divide by 4.
2697 * Our minimum delay is 150uS, maximum is 10ms. If no SMI
2698 * interrupt was detected we use the full 10ms.
2702 else if (smibest
< 150 * 4 + 350)
2704 else if ((smibest
- 350) / 4 < 10000)
2705 u_sleep((smibest
- 350) / 4);
2710 * Do an INIT IPI: deassert RESET
2712 * Use level triggered mode to deassert. It is unclear
2713 * why we need to do this.
2715 lapic
->icr_lo
= icr_lo
| 0x00008500;
2716 while (lapic
->icr_lo
& APIC_DELSTAT_MASK
)
2718 u_sleep(150); /* wait 150us */
2721 * Next we do a STARTUP IPI: the previous INIT IPI might still be
2722 * latched, (P5 bug) this 1st STARTUP would then terminate
2723 * immediately, and the previously started INIT IPI would continue. OR
2724 * the previous INIT IPI has already run. and this STARTUP IPI will
2725 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
2728 lapic
->icr_lo
= icr_lo
| 0x00000600 | vector
;
2729 while (lapic
->icr_lo
& APIC_DELSTAT_MASK
)
2731 u_sleep(200); /* wait ~200uS */
2734 * Finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
2735 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
2736 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
2737 * recognized after hardware RESET or INIT IPI.
2739 lapic
->icr_lo
= icr_lo
| 0x00000600 | vector
;
2740 while (lapic
->icr_lo
& APIC_DELSTAT_MASK
)
2743 /* Resume normal operation */
2746 /* wait for it to start, see ap_init() */
2747 set_apic_timer(5000000);/* == 5 seconds */
2748 while (read_apic_timer()) {
2749 if (smp_startup_mask
& (1 << gd
->mi
.gd_cpuid
))
2750 return 1; /* return SUCCESS */
2753 return 0; /* return FAILURE */
2768 while (read_apic_timer()) {
2770 for (count
= 0; count
< 100; ++count
)
2771 ntsc
= rdtsc(); /* force loop to occur */
2773 ndelta
= ntsc
- ltsc
;
2774 if (ldelta
> ndelta
)
2776 if (ndelta
> ldelta
* 2)
2779 ldelta
= ntsc
- ltsc
;
2782 return(read_apic_timer());
2786 * Synchronously flush the TLB on all other CPU's. The current cpu's
2787 * TLB is not flushed. If the caller wishes to flush the current cpu's
2788 * TLB the caller must call cpu_invltlb() in addition to smp_invltlb().
2790 * NOTE: If for some reason we were unable to start all cpus we cannot
2791 * safely use broadcast IPIs.
2794 static cpumask_t smp_invltlb_req
;
2796 #define SMP_INVLTLB_DEBUG
2802 struct mdglobaldata
*md
= mdcpu
;
2803 #ifdef SMP_INVLTLB_DEBUG
2808 crit_enter_gd(&md
->mi
);
2809 md
->gd_invltlb_ret
= 0;
2810 ++md
->mi
.gd_cnt
.v_smpinvltlb
;
2811 atomic_set_int(&smp_invltlb_req
, md
->mi
.gd_cpumask
);
2812 #ifdef SMP_INVLTLB_DEBUG
2815 if (smp_startup_mask
== smp_active_mask
) {
2816 all_but_self_ipi(XINVLTLB_OFFSET
);
2818 selected_apic_ipi(smp_active_mask
& ~md
->mi
.gd_cpumask
,
2819 XINVLTLB_OFFSET
, APIC_DELMODE_FIXED
);
2822 #ifdef SMP_INVLTLB_DEBUG
2824 kprintf("smp_invltlb: ipi sent\n");
2826 while ((md
->gd_invltlb_ret
& smp_active_mask
& ~md
->mi
.gd_cpumask
) !=
2827 (smp_active_mask
& ~md
->mi
.gd_cpumask
)) {
2830 #ifdef SMP_INVLTLB_DEBUG
2832 if (++count
== 400000000) {
2833 print_backtrace(-1);
2834 kprintf("smp_invltlb: endless loop %08lx %08lx, "
2835 "rflags %016jx retry",
2836 (long)md
->gd_invltlb_ret
,
2837 (long)smp_invltlb_req
,
2838 (intmax_t)read_rflags());
2839 __asm
__volatile ("sti");
2842 lwkt_process_ipiq();
2844 int bcpu
= bsfl(~md
->gd_invltlb_ret
& ~md
->mi
.gd_cpumask
& smp_active_mask
);
2847 kprintf("bcpu %d\n", bcpu
);
2848 xgd
= globaldata_find(bcpu
);
2849 kprintf("thread %p %s\n", xgd
->gd_curthread
, xgd
->gd_curthread
->td_comm
);
2852 Debugger("giving up");
2858 atomic_clear_int(&smp_invltlb_req
, md
->mi
.gd_cpumask
);
2859 crit_exit_gd(&md
->mi
);
2866 * Called from Xinvltlb assembly with interrupts disabled. We didn't
2867 * bother to bump the critical section count or nested interrupt count
2868 * so only do very low level operations here.
2871 smp_invltlb_intr(void)
2873 struct mdglobaldata
*md
= mdcpu
;
2874 struct mdglobaldata
*omd
;
2879 mask
= smp_invltlb_req
;
2883 mask
&= ~(1 << cpu
);
2884 omd
= (struct mdglobaldata
*)globaldata_find(cpu
);
2885 atomic_set_int(&omd
->gd_invltlb_ret
, md
->mi
.gd_cpumask
);
2892 * When called the executing CPU will send an IPI to all other CPUs
2893 * requesting that they halt execution.
2895 * Usually (but not necessarily) called with 'other_cpus' as its arg.
2897 * - Signals all CPUs in map to stop.
2898 * - Waits for each to stop.
2905 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
2906 * from executing at same time.
2909 stop_cpus(u_int map
)
2911 map
&= smp_active_mask
;
2913 /* send the Xcpustop IPI to all CPUs in map */
2914 selected_apic_ipi(map
, XCPUSTOP_OFFSET
, APIC_DELMODE_FIXED
);
2916 while ((stopped_cpus
& map
) != map
)
2924 * Called by a CPU to restart stopped CPUs.
2926 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
2928 * - Signals all CPUs in map to restart.
2929 * - Waits for each to restart.
2937 restart_cpus(u_int map
)
2939 /* signal other cpus to restart */
2940 started_cpus
= map
& smp_active_mask
;
2942 while ((stopped_cpus
& map
) != 0) /* wait for each to clear its bit */
2949 * This is called once the mpboot code has gotten us properly relocated
2950 * and the MMU turned on, etc. ap_init() is actually the idle thread,
2951 * and when it returns the scheduler will call the real cpu_idle() main
2952 * loop for the idlethread. Interrupts are disabled on entry and should
2953 * remain disabled at return.
2961 * Adjust smp_startup_mask to signal the BSP that we have started
2962 * up successfully. Note that we do not yet hold the BGL. The BSP
2963 * is waiting for our signal.
2965 * We can't set our bit in smp_active_mask yet because we are holding
2966 * interrupts physically disabled and remote cpus could deadlock
2967 * trying to send us an IPI.
2969 smp_startup_mask
|= 1 << mycpu
->gd_cpuid
;
2973 * Interlock for finalization. Wait until mp_finish is non-zero,
2974 * then get the MP lock.
2976 * Note: We are in a critical section.
2978 * Note: We have to synchronize td_mpcount to our desired MP state
2979 * before calling cpu_try_mplock().
2981 * Note: we are the idle thread, we can only spin.
2983 * Note: The load fence is memory volatile and prevents the compiler
2984 * from improperly caching mp_finish, and the cpu from improperly
2987 while (mp_finish
== 0)
2989 ++curthread
->td_mpcount
;
2990 while (cpu_try_mplock() == 0)
2993 if (cpu_feature
& CPUID_TSC
) {
2995 * The BSP is constantly updating tsc0_offset, figure out the
2996 * relative difference to synchronize ktrdump.
2998 tsc_offsets
[mycpu
->gd_cpuid
] = rdtsc() - tsc0_offset
;
3001 /* BSP may have changed PTD while we're waiting for the lock */
3004 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
3008 /* Build our map of 'other' CPUs. */
3009 mycpu
->gd_other_cpus
= smp_startup_mask
& ~(1 << mycpu
->gd_cpuid
);
3011 kprintf("SMP: AP CPU #%d Launched!\n", mycpu
->gd_cpuid
);
3013 /* A quick check from sanity claus */
3014 apic_id
= (apic_id_to_logical
[(lapic
->id
& 0x0f000000) >> 24]);
3015 if (mycpu
->gd_cpuid
!= apic_id
) {
3016 kprintf("SMP: cpuid = %d\n", mycpu
->gd_cpuid
);
3017 kprintf("SMP: apic_id = %d\n", apic_id
);
3019 kprintf("PTD[MPPTDI] = %p\n", (void *)PTD
[MPPTDI
]);
3021 panic("cpuid mismatch! boom!!");
3024 /* Initialize AP's local APIC for irq's */
3025 apic_initialize(FALSE
);
3027 /* Set memory range attributes for this CPU to match the BSP */
3028 mem_range_AP_init();
3031 * Once we go active we must process any IPIQ messages that may
3032 * have been queued, because no actual IPI will occur until we
3033 * set our bit in the smp_active_mask. If we don't the IPI
3034 * message interlock could be left set which would also prevent
3037 * The idle loop doesn't expect the BGL to be held and while
3038 * lwkt_switch() normally cleans things up this is a special case
3039 * because we returning almost directly into the idle loop.
3041 * The idle thread is never placed on the runq, make sure
3042 * nothing we've done put it there.
3044 KKASSERT(curthread
->td_mpcount
== 1);
3045 smp_active_mask
|= 1 << mycpu
->gd_cpuid
;
3048 * Enable interrupts here. idle_restore will also do it, but
3049 * doing it here lets us clean up any strays that got posted to
3050 * the CPU during the AP boot while we are still in a critical
3053 __asm
__volatile("sti; pause; pause"::);
3054 mdcpu
->gd_fpending
= 0;
3056 initclocks_pcpu(); /* clock interrupts (via IPIs) */
3057 lwkt_process_ipiq();
3060 * Releasing the mp lock lets the BSP finish up the SMP init
3063 KKASSERT((curthread
->td_flags
& TDF_RUNQ
) == 0);
3067 * Get SMP fully working before we start initializing devices.
3075 kprintf("Finish MP startup\n");
3076 if (cpu_feature
& CPUID_TSC
)
3077 tsc0_offset
= rdtsc();
3080 while (smp_active_mask
!= smp_startup_mask
) {
3082 if (cpu_feature
& CPUID_TSC
)
3083 tsc0_offset
= rdtsc();
3085 while (try_mplock() == 0)
3088 kprintf("Active CPU Mask: %08x\n", smp_active_mask
);
3091 SYSINIT(finishsmp
, SI_BOOT2_FINISH_SMP
, SI_ORDER_FIRST
, ap_finish
, NULL
)
3094 cpu_send_ipiq(int dcpu
)
3096 if ((1 << dcpu
) & smp_active_mask
)
3097 single_apic_ipi(dcpu
, XIPIQ_OFFSET
, APIC_DELMODE_FIXED
);
3100 #if 0 /* single_apic_ipi_passive() not working yet */
3102 * Returns 0 on failure, 1 on success
3105 cpu_send_ipiq_passive(int dcpu
)
3108 if ((1 << dcpu
) & smp_active_mask
) {
3109 r
= single_apic_ipi_passive(dcpu
, XIPIQ_OFFSET
,
3110 APIC_DELMODE_FIXED
);