2 * Copyright (c) 1996, by Steve Passe
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 * derived from this software without specific prior written permission.
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * $FreeBSD: src/sys/i386/i386/mp_machdep.c,v 1.115.2.15 2003/03/14 21:22:35 jhb Exp $
26 * $DragonFly: src/sys/platform/pc32/i386/mp_machdep.c,v 1.60 2008/06/07 12:03:52 mneumann Exp $
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/sysctl.h>
35 #include <sys/malloc.h>
36 #include <sys/memrange.h>
37 #include <sys/cons.h> /* cngetc() */
38 #include <sys/machintr.h>
40 #include <sys/mplock2.h>
43 #include <vm/vm_param.h>
45 #include <vm/vm_kern.h>
46 #include <vm/vm_extern.h>
48 #include <vm/vm_map.h>
54 #include <machine/smp.h>
55 #include <machine_base/apic/apicreg.h>
56 #include <machine/atomic.h>
57 #include <machine/cpufunc.h>
58 #include <machine_base/apic/mpapic.h>
59 #include <machine/psl.h>
60 #include <machine/segments.h>
61 #include <machine/tss.h>
62 #include <machine/specialreg.h>
63 #include <machine/globaldata.h>
65 #include <machine/md_var.h> /* setidt() */
66 #include <machine_base/icu/icu.h> /* IPIs */
67 #include <machine_base/isa/intr_machdep.h> /* IPIs */
69 #define FIXUP_EXTRA_APIC_INTS 8 /* additional entries we may create */
71 #define WARMBOOT_TARGET 0
72 #define WARMBOOT_OFF (KERNBASE + 0x0467)
73 #define WARMBOOT_SEG (KERNBASE + 0x0469)
75 #define BIOS_BASE (0xf0000)
76 #define BIOS_SIZE (0x10000)
77 #define BIOS_COUNT (BIOS_SIZE/4)
79 #define CMOS_REG (0x70)
80 #define CMOS_DATA (0x71)
81 #define BIOS_RESET (0x0f)
82 #define BIOS_WARM (0x0a)
84 #define PROCENTRY_FLAG_EN 0x01
85 #define PROCENTRY_FLAG_BP 0x02
86 #define IOAPICENTRY_FLAG_EN 0x01
89 /* MP Floating Pointer Structure */
90 typedef struct MPFPS
{
103 /* MP Configuration Table Header */
104 typedef struct MPCTH
{
106 u_short base_table_length
;
110 u_char product_id
[12];
111 u_int32_t oem_table_pointer
;
112 u_short oem_table_size
;
114 u_int32_t apic_address
;
115 u_short extended_table_length
;
116 u_char extended_table_checksum
;
121 typedef struct PROCENTRY
{
126 u_int32_t cpu_signature
;
127 u_int32_t feature_flags
;
132 typedef struct BUSENTRY
{
138 typedef struct IOAPICENTRY
{
143 u_int32_t apic_address
;
144 } *io_apic_entry_ptr
;
146 typedef struct INTENTRY
{
156 /* descriptions of MP basetable entries */
157 typedef struct BASETABLE_ENTRY
{
166 vm_size_t mp_cth_mapsz
;
169 typedef int (*mptable_iter_func
)(void *, const void *, int);
172 * this code MUST be enabled here and in mpboot.s.
173 * it follows the very early stages of AP boot by placing values in CMOS ram.
174 * it NORMALLY will never be needed and thus the primitive method for enabling.
177 #if defined(CHECK_POINTS)
178 #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA))
179 #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
181 #define CHECK_INIT(D); \
182 CHECK_WRITE(0x34, (D)); \
183 CHECK_WRITE(0x35, (D)); \
184 CHECK_WRITE(0x36, (D)); \
185 CHECK_WRITE(0x37, (D)); \
186 CHECK_WRITE(0x38, (D)); \
187 CHECK_WRITE(0x39, (D));
189 #define CHECK_PRINT(S); \
190 kprintf("%s: %d, %d, %d, %d, %d, %d\n", \
199 #else /* CHECK_POINTS */
201 #define CHECK_INIT(D)
202 #define CHECK_PRINT(S)
204 #endif /* CHECK_POINTS */
207 * Values to send to the POST hardware.
209 #define MP_BOOTADDRESS_POST 0x10
210 #define MP_PROBE_POST 0x11
211 #define MPTABLE_PASS1_POST 0x12
213 #define MP_START_POST 0x13
214 #define MP_ENABLE_POST 0x14
215 #define MPTABLE_PASS2_POST 0x15
217 #define START_ALL_APS_POST 0x16
218 #define INSTALL_AP_TRAMP_POST 0x17
219 #define START_AP_POST 0x18
221 #define MP_ANNOUNCE_POST 0x19
223 static int need_hyperthreading_fixup
;
224 static u_int logical_cpus
;
225 u_int logical_cpus_mask
;
227 static int madt_probe_test
;
228 TUNABLE_INT("hw.madt_probe_test", &madt_probe_test
);
230 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
231 int current_postcode
;
233 /** XXX FIXME: what system files declare these??? */
234 extern struct region_descriptor r_gdt
, r_idt
;
236 int mp_naps
; /* # of Applications processors */
238 static int mp_nbusses
; /* # of busses */
239 int mp_napics
; /* # of IO APICs */
241 static vm_offset_t cpu_apic_address
;
243 vm_offset_t io_apic_address
[NAPICID
]; /* NAPICID is more than enough */
244 u_int32_t
*io_apic_versions
;
248 u_int32_t cpu_apic_versions
[MAXCPU
];
250 extern int64_t tsc_offsets
[];
252 extern u_long ebda_addr
;
255 struct apic_intmapinfo int_to_apicintpin
[APIC_INTMAPSIZE
];
259 * APIC ID logical/physical mapping structures.
260 * We oversize these to simplify boot-time config.
262 int cpu_num_to_apic_id
[NAPICID
];
264 int io_num_to_apic_id
[NAPICID
];
266 int apic_id_to_logical
[NAPICID
];
268 /* AP uses this during bootstrap. Do not staticize. */
273 * SMP page table page. Setup by locore to point to a page table
274 * page from which we allocate per-cpu privatespace areas io_apics,
278 #define IO_MAPPING_START_INDEX \
279 (SMP_MAXCPU * sizeof(struct privatespace) / PAGE_SIZE)
281 extern pt_entry_t
*SMPpt
;
283 struct pcb stoppcbs
[MAXCPU
];
285 extern inthand_t
IDTVEC(fast_syscall
), IDTVEC(fast_syscall32
);
287 static basetable_entry basetable_entry_types
[] =
289 {0, 20, "Processor"},
297 * Local data and functions.
300 static u_int boot_address
;
301 static u_int base_memory
;
302 static int mp_finish
;
304 static void mp_enable(u_int boot_addr
);
306 static int mptable_iterate_entries(const mpcth_t
,
307 mptable_iter_func
, void *);
308 static int mptable_probe(void);
309 static int mptable_check(vm_paddr_t
);
310 static long mptable_search_sig(u_int32_t target
, int count
);
311 static void mptable_hyperthread_fixup(u_int id_mask
);
312 static void mptable_pass1(struct mptable_pos
*);
313 static int mptable_pass2(struct mptable_pos
*);
314 static void mptable_default(int type
);
315 static void mptable_fix(void);
316 static int mptable_map(struct mptable_pos
*, vm_paddr_t
);
317 static void mptable_unmap(struct mptable_pos
*);
318 static void mptable_lapic_enumerate(struct mptable_pos
*);
321 static void setup_apic_irq_mapping(void);
322 static int apic_int_is_bus_type(int intr
, int bus_type
);
324 static int start_all_aps(u_int boot_addr
);
326 static void install_ap_tramp(u_int boot_addr
);
328 static int start_ap(struct mdglobaldata
*gd
, u_int boot_addr
, int smibest
);
329 static void lapic_init(vm_offset_t
);
330 static int smitest(void);
332 static cpumask_t smp_startup_mask
= 1; /* which cpus have been started */
333 cpumask_t smp_active_mask
= 1; /* which cpus are ready for IPIs etc? */
334 SYSCTL_INT(_machdep
, OID_AUTO
, smp_active
, CTLFLAG_RD
, &smp_active_mask
, 0, "");
335 static u_int bootMP_size
;
338 * Calculate usable address in base memory for AP trampoline code.
341 mp_bootaddress(u_int basemem
)
343 POSTCODE(MP_BOOTADDRESS_POST
);
345 base_memory
= basemem
;
347 bootMP_size
= mptramp_end
- mptramp_start
;
348 boot_address
= trunc_page(basemem
* 1024); /* round down to 4k boundary */
349 if (((basemem
* 1024) - boot_address
) < bootMP_size
)
350 boot_address
-= PAGE_SIZE
; /* not enough, lower by 4k */
351 /* 3 levels of page table pages */
352 mptramp_pagetables
= boot_address
- (PAGE_SIZE
* 3);
354 return mptramp_pagetables
;
359 * Look for an Intel MP spec table (ie, SMP capable hardware).
368 * Make sure our SMPpt[] page table is big enough to hold all the
371 KKASSERT(IO_MAPPING_START_INDEX
< NPTEPG
- 2);
373 POSTCODE(MP_PROBE_POST
);
375 /* see if EBDA exists */
376 if (ebda_addr
!= 0) {
377 /* search first 1K of EBDA */
378 target
= (u_int32_t
)ebda_addr
;
379 if ((x
= mptable_search_sig(target
, 1024 / 4)) > 0)
382 /* last 1K of base memory, effective 'top of base' passed in */
383 target
= (u_int32_t
)(base_memory
- 0x400);
384 if ((x
= mptable_search_sig(target
, 1024 / 4)) > 0)
388 /* search the BIOS */
389 target
= (u_int32_t
)BIOS_BASE
;
390 if ((x
= mptable_search_sig(target
, BIOS_COUNT
)) > 0)
397 struct mptable_check_cbarg
{
403 mptable_check_callback(void *xarg
, const void *pos
, int type
)
405 const struct PROCENTRY
*ent
;
406 struct mptable_check_cbarg
*arg
= xarg
;
412 if ((ent
->cpu_flags
& PROCENTRY_FLAG_EN
) == 0)
416 if (ent
->cpu_flags
& PROCENTRY_FLAG_BP
) {
417 if (arg
->found_bsp
) {
418 kprintf("more than one BSP in base MP table\n");
427 mptable_check(vm_paddr_t mpfps_paddr
)
429 struct mptable_pos mpt
;
430 struct mptable_check_cbarg arg
;
434 if (mpfps_paddr
== 0)
437 error
= mptable_map(&mpt
, mpfps_paddr
);
441 if (mpt
.mp_fps
->mpfb1
!= 0)
449 if (cth
->apic_address
== 0)
452 bzero(&arg
, sizeof(arg
));
453 error
= mptable_iterate_entries(cth
, mptable_check_callback
, &arg
);
455 if (arg
.cpu_count
== 0) {
456 kprintf("MP table contains no processor entries\n");
458 } else if (!arg
.found_bsp
) {
459 kprintf("MP table does not contains BSP entry\n");
469 mptable_iterate_entries(const mpcth_t cth
, mptable_iter_func func
, void *arg
)
471 int count
, total_size
;
472 const void *position
;
474 KKASSERT(cth
->base_table_length
>= sizeof(struct MPCTH
));
475 total_size
= cth
->base_table_length
- sizeof(struct MPCTH
);
476 position
= (const uint8_t *)cth
+ sizeof(struct MPCTH
);
477 count
= cth
->entry_count
;
482 KKASSERT(total_size
>= 0);
483 if (total_size
== 0) {
484 kprintf("invalid base MP table, "
485 "entry count and length mismatch\n");
489 type
= *(const uint8_t *)position
;
491 case 0: /* processor_entry */
492 case 1: /* bus_entry */
493 case 2: /* io_apic_entry */
494 case 3: /* int_entry */
495 case 4: /* int_entry */
498 kprintf("unknown base MP table entry type %d\n", type
);
502 if (total_size
< basetable_entry_types
[type
].length
) {
503 kprintf("invalid base MP table length, "
504 "does not contain all entries\n");
507 total_size
-= basetable_entry_types
[type
].length
;
509 error
= func(arg
, position
, type
);
513 position
= (const uint8_t *)position
+
514 basetable_entry_types
[type
].length
;
521 * Startup the SMP processors.
526 POSTCODE(MP_START_POST
);
527 mp_enable(boot_address
);
532 * Print various information about the SMP system hardware and setup.
539 POSTCODE(MP_ANNOUNCE_POST
);
541 kprintf("DragonFly/MP: Multiprocessor motherboard\n");
542 kprintf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
543 kprintf(", version: 0x%08x", cpu_apic_versions
[0]);
544 kprintf(", at 0x%08jx\n", (intmax_t)cpu_apic_address
);
545 for (x
= 1; x
<= mp_naps
; ++x
) {
546 kprintf(" cpu%d (AP): apic id: %2d", x
, CPU_TO_ID(x
));
547 kprintf(", version: 0x%08x", cpu_apic_versions
[x
]);
548 kprintf(", at 0x%08jx\n", (intmax_t)cpu_apic_address
);
552 for (x
= 0; x
< mp_napics
; ++x
) {
553 kprintf(" io%d (APIC): apic id: %2d", x
, IO_TO_ID(x
));
554 kprintf(", version: 0x%08x", io_apic_versions
[x
]);
555 kprintf(", at 0x%08lx\n", io_apic_address
[x
]);
558 kprintf(" Warning: APIC I/O disabled\n");
563 * AP cpu's call this to sync up protected mode.
565 * WARNING! %gs is not set up on entry. This routine sets up %gs.
571 int x
, myid
= bootAP
;
573 struct mdglobaldata
*md
;
574 struct privatespace
*ps
;
576 ps
= &CPU_prvspace
[myid
];
578 gdt_segs
[GPROC0_SEL
].ssd_base
=
579 (long) &ps
->mdglobaldata
.gd_common_tss
;
580 ps
->mdglobaldata
.mi
.gd_prvspace
= ps
;
582 /* We fill the 32-bit segment descriptors */
583 for (x
= 0; x
< NGDT
; x
++) {
584 if (x
!= GPROC0_SEL
&& x
!= (GPROC0_SEL
+ 1))
585 ssdtosd(&gdt_segs
[x
], &gdt
[myid
* NGDT
+ x
]);
587 /* And now a 64-bit one */
588 ssdtosyssd(&gdt_segs
[GPROC0_SEL
],
589 (struct system_segment_descriptor
*)&gdt
[myid
* NGDT
+ GPROC0_SEL
]);
591 r_gdt
.rd_limit
= NGDT
* sizeof(gdt
[0]) - 1;
592 r_gdt
.rd_base
= (long) &gdt
[myid
* NGDT
];
593 lgdt(&r_gdt
); /* does magic intra-segment return */
595 /* lgdt() destroys the GSBASE value, so we load GSBASE after lgdt() */
596 wrmsr(MSR_FSBASE
, 0); /* User value */
597 wrmsr(MSR_GSBASE
, (u_int64_t
)ps
);
598 wrmsr(MSR_KGSBASE
, 0); /* XXX User value while we're in the kernel */
604 mdcpu
->gd_currentldt
= _default_ldt
;
607 gsel_tss
= GSEL(GPROC0_SEL
, SEL_KPL
);
608 gdt
[myid
* NGDT
+ GPROC0_SEL
].sd_type
= SDT_SYSTSS
;
610 md
= mdcpu
; /* loaded through %gs:0 (mdglobaldata.mi.gd_prvspace)*/
612 md
->gd_common_tss
.tss_rsp0
= 0; /* not used until after switch */
614 md
->gd_common_tss
.tss_ioopt
= (sizeof md
->gd_common_tss
) << 16;
616 md
->gd_tss_gdt
= &gdt
[myid
* NGDT
+ GPROC0_SEL
];
617 md
->gd_common_tssd
= *md
->gd_tss_gdt
;
619 md
->gd_common_tss
.tss_ist1
= (long)&doublefault_stack
[PAGE_SIZE
];
624 * Set to a known state:
625 * Set by mpboot.s: CR0_PG, CR0_PE
626 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM
629 cr0
&= ~(CR0_CD
| CR0_NW
| CR0_EM
);
632 /* Set up the fast syscall stuff */
633 msr
= rdmsr(MSR_EFER
) | EFER_SCE
;
634 wrmsr(MSR_EFER
, msr
);
635 wrmsr(MSR_LSTAR
, (u_int64_t
)IDTVEC(fast_syscall
));
636 wrmsr(MSR_CSTAR
, (u_int64_t
)IDTVEC(fast_syscall32
));
637 msr
= ((u_int64_t
)GSEL(GCODE_SEL
, SEL_KPL
) << 32) |
638 ((u_int64_t
)GSEL(GUCODE32_SEL
, SEL_UPL
) << 48);
639 wrmsr(MSR_STAR
, msr
);
640 wrmsr(MSR_SF_MASK
, PSL_NT
|PSL_T
|PSL_I
|PSL_C
|PSL_D
);
642 pmap_set_opt(); /* PSE/4MB pages, etc */
644 /* Initialize the PAT MSR. */
648 /* set up CPU registers and state */
651 /* set up SSE/NX registers */
654 /* set up FPU state on the AP */
655 npxinit(__INITIAL_NPXCW__
);
657 /* disable the APIC, just to be SURE */
658 lapic
->svr
&= ~APIC_SVR_ENABLE
;
660 /* data returned to BSP */
661 cpu_apic_versions
[0] = lapic
->version
;
664 /*******************************************************************
665 * local functions and data
669 * start the SMP system
672 mp_enable(u_int boot_addr
)
679 vm_paddr_t mpfps_paddr
;
681 POSTCODE(MP_ENABLE_POST
);
683 if (madt_probe_test
) {
686 mpfps_paddr
= mptable_probe();
687 if (mptable_check(mpfps_paddr
))
692 struct mptable_pos mpt
;
694 mptable_map(&mpt
, mpfps_paddr
);
696 mptable_lapic_enumerate(&mpt
);
700 * We can safely map physical memory into SMPpt after
701 * mptable_pass1() completes.
706 * Examine the MP table for needed info
708 x
= mptable_pass2(&mpt
);
713 * Can't process default configs till the
714 * CPU APIC is pmapped
719 /* post scan cleanup */
723 vm_paddr_t madt_paddr
;
724 vm_offset_t lapic_addr
;
727 madt_paddr
= madt_probe();
729 panic("mp_enable: madt_probe failed\n");
731 lapic_addr
= madt_pass1(madt_paddr
);
733 panic("mp_enable: no local apic (madt)!\n");
735 lapic_init(lapic_addr
);
737 bsp_apic_id
= (lapic
->id
& 0xff000000) >> 24;
738 if (madt_pass2(madt_paddr
, bsp_apic_id
))
739 panic("mp_enable: madt_pass2 failed\n");
744 setup_apic_irq_mapping();
746 /* fill the LOGICAL io_apic_versions table */
747 for (apic
= 0; apic
< mp_napics
; ++apic
) {
748 ux
= io_apic_read(apic
, IOAPIC_VER
);
749 io_apic_versions
[apic
] = ux
;
750 io_apic_set_id(apic
, IO_TO_ID(apic
));
753 /* program each IO APIC in the system */
754 for (apic
= 0; apic
< mp_napics
; ++apic
)
755 if (io_apic_setup(apic
) < 0)
756 panic("IO APIC setup failure");
761 * These are required for SMP operation
764 /* install a 'Spurious INTerrupt' vector */
765 setidt(XSPURIOUSINT_OFFSET
, Xspuriousint
,
766 SDT_SYSIGT
, SEL_KPL
, 0);
768 /* install an inter-CPU IPI for TLB invalidation */
769 setidt(XINVLTLB_OFFSET
, Xinvltlb
,
770 SDT_SYSIGT
, SEL_KPL
, 0);
772 /* install an inter-CPU IPI for IPIQ messaging */
773 setidt(XIPIQ_OFFSET
, Xipiq
,
774 SDT_SYSIGT
, SEL_KPL
, 0);
776 /* install a timer vector */
777 setidt(XTIMER_OFFSET
, Xtimer
,
778 SDT_SYSIGT
, SEL_KPL
, 0);
780 /* install an inter-CPU IPI for CPU stop/restart */
781 setidt(XCPUSTOP_OFFSET
, Xcpustop
,
782 SDT_SYSIGT
, SEL_KPL
, 0);
784 /* start each Application Processor */
785 start_all_aps(boot_addr
);
790 * look for the MP spec signature
793 /* string defined by the Intel MP Spec as identifying the MP table */
794 #define MP_SIG 0x5f504d5f /* _MP_ */
795 #define NEXT(X) ((X) += 4)
797 mptable_search_sig(u_int32_t target
, int count
)
803 KKASSERT(target
!= 0);
805 map_size
= count
* sizeof(u_int32_t
);
806 addr
= pmap_mapdev((vm_paddr_t
)target
, map_size
);
809 for (x
= 0; x
< count
; NEXT(x
)) {
810 if (addr
[x
] == MP_SIG
) {
811 /* make array index a byte index */
812 ret
= target
+ (x
* sizeof(u_int32_t
));
817 pmap_unmapdev((vm_offset_t
)addr
, map_size
);
822 typedef struct BUSDATA
{
824 enum busTypes bus_type
;
827 typedef struct INTDATA
{
837 typedef struct BUSTYPENAME
{
844 static bus_type_name bus_type_table
[] =
850 {UNKNOWN_BUSTYPE
, "---"},
853 {UNKNOWN_BUSTYPE
, "---"},
854 {UNKNOWN_BUSTYPE
, "---"},
855 {UNKNOWN_BUSTYPE
, "---"},
856 {UNKNOWN_BUSTYPE
, "---"},
857 {UNKNOWN_BUSTYPE
, "---"},
859 {UNKNOWN_BUSTYPE
, "---"},
860 {UNKNOWN_BUSTYPE
, "---"},
861 {UNKNOWN_BUSTYPE
, "---"},
862 {UNKNOWN_BUSTYPE
, "---"},
864 {UNKNOWN_BUSTYPE
, "---"}
867 /* from MP spec v1.4, table 5-1 */
868 static int default_data
[7][5] =
870 /* nbus, id0, type0, id1, type1 */
871 {1, 0, ISA
, 255, 255},
872 {1, 0, EISA
, 255, 255},
873 {1, 0, EISA
, 255, 255},
874 {1, 0, MCA
, 255, 255},
876 {2, 0, EISA
, 1, PCI
},
881 static bus_datum
*bus_data
;
883 /* the IO INT data, one entry per possible APIC INTerrupt */
884 static io_int
*io_apic_ints
;
889 static int processor_entry (const struct PROCENTRY
*entry
, int cpu
);
891 static int bus_entry (bus_entry_ptr entry
, int bus
);
892 static int io_apic_entry (io_apic_entry_ptr entry
, int apic
);
893 static int int_entry (int_entry_ptr entry
, int intr
);
894 static int lookup_bus_type (char *name
);
899 * 1st pass on motherboard's Intel MP specification table.
908 mptable_pass1(struct mptable_pos
*mpt
)
920 POSTCODE(MPTABLE_PASS1_POST
);
923 KKASSERT(fps
!= NULL
);
926 /* clear various tables */
927 for (x
= 0; x
< NAPICID
; ++x
) {
928 io_apic_address
[x
] = ~0; /* IO APIC address table */
938 /* check for use of 'default' configuration */
939 if (fps
->mpfb1
!= 0) {
941 io_apic_address
[0] = DEFAULT_IO_APIC_BASE
;
942 mp_nbusses
= default_data
[fps
->mpfb1
- 1][0];
949 KKASSERT(cth
!= NULL
);
951 /* walk the table, recording info of interest */
952 totalSize
= cth
->base_table_length
- sizeof(struct MPCTH
);
953 position
= (u_char
*) cth
+ sizeof(struct MPCTH
);
954 count
= cth
->entry_count
;
957 switch (type
= *(u_char
*) position
) {
958 case 0: /* processor_entry */
960 case 1: /* bus_entry */
965 case 2: /* io_apic_entry */
967 if (((io_apic_entry_ptr
)position
)->apic_flags
968 & IOAPICENTRY_FLAG_EN
)
969 io_apic_address
[mp_napics
++] =
970 (vm_offset_t
)((io_apic_entry_ptr
)
971 position
)->apic_address
;
974 case 3: /* int_entry */
979 case 4: /* int_entry */
982 panic("mpfps Base Table HOSED!");
986 totalSize
-= basetable_entry_types
[type
].length
;
987 position
= (uint8_t *)position
+
988 basetable_entry_types
[type
].length
;
995 * 2nd pass on motherboard's Intel MP specification table.
998 * ID_TO_IO(N), phy APIC ID to log CPU/IO table
999 * IO_TO_ID(N), logical IO to APIC ID table
1004 mptable_pass2(struct mptable_pos
*mpt
)
1013 int apic
, bus
, intr
;
1018 POSTCODE(MPTABLE_PASS2_POST
);
1021 KKASSERT(fps
!= NULL
);
1024 MALLOC(io_apic_versions
, u_int32_t
*, sizeof(u_int32_t
) * mp_napics
,
1025 M_DEVBUF
, M_WAITOK
);
1026 MALLOC(ioapic
, volatile ioapic_t
**, sizeof(ioapic_t
*) * mp_napics
,
1027 M_DEVBUF
, M_WAITOK
| M_ZERO
);
1028 MALLOC(io_apic_ints
, io_int
*, sizeof(io_int
) * (nintrs
+ FIXUP_EXTRA_APIC_INTS
),
1029 M_DEVBUF
, M_WAITOK
);
1030 MALLOC(bus_data
, bus_datum
*, sizeof(bus_datum
) * mp_nbusses
,
1031 M_DEVBUF
, M_WAITOK
);
1035 for (i
= 0; i
< mp_napics
; i
++) {
1036 ioapic
[i
] = permanent_io_mapping(io_apic_address
[i
]);
1040 /* clear various tables */
1041 for (x
= 0; x
< NAPICID
; ++x
) {
1043 ID_TO_IO(x
) = -1; /* phy APIC ID to log CPU/IO table */
1044 IO_TO_ID(x
) = -1; /* logical IO to APIC ID table */
1049 /* clear bus data table */
1050 for (x
= 0; x
< mp_nbusses
; ++x
)
1051 bus_data
[x
].bus_id
= 0xff;
1053 /* clear IO APIC INT table */
1054 for (x
= 0; x
< (nintrs
+ 1); ++x
) {
1055 io_apic_ints
[x
].int_type
= 0xff;
1056 io_apic_ints
[x
].int_vector
= 0xff;
1060 /* record whether PIC or virtual-wire mode */
1061 machintr_setvar_simple(MACHINTR_VAR_IMCR_PRESENT
, fps
->mpfb2
& 0x80);
1063 /* check for use of 'default' configuration */
1064 if (fps
->mpfb1
!= 0)
1065 return fps
->mpfb1
; /* return default configuration type */
1068 KKASSERT(cth
!= NULL
);
1070 /* walk the table, recording info of interest */
1071 totalSize
= cth
->base_table_length
- sizeof(struct MPCTH
);
1072 position
= (u_char
*) cth
+ sizeof(struct MPCTH
);
1073 count
= cth
->entry_count
;
1074 apic
= bus
= intr
= 0;
1077 switch (type
= *(u_char
*) position
) {
1082 if (bus_entry(position
, bus
))
1088 if (io_apic_entry(position
, apic
))
1094 if (int_entry(position
, intr
))
1099 /* int_entry(position); */
1102 panic("mpfps Base Table HOSED!");
1106 totalSize
-= basetable_entry_types
[type
].length
;
1107 position
= (uint8_t *)position
+ basetable_entry_types
[type
].length
;
1110 /* report fact that its NOT a default configuration */
1116 * Check if we should perform a hyperthreading "fix-up" to
1117 * enumerate any logical CPU's that aren't already listed
1120 * XXX: We assume that all of the physical CPUs in the
1121 * system have the same number of logical CPUs.
1123 * XXX: We assume that APIC ID's are allocated such that
1124 * the APIC ID's for a physical processor are aligned
1125 * with the number of logical CPU's in the processor.
1128 mptable_hyperthread_fixup(u_int id_mask
)
1130 int i
, id
, lcpus_max
;
1132 if ((cpu_feature
& CPUID_HTT
) == 0)
1135 lcpus_max
= (cpu_procinfo
& CPUID_HTT_CORES
) >> 16;
1139 if (strcmp(cpu_vendor
, "GenuineIntel") == 0) {
1141 * INSTRUCTION SET REFERENCE, A-M (#253666)
1142 * Page 3-181, Table 3-20
1143 * "The nearest power-of-2 integer that is not smaller
1144 * than EBX[23:16] is the number of unique initial APIC
1145 * IDs reserved for addressing different logical
1146 * processors in a physical package."
1148 for (i
= 0; ; ++i
) {
1149 if ((1 << i
) >= lcpus_max
) {
1156 if (mp_naps
== lcpus_max
) {
1157 /* We have nothing to fix */
1159 } else if (mp_naps
== 1) {
1160 /* XXX this may be incorrect */
1161 logical_cpus
= lcpus_max
;
1163 int cur
, prev
, dist
;
1166 * Calculate the distances between two nearest
1167 * APIC IDs. If all such distances are same,
1168 * then it is the number of missing cpus that
1169 * we are going to fill later.
1171 dist
= cur
= prev
= -1;
1172 for (id
= 0; id
< MAXCPU
; ++id
) {
1173 if ((id_mask
& 1 << id
) == 0)
1178 int new_dist
= cur
- prev
;
1184 * Make sure that all distances
1185 * between two nearest APIC IDs
1188 if (dist
!= new_dist
)
1196 /* Must be power of 2 */
1197 if (dist
& (dist
- 1))
1200 /* Can't exceed CPU package capacity */
1201 if (dist
> lcpus_max
)
1202 logical_cpus
= lcpus_max
;
1204 logical_cpus
= dist
;
1208 * For each APIC ID of a CPU that is set in the mask,
1209 * scan the other candidate APIC ID's for this
1210 * physical processor. If any of those ID's are
1211 * already in the table, then kill the fixup.
1213 for (id
= 0; id
< MAXCPU
; id
++) {
1214 if ((id_mask
& 1 << id
) == 0)
1216 /* First, make sure we are on a logical_cpus boundary. */
1217 if (id
% logical_cpus
!= 0)
1219 for (i
= id
+ 1; i
< id
+ logical_cpus
; i
++)
1220 if ((id_mask
& 1 << i
) != 0)
1225 * Ok, the ID's checked out, so enable the fixup. We have to fixup
1226 * mp_naps right now.
1228 need_hyperthreading_fixup
= 1;
1229 mp_naps
*= logical_cpus
;
1233 mptable_map(struct mptable_pos
*mpt
, vm_paddr_t mpfps_paddr
)
1237 vm_size_t cth_mapsz
= 0;
1239 bzero(mpt
, sizeof(*mpt
));
1241 fps
= pmap_mapdev(mpfps_paddr
, sizeof(*fps
));
1242 if (fps
->pap
!= 0) {
1244 * Map configuration table header to get
1245 * the base table size
1247 cth
= pmap_mapdev(fps
->pap
, sizeof(*cth
));
1248 cth_mapsz
= cth
->base_table_length
;
1249 pmap_unmapdev((vm_offset_t
)cth
, sizeof(*cth
));
1251 if (cth_mapsz
< sizeof(*cth
)) {
1252 kprintf("invalid base MP table length %d\n",
1254 pmap_unmapdev((vm_offset_t
)fps
, sizeof(*fps
));
1259 * Map the base table
1261 cth
= pmap_mapdev(fps
->pap
, cth_mapsz
);
1266 mpt
->mp_cth_mapsz
= cth_mapsz
;
1272 mptable_unmap(struct mptable_pos
*mpt
)
1274 if (mpt
->mp_cth
!= NULL
) {
1275 pmap_unmapdev((vm_offset_t
)mpt
->mp_cth
, mpt
->mp_cth_mapsz
);
1277 mpt
->mp_cth_mapsz
= 0;
1279 if (mpt
->mp_fps
!= NULL
) {
1280 pmap_unmapdev((vm_offset_t
)mpt
->mp_fps
, sizeof(*mpt
->mp_fps
));
1288 assign_apic_irq(int apic
, int intpin
, int irq
)
1292 if (int_to_apicintpin
[irq
].ioapic
!= -1)
1293 panic("assign_apic_irq: inconsistent table");
1295 int_to_apicintpin
[irq
].ioapic
= apic
;
1296 int_to_apicintpin
[irq
].int_pin
= intpin
;
1297 int_to_apicintpin
[irq
].apic_address
= ioapic
[apic
];
1298 int_to_apicintpin
[irq
].redirindex
= IOAPIC_REDTBL
+ 2 * intpin
;
1300 for (x
= 0; x
< nintrs
; x
++) {
1301 if ((io_apic_ints
[x
].int_type
== 0 ||
1302 io_apic_ints
[x
].int_type
== 3) &&
1303 io_apic_ints
[x
].int_vector
== 0xff &&
1304 io_apic_ints
[x
].dst_apic_id
== IO_TO_ID(apic
) &&
1305 io_apic_ints
[x
].dst_apic_int
== intpin
)
1306 io_apic_ints
[x
].int_vector
= irq
;
1311 revoke_apic_irq(int irq
)
1317 if (int_to_apicintpin
[irq
].ioapic
== -1)
1318 panic("revoke_apic_irq: inconsistent table");
1320 oldapic
= int_to_apicintpin
[irq
].ioapic
;
1321 oldintpin
= int_to_apicintpin
[irq
].int_pin
;
1323 int_to_apicintpin
[irq
].ioapic
= -1;
1324 int_to_apicintpin
[irq
].int_pin
= 0;
1325 int_to_apicintpin
[irq
].apic_address
= NULL
;
1326 int_to_apicintpin
[irq
].redirindex
= 0;
1328 for (x
= 0; x
< nintrs
; x
++) {
1329 if ((io_apic_ints
[x
].int_type
== 0 ||
1330 io_apic_ints
[x
].int_type
== 3) &&
1331 io_apic_ints
[x
].int_vector
!= 0xff &&
1332 io_apic_ints
[x
].dst_apic_id
== IO_TO_ID(oldapic
) &&
1333 io_apic_ints
[x
].dst_apic_int
== oldintpin
)
1334 io_apic_ints
[x
].int_vector
= 0xff;
1342 allocate_apic_irq(int intr
)
1348 if (io_apic_ints
[intr
].int_vector
!= 0xff)
1349 return; /* Interrupt handler already assigned */
1351 if (io_apic_ints
[intr
].int_type
!= 0 &&
1352 (io_apic_ints
[intr
].int_type
!= 3 ||
1353 (io_apic_ints
[intr
].dst_apic_id
== IO_TO_ID(0) &&
1354 io_apic_ints
[intr
].dst_apic_int
== 0)))
1355 return; /* Not INT or ExtInt on != (0, 0) */
1358 while (irq
< APIC_INTMAPSIZE
&&
1359 int_to_apicintpin
[irq
].ioapic
!= -1)
1362 if (irq
>= APIC_INTMAPSIZE
)
1363 return; /* No free interrupt handlers */
1365 apic
= ID_TO_IO(io_apic_ints
[intr
].dst_apic_id
);
1366 intpin
= io_apic_ints
[intr
].dst_apic_int
;
1368 assign_apic_irq(apic
, intpin
, irq
);
1373 swap_apic_id(int apic
, int oldid
, int newid
)
1380 return; /* Nothing to do */
1382 kprintf("Changing APIC ID for IO APIC #%d from %d to %d in MP table\n",
1383 apic
, oldid
, newid
);
1385 /* Swap physical APIC IDs in interrupt entries */
1386 for (x
= 0; x
< nintrs
; x
++) {
1387 if (io_apic_ints
[x
].dst_apic_id
== oldid
)
1388 io_apic_ints
[x
].dst_apic_id
= newid
;
1389 else if (io_apic_ints
[x
].dst_apic_id
== newid
)
1390 io_apic_ints
[x
].dst_apic_id
= oldid
;
1393 /* Swap physical APIC IDs in IO_TO_ID mappings */
1394 for (oapic
= 0; oapic
< mp_napics
; oapic
++)
1395 if (IO_TO_ID(oapic
) == newid
)
1398 if (oapic
< mp_napics
) {
1399 kprintf("Changing APIC ID for IO APIC #%d from "
1400 "%d to %d in MP table\n",
1401 oapic
, newid
, oldid
);
1402 IO_TO_ID(oapic
) = oldid
;
1404 IO_TO_ID(apic
) = newid
;
1409 fix_id_to_io_mapping(void)
1413 for (x
= 0; x
< NAPICID
; x
++)
1416 for (x
= 0; x
<= mp_naps
; x
++)
1417 if (CPU_TO_ID(x
) < NAPICID
)
1418 ID_TO_IO(CPU_TO_ID(x
)) = x
;
1420 for (x
= 0; x
< mp_napics
; x
++)
1421 if (IO_TO_ID(x
) < NAPICID
)
1422 ID_TO_IO(IO_TO_ID(x
)) = x
;
1427 first_free_apic_id(void)
1431 for (freeid
= 0; freeid
< NAPICID
; freeid
++) {
1432 for (x
= 0; x
<= mp_naps
; x
++)
1433 if (CPU_TO_ID(x
) == freeid
)
1437 for (x
= 0; x
< mp_napics
; x
++)
1438 if (IO_TO_ID(x
) == freeid
)
1449 io_apic_id_acceptable(int apic
, int id
)
1451 int cpu
; /* Logical CPU number */
1452 int oapic
; /* Logical IO APIC number for other IO APIC */
1455 return 0; /* Out of range */
1457 for (cpu
= 0; cpu
<= mp_naps
; cpu
++)
1458 if (CPU_TO_ID(cpu
) == id
)
1459 return 0; /* Conflict with CPU */
1461 for (oapic
= 0; oapic
< mp_napics
&& oapic
< apic
; oapic
++)
1462 if (IO_TO_ID(oapic
) == id
)
1463 return 0; /* Conflict with other APIC */
1465 return 1; /* ID is acceptable for IO APIC */
1470 io_apic_find_int_entry(int apic
, int pin
)
1474 /* search each of the possible INTerrupt sources */
1475 for (x
= 0; x
< nintrs
; ++x
) {
1476 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
1477 (pin
== io_apic_ints
[x
].dst_apic_int
))
1478 return (&io_apic_ints
[x
]);
1486 * parse an Intel MP specification table
1494 int apic
; /* IO APIC unit number */
1495 int freeid
; /* Free physical APIC ID */
1496 int physid
; /* Current physical IO APIC ID */
1498 int bus_0
= 0; /* Stop GCC warning */
1499 int bus_pci
= 0; /* Stop GCC warning */
1503 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
1504 * did it wrong. The MP spec says that when more than 1 PCI bus
1505 * exists the BIOS must begin with bus entries for the PCI bus and use
1506 * actual PCI bus numbering. This implies that when only 1 PCI bus
1507 * exists the BIOS can choose to ignore this ordering, and indeed many
1508 * MP motherboards do ignore it. This causes a problem when the PCI
1509 * sub-system makes requests of the MP sub-system based on PCI bus
1510 * numbers. So here we look for the situation and renumber the
1511 * busses and associated INTs in an effort to "make it right".
1514 /* find bus 0, PCI bus, count the number of PCI busses */
1515 for (num_pci_bus
= 0, x
= 0; x
< mp_nbusses
; ++x
) {
1516 if (bus_data
[x
].bus_id
== 0) {
1519 if (bus_data
[x
].bus_type
== PCI
) {
1525 * bus_0 == slot of bus with ID of 0
1526 * bus_pci == slot of last PCI bus encountered
1529 /* check the 1 PCI bus case for sanity */
1530 /* if it is number 0 all is well */
1531 if (num_pci_bus
== 1 &&
1532 bus_data
[bus_pci
].bus_id
!= 0) {
1534 /* mis-numbered, swap with whichever bus uses slot 0 */
1536 /* swap the bus entry types */
1537 bus_data
[bus_pci
].bus_type
= bus_data
[bus_0
].bus_type
;
1538 bus_data
[bus_0
].bus_type
= PCI
;
1540 /* swap each relavant INTerrupt entry */
1541 id
= bus_data
[bus_pci
].bus_id
;
1542 for (x
= 0; x
< nintrs
; ++x
) {
1543 if (io_apic_ints
[x
].src_bus_id
== id
) {
1544 io_apic_ints
[x
].src_bus_id
= 0;
1546 else if (io_apic_ints
[x
].src_bus_id
== 0) {
1547 io_apic_ints
[x
].src_bus_id
= id
;
1552 /* Assign IO APIC IDs.
1554 * First try the existing ID. If a conflict is detected, try
1555 * the ID in the MP table. If a conflict is still detected, find
1558 * We cannot use the ID_TO_IO table before all conflicts has been
1559 * resolved and the table has been corrected.
1561 for (apic
= 0; apic
< mp_napics
; ++apic
) { /* For all IO APICs */
1563 /* First try to use the value set by the BIOS */
1564 physid
= io_apic_get_id(apic
);
1565 if (io_apic_id_acceptable(apic
, physid
)) {
1566 if (IO_TO_ID(apic
) != physid
)
1567 swap_apic_id(apic
, IO_TO_ID(apic
), physid
);
1571 /* Then check if the value in the MP table is acceptable */
1572 if (io_apic_id_acceptable(apic
, IO_TO_ID(apic
)))
1575 /* Last resort, find a free APIC ID and use it */
1576 freeid
= first_free_apic_id();
1577 if (freeid
>= NAPICID
)
1578 panic("No free physical APIC IDs found");
1580 if (io_apic_id_acceptable(apic
, freeid
)) {
1581 swap_apic_id(apic
, IO_TO_ID(apic
), freeid
);
1584 panic("Free physical APIC ID not usable");
1586 fix_id_to_io_mapping();
1588 /* detect and fix broken Compaq MP table */
1589 if (apic_int_type(0, 0) == -1) {
1590 kprintf("APIC_IO: MP table broken: 8259->APIC entry missing!\n");
1591 io_apic_ints
[nintrs
].int_type
= 3; /* ExtInt */
1592 io_apic_ints
[nintrs
].int_vector
= 0xff; /* Unassigned */
1593 /* XXX fixme, set src bus id etc, but it doesn't seem to hurt */
1594 io_apic_ints
[nintrs
].dst_apic_id
= IO_TO_ID(0);
1595 io_apic_ints
[nintrs
].dst_apic_int
= 0; /* Pin 0 */
1597 } else if (apic_int_type(0, 0) == 0) {
1598 kprintf("APIC_IO: MP table broken: ExtINT entry corrupt!\n");
1599 for (x
= 0; x
< nintrs
; ++x
)
1600 if ((0 == ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
1601 (0 == io_apic_ints
[x
].dst_apic_int
)) {
1602 io_apic_ints
[x
].int_type
= 3;
1603 io_apic_ints
[x
].int_vector
= 0xff;
1609 * Fix missing IRQ 15 when IRQ 14 is an ISA interrupt. IDE
1610 * controllers universally come in pairs. If IRQ 14 is specified
1611 * as an ISA interrupt, then IRQ 15 had better be too.
1613 * [ Shuttle XPC / AMD Athlon X2 ]
1614 * The MPTable is missing an entry for IRQ 15. Note that the
1615 * ACPI table has an entry for both 14 and 15.
1617 if (apic_int_type(0, 14) == 0 && apic_int_type(0, 15) == -1) {
1618 kprintf("APIC_IO: MP table broken: IRQ 15 not ISA when IRQ 14 is!\n");
1619 io14
= io_apic_find_int_entry(0, 14);
1620 io_apic_ints
[nintrs
] = *io14
;
1621 io_apic_ints
[nintrs
].src_bus_irq
= 15;
1622 io_apic_ints
[nintrs
].dst_apic_int
= 15;
1630 /* Assign low level interrupt handlers */
1632 setup_apic_irq_mapping(void)
1638 for (x
= 0; x
< APIC_INTMAPSIZE
; x
++) {
1639 int_to_apicintpin
[x
].ioapic
= -1;
1640 int_to_apicintpin
[x
].int_pin
= 0;
1641 int_to_apicintpin
[x
].apic_address
= NULL
;
1642 int_to_apicintpin
[x
].redirindex
= 0;
1645 /* First assign ISA/EISA interrupts */
1646 for (x
= 0; x
< nintrs
; x
++) {
1647 int_vector
= io_apic_ints
[x
].src_bus_irq
;
1648 if (int_vector
< APIC_INTMAPSIZE
&&
1649 io_apic_ints
[x
].int_vector
== 0xff &&
1650 int_to_apicintpin
[int_vector
].ioapic
== -1 &&
1651 (apic_int_is_bus_type(x
, ISA
) ||
1652 apic_int_is_bus_type(x
, EISA
)) &&
1653 io_apic_ints
[x
].int_type
== 0) {
1654 assign_apic_irq(ID_TO_IO(io_apic_ints
[x
].dst_apic_id
),
1655 io_apic_ints
[x
].dst_apic_int
,
1660 /* Assign ExtInt entry if no ISA/EISA interrupt 0 entry */
1661 for (x
= 0; x
< nintrs
; x
++) {
1662 if (io_apic_ints
[x
].dst_apic_int
== 0 &&
1663 io_apic_ints
[x
].dst_apic_id
== IO_TO_ID(0) &&
1664 io_apic_ints
[x
].int_vector
== 0xff &&
1665 int_to_apicintpin
[0].ioapic
== -1 &&
1666 io_apic_ints
[x
].int_type
== 3) {
1667 assign_apic_irq(0, 0, 0);
1672 /* Assign PCI interrupts */
1673 for (x
= 0; x
< nintrs
; ++x
) {
1674 if (io_apic_ints
[x
].int_type
== 0 &&
1675 io_apic_ints
[x
].int_vector
== 0xff &&
1676 apic_int_is_bus_type(x
, PCI
))
1677 allocate_apic_irq(x
);
1684 mp_set_cpuids(int cpu_id
, int apic_id
)
1686 CPU_TO_ID(cpu_id
) = apic_id
;
1687 ID_TO_CPU(apic_id
) = cpu_id
;
1691 processor_entry(const struct PROCENTRY
*entry
, int cpu
)
1695 /* check for usability */
1696 if (!(entry
->cpu_flags
& PROCENTRY_FLAG_EN
))
1699 if(entry
->apic_id
>= NAPICID
)
1700 panic("CPU APIC ID out of range (0..%d)", NAPICID
- 1);
1701 /* check for BSP flag */
1702 if (entry
->cpu_flags
& PROCENTRY_FLAG_BP
) {
1703 mp_set_cpuids(0, entry
->apic_id
);
1704 return 0; /* its already been counted */
1707 /* add another AP to list, if less than max number of CPUs */
1708 else if (cpu
< MAXCPU
) {
1709 mp_set_cpuids(cpu
, entry
->apic_id
);
1719 bus_entry(bus_entry_ptr entry
, int bus
)
1724 /* encode the name into an index */
1725 for (x
= 0; x
< 6; ++x
) {
1726 if ((c
= entry
->bus_type
[x
]) == ' ')
1732 if ((x
= lookup_bus_type(name
)) == UNKNOWN_BUSTYPE
)
1733 panic("unknown bus type: '%s'", name
);
1735 bus_data
[bus
].bus_id
= entry
->bus_id
;
1736 bus_data
[bus
].bus_type
= x
;
1742 io_apic_entry(io_apic_entry_ptr entry
, int apic
)
1744 if (!(entry
->apic_flags
& IOAPICENTRY_FLAG_EN
))
1747 IO_TO_ID(apic
) = entry
->apic_id
;
1748 if (entry
->apic_id
< NAPICID
)
1749 ID_TO_IO(entry
->apic_id
) = apic
;
1755 lookup_bus_type(char *name
)
1759 for (x
= 0; x
< MAX_BUSTYPE
; ++x
)
1760 if (strcmp(bus_type_table
[x
].name
, name
) == 0)
1761 return bus_type_table
[x
].type
;
1763 return UNKNOWN_BUSTYPE
;
1767 int_entry(int_entry_ptr entry
, int intr
)
1771 io_apic_ints
[intr
].int_type
= entry
->int_type
;
1772 io_apic_ints
[intr
].int_flags
= entry
->int_flags
;
1773 io_apic_ints
[intr
].src_bus_id
= entry
->src_bus_id
;
1774 io_apic_ints
[intr
].src_bus_irq
= entry
->src_bus_irq
;
1775 if (entry
->dst_apic_id
== 255) {
1776 /* This signal goes to all IO APICS. Select an IO APIC
1777 with sufficient number of interrupt pins */
1778 for (apic
= 0; apic
< mp_napics
; apic
++)
1779 if (((io_apic_read(apic
, IOAPIC_VER
) &
1780 IOART_VER_MAXREDIR
) >> MAXREDIRSHIFT
) >=
1781 entry
->dst_apic_int
)
1783 if (apic
< mp_napics
)
1784 io_apic_ints
[intr
].dst_apic_id
= IO_TO_ID(apic
);
1786 io_apic_ints
[intr
].dst_apic_id
= entry
->dst_apic_id
;
1788 io_apic_ints
[intr
].dst_apic_id
= entry
->dst_apic_id
;
1789 io_apic_ints
[intr
].dst_apic_int
= entry
->dst_apic_int
;
1795 apic_int_is_bus_type(int intr
, int bus_type
)
1799 for (bus
= 0; bus
< mp_nbusses
; ++bus
)
1800 if ((bus_data
[bus
].bus_id
== io_apic_ints
[intr
].src_bus_id
)
1801 && ((int) bus_data
[bus
].bus_type
== bus_type
))
1808 * Given a traditional ISA INT mask, return an APIC mask.
1811 isa_apic_mask(u_int isa_mask
)
1816 #if defined(SKIP_IRQ15_REDIRECT)
1817 if (isa_mask
== (1 << 15)) {
1818 kprintf("skipping ISA IRQ15 redirect\n");
1821 #endif /* SKIP_IRQ15_REDIRECT */
1823 isa_irq
= ffs(isa_mask
); /* find its bit position */
1824 if (isa_irq
== 0) /* doesn't exist */
1826 --isa_irq
; /* make it zero based */
1828 apic_pin
= isa_apic_irq(isa_irq
); /* look for APIC connection */
1832 return (1 << apic_pin
); /* convert pin# to a mask */
1836 * Determine which APIC pin an ISA/EISA INT is attached to.
1838 #define INTTYPE(I) (io_apic_ints[(I)].int_type)
1839 #define INTPIN(I) (io_apic_ints[(I)].dst_apic_int)
1840 #define INTIRQ(I) (io_apic_ints[(I)].int_vector)
1841 #define INTAPIC(I) (ID_TO_IO(io_apic_ints[(I)].dst_apic_id))
1843 #define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq)
1845 isa_apic_irq(int isa_irq
)
1849 for (intr
= 0; intr
< nintrs
; ++intr
) { /* check each record */
1850 if (INTTYPE(intr
) == 0) { /* standard INT */
1851 if (SRCBUSIRQ(intr
) == isa_irq
) {
1852 if (apic_int_is_bus_type(intr
, ISA
) ||
1853 apic_int_is_bus_type(intr
, EISA
)) {
1854 if (INTIRQ(intr
) == 0xff)
1855 return -1; /* unassigned */
1856 return INTIRQ(intr
); /* found */
1861 return -1; /* NOT found */
1866 * Determine which APIC pin a PCI INT is attached to.
1868 #define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id)
1869 #define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1870 #define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03)
1872 pci_apic_irq(int pciBus
, int pciDevice
, int pciInt
)
1876 --pciInt
; /* zero based */
1878 for (intr
= 0; intr
< nintrs
; ++intr
) { /* check each record */
1879 if ((INTTYPE(intr
) == 0) /* standard INT */
1880 && (SRCBUSID(intr
) == pciBus
)
1881 && (SRCBUSDEVICE(intr
) == pciDevice
)
1882 && (SRCBUSLINE(intr
) == pciInt
)) { /* a candidate IRQ */
1883 if (apic_int_is_bus_type(intr
, PCI
)) {
1884 if (INTIRQ(intr
) == 0xff) {
1885 kprintf("IOAPIC: pci_apic_irq() "
1887 return -1; /* unassigned */
1889 return INTIRQ(intr
); /* exact match */
1894 return -1; /* NOT found */
1898 next_apic_irq(int irq
)
1905 for (intr
= 0; intr
< nintrs
; intr
++) {
1906 if (INTIRQ(intr
) != irq
|| INTTYPE(intr
) != 0)
1908 bus
= SRCBUSID(intr
);
1909 bustype
= apic_bus_type(bus
);
1910 if (bustype
!= ISA
&&
1916 if (intr
>= nintrs
) {
1919 for (ointr
= intr
+ 1; ointr
< nintrs
; ointr
++) {
1920 if (INTTYPE(ointr
) != 0)
1922 if (bus
!= SRCBUSID(ointr
))
1924 if (bustype
== PCI
) {
1925 if (SRCBUSDEVICE(intr
) != SRCBUSDEVICE(ointr
))
1927 if (SRCBUSLINE(intr
) != SRCBUSLINE(ointr
))
1930 if (bustype
== ISA
|| bustype
== EISA
) {
1931 if (SRCBUSIRQ(intr
) != SRCBUSIRQ(ointr
))
1934 if (INTPIN(intr
) == INTPIN(ointr
))
1938 if (ointr
>= nintrs
) {
1941 return INTIRQ(ointr
);
1956 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
1959 * Exactly what this means is unclear at this point. It is a solution
1960 * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard
1961 * could route any of the ISA INTs to upper (>15) IRQ values. But most would
1962 * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
1966 undirect_isa_irq(int rirq
)
1970 kprintf("Freeing redirected ISA irq %d.\n", rirq
);
1971 /** FIXME: tickle the MB redirector chip */
1975 kprintf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq
);
1982 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
1985 undirect_pci_irq(int rirq
)
1989 kprintf("Freeing redirected PCI irq %d.\n", rirq
);
1991 /** FIXME: tickle the MB redirector chip */
1995 kprintf("Freeing (NOT implemented) redirected PCI irq %d.\n",
2005 * given a bus ID, return:
2006 * the bus type if found
2010 apic_bus_type(int id
)
2014 for (x
= 0; x
< mp_nbusses
; ++x
)
2015 if (bus_data
[x
].bus_id
== id
)
2016 return bus_data
[x
].bus_type
;
2022 * given a LOGICAL APIC# and pin#, return:
2023 * the associated src bus ID if found
2027 apic_src_bus_id(int apic
, int pin
)
2031 /* search each of the possible INTerrupt sources */
2032 for (x
= 0; x
< nintrs
; ++x
)
2033 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2034 (pin
== io_apic_ints
[x
].dst_apic_int
))
2035 return (io_apic_ints
[x
].src_bus_id
);
2037 return -1; /* NOT found */
2041 * given a LOGICAL APIC# and pin#, return:
2042 * the associated src bus IRQ if found
2046 apic_src_bus_irq(int apic
, int pin
)
2050 for (x
= 0; x
< nintrs
; x
++)
2051 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2052 (pin
== io_apic_ints
[x
].dst_apic_int
))
2053 return (io_apic_ints
[x
].src_bus_irq
);
2055 return -1; /* NOT found */
2060 * given a LOGICAL APIC# and pin#, return:
2061 * the associated INTerrupt type if found
2065 apic_int_type(int apic
, int pin
)
2069 /* search each of the possible INTerrupt sources */
2070 for (x
= 0; x
< nintrs
; ++x
) {
2071 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2072 (pin
== io_apic_ints
[x
].dst_apic_int
))
2073 return (io_apic_ints
[x
].int_type
);
2075 return -1; /* NOT found */
2079 * Return the IRQ associated with an APIC pin
2082 apic_irq(int apic
, int pin
)
2087 for (x
= 0; x
< nintrs
; ++x
) {
2088 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2089 (pin
== io_apic_ints
[x
].dst_apic_int
)) {
2090 res
= io_apic_ints
[x
].int_vector
;
2093 if (apic
!= int_to_apicintpin
[res
].ioapic
)
2094 panic("apic_irq: inconsistent table %d/%d", apic
, int_to_apicintpin
[res
].ioapic
);
2095 if (pin
!= int_to_apicintpin
[res
].int_pin
)
2096 panic("apic_irq inconsistent table (2)");
2105 * given a LOGICAL APIC# and pin#, return:
2106 * the associated trigger mode if found
2110 apic_trigger(int apic
, int pin
)
2114 /* search each of the possible INTerrupt sources */
2115 for (x
= 0; x
< nintrs
; ++x
)
2116 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2117 (pin
== io_apic_ints
[x
].dst_apic_int
))
2118 return ((io_apic_ints
[x
].int_flags
>> 2) & 0x03);
2120 return -1; /* NOT found */
2125 * given a LOGICAL APIC# and pin#, return:
2126 * the associated 'active' level if found
2130 apic_polarity(int apic
, int pin
)
2134 /* search each of the possible INTerrupt sources */
2135 for (x
= 0; x
< nintrs
; ++x
)
2136 if ((apic
== ID_TO_IO(io_apic_ints
[x
].dst_apic_id
)) &&
2137 (pin
== io_apic_ints
[x
].dst_apic_int
))
2138 return (io_apic_ints
[x
].int_flags
& 0x03);
2140 return -1; /* NOT found */
2146 * set data according to MP defaults
2147 * FIXME: probably not complete yet...
2150 mptable_default(int type
)
2152 #if defined(APIC_IO)
2155 #endif /* APIC_IO */
2158 kprintf(" MP default config type: %d\n", type
);
2161 kprintf(" bus: ISA, APIC: 82489DX\n");
2164 kprintf(" bus: EISA, APIC: 82489DX\n");
2167 kprintf(" bus: EISA, APIC: 82489DX\n");
2170 kprintf(" bus: MCA, APIC: 82489DX\n");
2173 kprintf(" bus: ISA+PCI, APIC: Integrated\n");
2176 kprintf(" bus: EISA+PCI, APIC: Integrated\n");
2179 kprintf(" bus: MCA+PCI, APIC: Integrated\n");
2182 kprintf(" future type\n");
2188 #if defined(APIC_IO)
2189 /* one and only IO APIC */
2190 io_apic_id
= (io_apic_read(0, IOAPIC_ID
) & APIC_ID_MASK
) >> 24;
2193 * sanity check, refer to MP spec section 3.6.6, last paragraph
2194 * necessary as some hardware isn't properly setting up the IO APIC
2196 #if defined(REALLY_ANAL_IOAPICID_VALUE)
2197 if (io_apic_id
!= 2) {
2199 if ((io_apic_id
== 0) || (io_apic_id
== 1) || (io_apic_id
== 15)) {
2200 #endif /* REALLY_ANAL_IOAPICID_VALUE */
2201 io_apic_set_id(0, 2);
2204 IO_TO_ID(0) = io_apic_id
;
2205 ID_TO_IO(io_apic_id
) = 0;
2206 #endif /* APIC_IO */
2208 /* fill out bus entries */
2218 bus_data
[0].bus_id
= default_data
[type
- 1][1];
2219 bus_data
[0].bus_type
= default_data
[type
- 1][2];
2220 bus_data
[1].bus_id
= default_data
[type
- 1][3];
2221 bus_data
[1].bus_type
= default_data
[type
- 1][4];
2225 /* case 4: case 7: MCA NOT supported */
2226 default: /* illegal/reserved */
2227 panic("BAD default MP config: %d", type
);
2231 #if defined(APIC_IO)
2232 /* general cases from MP v1.4, table 5-2 */
2233 for (pin
= 0; pin
< 16; ++pin
) {
2234 io_apic_ints
[pin
].int_type
= 0;
2235 io_apic_ints
[pin
].int_flags
= 0x05; /* edge/active-hi */
2236 io_apic_ints
[pin
].src_bus_id
= 0;
2237 io_apic_ints
[pin
].src_bus_irq
= pin
; /* IRQ2 caught below */
2238 io_apic_ints
[pin
].dst_apic_id
= io_apic_id
;
2239 io_apic_ints
[pin
].dst_apic_int
= pin
; /* 1-to-1 */
2242 /* special cases from MP v1.4, table 5-2 */
2244 io_apic_ints
[2].int_type
= 0xff; /* N/C */
2245 io_apic_ints
[13].int_type
= 0xff; /* N/C */
2246 #if !defined(APIC_MIXED_MODE)
2248 panic("sorry, can't support type 2 default yet");
2249 #endif /* APIC_MIXED_MODE */
2252 io_apic_ints
[2].src_bus_irq
= 0; /* ISA IRQ0 is on APIC INT 2 */
2255 io_apic_ints
[0].int_type
= 0xff; /* N/C */
2257 io_apic_ints
[0].int_type
= 3; /* vectored 8259 */
2258 #endif /* APIC_IO */
2262 * Map a physical memory address representing I/O into KVA. The I/O
2263 * block is assumed not to cross a page boundary.
2266 permanent_io_mapping(vm_paddr_t pa
)
2268 KKASSERT(pa
< 0x100000000LL
);
2270 return pmap_mapdev_uncacheable(pa
, PAGE_SIZE
);
2274 * start each AP in our list
2277 start_all_aps(u_int boot_addr
)
2279 vm_offset_t va
= boot_address
+ KERNBASE
;
2280 u_int64_t
*pt4
, *pt3
, *pt2
;
2286 u_char mpbiosreason
;
2287 u_long mpbioswarmvec
;
2288 struct mdglobaldata
*gd
;
2289 struct privatespace
*ps
;
2291 POSTCODE(START_ALL_APS_POST
);
2293 /* Initialize BSP's local APIC */
2294 apic_initialize(TRUE
);
2296 /* install the AP 1st level boot code */
2297 pmap_kenter(va
, boot_address
);
2298 cpu_invlpg((void *)va
); /* JG XXX */
2299 bcopy(mptramp_start
, (void *)va
, bootMP_size
);
2301 /* Locate the page tables, they'll be below the trampoline */
2302 pt4
= (u_int64_t
*)(uintptr_t)(mptramp_pagetables
+ KERNBASE
);
2303 pt3
= pt4
+ (PAGE_SIZE
) / sizeof(u_int64_t
);
2304 pt2
= pt3
+ (PAGE_SIZE
) / sizeof(u_int64_t
);
2306 /* Create the initial 1GB replicated page tables */
2307 for (i
= 0; i
< 512; i
++) {
2308 /* Each slot of the level 4 pages points to the same level 3 page */
2309 pt4
[i
] = (u_int64_t
)(uintptr_t)(mptramp_pagetables
+ PAGE_SIZE
);
2310 pt4
[i
] |= PG_V
| PG_RW
| PG_U
;
2312 /* Each slot of the level 3 pages points to the same level 2 page */
2313 pt3
[i
] = (u_int64_t
)(uintptr_t)(mptramp_pagetables
+ (2 * PAGE_SIZE
));
2314 pt3
[i
] |= PG_V
| PG_RW
| PG_U
;
2316 /* The level 2 page slots are mapped with 2MB pages for 1GB. */
2317 pt2
[i
] = i
* (2 * 1024 * 1024);
2318 pt2
[i
] |= PG_V
| PG_RW
| PG_PS
| PG_U
;
2321 /* save the current value of the warm-start vector */
2322 mpbioswarmvec
= *((u_int32_t
*) WARMBOOT_OFF
);
2323 outb(CMOS_REG
, BIOS_RESET
);
2324 mpbiosreason
= inb(CMOS_DATA
);
2326 /* setup a vector to our boot code */
2327 *((volatile u_short
*) WARMBOOT_OFF
) = WARMBOOT_TARGET
;
2328 *((volatile u_short
*) WARMBOOT_SEG
) = (boot_address
>> 4);
2329 outb(CMOS_REG
, BIOS_RESET
);
2330 outb(CMOS_DATA
, BIOS_WARM
); /* 'warm-start' */
2333 * If we have a TSC we can figure out the SMI interrupt rate.
2334 * The SMI does not necessarily use a constant rate. Spend
2335 * up to 250ms trying to figure it out.
2338 if (cpu_feature
& CPUID_TSC
) {
2339 set_apic_timer(275000);
2340 smilast
= read_apic_timer();
2341 for (x
= 0; x
< 20 && read_apic_timer(); ++x
) {
2342 smicount
= smitest();
2343 if (smibest
== 0 || smilast
- smicount
< smibest
)
2344 smibest
= smilast
- smicount
;
2347 if (smibest
> 250000)
2350 smibest
= smibest
* (int64_t)1000000 /
2351 get_apic_timer_frequency();
2355 kprintf("SMI Frequency (worst case): %d Hz (%d us)\n",
2356 1000000 / smibest
, smibest
);
2359 for (x
= 1; x
<= mp_naps
; ++x
) {
2361 /* This is a bit verbose, it will go away soon. */
2363 /* first page of AP's private space */
2364 pg
= x
* x86_64_btop(sizeof(struct privatespace
));
2366 /* allocate new private data page(s) */
2367 gd
= (struct mdglobaldata
*)kmem_alloc(&kernel_map
,
2368 MDGLOBALDATA_BASEALLOC_SIZE
);
2370 gd
= &CPU_prvspace
[x
].mdglobaldata
; /* official location */
2371 bzero(gd
, sizeof(*gd
));
2372 gd
->mi
.gd_prvspace
= ps
= &CPU_prvspace
[x
];
2374 /* prime data page for it to use */
2375 mi_gdinit(&gd
->mi
, x
);
2377 gd
->gd_CMAP1
= &SMPpt
[pg
+ 0];
2378 gd
->gd_CMAP2
= &SMPpt
[pg
+ 1];
2379 gd
->gd_CMAP3
= &SMPpt
[pg
+ 2];
2380 gd
->gd_PMAP1
= &SMPpt
[pg
+ 3];
2381 gd
->gd_CADDR1
= ps
->CPAGE1
;
2382 gd
->gd_CADDR2
= ps
->CPAGE2
;
2383 gd
->gd_CADDR3
= ps
->CPAGE3
;
2384 gd
->gd_PADDR1
= (pt_entry_t
*)ps
->PPAGE1
;
2385 gd
->mi
.gd_ipiq
= (void *)kmem_alloc(&kernel_map
, sizeof(lwkt_ipiq
) * (mp_naps
+ 1));
2386 bzero(gd
->mi
.gd_ipiq
, sizeof(lwkt_ipiq
) * (mp_naps
+ 1));
2388 /* setup a vector to our boot code */
2389 *((volatile u_short
*) WARMBOOT_OFF
) = WARMBOOT_TARGET
;
2390 *((volatile u_short
*) WARMBOOT_SEG
) = (boot_addr
>> 4);
2391 outb(CMOS_REG
, BIOS_RESET
);
2392 outb(CMOS_DATA
, BIOS_WARM
); /* 'warm-start' */
2395 * Setup the AP boot stack
2397 bootSTK
= &ps
->idlestack
[UPAGES
*PAGE_SIZE
/2];
2400 /* attempt to start the Application Processor */
2401 CHECK_INIT(99); /* setup checkpoints */
2402 if (!start_ap(gd
, boot_addr
, smibest
)) {
2403 kprintf("AP #%d (PHY# %d) failed!\n", x
, CPU_TO_ID(x
));
2404 CHECK_PRINT("trace"); /* show checkpoints */
2405 /* better panic as the AP may be running loose */
2406 kprintf("panic y/n? [y] ");
2407 if (cngetc() != 'n')
2410 CHECK_PRINT("trace"); /* show checkpoints */
2412 /* record its version info */
2413 cpu_apic_versions
[x
] = cpu_apic_versions
[0];
2416 /* set ncpus to 1 + highest logical cpu. Not all may have come up */
2419 /* ncpus2 -- ncpus rounded down to the nearest power of 2 */
2420 for (shift
= 0; (1 << shift
) <= ncpus
; ++shift
)
2423 ncpus2_shift
= shift
;
2424 ncpus2
= 1 << shift
;
2425 ncpus2_mask
= ncpus2
- 1;
2427 /* ncpus_fit -- ncpus rounded up to the nearest power of 2 */
2428 if ((1 << shift
) < ncpus
)
2430 ncpus_fit
= 1 << shift
;
2431 ncpus_fit_mask
= ncpus_fit
- 1;
2433 /* build our map of 'other' CPUs */
2434 mycpu
->gd_other_cpus
= smp_startup_mask
& ~(1 << mycpu
->gd_cpuid
);
2435 mycpu
->gd_ipiq
= (void *)kmem_alloc(&kernel_map
, sizeof(lwkt_ipiq
) * ncpus
);
2436 bzero(mycpu
->gd_ipiq
, sizeof(lwkt_ipiq
) * ncpus
);
2438 /* fill in our (BSP) APIC version */
2439 cpu_apic_versions
[0] = lapic
->version
;
2441 /* restore the warmstart vector */
2442 *(u_long
*) WARMBOOT_OFF
= mpbioswarmvec
;
2443 outb(CMOS_REG
, BIOS_RESET
);
2444 outb(CMOS_DATA
, mpbiosreason
);
2447 * NOTE! The idlestack for the BSP was setup by locore. Finish
2448 * up, clean out the P==V mapping we did earlier.
2452 /* number of APs actually started */
2458 * load the 1st level AP boot code into base memory.
2461 /* targets for relocation */
2462 extern void bigJump(void);
2463 extern void bootCodeSeg(void);
2464 extern void bootDataSeg(void);
2465 extern void MPentry(void);
2466 extern u_int MP_GDT
;
2467 extern u_int mp_gdtbase
;
2472 install_ap_tramp(u_int boot_addr
)
2475 int size
= *(int *) ((u_long
) & bootMP_size
);
2476 u_char
*src
= (u_char
*) ((u_long
) bootMP
);
2477 u_char
*dst
= (u_char
*) boot_addr
+ KERNBASE
;
2478 u_int boot_base
= (u_int
) bootMP
;
2483 POSTCODE(INSTALL_AP_TRAMP_POST
);
2485 for (x
= 0; x
< size
; ++x
)
2489 * modify addresses in code we just moved to basemem. unfortunately we
2490 * need fairly detailed info about mpboot.s for this to work. changes
2491 * to mpboot.s might require changes here.
2494 /* boot code is located in KERNEL space */
2495 dst
= (u_char
*) boot_addr
+ KERNBASE
;
2497 /* modify the lgdt arg */
2498 dst32
= (u_int32_t
*) (dst
+ ((u_int
) & mp_gdtbase
- boot_base
));
2499 *dst32
= boot_addr
+ ((u_int
) & MP_GDT
- boot_base
);
2501 /* modify the ljmp target for MPentry() */
2502 dst32
= (u_int32_t
*) (dst
+ ((u_int
) bigJump
- boot_base
) + 1);
2503 *dst32
= ((u_int
) MPentry
- KERNBASE
);
2505 /* modify the target for boot code segment */
2506 dst16
= (u_int16_t
*) (dst
+ ((u_int
) bootCodeSeg
- boot_base
));
2507 dst8
= (u_int8_t
*) (dst16
+ 1);
2508 *dst16
= (u_int
) boot_addr
& 0xffff;
2509 *dst8
= ((u_int
) boot_addr
>> 16) & 0xff;
2511 /* modify the target for boot data segment */
2512 dst16
= (u_int16_t
*) (dst
+ ((u_int
) bootDataSeg
- boot_base
));
2513 dst8
= (u_int8_t
*) (dst16
+ 1);
2514 *dst16
= (u_int
) boot_addr
& 0xffff;
2515 *dst8
= ((u_int
) boot_addr
>> 16) & 0xff;
2521 * This function starts the AP (application processor) identified
2522 * by the APIC ID 'physicalCpu'. It does quite a "song and dance"
2523 * to accomplish this. This is necessary because of the nuances
2524 * of the different hardware we might encounter. It ain't pretty,
2525 * but it seems to work.
2527 * NOTE: eventually an AP gets to ap_init(), which is called just
2528 * before the AP goes into the LWKT scheduler's idle loop.
2531 start_ap(struct mdglobaldata
*gd
, u_int boot_addr
, int smibest
)
2535 u_long icr_lo
, icr_hi
;
2537 POSTCODE(START_AP_POST
);
2539 /* get the PHYSICAL APIC ID# */
2540 physical_cpu
= CPU_TO_ID(gd
->mi
.gd_cpuid
);
2542 /* calculate the vector */
2543 vector
= (boot_addr
>> 12) & 0xff;
2545 /* We don't want anything interfering */
2548 /* Make sure the target cpu sees everything */
2552 * Try to detect when a SMI has occurred, wait up to 200ms.
2554 * If a SMI occurs during an AP reset but before we issue
2555 * the STARTUP command, the AP may brick. To work around
2556 * this problem we hold off doing the AP startup until
2557 * after we have detected the SMI. Hopefully another SMI
2558 * will not occur before we finish the AP startup.
2560 * Retries don't seem to help. SMIs have a window of opportunity
2561 * and if USB->legacy keyboard emulation is enabled in the BIOS
2562 * the interrupt rate can be quite high.
2564 * NOTE: Don't worry about the L1 cache load, it might bloat
2565 * ldelta a little but ndelta will be so huge when the SMI
2566 * occurs the detection logic will still work fine.
2569 set_apic_timer(200000);
2574 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
2575 * and running the target CPU. OR this INIT IPI might be latched (P5
2576 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
2579 * see apic/apicreg.h for icr bit definitions.
2581 * TIME CRITICAL CODE, DO NOT DO ANY KPRINTFS IN THE HOT PATH.
2585 * Setup the address for the target AP. We can setup
2586 * icr_hi once and then just trigger operations with
2589 icr_hi
= lapic
->icr_hi
& ~APIC_ID_MASK
;
2590 icr_hi
|= (physical_cpu
<< 24);
2591 icr_lo
= lapic
->icr_lo
& 0xfff00000;
2592 lapic
->icr_hi
= icr_hi
;
2595 * Do an INIT IPI: assert RESET
2597 * Use edge triggered mode to assert INIT
2599 lapic
->icr_lo
= icr_lo
| 0x00004500;
2600 while (lapic
->icr_lo
& APIC_DELSTAT_MASK
)
2604 * The spec calls for a 10ms delay but we may have to use a
2605 * MUCH lower delay to avoid bricking an AP due to a fast SMI
2606 * interrupt. We have other loops here too and dividing by 2
2607 * doesn't seem to be enough even after subtracting 350us,
2608 * so we divide by 4.
2610 * Our minimum delay is 150uS, maximum is 10ms. If no SMI
2611 * interrupt was detected we use the full 10ms.
2615 else if (smibest
< 150 * 4 + 350)
2617 else if ((smibest
- 350) / 4 < 10000)
2618 u_sleep((smibest
- 350) / 4);
2623 * Do an INIT IPI: deassert RESET
2625 * Use level triggered mode to deassert. It is unclear
2626 * why we need to do this.
2628 lapic
->icr_lo
= icr_lo
| 0x00008500;
2629 while (lapic
->icr_lo
& APIC_DELSTAT_MASK
)
2631 u_sleep(150); /* wait 150us */
2634 * Next we do a STARTUP IPI: the previous INIT IPI might still be
2635 * latched, (P5 bug) this 1st STARTUP would then terminate
2636 * immediately, and the previously started INIT IPI would continue. OR
2637 * the previous INIT IPI has already run. and this STARTUP IPI will
2638 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
2641 lapic
->icr_lo
= icr_lo
| 0x00000600 | vector
;
2642 while (lapic
->icr_lo
& APIC_DELSTAT_MASK
)
2644 u_sleep(200); /* wait ~200uS */
2647 * Finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
2648 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
2649 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
2650 * recognized after hardware RESET or INIT IPI.
2652 lapic
->icr_lo
= icr_lo
| 0x00000600 | vector
;
2653 while (lapic
->icr_lo
& APIC_DELSTAT_MASK
)
2656 /* Resume normal operation */
2659 /* wait for it to start, see ap_init() */
2660 set_apic_timer(5000000);/* == 5 seconds */
2661 while (read_apic_timer()) {
2662 if (smp_startup_mask
& (1 << gd
->mi
.gd_cpuid
))
2663 return 1; /* return SUCCESS */
2666 return 0; /* return FAILURE */
2681 while (read_apic_timer()) {
2683 for (count
= 0; count
< 100; ++count
)
2684 ntsc
= rdtsc(); /* force loop to occur */
2686 ndelta
= ntsc
- ltsc
;
2687 if (ldelta
> ndelta
)
2689 if (ndelta
> ldelta
* 2)
2692 ldelta
= ntsc
- ltsc
;
2695 return(read_apic_timer());
2699 * Lazy flush the TLB on all other CPU's. DEPRECATED.
2701 * If for some reason we were unable to start all cpus we cannot safely
2702 * use broadcast IPIs.
2708 if (smp_startup_mask
== smp_active_mask
) {
2709 all_but_self_ipi(XINVLTLB_OFFSET
);
2711 selected_apic_ipi(smp_active_mask
, XINVLTLB_OFFSET
,
2712 APIC_DELMODE_FIXED
);
2718 * When called the executing CPU will send an IPI to all other CPUs
2719 * requesting that they halt execution.
2721 * Usually (but not necessarily) called with 'other_cpus' as its arg.
2723 * - Signals all CPUs in map to stop.
2724 * - Waits for each to stop.
2731 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
2732 * from executing at same time.
2735 stop_cpus(u_int map
)
2737 map
&= smp_active_mask
;
2739 /* send the Xcpustop IPI to all CPUs in map */
2740 selected_apic_ipi(map
, XCPUSTOP_OFFSET
, APIC_DELMODE_FIXED
);
2742 while ((stopped_cpus
& map
) != map
)
2750 * Called by a CPU to restart stopped CPUs.
2752 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
2754 * - Signals all CPUs in map to restart.
2755 * - Waits for each to restart.
2763 restart_cpus(u_int map
)
2765 /* signal other cpus to restart */
2766 started_cpus
= map
& smp_active_mask
;
2768 while ((stopped_cpus
& map
) != 0) /* wait for each to clear its bit */
2775 * This is called once the mpboot code has gotten us properly relocated
2776 * and the MMU turned on, etc. ap_init() is actually the idle thread,
2777 * and when it returns the scheduler will call the real cpu_idle() main
2778 * loop for the idlethread. Interrupts are disabled on entry and should
2779 * remain disabled at return.
2787 * Adjust smp_startup_mask to signal the BSP that we have started
2788 * up successfully. Note that we do not yet hold the BGL. The BSP
2789 * is waiting for our signal.
2791 * We can't set our bit in smp_active_mask yet because we are holding
2792 * interrupts physically disabled and remote cpus could deadlock
2793 * trying to send us an IPI.
2795 smp_startup_mask
|= 1 << mycpu
->gd_cpuid
;
2799 * Interlock for finalization. Wait until mp_finish is non-zero,
2800 * then get the MP lock.
2802 * Note: We are in a critical section.
2804 * Note: We have to synchronize td_mpcount to our desired MP state
2805 * before calling cpu_try_mplock().
2807 * Note: we are the idle thread, we can only spin.
2809 * Note: The load fence is memory volatile and prevents the compiler
2810 * from improperly caching mp_finish, and the cpu from improperly
2813 while (mp_finish
== 0)
2815 ++curthread
->td_mpcount
;
2816 while (cpu_try_mplock() == 0)
2819 if (cpu_feature
& CPUID_TSC
) {
2821 * The BSP is constantly updating tsc0_offset, figure out the
2822 * relative difference to synchronize ktrdump.
2824 tsc_offsets
[mycpu
->gd_cpuid
] = rdtsc() - tsc0_offset
;
2827 /* BSP may have changed PTD while we're waiting for the lock */
2830 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
2834 /* Build our map of 'other' CPUs. */
2835 mycpu
->gd_other_cpus
= smp_startup_mask
& ~(1 << mycpu
->gd_cpuid
);
2837 kprintf("SMP: AP CPU #%d Launched!\n", mycpu
->gd_cpuid
);
2839 /* A quick check from sanity claus */
2840 apic_id
= (apic_id_to_logical
[(lapic
->id
& 0x0f000000) >> 24]);
2841 if (mycpu
->gd_cpuid
!= apic_id
) {
2842 kprintf("SMP: cpuid = %d\n", mycpu
->gd_cpuid
);
2843 kprintf("SMP: apic_id = %d\n", apic_id
);
2845 kprintf("PTD[MPPTDI] = %p\n", (void *)PTD
[MPPTDI
]);
2847 panic("cpuid mismatch! boom!!");
2850 /* Initialize AP's local APIC for irq's */
2851 apic_initialize(FALSE
);
2853 /* Set memory range attributes for this CPU to match the BSP */
2854 mem_range_AP_init();
2857 * Once we go active we must process any IPIQ messages that may
2858 * have been queued, because no actual IPI will occur until we
2859 * set our bit in the smp_active_mask. If we don't the IPI
2860 * message interlock could be left set which would also prevent
2863 * The idle loop doesn't expect the BGL to be held and while
2864 * lwkt_switch() normally cleans things up this is a special case
2865 * because we returning almost directly into the idle loop.
2867 * The idle thread is never placed on the runq, make sure
2868 * nothing we've done put it there.
2870 KKASSERT(curthread
->td_mpcount
== 1);
2871 smp_active_mask
|= 1 << mycpu
->gd_cpuid
;
2874 * Enable interrupts here. idle_restore will also do it, but
2875 * doing it here lets us clean up any strays that got posted to
2876 * the CPU during the AP boot while we are still in a critical
2879 __asm
__volatile("sti; pause; pause"::);
2880 mdcpu
->gd_fpending
= 0;
2882 initclocks_pcpu(); /* clock interrupts (via IPIs) */
2883 lwkt_process_ipiq();
2886 * Releasing the mp lock lets the BSP finish up the SMP init
2889 KKASSERT((curthread
->td_flags
& TDF_RUNQ
) == 0);
2893 * Get SMP fully working before we start initializing devices.
2901 kprintf("Finish MP startup\n");
2902 if (cpu_feature
& CPUID_TSC
)
2903 tsc0_offset
= rdtsc();
2906 while (smp_active_mask
!= smp_startup_mask
) {
2908 if (cpu_feature
& CPUID_TSC
)
2909 tsc0_offset
= rdtsc();
2911 while (try_mplock() == 0)
2914 kprintf("Active CPU Mask: %08x\n", smp_active_mask
);
2917 SYSINIT(finishsmp
, SI_BOOT2_FINISH_SMP
, SI_ORDER_FIRST
, ap_finish
, NULL
)
2920 cpu_send_ipiq(int dcpu
)
2922 if ((1 << dcpu
) & smp_active_mask
)
2923 single_apic_ipi(dcpu
, XIPIQ_OFFSET
, APIC_DELMODE_FIXED
);
2926 #if 0 /* single_apic_ipi_passive() not working yet */
2928 * Returns 0 on failure, 1 on success
2931 cpu_send_ipiq_passive(int dcpu
)
2934 if ((1 << dcpu
) & smp_active_mask
) {
2935 r
= single_apic_ipi_passive(dcpu
, XIPIQ_OFFSET
,
2936 APIC_DELMODE_FIXED
);
2942 struct mptable_lapic_cbarg1
{
2948 mptable_lapic_pass1_callback(void *xarg
, const void *pos
, int type
)
2950 const struct PROCENTRY
*ent
;
2951 struct mptable_lapic_cbarg1
*arg
= xarg
;
2957 if ((ent
->cpu_flags
& PROCENTRY_FLAG_EN
) == 0)
2961 arg
->id_mask
|= 1 << ent
->apic_id
;
2965 struct mptable_lapic_cbarg2
{
2971 mptable_lapic_pass2_callback(void *xarg
, const void *pos
, int type
)
2973 const struct PROCENTRY
*ent
;
2974 struct mptable_lapic_cbarg2
*arg
= xarg
;
2980 if (ent
->cpu_flags
& PROCENTRY_FLAG_BP
) {
2981 KKASSERT(!arg
->found_bsp
);
2985 if (processor_entry(ent
, arg
->cpu
))
2988 if (need_hyperthreading_fixup
) {
2989 struct PROCENTRY proc
;
2993 * Create fake mptable processor entries
2994 * and feed them to processor_entry() to
2995 * enumerate the logical CPUs.
2997 bzero(&proc
, sizeof(proc
));
2999 proc
.cpu_flags
= PROCENTRY_FLAG_EN
;
3000 proc
.apic_id
= ent
->apic_id
;
3002 for (i
= 1; i
< logical_cpus
; i
++) {
3004 processor_entry(&proc
, arg
->cpu
);
3005 logical_cpus_mask
|= (1 << arg
->cpu
);
3014 * cpu_apic_address (common to all CPUs)
3016 * need_hyperthreading_fixup
3019 * ID_TO_IO(N), phy APIC ID to log CPU/IO table
3020 * CPU_TO_ID(N), logical CPU to APIC ID table
3023 mptable_lapic_enumerate(struct mptable_pos
*mpt
)
3027 vm_offset_t lapic_addr
;
3030 KKASSERT(fps
!= NULL
);
3032 /* init everything to empty */
3035 /* check for use of 'default' configuration */
3036 if (fps
->mpfb1
!= 0) {
3037 /* use default addresses */
3038 lapic_addr
= DEFAULT_APIC_BASE
;
3040 /* fill in with defaults */
3041 mp_naps
= 1; /* exclude BSP */
3043 struct mptable_lapic_cbarg1 arg
;
3047 KKASSERT(cth
!= NULL
);
3049 lapic_addr
= (vm_offset_t
)cth
->apic_address
;
3050 KKASSERT(lapic_addr
!= 0);
3052 bzero(&arg
, sizeof(arg
));
3053 error
= mptable_iterate_entries(cth
,
3054 mptable_lapic_pass1_callback
, &arg
);
3056 panic("mptable_iterate_entries(lapic_pass1) failed\n");
3058 KKASSERT(arg
.cpu_count
!= 0);
3059 mp_naps
= arg
.cpu_count
;
3061 /* Qualify the numbers */
3062 if (mp_naps
> MAXCPU
) {
3063 kprintf("Warning: only using %d of %d available CPUs!\n",
3068 /* See if we need to fixup HT logical CPUs. */
3069 mptable_hyperthread_fixup(arg
.id_mask
);
3071 /* Qualify the numbers again, after hyperthreading fixup */
3072 if (mp_naps
> MAXCPU
) {
3073 kprintf("Warning: only using %d of %d available CPUs!\n",
3078 --mp_naps
; /* subtract the BSP */
3081 lapic_init(lapic_addr
);
3083 if (fps
->mpfb1
!= 0) {
3084 int ap_cpu_id
, boot_cpu_id
;
3086 boot_cpu_id
= (lapic
->id
& APIC_ID_MASK
) >> 24;
3087 ap_cpu_id
= (boot_cpu_id
== 0) ? 1 : 0;
3090 CPU_TO_ID(0) = boot_cpu_id
;
3091 ID_TO_CPU(boot_cpu_id
) = 0;
3093 /* one and only AP */
3094 CPU_TO_ID(1) = ap_cpu_id
;
3095 ID_TO_CPU(ap_cpu_id
) = 1;
3097 struct mptable_lapic_cbarg2 arg
;
3099 bzero(&arg
, sizeof(arg
));
3102 error
= mptable_iterate_entries(mpt
->mp_cth
,
3103 mptable_lapic_pass2_callback
, &arg
);
3105 panic("mptable_iterate_entries(lapic_pass2) failed\n");
3106 KKASSERT(arg
.found_bsp
);
3111 lapic_init(vm_offset_t lapic_addr
)
3114 * lapic not mapped yet (pmap_init is called too late)
3116 lapic
= pmap_mapdev_uncacheable(lapic_addr
, sizeof(struct LAPIC
));
3119 /* Local apic is mapped on last page */
3120 SMPpt
[NPTEPG
- 1] = (pt_entry_t
)(PG_V
| PG_RW
| PG_N
|
3121 pmap_get_pgeflag() | (lapic_addr
& PG_FRAME
));
3124 /* Just for printing */
3125 cpu_apic_address
= lapic_addr
;