1 /* $NetBSD: cpu.c,v 1.65 2009/11/21 03:11:01 rmind Exp $ */
4 * Copyright (c) 2000, 2006, 2007, 2008 The NetBSD Foundation, Inc.
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
33 * Copyright (c) 1999 Stefan Grefen
35 * Redistribution and use in source and binary forms, with or without
36 * modification, are permitted provided that the following conditions
38 * 1. Redistributions of source code must retain the above copyright
39 * notice, this list of conditions and the following disclaimer.
40 * 2. Redistributions in binary form must reproduce the above copyright
41 * notice, this list of conditions and the following disclaimer in the
42 * documentation and/or other materials provided with the distribution.
43 * 3. All advertising materials mentioning features or use of this software
44 * must display the following acknowledgement:
45 * This product includes software developed by the NetBSD
46 * Foundation, Inc. and its contributors.
47 * 4. Neither the name of The NetBSD Foundation nor the names of its
48 * contributors may be used to endorse or promote products derived
49 * from this software without specific prior written permission.
51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 #include <sys/cdefs.h>
65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.65 2009/11/21 03:11:01 rmind Exp $");
68 #include "opt_mpbios.h" /* for MPDEBUG */
78 #include <sys/param.h>
80 #include <sys/systm.h>
81 #include <sys/device.h>
84 #include <sys/atomic.h>
85 #include <sys/reboot.h>
87 #include <uvm/uvm_extern.h>
89 #include <machine/cpufunc.h>
90 #include <machine/cpuvar.h>
91 #include <machine/pmap.h>
92 #include <machine/vmparam.h>
93 #include <machine/mpbiosvar.h>
94 #include <machine/pcb.h>
95 #include <machine/specialreg.h>
96 #include <machine/segments.h>
97 #include <machine/gdt.h>
98 #include <machine/mtrr.h>
99 #include <machine/pio.h>
100 #include <machine/cpu_counter.h>
103 #include <machine/tlog.h>
106 #include <machine/apicvar.h>
107 #include <machine/i82489reg.h>
108 #include <machine/i82489var.h>
110 #include <dev/ic/mc146818reg.h>
111 #include <i386/isa/nvram.h>
112 #include <dev/isa/isareg.h>
117 #error cpu_info contains 32bit bitmasks
120 int cpu_match(device_t
, cfdata_t
, void *);
121 void cpu_attach(device_t
, device_t
, void *);
123 static bool cpu_suspend(device_t
, pmf_qual_t
);
124 static bool cpu_resume(device_t
, pmf_qual_t
);
127 device_t sc_dev
; /* device tree glue */
128 struct cpu_info
*sc_info
; /* pointer to CPU info */
132 int mp_cpu_start(struct cpu_info
*, paddr_t
);
133 void mp_cpu_start_cleanup(struct cpu_info
*);
134 const struct cpu_functions mp_cpu_funcs
= { mp_cpu_start
, NULL
,
135 mp_cpu_start_cleanup
};
138 CFATTACH_DECL_NEW(cpu
, sizeof(struct cpu_softc
),
139 cpu_match
, cpu_attach
, NULL
, NULL
);
142 * Statically-allocated CPU info for the primary CPU (or the only
143 * CPU, on uniprocessors). The CPU info list is initialized to
147 struct tlog tlog_primary
;
149 struct cpu_info cpu_info_primary
__aligned(CACHE_LINE_SIZE
) = {
151 .ci_self
= &cpu_info_primary
,
156 .ci_tlog_base
= &tlog_primary
,
157 #endif /* !TRAPLOG */
160 struct cpu_info
*cpu_info_list
= &cpu_info_primary
;
162 static void cpu_set_tss_gates(struct cpu_info
*);
165 static void tss_init(struct i386tss
*, void *, void *);
168 static void cpu_init_idle_lwp(struct cpu_info
*);
170 uint32_t cpus_attached
= 0;
171 uint32_t cpus_running
= 0;
173 extern char x86_64_doubleflt_stack
[];
176 paddr_t mp_trampoline_paddr
= MP_TRAMPOLINE
;
177 static vaddr_t cmos_data_mapping
;
178 struct cpu_info
*cpu_starting
;
180 void cpu_hatch(void *);
181 static void cpu_boot_secondary(struct cpu_info
*ci
);
182 static void cpu_start_secondary(struct cpu_info
*ci
);
183 static void cpu_copy_trampoline(void);
186 * Runs once per boot once multiprocessor goo has been detected and
187 * the local APIC on the boot processor has been mapped.
189 * Called from lapic_boot_init() (from mpbios_scan()).
195 cpu_info_primary
.ci_cpuid
= lapic_cpu_number();
196 cpu_copy_trampoline();
198 cmos_data_mapping
= uvm_km_alloc(kernel_map
, PAGE_SIZE
, 0, UVM_KMF_VAONLY
);
199 if (cmos_data_mapping
== 0)
200 panic("No KVA for page 0");
201 pmap_kenter_pa(cmos_data_mapping
, 0, VM_PROT_READ
|VM_PROT_WRITE
, 0);
202 pmap_update(pmap_kernel());
206 cpu_match(device_t parent
, cfdata_t match
, void *aux
)
213 cpu_vm_init(struct cpu_info
*ci
)
217 for (i
= CAI_ICACHE
; i
<= CAI_L2CACHE
; i
++) {
218 struct x86_cache_info
*cai
;
221 cai
= &ci
->ci_cinfo
[i
];
223 tcolors
= atop(cai
->cai_totalsize
);
224 switch(cai
->cai_associativity
) {
226 tcolors
= 1; /* fully associative */
232 tcolors
/= cai
->cai_associativity
;
234 ncolors
= max(ncolors
, tcolors
);
236 * If the desired number of colors is not a power of
237 * two, it won't be good. Find the greatest power of
238 * two which is an even divisor of the number of colors,
239 * to preserve even coloring of pages.
241 if (ncolors
& (ncolors
- 1) ) {
243 for (try = 1; try < ncolors
; try *= 2) {
244 if (ncolors
% try == 0) picked
= try;
247 panic("desired number of cache colors %d is "
248 " > 1, but not even!", ncolors
);
255 * Knowing the size of the largest cache on this CPU, re-color
258 if (ncolors
<= uvmexp
.ncolors
)
260 aprint_debug_dev(ci
->ci_dev
, "%d page colors\n", ncolors
);
261 uvm_page_recolor(ncolors
);
266 cpu_attach(device_t parent
, device_t self
, void *aux
)
268 struct cpu_softc
*sc
= device_private(self
);
269 struct cpu_attach_args
*caa
= aux
;
272 int cpunum
= caa
->cpu_number
;
277 if (cpus_attached
== ~0) {
278 aprint_error(": increase MAXCPUS\n");
283 * If we're an Application Processor, allocate a cpu_info
284 * structure, otherwise use the primary's.
286 if (caa
->cpu_role
== CPU_ROLE_AP
) {
287 if ((boothowto
& RB_MD1
) != 0) {
288 aprint_error(": multiprocessor boot disabled\n");
289 if (!pmf_device_register(self
, NULL
, NULL
))
290 aprint_error_dev(self
,
291 "couldn't establish power handler\n");
294 aprint_naive(": Application Processor\n");
295 ptr
= (uintptr_t)kmem_alloc(sizeof(*ci
) + CACHE_LINE_SIZE
- 1,
297 ci
= (struct cpu_info
*)((ptr
+ CACHE_LINE_SIZE
- 1) &
298 ~(CACHE_LINE_SIZE
- 1));
299 memset(ci
, 0, sizeof(*ci
));
302 ci
->ci_tlog_base
= kmem_zalloc(sizeof(struct tlog
), KM_SLEEP
);
305 aprint_naive(": %s Processor\n",
306 caa
->cpu_role
== CPU_ROLE_SP
? "Single" : "Boot");
307 ci
= &cpu_info_primary
;
308 if (cpunum
!= lapic_cpu_number()) {
309 /* XXX should be done earlier. */
311 aprint_verbose("\n");
312 aprint_verbose_dev(self
, "running CPU at apic %d"
313 " instead of at expected %d", lapic_cpu_number(),
315 reg
= i82489_readreg(LAPIC_ID
);
316 i82489_writereg(LAPIC_ID
, (reg
& ~LAPIC_ID_MASK
) |
317 (cpunum
<< LAPIC_ID_SHIFT
));
319 if (cpunum
!= lapic_cpu_number()) {
320 aprint_error_dev(self
, "unable to reset apic id\n");
327 ci
->ci_cpuid
= caa
->cpu_number
;
328 ci
->ci_func
= caa
->cpu_func
;
330 /* Must be before mi_cpu_attach(). */
333 if (caa
->cpu_role
== CPU_ROLE_AP
) {
336 error
= mi_cpu_attach(ci
);
339 aprint_error_dev(self
,
340 "mi_cpu_attach failed with %d\n", error
);
345 KASSERT(ci
->ci_data
.cpu_idlelwp
!= NULL
);
348 ci
->ci_cpumask
= (1 << cpu_index(ci
));
349 pmap_reference(pmap_kernel());
350 ci
->ci_pmap
= pmap_kernel();
351 ci
->ci_tlbstate
= TLBSTATE_STALE
;
354 * Boot processor may not be attached first, but the below
355 * must be done to allow booting other processors.
358 atomic_or_32(&ci
->ci_flags
, CPUF_PRESENT
| CPUF_PRIMARY
);
361 cpu_get_tsc_freq(ci
);
363 cpu_set_tss_gates(ci
);
364 pmap_cpu_init_late(ci
);
365 if (caa
->cpu_role
!= CPU_ROLE_SP
) {
369 lapic_calibrate_timer(ci
);
371 /* Make sure DELAY() is initialized. */
376 /* further PCB init done later. */
378 switch (caa
->cpu_role
) {
380 atomic_or_32(&ci
->ci_flags
, CPUF_SP
);
387 atomic_or_32(&ci
->ci_flags
, CPUF_BSP
);
399 cpu_set_tss_gates(ci
);
400 pmap_cpu_init_early(ci
);
401 pmap_cpu_init_late(ci
);
402 cpu_start_secondary(ci
);
403 if (ci
->ci_flags
& CPUF_PRESENT
) {
404 struct cpu_info
*tmp
;
417 panic("unknown processor type??\n");
420 atomic_or_32(&cpus_attached
, ci
->ci_cpumask
);
422 if (!pmf_device_register(self
, cpu_suspend
, cpu_resume
))
423 aprint_error_dev(self
, "couldn't establish power handler\n");
426 struct lwp
*l
= ci
->ci_data
.cpu_idlelwp
;
427 struct pcb
*pcb
= lwp_getpcb(l
);
429 aprint_verbose_dev(self
,
430 "idle lwp at %p, idle sp at %p\n",
442 * Initialize the processor appropriately.
446 cpu_init(struct cpu_info
*ci
)
449 lcr0(rcr0() | CR0_WP
);
452 * On a P6 or above, enable global TLB caching if the
453 * hardware supports it.
455 if (cpu_feature
& CPUID_PGE
)
456 lcr4(rcr4() | CR4_PGE
); /* enable global TLB caching */
459 * If we have FXSAVE/FXRESTOR, use them.
461 if (cpu_feature
& CPUID_FXSR
) {
462 lcr4(rcr4() | CR4_OSFXSR
);
465 * If we have SSE/SSE2, enable XMM exceptions.
467 if (cpu_feature
& (CPUID_SSE
|CPUID_SSE2
))
468 lcr4(rcr4() | CR4_OSXMMEXCPT
);
473 * On a P6 or above, initialize MTRR's if the hardware supports them.
475 if (cpu_feature
& CPUID_MTRR
) {
476 if ((ci
->ci_flags
& CPUF_AP
) == 0)
477 i686_mtrr_init_first();
482 if (strcmp((char *)(ci
->ci_vendor
), "AuthenticAMD") == 0) {
484 * Must be a K6-2 Step >= 7 or a K6-III.
486 if (CPUID2FAMILY(ci
->ci_signature
) == 5) {
487 if (CPUID2MODEL(ci
->ci_signature
) > 8 ||
488 (CPUID2MODEL(ci
->ci_signature
) == 8 &&
489 CPUID2STEPPING(ci
->ci_signature
) >= 7)) {
490 mtrr_funcs
= &k6_mtrr_funcs
;
491 k6_mtrr_init_first();
499 atomic_or_32(&cpus_running
, ci
->ci_cpumask
);
501 if (ci
!= &cpu_info_primary
) {
502 /* Synchronize TSC again, and check for drift. */
504 atomic_or_32(&ci
->ci_flags
, CPUF_RUNNING
);
507 atomic_or_32(&ci
->ci_flags
, CPUF_RUNNING
);
512 cpu_boot_secondary_processors(void)
517 /* Now that we know the number of CPUs, patch the text segment. */
520 for (i
=0; i
< maxcpus
; i
++) {
524 if (ci
->ci_data
.cpu_idlelwp
== NULL
)
526 if ((ci
->ci_flags
& CPUF_PRESENT
) == 0)
528 if (ci
->ci_flags
& (CPUF_BSP
|CPUF_SP
|CPUF_PRIMARY
))
530 cpu_boot_secondary(ci
);
533 x86_mp_online
= true;
535 /* Now that we know about the TSC, attach the timecounter. */
538 /* Enable zeroing of pages in the idle loop if we have SSE2. */
539 vm_page_zero_enable
= ((cpu_feature
& CPUID_SSE2
) != 0);
543 cpu_init_idle_lwp(struct cpu_info
*ci
)
545 struct lwp
*l
= ci
->ci_data
.cpu_idlelwp
;
546 struct pcb
*pcb
= lwp_getpcb(l
);
548 pcb
->pcb_cr0
= rcr0();
552 cpu_init_idle_lwps(void)
557 for (i
= 0; i
< maxcpus
; i
++) {
561 if (ci
->ci_data
.cpu_idlelwp
== NULL
)
563 if ((ci
->ci_flags
& CPUF_PRESENT
) == 0)
565 cpu_init_idle_lwp(ci
);
570 cpu_start_secondary(struct cpu_info
*ci
)
572 extern paddr_t mp_pdirpa
;
576 mp_pdirpa
= pmap_init_tmp_pgtbl(mp_trampoline_paddr
);
577 atomic_or_32(&ci
->ci_flags
, CPUF_AP
);
578 ci
->ci_curlwp
= ci
->ci_data
.cpu_idlelwp
;
579 if (CPU_STARTUP(ci
, mp_trampoline_paddr
) != 0) {
584 * Wait for it to become ready. Setting cpu_starting opens the
585 * initial gate and allows the AP to start soft initialization.
587 KASSERT(cpu_starting
== NULL
);
589 for (i
= 100000; (!(ci
->ci_flags
& CPUF_PRESENT
)) && i
> 0; i
--) {
591 extern int cpu_trace
[3];
592 static int otrace
[3];
593 if (memcmp(otrace
, cpu_trace
, sizeof(otrace
)) != 0) {
594 aprint_debug_dev(ci
->ci_dev
, "trace %02x %02x %02x\n",
595 cpu_trace
[0], cpu_trace
[1], cpu_trace
[2]);
596 memcpy(otrace
, cpu_trace
, sizeof(otrace
));
602 if ((ci
->ci_flags
& CPUF_PRESENT
) == 0) {
603 aprint_error_dev(ci
->ci_dev
, "failed to become ready\n");
604 #if defined(MPDEBUG) && defined(DDB)
605 printf("dropping into debugger; continue from here to resume boot\n");
610 * Synchronize time stamp counters. Invalidate cache and do twice
611 * to try and minimize possible cache effects. Disable interrupts
612 * to try and rule out any external interference.
614 psl
= x86_read_psl();
621 CPU_START_CLEANUP(ci
);
626 cpu_boot_secondary(struct cpu_info
*ci
)
632 atomic_or_32(&ci
->ci_flags
, CPUF_GO
);
633 for (i
= 100000; (!(ci
->ci_flags
& CPUF_RUNNING
)) && i
> 0; i
--) {
636 if ((ci
->ci_flags
& CPUF_RUNNING
) == 0) {
637 aprint_error_dev(ci
->ci_dev
, "failed to start\n");
638 #if defined(MPDEBUG) && defined(DDB)
639 printf("dropping into debugger; continue from here to resume boot\n");
643 /* Synchronize TSC again, check for drift. */
644 drift
= ci
->ci_data
.cpu_cc_skew
;
645 psl
= x86_read_psl();
650 drift
-= ci
->ci_data
.cpu_cc_skew
;
651 aprint_debug_dev(ci
->ci_dev
, "TSC skew=%lld drift=%lld\n",
652 (long long)ci
->ci_data
.cpu_cc_skew
, (long long)drift
);
653 tsc_sync_drift(drift
);
658 * The CPU ends up here when its ready to run
659 * This is called from code in mptramp.s; at this point, we are running
660 * in the idle pcb/idle stack of the new CPU. When this function returns,
661 * this processor will enter the idle loop and start looking for work.
666 struct cpu_info
*ci
= (struct cpu_info
*)v
;
671 cpu_init_msrs(ci
, true);
675 ci
->ci_data
.cpu_cc_freq
= cpu_info_primary
.ci_data
.cpu_cc_freq
;
676 /* cpu_get_tsc_freq(ci); */
678 KDASSERT((ci
->ci_flags
& CPUF_PRESENT
) == 0);
681 * Synchronize time stamp counters. Invalidate cache and do twice
682 * to try and minimize possible cache effects. Note that interrupts
683 * are off at this point.
686 atomic_or_32(&ci
->ci_flags
, CPUF_PRESENT
);
690 * Wait to be brought online. Use 'monitor/mwait' if available,
691 * in order to make the TSC drift as much as possible. so that
692 * we can detect it later. If not available, try 'pause'.
693 * We'd like to use 'hlt', but we have interrupts off.
695 while ((ci
->ci_flags
& CPUF_GO
) == 0) {
696 if ((ci
->ci_feature2_flags
& CPUID2_MONITOR
) != 0) {
697 x86_monitor(&ci
->ci_flags
, 0, 0);
698 if ((ci
->ci_flags
& CPUF_GO
) != 0) {
703 for (i
= 10000; i
!= 0; i
--) {
709 /* Because the text may have been patched in x86_patch(). */
713 KASSERT((ci
->ci_flags
& CPUF_RUNNING
) == 0);
715 lcr3(pmap_kernel()->pm_pdirpa
);
716 pcb
= lwp_getpcb(curlwp
);
717 pcb
->pcb_cr3
= pmap_kernel()->pm_pdirpa
;
718 pcb
= lwp_getpcb(ci
->ci_data
.cpu_idlelwp
);
734 lldt(GSYSSEL(GLDT_SEL
, SEL_KPL
));
738 cpu_get_tsc_freq(ci
);
750 aprint_debug_dev(ci
->ci_dev
, "running\n");
755 #include <ddb/db_output.h>
756 #include <machine/db_machdep.h>
759 * Dump CPU information from ddb.
765 CPU_INFO_ITERATOR cii
;
767 db_printf("addr dev id flags ipis curlwp fpcurlwp\n");
768 for (CPU_INFO_FOREACH(cii
, ci
)) {
769 db_printf("%p %s %ld %x %x %10p %10p\n",
771 ci
->ci_dev
== NULL
? "BOOT" : device_xname(ci
->ci_dev
),
773 ci
->ci_flags
, ci
->ci_ipis
,
781 cpu_copy_trampoline(void)
786 extern u_char cpu_spinup_trampoline
[];
787 extern u_char cpu_spinup_trampoline_end
[];
789 vaddr_t mp_trampoline_vaddr
;
791 mp_trampoline_vaddr
= uvm_km_alloc(kernel_map
, PAGE_SIZE
, 0,
794 pmap_kenter_pa(mp_trampoline_vaddr
, mp_trampoline_paddr
,
795 VM_PROT_READ
| VM_PROT_WRITE
, 0);
796 pmap_update(pmap_kernel());
797 memcpy((void *)mp_trampoline_vaddr
,
798 cpu_spinup_trampoline
,
799 cpu_spinup_trampoline_end
- cpu_spinup_trampoline
);
801 pmap_kremove(mp_trampoline_vaddr
, PAGE_SIZE
);
802 pmap_update(pmap_kernel());
803 uvm_km_free(kernel_map
, mp_trampoline_vaddr
, PAGE_SIZE
, UVM_KMF_VAONLY
);
808 tss_init(struct i386tss
*tss
, void *stack
, void *func
)
810 memset(tss
, 0, sizeof *tss
);
811 tss
->tss_esp0
= tss
->tss_esp
= (int)((char *)stack
+ USPACE
- 16);
812 tss
->tss_ss0
= GSEL(GDATA_SEL
, SEL_KPL
);
813 tss
->__tss_cs
= GSEL(GCODE_SEL
, SEL_KPL
);
814 tss
->tss_fs
= GSEL(GCPU_SEL
, SEL_KPL
);
815 tss
->tss_gs
= tss
->__tss_es
= tss
->__tss_ds
=
816 tss
->__tss_ss
= GSEL(GDATA_SEL
, SEL_KPL
);
817 tss
->tss_cr3
= pmap_kernel()->pm_pdirpa
;
818 tss
->tss_esp
= (int)((char *)stack
+ USPACE
- 16);
819 tss
->tss_ldt
= GSEL(GLDT_SEL
, SEL_KPL
);
820 tss
->__tss_eflags
= PSL_MBO
| PSL_NT
; /* XXX not needed? */
821 tss
->__tss_eip
= (int)func
;
825 #define IDTVEC(name) __CONCAT(X, name)
826 typedef void (vector
)(void);
827 extern vector
IDTVEC(tss_trap08
);
829 extern vector Xintrddbipi
;
834 cpu_set_tss_gates(struct cpu_info
*ci
)
836 struct segment_descriptor sd
;
838 ci
->ci_doubleflt_stack
= (char *)uvm_km_alloc(kernel_map
, USPACE
, 0,
840 tss_init(&ci
->ci_doubleflt_tss
, ci
->ci_doubleflt_stack
,
842 setsegment(&sd
, &ci
->ci_doubleflt_tss
, sizeof(struct i386tss
) - 1,
843 SDT_SYS386TSS
, SEL_KPL
, 0, 0);
844 ci
->ci_gdt
[GTRAPTSS_SEL
].sd
= sd
;
845 setgate(&idt
[8], NULL
, 0, SDT_SYSTASKGT
, SEL_KPL
,
846 GSEL(GTRAPTSS_SEL
, SEL_KPL
));
850 * Set up separate handler for the DDB IPI, so that it doesn't
851 * stomp on a possibly corrupted stack.
853 * XXX overwriting the gate set in db_machine_init.
854 * Should rearrange the code so that it's set only once.
856 ci
->ci_ddbipi_stack
= (char *)uvm_km_alloc(kernel_map
, USPACE
, 0,
858 tss_init(&ci
->ci_ddbipi_tss
, ci
->ci_ddbipi_stack
, Xintrddbipi
);
860 setsegment(&sd
, &ci
->ci_ddbipi_tss
, sizeof(struct i386tss
) - 1,
861 SDT_SYS386TSS
, SEL_KPL
, 0, 0);
862 ci
->ci_gdt
[GIPITSS_SEL
].sd
= sd
;
864 setgate(&idt
[ddb_vec
], NULL
, 0, SDT_SYSTASKGT
, SEL_KPL
,
865 GSEL(GIPITSS_SEL
, SEL_KPL
));
870 cpu_set_tss_gates(struct cpu_info
*ci
)
877 mp_cpu_start(struct cpu_info
*ci
, paddr_t target
)
879 unsigned short dwordptr
[2];
883 * Bootstrap code must be addressable in real mode
884 * and it must be page aligned.
886 KASSERT(target
< 0x10000 && target
% PAGE_SIZE
== 0);
889 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
892 outb(IO_RTC
, NVRAM_RESET
);
893 outb(IO_RTC
+1, NVRAM_RESET_JUMP
);
896 * "and the warm reset vector (DWORD based at 40:67) to point
897 * to the AP startup code ..."
901 dwordptr
[1] = target
>> 4;
903 memcpy((uint8_t *)cmos_data_mapping
+ 0x467, dwordptr
, 4);
905 if ((cpu_feature
& CPUID_APIC
) == 0) {
906 aprint_error("mp_cpu_start: CPU does not have APIC\n");
911 * ... prior to executing the following sequence:". We'll also add in
912 * local cache flush, in case the BIOS has left the AP with its cache
913 * disabled. It may not be able to cope with MP coherency.
917 if (ci
->ci_flags
& CPUF_AP
) {
918 error
= x86_ipi_init(ci
->ci_cpuid
);
920 aprint_error_dev(ci
->ci_dev
, "%s: IPI not taken (1)\n",
926 error
= x86_ipi_startup(ci
->ci_cpuid
, target
/ PAGE_SIZE
);
928 aprint_error_dev(ci
->ci_dev
, "%s: IPI not taken (2)\n",
934 error
= x86_ipi_startup(ci
->ci_cpuid
, target
/ PAGE_SIZE
);
936 aprint_error_dev(ci
->ci_dev
, "%s: IPI not taken (3)\n",
947 mp_cpu_start_cleanup(struct cpu_info
*ci
)
950 * Ensure the NVRAM reset byte contains something vaguely sane.
953 outb(IO_RTC
, NVRAM_RESET
);
954 outb(IO_RTC
+1, NVRAM_RESET_RST
);
958 typedef void (vector
)(void);
959 extern vector Xsyscall
, Xsyscall32
;
962 cpu_init_msrs(struct cpu_info
*ci
, bool full
)
965 ((uint64_t)GSEL(GCODE_SEL
, SEL_KPL
) << 32) |
966 ((uint64_t)LSEL(LSYSRETBASE_SEL
, SEL_UPL
) << 48));
967 wrmsr(MSR_LSTAR
, (uint64_t)Xsyscall
);
968 wrmsr(MSR_CSTAR
, (uint64_t)Xsyscall32
);
969 wrmsr(MSR_SFMASK
, PSL_NT
|PSL_T
|PSL_I
|PSL_C
);
972 wrmsr(MSR_FSBASE
, 0);
973 wrmsr(MSR_GSBASE
, (uint64_t)ci
);
974 wrmsr(MSR_KERNELGSBASE
, 0);
977 if (cpu_feature
& CPUID_NOX
)
978 wrmsr(MSR_EFER
, rdmsr(MSR_EFER
) | EFER_NXE
);
980 #endif /* __x86_64__ */
998 /* XXX joerg restructure and restart CPUs individually */
1000 cpu_suspend(device_t dv
, pmf_qual_t qual
)
1002 struct cpu_softc
*sc
= device_private(dv
);
1003 struct cpu_info
*ci
= sc
->sc_info
;
1006 if (ci
->ci_flags
& CPUF_PRIMARY
)
1008 if (ci
->ci_data
.cpu_idlelwp
== NULL
)
1010 if ((ci
->ci_flags
& CPUF_PRESENT
) == 0)
1013 sc
->sc_wasonline
= !(ci
->ci_schedstate
.spc_flags
& SPCF_OFFLINE
);
1015 if (sc
->sc_wasonline
) {
1016 mutex_enter(&cpu_lock
);
1017 err
= cpu_setstate(ci
, false);
1018 mutex_exit(&cpu_lock
);
1028 cpu_resume(device_t dv
, pmf_qual_t qual
)
1030 struct cpu_softc
*sc
= device_private(dv
);
1031 struct cpu_info
*ci
= sc
->sc_info
;
1034 if (ci
->ci_flags
& CPUF_PRIMARY
)
1036 if (ci
->ci_data
.cpu_idlelwp
== NULL
)
1038 if ((ci
->ci_flags
& CPUF_PRESENT
) == 0)
1041 if (sc
->sc_wasonline
) {
1042 mutex_enter(&cpu_lock
);
1043 err
= cpu_setstate(ci
, true);
1044 mutex_exit(&cpu_lock
);
1051 cpu_get_tsc_freq(struct cpu_info
*ci
)
1055 if (ci
->ci_feature_flags
& CPUID_TSC
) {
1056 last_tsc
= rdmsr(MSR_TSC
);
1057 i8254_delay(100000);
1058 ci
->ci_data
.cpu_cc_freq
= (rdmsr(MSR_TSC
) - last_tsc
) * 10;
1063 x86_cpu_idle_mwait(void)
1065 struct cpu_info
*ci
= curcpu();
1067 KASSERT(ci
->ci_ilevel
== IPL_NONE
);
1069 x86_monitor(&ci
->ci_want_resched
, 0, 0);
1070 if (__predict_false(ci
->ci_want_resched
)) {
1077 x86_cpu_idle_halt(void)
1079 struct cpu_info
*ci
= curcpu();
1081 KASSERT(ci
->ci_ilevel
== IPL_NONE
);
1084 if (!__predict_false(ci
->ci_want_resched
)) {