4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2012 Gary Mills
24 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2011 by Delphix. All rights reserved.
28 * Copyright (c) 2010, Intel Corporation.
29 * All rights reserved.
32 #include <sys/types.h>
33 #include <sys/sysmacros.h>
35 #include <sys/promif.h>
36 #include <sys/clock.h>
37 #include <sys/cpuvar.h>
38 #include <sys/stack.h>
41 #include <sys/reboot.h>
42 #include <sys/avintr.h>
43 #include <sys/vtrace.h>
45 #include <sys/thread.h>
46 #include <sys/cpupart.h>
48 #include <sys/copyops.h>
51 #include <sys/debug.h>
52 #include <sys/sunddi.h>
53 #include <sys/x86_archext.h>
54 #include <sys/privregs.h>
55 #include <sys/machsystm.h>
56 #include <sys/ontrap.h>
57 #include <sys/bootconf.h>
58 #include <sys/boot_console.h>
59 #include <sys/kdi_machimpl.h>
60 #include <sys/archsystm.h>
61 #include <sys/promif.h>
62 #include <sys/pci_cfgspace.h>
64 #include <sys/hypervisor.h>
66 #include <sys/xpv_support.h>
70 * some globals for patching the result of cpuid
71 * to solve problems w/ creative cpu vendors
74 extern uint32_t cpuid_feature_ecx_include
;
75 extern uint32_t cpuid_feature_ecx_exclude
;
76 extern uint32_t cpuid_feature_edx_include
;
77 extern uint32_t cpuid_feature_edx_exclude
;
80 * Dummy spl priority masks
82 static unsigned char dummy_cpu_pri
[MAXIPL
+ 1] = {
83 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
84 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf
91 set_console_mode(uint8_t val
)
93 struct bop_regs rp
= {0};
99 BOP_DOINT(bootops
, 0x10, &rp
);
104 * Setup routine called right before main(). Interposing this function
105 * before main() allows us to call it in a machine-independent fashion.
108 mlsetup(struct regs
*rp
)
110 u_longlong_t prop_value
;
111 extern struct classfuncs sys_classfuncs
;
112 extern disp_t cpu0_disp
;
113 extern char t0stack
[];
114 extern int post_fastreboot
;
115 extern uint64_t plat_dr_options
;
117 ASSERT_STACK_ALIGNED();
120 * initialize cpu_self
122 cpu
[0]->cpu_self
= cpu
[0];
126 * Point at the hypervisor's virtual cpu structure
128 cpu
[0]->cpu_m
.mcpu_vcpu_info
= &HYPERVISOR_shared_info
->vcpu_info
[0];
132 * Set up dummy cpu_pri_data values till psm spl code is
133 * installed. This allows splx() to work on amd64.
136 cpu
[0]->cpu_pri_data
= dummy_cpu_pri
;
139 * check if we've got special bits to clear or set
140 * when checking cpu features
143 if (bootprop_getval("cpuid_feature_ecx_include", &prop_value
) != 0)
144 cpuid_feature_ecx_include
= 0;
146 cpuid_feature_ecx_include
= (uint32_t)prop_value
;
148 if (bootprop_getval("cpuid_feature_ecx_exclude", &prop_value
) != 0)
149 cpuid_feature_ecx_exclude
= 0;
151 cpuid_feature_ecx_exclude
= (uint32_t)prop_value
;
153 if (bootprop_getval("cpuid_feature_edx_include", &prop_value
) != 0)
154 cpuid_feature_edx_include
= 0;
156 cpuid_feature_edx_include
= (uint32_t)prop_value
;
158 if (bootprop_getval("cpuid_feature_edx_exclude", &prop_value
) != 0)
159 cpuid_feature_edx_exclude
= 0;
161 cpuid_feature_edx_exclude
= (uint32_t)prop_value
;
164 * Initialize idt0, gdt0, ldt0_default, ktss0 and dftss.
169 * lgrp_init() and possibly cpuid_pass1() need PCI config
173 if (DOMAIN_IS_INITDOMAIN(xen_info
))
178 * Initialize the platform type from CPU 0 to ensure that
179 * determine_platform() is only ever called once.
181 determine_platform();
185 * The first lightweight pass (pass0) through the cpuid data
186 * was done in locore before mlsetup was called. Do the next
189 * The x86_featureset is initialized here based on the capabilities
190 * of the boot CPU. Note that if we choose to support CPUs that have
191 * different feature sets (at which point we would almost certainly
192 * want to set the feature bits to correspond to the feature
193 * minimum) this value may be altered.
195 cpuid_pass1(cpu
[0], x86_featureset
);
198 if ((get_hwenv() & HW_XEN_HVM
) != 0)
202 * Patch the tsc_read routine with appropriate set of instructions,
203 * depending on the processor family and architecure, to read the
204 * time-stamp counter while ensuring no out-of-order execution.
205 * Patch it while the kernel text is still writable.
207 * Note: tsc_read is not patched for intel processors whose family
208 * is >6 and for amd whose family >f (in case they don't support rdtscp
209 * instruction, unlikely). By default tsc_read will use cpuid for
210 * serialization in such cases. The following code needs to be
211 * revisited if intel processors of family >= f retains the
212 * instruction serialization nature of mfence instruction.
213 * Note: tsc_read is not patched for x86 processors which do
214 * not support "mfence". By default tsc_read will use cpuid for
215 * serialization in such cases.
217 * The Xen hypervisor does not correctly report whether rdtscp is
218 * supported or not, so we must assume that it is not.
220 if ((get_hwenv() & HW_XEN_HVM
) == 0 &&
221 is_x86_feature(x86_featureset
, X86FSET_TSCP
))
222 patch_tsc_read(X86_HAVE_TSCP
);
223 else if (cpuid_getvendor(CPU
) == X86_VENDOR_AMD
&&
224 cpuid_getfamily(CPU
) <= 0xf &&
225 is_x86_feature(x86_featureset
, X86FSET_SSE2
))
226 patch_tsc_read(X86_TSC_MFENCE
);
227 else if (cpuid_getvendor(CPU
) == X86_VENDOR_Intel
&&
228 cpuid_getfamily(CPU
) <= 6 &&
229 is_x86_feature(x86_featureset
, X86FSET_SSE2
))
230 patch_tsc_read(X86_TSC_LFENCE
);
234 #if defined(__i386) && !defined(__xpv)
236 * Some i386 processors do not implement the rdtsc instruction,
237 * or at least they do not implement it correctly. Patch them to
240 if (!is_x86_feature(x86_featureset
, X86FSET_TSC
))
241 patch_tsc_read(X86_NO_TSC
);
242 #endif /* __i386 && !__xpv */
244 #if defined(__amd64) && !defined(__xpv)
245 patch_memops(cpuid_getvendor(CPU
));
246 #endif /* __amd64 && !__xpv */
249 /* XXPV what, if anything, should be dorked with here under xen? */
252 * While we're thinking about the TSC, let's set up %cr4 so that
253 * userland can issue rdtsc, and initialize the TSC_AUX value
254 * (the cpuid) for the rdtscp instruction on appropriately
257 if (is_x86_feature(x86_featureset
, X86FSET_TSC
))
258 setcr4(getcr4() & ~CR4_TSD
);
260 if (is_x86_feature(x86_featureset
, X86FSET_TSCP
))
261 (void) wrmsr(MSR_AMD_TSCAUX
, 0);
263 if (is_x86_feature(x86_featureset
, X86FSET_DE
))
264 setcr4(getcr4() | CR4_DE
);
270 t0
.t_stk
= (caddr_t
)rp
- MINFRAME
;
271 t0
.t_stkbase
= t0stack
;
272 t0
.t_pri
= maxclsyspri
- 3;
273 t0
.t_schedflag
= TS_LOAD
| TS_DONT_SWAP
;
275 t0
.t_plockp
= &p0lock
.pl_lock
;
282 t0
.t_disp_queue
= &cpu0_disp
;
283 t0
.t_bind_cpu
= PBIND_NONE
;
284 t0
.t_bind_pset
= PS_NONE
;
285 t0
.t_bindflag
= (uchar_t
)default_binding_mode
;
286 t0
.t_cpupart
= &cp_default
;
287 t0
.t_clfuncs
= &sys_classfuncs
.thread
;
289 THREAD_ONPROC(&t0
, CPU
);
291 lwp0
.lwp_thread
= &t0
;
292 lwp0
.lwp_regs
= (void *)rp
;
293 lwp0
.lwp_procp
= &p0
;
294 t0
.t_tid
= p0
.p_lwpcnt
= p0
.p_lwprcnt
= p0
.p_lwpid
= 1;
300 p0
.p_stksize
= 2*PAGESIZE
;
303 p0
.p_lockp
= &p0lock
;
305 p0
.p_t1_lgrpid
= LGRP_NONE
;
306 p0
.p_tr_lgrpid
= LGRP_NONE
;
307 sigorset(&p0
.p_ignore
, &ignoredefault
);
309 CPU
->cpu_thread
= &t0
;
310 bzero(&cpu0_disp
, sizeof (disp_t
));
311 CPU
->cpu_disp
= &cpu0_disp
;
312 CPU
->cpu_disp
->disp_cpu
= CPU
;
313 CPU
->cpu_dispthread
= &t0
;
314 CPU
->cpu_idle_thread
= &t0
;
315 CPU
->cpu_flags
= CPU_READY
| CPU_RUNNING
| CPU_EXISTS
| CPU_ENABLE
;
316 CPU
->cpu_dispatch_pri
= t0
.t_pri
;
320 CPU
->cpu_pri
= 12; /* initial PIL for the boot CPU */
323 * The kernel doesn't use LDTs unless a process explicitly requests one.
325 p0
.p_ldt_desc
= null_sdesc
;
328 * Initialize thread/cpu microstate accounting
330 init_mstate(&t0
, LMS_SYSTEM
);
331 init_cpu_mstate(CPU
, CMS_SYSTEM
);
334 * Initialize lists of available and active CPUs.
338 pg_cpu_bootstrap(CPU
);
341 * Now that we have taken over the GDT, IDT and have initialized
342 * active CPU list it's time to inform kmdb if present.
344 if (boothowto
& RB_DEBUG
)
348 * Explicitly set console to text mode (0x3) if this is a boot
349 * post Fast Reboot, and the console is set to CONS_SCREEN_TEXT.
351 if (post_fastreboot
&& boot_console_type(NULL
) == CONS_SCREEN_TEXT
)
352 set_console_mode(0x3);
355 * If requested (boot -d) drop into kmdb.
357 * This must be done after cpu_list_init() on the 64-bit kernel
358 * since taking a trap requires that we re-compute gsbase based
361 if (boothowto
& RB_DEBUGENTER
)
364 cpu_vm_data_init(CPU
);
366 rp
->r_fp
= 0; /* terminate kernel stack traces! */
368 prom_init("kernel", (void *)NULL
);
370 /* User-set option overrides firmware value. */
371 if (bootprop_getval(PLAT_DR_OPTIONS_NAME
, &prop_value
) == 0) {
372 plat_dr_options
= (uint64_t)prop_value
;
375 /* No support of DR operations on xpv */
378 /* Flag PLAT_DR_FEATURE_ENABLED should only be set by DR driver. */
379 plat_dr_options
&= ~PLAT_DR_FEATURE_ENABLED
;
381 /* Only enable CPU/memory DR on 64 bits kernel. */
382 plat_dr_options
&= ~PLAT_DR_FEATURE_MEMORY
;
383 plat_dr_options
&= ~PLAT_DR_FEATURE_CPU
;
388 * Get value of "plat_dr_physmax" boot option.
389 * It overrides values calculated from MSCT or SRAT table.
391 if (bootprop_getval(PLAT_DR_PHYSMAX_NAME
, &prop_value
) == 0) {
392 plat_dr_physmax
= ((uint64_t)prop_value
) >> PAGESHIFT
;
395 /* Get value of boot_ncpus. */
396 if (bootprop_getval(BOOT_NCPUS_NAME
, &prop_value
) != 0) {
399 boot_ncpus
= (int)prop_value
;
400 if (boot_ncpus
<= 0 || boot_ncpus
> NCPU
)
405 * Set max_ncpus and boot_max_ncpus to boot_ncpus if platform doesn't
406 * support CPU DR operations.
408 if (plat_dr_support_cpu() == 0) {
409 max_ncpus
= boot_max_ncpus
= boot_ncpus
;
411 if (bootprop_getval(PLAT_MAX_NCPUS_NAME
, &prop_value
) != 0) {
414 max_ncpus
= (int)prop_value
;
415 if (max_ncpus
<= 0 || max_ncpus
> NCPU
) {
418 if (boot_ncpus
> max_ncpus
) {
419 boot_ncpus
= max_ncpus
;
423 if (bootprop_getval(BOOT_MAX_NCPUS_NAME
, &prop_value
) != 0) {
424 boot_max_ncpus
= boot_ncpus
;
426 boot_max_ncpus
= (int)prop_value
;
427 if (boot_max_ncpus
<= 0 || boot_max_ncpus
> NCPU
) {
428 boot_max_ncpus
= boot_ncpus
;
429 } else if (boot_max_ncpus
> max_ncpus
) {
430 boot_max_ncpus
= max_ncpus
;
436 * Initialize the lgrp framework
438 lgrp_init(LGRP_INIT_STAGE1
);
440 if (boothowto
& RB_HALT
) {
441 prom_printf("unix: kernel halted by -h flag\n");
445 ASSERT_STACK_ALIGNED();
448 * Fill out cpu_ucode_info. Update microcode if necessary.
452 if (workaround_errata(CPU
) != 0)
453 panic("critical workaround(s) missing for boot cpu");
458 mach_modpath(char *path
, const char *filename
)
461 * Construct the directory path from the filename.
466 const char isastr
[] = "/amd64";
467 size_t isalen
= strlen(isastr
);
469 if ((p
= strrchr(filename
, '/')) == NULL
)
472 while (p
> filename
&& *(p
- 1) == '/')
473 p
--; /* remove trailing '/' characters */
475 p
++; /* so "/" -is- the modpath in this case */
478 * Remove optional isa-dependent directory name - the module
479 * subsystem will put this back again (!)
483 strncmp(&filename
[len
- isalen
], isastr
, isalen
) == 0)
487 * "/platform/mumblefrotz" + " " + MOD_DEFPATH
489 len
+= (p
- filename
) + 1 + strlen(MOD_DEFPATH
) + 1;
490 (void) strncpy(path
, filename
, p
- filename
);