?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge commit '15aeb4d1148772724cf568e1f7a13fbb99f11ab8'
[unleashed.git] / arch / x86 / kernel / os / cpuid.c
blob49983625a5b044224cd7cb1c5e01d843e6a07716
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
26 */
27 /*
28 * Copyright (c) 2010, Intel Corporation.
29 * All rights reserved.
30 */
31 /*
32 * Portions Copyright 2009 Advanced Micro Devices, Inc.
33 */
34 /*
35 * Copyright 2018 Joyent, Inc.
36 */
37 /*
38 * Various routines to handle identification
39 * and classification of x86 processors.
40 */
41
42 #include <sys/types.h>
43 #include <sys/archsystm.h>
44 #include <sys/x86_archext.h>
45 #include <sys/kmem.h>
46 #include <sys/systm.h>
47 #include <sys/cmn_err.h>
48 #include <sys/sunddi.h>
49 #include <sys/sunndi.h>
50 #include <sys/cpuvar.h>
51 #include <sys/processor.h>
52 #include <sys/sysmacros.h>
53 #include <sys/pg.h>
54 #include <sys/fp.h>
55 #include <sys/controlregs.h>
56 #include <sys/bitmap.h>
57 #include <sys/auxv_386.h>
58 #include <sys/memnode.h>
59 #include <sys/pci_cfgspace.h>
60 #include <sys/comm_page.h>
61 #include <sys/mach_mmu.h>
62 #include <sys/ucode.h>
63 #include <sys/tsc.h>
64
65 #include <sys/ontrap.h>
66
67 /*
68 * Pass 0 of cpuid feature analysis happens in locore. It contains special code
69 * to recognize Cyrix processors that are not cpuid-compliant, and to deal with
70 * them accordingly. For most modern processors, feature detection occurs here
71 * in pass 1.
72 *
73 * Pass 1 of cpuid feature analysis happens just at the beginning of mlsetup()
74 * for the boot CPU and does the basic analysis that the early kernel needs.
75 * x86_featureset is set based on the return value of cpuid_pass1() of the boot
76 * CPU.
77 *
78 * Pass 1 includes:
79 *
80 * o Determining vendor/model/family/stepping and setting x86_type and
81 * x86_vendor accordingly.
82 * o Processing the feature flags returned by the cpuid instruction while
83 * applying any workarounds or tricks for the specific processor.
84 * o Mapping the feature flags into illumos feature bits (X86_*).
85 * o Processing extended feature flags if supported by the processor,
86 * again while applying specific processor knowledge.
87 * o Determining the CMT characteristics of the system.
88 *
89 * Pass 1 is done on non-boot CPUs during their initialization and the results
90 * are used only as a meager attempt at ensuring that all processors within the
91 * system support the same features.
92 *
93 * Pass 2 of cpuid feature analysis happens just at the beginning
94 * of startup(). It just copies in and corrects the remainder
95 * of the cpuid data we depend on: standard cpuid functions that we didn't
96 * need for pass1 feature analysis, and extended cpuid functions beyond the
97 * simple feature processing done in pass1.
98 *
99 * Pass 3 of cpuid analysis is invoked after basic kernel services; in
100 * particular kernel memory allocation has been made available. It creates a
101 * readable brand string based on the data collected in the first two passes.
102 *
103 * Pass 4 of cpuid analysis is invoked after post_startup() when all
104 * the support infrastructure for various hardware features has been
105 * initialized. It determines which processor features will be reported
106 * to userland via the aux vector.
107 *
108 * All passes are executed on all CPUs, but only the boot CPU determines what
109 * features the kernel will use.
110 *
111 * Much of the worst junk in this file is for the support of processors
112 * that didn't really implement the cpuid instruction properly.
113 *
114 * NOTE: The accessor functions (cpuid_get*) are aware of, and ASSERT upon,
115 * the pass numbers. Accordingly, changes to the pass code may require changes
116 * to the accessor code.
117 */
118
119 uint_t x86_vendor = X86_VENDOR_IntelClone;
120 uint_t x86_type = X86_TYPE_OTHER;
121 uint_t x86_clflush_size = 0;
122
123 int x86_use_pcid = -1;
124 int x86_use_invpcid = -1;
125
126 uint_t pentiumpro_bug4046376;
127
128 uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
129
130 static char *x86_feature_names[NUM_X86_FEATURES] = {
131 "lgpg",
132 "tsc",
133 "msr",
134 "mtrr",
135 "pge",
136 "de",
137 "cmov",
138 "mmx",
139 "mca",
140 "pae",
141 "cv8",
142 "pat",
143 "sep",
144 "sse",
145 "sse2",
146 "htt",
147 "asysc",
148 "nx",
149 "sse3",
150 "cx16",
151 "cmp",
152 "tscp",
153 "mwait",
154 "sse4a",
155 "cpuid",
156 "ssse3",
157 "sse4_1",
158 "sse4_2",
159 "1gpg",
160 "clfsh",
161 "64",
162 "aes",
163 "pclmulqdq",
164 "xsave",
165 "avx",
166 "vmx",
167 "svm",
168 "topoext",
169 "f16c",
170 "rdrand",
171 "x2apic",
172 "avx2",
173 "bmi1",
174 "bmi2",
175 "fma",
176 "smep",
177 "smap",
178 "adx",
179 "rdseed",
180 "mpx",
181 "avx512f",
182 "avx512dq",
183 "avx512pf",
184 "avx512er",
185 "avx512cd",
186 "avx512bw",
187 "avx512vl",
188 "avx512fma",
189 "avx512vbmi",
190 "avx512_vpopcntdq",
191 "avx512_4vnniw",
192 "avx512_4fmaps",
193 "xsaveopt",
194 "xsavec",
195 "xsaves",
196 "sha",
197 "umip",
198 "pku",
199 "ospke",
200 "pcid",
201 "invpcid",
202 "ibrs",
203 "ibpb",
204 "stibp",
205 "ssbd",
206 "ssbd_virt",
207 "rdcl_no",
208 "ibrs_all",
209 "rsba",
210 "ssb_no",
211 "stibp_all"
212 };
213
214 boolean_t
215 is_x86_feature(void *featureset, uint_t feature)
216 {
217 ASSERT(feature < NUM_X86_FEATURES);
218 return (BT_TEST((ulong_t *)featureset, feature));
219 }
220
221 void
222 add_x86_feature(void *featureset, uint_t feature)
223 {
224 ASSERT(feature < NUM_X86_FEATURES);
225 BT_SET((ulong_t *)featureset, feature);
226 }
227
228 void
229 remove_x86_feature(void *featureset, uint_t feature)
230 {
231 ASSERT(feature < NUM_X86_FEATURES);
232 BT_CLEAR((ulong_t *)featureset, feature);
233 }
234
235 boolean_t
236 compare_x86_featureset(void *setA, void *setB)
237 {
238 /*
239 * We assume that the unused bits of the bitmap are always zero.
240 */
241 if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
242 return (B_TRUE);
243 } else {
244 return (B_FALSE);
245 }
246 }
247
248 void
249 print_x86_featureset(void *featureset)
250 {
251 uint_t i;
252
253 for (i = 0; i < NUM_X86_FEATURES; i++) {
254 if (is_x86_feature(featureset, i)) {
255 cmn_err(CE_CONT, "?x86_feature: %s\n",
256 x86_feature_names[i]);
257 }
258 }
259 }
260
261 /* Note: This is the maximum size for the CPU, not the size of the structure. */
262 static size_t xsave_state_size = 0;
263 uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
264 boolean_t xsave_force_disable = B_FALSE;
265 extern int disable_smap;
266
267 /*
268 * This is set to platform type we are running on.
269 */
270 static int platform_type = -1;
271
272 /*
273 * Variable to patch if hypervisor platform detection needs to be
274 * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
275 */
276 int enable_platform_detection = 1;
277
278 /*
279 * monitor/mwait info.
280 *
281 * size_actual and buf_actual are the real address and size allocated to get
282 * proper mwait_buf alignement. buf_actual and size_actual should be passed
283 * to kmem_free(). Currently kmem_alloc() and mwait happen to both use
284 * processor cache-line alignment, but this is not guarantied in the furture.
285 */
286 struct mwait_info {
287 size_t mon_min; /* min size to avoid missed wakeups */
288 size_t mon_max; /* size to avoid false wakeups */
289 size_t size_actual; /* size actually allocated */
290 void *buf_actual; /* memory actually allocated */
291 uint32_t support; /* processor support of monitor/mwait */
292 };
293
294 /*
295 * xsave/xrestor info.
296 *
297 * This structure contains HW feature bits and the size of the xsave save area.
298 * Note: the kernel declares a fixed size (AVX_XSAVE_SIZE) structure
299 * (xsave_state) to describe the xsave layout. However, at runtime the
300 * per-lwp xsave area is dynamically allocated based on xsav_max_size. The
301 * xsave_state structure simply represents the legacy layout of the beginning
302 * of the xsave area.
303 */
304 struct xsave_info {
305 uint32_t xsav_hw_features_low; /* Supported HW features */
306 uint32_t xsav_hw_features_high; /* Supported HW features */
307 size_t xsav_max_size; /* max size save area for HW features */
308 size_t ymm_size; /* AVX: size of ymm save area */
309 size_t ymm_offset; /* AVX: offset for ymm save area */
310 size_t bndregs_size; /* MPX: size of bndregs save area */
311 size_t bndregs_offset; /* MPX: offset for bndregs save area */
312 size_t bndcsr_size; /* MPX: size of bndcsr save area */
313 size_t bndcsr_offset; /* MPX: offset for bndcsr save area */
314 size_t opmask_size; /* AVX512: size of opmask save */
315 size_t opmask_offset; /* AVX512: offset for opmask save */
316 size_t zmmlo_size; /* AVX512: size of zmm 256 save */
317 size_t zmmlo_offset; /* AVX512: offset for zmm 256 save */
318 size_t zmmhi_size; /* AVX512: size of zmm hi reg save */
319 size_t zmmhi_offset; /* AVX512: offset for zmm hi reg save */
320 };
321
322
323 /*
324 * These constants determine how many of the elements of the
325 * cpuid we cache in the cpuid_info data structure; the
326 * remaining elements are accessible via the cpuid instruction.
327 */
328
329 #define NMAX_CPI_STD 8 /* eax = 0 .. 7 */
330 #define NMAX_CPI_EXTD 0x1f /* eax = 0x80000000 .. 0x8000001e */
331
332 /*
333 * Some terminology needs to be explained:
334 * - Socket: Something that can be plugged into a motherboard.
335 * - Package: Same as socket
336 * - Chip: Same as socket. Note that AMD's documentation uses term "chip"
337 * differently: there, chip is the same as processor node (below)
338 * - Processor node: Some AMD processors have more than one
339 * "subprocessor" embedded in a package. These subprocessors (nodes)
340 * are fully-functional processors themselves with cores, caches,
341 * memory controllers, PCI configuration spaces. They are connected
342 * inside the package with Hypertransport links. On single-node
343 * processors, processor node is equivalent to chip/socket/package.
344 * - Compute Unit: Some AMD processors pair cores in "compute units" that
345 * share the FPU and the I$ and L2 caches.
346 */
347
348 struct cpuid_info {
349 uint_t cpi_pass; /* last pass completed */
350 /*
351 * standard function information
352 */
353 uint_t cpi_maxeax; /* fn 0: %eax */
354 char cpi_vendorstr[13]; /* fn 0: %ebx:%ecx:%edx */
355 uint_t cpi_vendor; /* enum of cpi_vendorstr */
356
357 uint_t cpi_family; /* fn 1: extended family */
358 uint_t cpi_model; /* fn 1: extended model */
359 uint_t cpi_step; /* fn 1: stepping */
360 chipid_t cpi_chipid; /* fn 1: %ebx: Intel: chip # */
361 /* AMD: package/socket # */
362 uint_t cpi_brandid; /* fn 1: %ebx: brand ID */
363 int cpi_clogid; /* fn 1: %ebx: thread # */
364 uint_t cpi_ncpu_per_chip; /* fn 1: %ebx: logical cpu count */
365 uint8_t cpi_cacheinfo[16]; /* fn 2: intel-style cache desc */
366 uint_t cpi_ncache; /* fn 2: number of elements */
367 uint_t cpi_ncpu_shr_last_cache; /* fn 4: %eax: ncpus sharing cache */
368 id_t cpi_last_lvl_cacheid; /* fn 4: %eax: derived cache id */
369 uint_t cpi_std_4_size; /* fn 4: number of fn 4 elements */
370 struct cpuid_regs **cpi_std_4; /* fn 4: %ecx == 0 .. fn4_size */
371 struct cpuid_regs cpi_std[NMAX_CPI_STD]; /* 0 .. 7 */
372 /*
373 * extended function information
374 */
375 uint_t cpi_xmaxeax; /* fn 0x80000000: %eax */
376 char cpi_brandstr[49]; /* fn 0x8000000[234] */
377 uint8_t cpi_pabits; /* fn 0x80000006: %eax */
378 uint8_t cpi_vabits; /* fn 0x80000006: %eax */
379 uint8_t cpi_fp_amd_save; /* AMD: FP error pointer save rqd. */
380 struct cpuid_regs cpi_extd[NMAX_CPI_EXTD]; /* 0x800000XX */
381
382 id_t cpi_coreid; /* same coreid => strands share core */
383 int cpi_pkgcoreid; /* core number within single package */
384 uint_t cpi_ncore_per_chip; /* AMD: fn 0x80000008: %ecx[7-0] */
385 /* Intel: fn 4: %eax[31-26] */
386 /*
387 * supported feature information
388 */
389 uint32_t cpi_support[6];
390 #define STD_EDX_FEATURES 0
391 #define AMD_EDX_FEATURES 1
392 #define TM_EDX_FEATURES 2
393 #define STD_ECX_FEATURES 3
394 #define AMD_ECX_FEATURES 4
395 #define STD_EBX_FEATURES 5
396 /*
397 * Synthesized information, where known.
398 */
399 uint32_t cpi_chiprev; /* See X86_CHIPREV_* in x86_archext.h */
400 const char *cpi_chiprevstr; /* May be NULL if chiprev unknown */
401 uint32_t cpi_socket; /* Chip package/socket type */
402
403 struct mwait_info cpi_mwait; /* fn 5: monitor/mwait info */
404 uint32_t cpi_apicid;
405 uint_t cpi_procnodeid; /* AMD: nodeID on HT, Intel: chipid */
406 uint_t cpi_procnodes_per_pkg; /* AMD: # of nodes in the package */
407 /* Intel: 1 */
408 uint_t cpi_compunitid; /* AMD: ComputeUnit ID, Intel: coreid */
409 uint_t cpi_cores_per_compunit; /* AMD: # of cores in the ComputeUnit */
410
411 struct xsave_info cpi_xsave; /* fn D: xsave/xrestor info */
412 };
413
414
415 static struct cpuid_info cpuid_info0;
416
417 /*
418 * These bit fields are defined by the Intel Application Note AP-485
419 * "Intel Processor Identification and the CPUID Instruction"
420 */
421 #define CPI_FAMILY_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
422 #define CPI_MODEL_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
423 #define CPI_TYPE(cpi) BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
424 #define CPI_FAMILY(cpi) BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
425 #define CPI_STEP(cpi) BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
426 #define CPI_MODEL(cpi) BITX((cpi)->cpi_std[1].cp_eax, 7, 4)
427
428 #define CPI_FEATURES_EDX(cpi) ((cpi)->cpi_std[1].cp_edx)
429 #define CPI_FEATURES_ECX(cpi) ((cpi)->cpi_std[1].cp_ecx)
430 #define CPI_FEATURES_XTD_EDX(cpi) ((cpi)->cpi_extd[1].cp_edx)
431 #define CPI_FEATURES_XTD_ECX(cpi) ((cpi)->cpi_extd[1].cp_ecx)
432 #define CPI_FEATURES_7_0_EBX(cpi) ((cpi)->cpi_std[7].cp_ebx)
433 #define CPI_FEATURES_7_0_ECX(cpi) ((cpi)->cpi_std[7].cp_ecx)
434 #define CPI_FEATURES_7_0_EDX(cpi) ((cpi)->cpi_std[7].cp_edx)
435
436 #define CPI_BRANDID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
437 #define CPI_CHUNKS(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
438 #define CPI_CPU_COUNT(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
439 #define CPI_APIC_ID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)
440
441 #define CPI_MAXEAX_MAX 0x100 /* sanity control */
442 #define CPI_XMAXEAX_MAX 0x80000100
443 #define CPI_FN4_ECX_MAX 0x20 /* sanity: max fn 4 levels */
444 #define CPI_FNB_ECX_MAX 0x20 /* sanity: max fn B levels */
445
446 /*
447 * Function 4 (Deterministic Cache Parameters) macros
448 * Defined by Intel Application Note AP-485
449 */
450 #define CPI_NUM_CORES(regs) BITX((regs)->cp_eax, 31, 26)
451 #define CPI_NTHR_SHR_CACHE(regs) BITX((regs)->cp_eax, 25, 14)
452 #define CPI_FULL_ASSOC_CACHE(regs) BITX((regs)->cp_eax, 9, 9)
453 #define CPI_SELF_INIT_CACHE(regs) BITX((regs)->cp_eax, 8, 8)
454 #define CPI_CACHE_LVL(regs) BITX((regs)->cp_eax, 7, 5)
455 #define CPI_CACHE_TYPE(regs) BITX((regs)->cp_eax, 4, 0)
456 #define CPI_CPU_LEVEL_TYPE(regs) BITX((regs)->cp_ecx, 15, 8)
457
458 #define CPI_CACHE_WAYS(regs) BITX((regs)->cp_ebx, 31, 22)
459 #define CPI_CACHE_PARTS(regs) BITX((regs)->cp_ebx, 21, 12)
460 #define CPI_CACHE_COH_LN_SZ(regs) BITX((regs)->cp_ebx, 11, 0)
461
462 #define CPI_CACHE_SETS(regs) BITX((regs)->cp_ecx, 31, 0)
463
464 #define CPI_PREFCH_STRIDE(regs) BITX((regs)->cp_edx, 9, 0)
465
466
467 /*
468 * A couple of shorthand macros to identify "later" P6-family chips
469 * like the Pentium M and Core. First, the "older" P6-based stuff
470 * (loosely defined as "pre-Pentium-4"):
471 * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
472 */
473 #define IS_LEGACY_P6(cpi) ( \
474 cpi->cpi_family == 6 && \
475 (cpi->cpi_model == 1 || \
476 cpi->cpi_model == 3 || \
477 cpi->cpi_model == 5 || \
478 cpi->cpi_model == 6 || \
479 cpi->cpi_model == 7 || \
480 cpi->cpi_model == 8 || \
481 cpi->cpi_model == 0xA || \
482 cpi->cpi_model == 0xB) \
483 )
484
485 /* A "new F6" is everything with family 6 that's not the above */
486 #define IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))
487
488 /* Extended family/model support */
489 #define IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
490 cpi->cpi_family >= 0xf)
491
492 /*
493 * Info for monitor/mwait idle loop.
494 *
495 * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
496 * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
497 * 2006.
498 * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
499 * Documentation Updates" #33633, Rev 2.05, December 2006.
500 */
501 #define MWAIT_SUPPORT (0x00000001) /* mwait supported */
502 #define MWAIT_EXTENSIONS (0x00000002) /* extenstion supported */
503 #define MWAIT_ECX_INT_ENABLE (0x00000004) /* ecx 1 extension supported */
504 #define MWAIT_SUPPORTED(cpi) ((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
505 #define MWAIT_INT_ENABLE(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x2)
506 #define MWAIT_EXTENSION(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x1)
507 #define MWAIT_SIZE_MIN(cpi) BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
508 #define MWAIT_SIZE_MAX(cpi) BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
509 /*
510 * Number of sub-cstates for a given c-state.
511 */
512 #define MWAIT_NUM_SUBC_STATES(cpi, c_state) \
513 BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)
514
515 /*
516 * XSAVE leaf 0xD enumeration
517 */
518 #define CPUID_LEAFD_2_YMM_OFFSET 576
519 #define CPUID_LEAFD_2_YMM_SIZE 256
520
521 /*
522 * Functions we consune from cpuid_subr.c; don't publish these in a header
523 * file to try and keep people using the expected cpuid_* interfaces.
524 */
525 extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
526 extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
527 extern uint32_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
528 extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
529 extern uint_t _cpuid_vendorstr_to_vendorcode(char *);
530
531 /*
532 * Apply up various platform-dependent restrictions where the
533 * underlying platform restrictions mean the CPU can be marked
534 * as less capable than its cpuid instruction would imply.
535 */
536 #define platform_cpuid_mangle(vendor, eax, cp) /* nothing */
537
538 /*
539 * Some undocumented ways of patching the results of the cpuid
540 * instruction to permit running Solaris 10 on future cpus that
541 * we don't currently support. Could be set to non-zero values
542 * via settings in eeprom.
543 */
544
545 uint32_t cpuid_feature_ecx_include;
546 uint32_t cpuid_feature_ecx_exclude;
547 uint32_t cpuid_feature_edx_include;
548 uint32_t cpuid_feature_edx_exclude;
549
550 /*
551 * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
552 */
553 void
554 cpuid_alloc_space(cpu_t *cpu)
555 {
556 /*
557 * By convention, cpu0 is the boot cpu, which is set up
558 * before memory allocation is available. All other cpus get
559 * their cpuid_info struct allocated here.
560 */
561 ASSERT(cpu->cpu_id != 0);
562 ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
563 cpu->cpu_m.mcpu_cpi =
564 kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
565 }
566
567 void
568 cpuid_free_space(cpu_t *cpu)
569 {
570 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
571 int i;
572
573 ASSERT(cpi != NULL);
574 ASSERT(cpi != &cpuid_info0);
575
576 /*
577 * Free up any function 4 related dynamic storage
578 */
579 for (i = 1; i < cpi->cpi_std_4_size; i++)
580 kmem_free(cpi->cpi_std_4[i], sizeof (struct cpuid_regs));
581 if (cpi->cpi_std_4_size > 0)
582 kmem_free(cpi->cpi_std_4,
583 cpi->cpi_std_4_size * sizeof (struct cpuid_regs *));
584
585 kmem_free(cpi, sizeof (*cpi));
586 cpu->cpu_m.mcpu_cpi = NULL;
587 }
588
589 /*
590 * Determine the type of the underlying platform. This is used to customize
591 * initialization of various subsystems (e.g. TSC). determine_platform() must
592 * only ever be called once to prevent two processors from seeing different
593 * values of platform_type. Must be called before cpuid_pass1(), the earliest
594 * consumer to execute (uses _cpuid_chiprev --> synth_amd_info --> get_hwenv).
595 */
596 void
597 determine_platform(void)
598 {
599 struct cpuid_regs cp;
600 uint32_t base;
601 uint32_t regs[4];
602 char *hvstr = (char *)regs;
603
604 ASSERT(platform_type == -1);
605
606 platform_type = HW_NATIVE;
607
608 if (!enable_platform_detection)
609 return;
610
611 /*
612 * If Hypervisor CPUID bit is set, try to determine hypervisor
613 * vendor signature, and set platform type accordingly.
614 *
615 * References:
616 * http://lkml.org/lkml/2008/10/1/246
617 * http://kb.vmware.com/kb/1009458
618 */
619 cp.cp_eax = 0x1;
620 (void) __cpuid_insn(&cp);
621 if ((cp.cp_ecx & CPUID_INTC_ECX_HV) != 0) {
622 cp.cp_eax = 0x40000000;
623 (void) __cpuid_insn(&cp);
624 regs[0] = cp.cp_ebx;
625 regs[1] = cp.cp_ecx;
626 regs[2] = cp.cp_edx;
627 regs[3] = 0;
628 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0) {
629 platform_type = HW_XEN_HVM;
630 return;
631 }
632 if (strcmp(hvstr, HVSIG_VMWARE) == 0) {
633 platform_type = HW_VMWARE;
634 return;
635 }
636 if (strcmp(hvstr, HVSIG_KVM) == 0) {
637 platform_type = HW_KVM;
638 return;
639 }
640 if (strcmp(hvstr, HVSIG_BHYVE) == 0) {
641 platform_type = HW_BHYVE;
642 return;
643 }
644 if (strcmp(hvstr, HVSIG_MICROSOFT) == 0)
645 platform_type = HW_MICROSOFT;
646 } else {
647 /*
648 * Check older VMware hardware versions. VMware hypervisor is
649 * detected by performing an IN operation to VMware hypervisor
650 * port and checking that value returned in %ebx is VMware
651 * hypervisor magic value.
652 *
653 * References: http://kb.vmware.com/kb/1009458
654 */
655 vmware_port(VMWARE_HVCMD_GETVERSION, regs);
656 if (regs[1] == VMWARE_HVMAGIC) {
657 platform_type = HW_VMWARE;
658 return;
659 }
660 }
661
662 /*
663 * Check Xen hypervisor. In a fully virtualized domain,
664 * Xen's pseudo-cpuid function returns a string representing the
665 * Xen signature in %ebx, %ecx, and %edx. %eax contains the maximum
666 * supported cpuid function. We need at least a (base + 2) leaf value
667 * to do what we want to do. Try different base values, since the
668 * hypervisor might use a different one depending on whether Hyper-V
669 * emulation is switched on by default or not.
670 */
671 for (base = 0x40000000; base < 0x40010000; base += 0x100) {
672 cp.cp_eax = base;
673 (void) __cpuid_insn(&cp);
674 regs[0] = cp.cp_ebx;
675 regs[1] = cp.cp_ecx;
676 regs[2] = cp.cp_edx;
677 regs[3] = 0;
678 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0 &&
679 cp.cp_eax >= (base + 2)) {
680 platform_type &= ~HW_NATIVE;
681 platform_type |= HW_XEN_HVM;
682 return;
683 }
684 }
685 }
686
687 int
688 get_hwenv(void)
689 {
690 ASSERT(platform_type != -1);
691 return (platform_type);
692 }
693
694 int
695 is_controldom(void)
696 {
697 return (0);
698 }
699
700
701 static void
702 cpuid_intel_getids(cpu_t *cpu, void *feature)
703 {
704 uint_t i;
705 uint_t chipid_shift = 0;
706 uint_t coreid_shift = 0;
707 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
708
709 for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
710 chipid_shift++;
711
712 cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
713 cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);
714
715 if (is_x86_feature(feature, X86FSET_CMP)) {
716 /*
717 * Multi-core (and possibly multi-threaded)
718 * processors.
719 */
720 uint_t ncpu_per_core;
721 if (cpi->cpi_ncore_per_chip == 1)
722 ncpu_per_core = cpi->cpi_ncpu_per_chip;
723 else if (cpi->cpi_ncore_per_chip > 1)
724 ncpu_per_core = cpi->cpi_ncpu_per_chip /
725 cpi->cpi_ncore_per_chip;
726 /*
727 * 8bit APIC IDs on dual core Pentiums
728 * look like this:
729 *
730 * +-----------------------+------+------+
731 * | Physical Package ID | MC | HT |
732 * +-----------------------+------+------+
733 * <------- chipid -------->
734 * <------- coreid --------------->
735 * <--- clogid -->
736 * <------>
737 * pkgcoreid
738 *
739 * Where the number of bits necessary to
740 * represent MC and HT fields together equals
741 * to the minimum number of bits necessary to
742 * store the value of cpi->cpi_ncpu_per_chip.
743 * Of those bits, the MC part uses the number
744 * of bits necessary to store the value of
745 * cpi->cpi_ncore_per_chip.
746 */
747 for (i = 1; i < ncpu_per_core; i <<= 1)
748 coreid_shift++;
749 cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
750 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
751 } else if (is_x86_feature(feature, X86FSET_HTT)) {
752 /*
753 * Single-core multi-threaded processors.
754 */
755 cpi->cpi_coreid = cpi->cpi_chipid;
756 cpi->cpi_pkgcoreid = 0;
757 }
758 cpi->cpi_procnodeid = cpi->cpi_chipid;
759 cpi->cpi_compunitid = cpi->cpi_coreid;
760 }
761
762 static void
763 cpuid_amd_getids(cpu_t *cpu)
764 {
765 int i, first_half, coreidsz;
766 uint32_t nb_caps_reg;
767 uint_t node2_1;
768 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
769 struct cpuid_regs *cp;
770
771 /*
772 * AMD CMP chips currently have a single thread per core.
773 *
774 * Since no two cpus share a core we must assign a distinct coreid
775 * per cpu, and we do this by using the cpu_id. This scheme does not,
776 * however, guarantee that sibling cores of a chip will have sequential
777 * coreids starting at a multiple of the number of cores per chip -
778 * that is usually the case, but if the ACPI MADT table is presented
779 * in a different order then we need to perform a few more gymnastics
780 * for the pkgcoreid.
781 *
782 * All processors in the system have the same number of enabled
783 * cores. Cores within a processor are always numbered sequentially
784 * from 0 regardless of how many or which are disabled, and there
785 * is no way for operating system to discover the real core id when some
786 * are disabled.
787 *
788 * In family 0x15, the cores come in pairs called compute units. They
789 * share I$ and L2 caches and the FPU. Enumeration of this feature is
790 * simplified by the new topology extensions CPUID leaf, indicated by
791 * the X86 feature X86FSET_TOPOEXT.
792 */
793
794 cpi->cpi_coreid = cpu->cpu_id;
795 cpi->cpi_compunitid = cpu->cpu_id;
796
797 if (cpi->cpi_xmaxeax >= 0x80000008) {
798
799 coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);
800
801 /*
802 * In AMD parlance chip is really a node while Solaris
803 * sees chip as equivalent to socket/package.
804 */
805 cpi->cpi_ncore_per_chip =
806 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
807 if (coreidsz == 0) {
808 /* Use legacy method */
809 for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
810 coreidsz++;
811 if (coreidsz == 0)
812 coreidsz = 1;
813 }
814 } else {
815 /* Assume single-core part */
816 cpi->cpi_ncore_per_chip = 1;
817 coreidsz = 1;
818 }
819
820 cpi->cpi_clogid = cpi->cpi_pkgcoreid =
821 cpi->cpi_apicid & ((1<<coreidsz) - 1);
822 cpi->cpi_ncpu_per_chip = cpi->cpi_ncore_per_chip;
823
824 /* Get node ID, compute unit ID */
825 if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
826 cpi->cpi_xmaxeax >= 0x8000001e) {
827 cp = &cpi->cpi_extd[0x1e];
828 cp->cp_eax = 0x8000001e;
829 (void) __cpuid_insn(cp);
830
831 cpi->cpi_procnodes_per_pkg = BITX(cp->cp_ecx, 10, 8) + 1;
832 cpi->cpi_procnodeid = BITX(cp->cp_ecx, 7, 0);
833 cpi->cpi_cores_per_compunit = BITX(cp->cp_ebx, 15, 8) + 1;
834 cpi->cpi_compunitid = BITX(cp->cp_ebx, 7, 0)
835 + (cpi->cpi_ncore_per_chip / cpi->cpi_cores_per_compunit)
836 * (cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg);
837 } else if (cpi->cpi_family == 0xf || cpi->cpi_family >= 0x11) {
838 cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
839 } else if (cpi->cpi_family == 0x10) {
840 /*
841 * See if we are a multi-node processor.
842 * All processors in the system have the same number of nodes
843 */
844 nb_caps_reg = pci_getl_func(0, 24, 3, 0xe8);
845 if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
846 /* Single-node */
847 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
848 coreidsz);
849 } else {
850
851 /*
852 * Multi-node revision D (2 nodes per package
853 * are supported)
854 */
855 cpi->cpi_procnodes_per_pkg = 2;
856
857 first_half = (cpi->cpi_pkgcoreid <=
858 (cpi->cpi_ncore_per_chip/2 - 1));
859
860 if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
861 /* We are BSP */
862 cpi->cpi_procnodeid = (first_half ? 0 : 1);
863 } else {
864
865 /* We are AP */
866 /* NodeId[2:1] bits to use for reading F3xe8 */
867 node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;
868
869 nb_caps_reg =
870 pci_getl_func(0, 24 + node2_1, 3, 0xe8);
871
872 /*
873 * Check IntNodeNum bit (31:30, but bit 31 is
874 * always 0 on dual-node processors)
875 */
876 if (BITX(nb_caps_reg, 30, 30) == 0)
877 cpi->cpi_procnodeid = node2_1 +
878 !first_half;
879 else
880 cpi->cpi_procnodeid = node2_1 +
881 first_half;
882 }
883 }
884 } else {
885 cpi->cpi_procnodeid = 0;
886 }
887
888 cpi->cpi_chipid =
889 cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg;
890 }
891
892 static void
893 cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
894 {
895 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
896
897 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
898 cpi->cpi_xmaxeax >= 0x80000008) {
899 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBPB)
900 add_x86_feature(featureset, X86FSET_IBPB);
901 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBRS)
902 add_x86_feature(featureset, X86FSET_IBRS);
903 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP)
904 add_x86_feature(featureset, X86FSET_STIBP);
905 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBRS_ALL)
906 add_x86_feature(featureset, X86FSET_IBRS_ALL);
907 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP_ALL)
908 add_x86_feature(featureset, X86FSET_STIBP_ALL);
909 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_PREFER_IBRS)
910 add_x86_feature(featureset, X86FSET_RSBA);
911 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSBD)
912 add_x86_feature(featureset, X86FSET_SSBD);
913 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_VIRT_SSBD)
914 add_x86_feature(featureset, X86FSET_SSBD_VIRT);
915 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSB_NO)
916 add_x86_feature(featureset, X86FSET_SSB_NO);
917 } else if (cpi->cpi_vendor == X86_VENDOR_Intel &&
918 cpi->cpi_maxeax >= 7) {
919 struct cpuid_regs *ecp;
920 ecp = &cpi->cpi_std[7];
921
922 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SPEC_CTRL) {
923 add_x86_feature(featureset, X86FSET_IBRS);
924 add_x86_feature(featureset, X86FSET_IBPB);
925 }
926
927 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_STIBP) {
928 add_x86_feature(featureset, X86FSET_STIBP);
929 }
930
931 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_ARCH_CAPS) {
932 on_trap_data_t otd;
933
934 /*
935 * Be paranoid and assume we'll get a #GP.
936 */
937 if (!on_trap(&otd, OT_DATA_ACCESS)) {
938 uint64_t reg;
939
940 reg = rdmsr(MSR_IA32_ARCH_CAPABILITIES);
941 if (reg & IA32_ARCH_CAP_RDCL_NO) {
942 add_x86_feature(featureset,
943 X86FSET_RDCL_NO);
944 }
945 if (reg & IA32_ARCH_CAP_IBRS_ALL) {
946 add_x86_feature(featureset,
947 X86FSET_IBRS_ALL);
948 }
949 if (reg & IA32_ARCH_CAP_RSBA) {
950 add_x86_feature(featureset,
951 X86FSET_RSBA);
952 }
953 if (reg & IA32_ARCH_CAP_SSB_NO) {
954 add_x86_feature(featureset,
955 X86FSET_SSB_NO);
956 }
957 }
958 no_trap();
959 }
960
961 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SSBD)
962 add_x86_feature(featureset, X86FSET_SSBD);
963 }
964 }
965
966 /*
967 * Setup XFeature_Enabled_Mask register. Required by xsave feature.
968 */
969 void
970 setup_xfem(void)
971 {
972 uint64_t flags = XFEATURE_LEGACY_FP;
973
974 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
975
976 if (is_x86_feature(x86_featureset, X86FSET_SSE))
977 flags |= XFEATURE_SSE;
978
979 if (is_x86_feature(x86_featureset, X86FSET_AVX))
980 flags |= XFEATURE_AVX;
981
982 if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
983 flags |= XFEATURE_AVX512;
984
985 set_xcr(XFEATURE_ENABLED_MASK, flags);
986
987 xsave_bv_all = flags;
988 }
989
990 void
991 cpuid_pass1(cpu_t *cpu, uchar_t *featureset)
992 {
993 uint32_t mask_ecx, mask_edx;
994 struct cpuid_info *cpi;
995 struct cpuid_regs *cp;
996 int xcpuid;
997 extern int idle_cpu_prefer_mwait;
998
999 /*
1000 * Space statically allocated for BSP, ensure pointer is set
1001 */
1002 if (cpu->cpu_id == 0) {
1003 if (cpu->cpu_m.mcpu_cpi == NULL)
1004 cpu->cpu_m.mcpu_cpi = &cpuid_info0;
1005 }
1006
1007 add_x86_feature(featureset, X86FSET_CPUID);
1008
1009 cpi = cpu->cpu_m.mcpu_cpi;
1010 ASSERT(cpi != NULL);
1011 cp = &cpi->cpi_std[0];
1012 cp->cp_eax = 0;
1013 cpi->cpi_maxeax = __cpuid_insn(cp);
1014 {
1015 uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
1016 *iptr++ = cp->cp_ebx;
1017 *iptr++ = cp->cp_edx;
1018 *iptr++ = cp->cp_ecx;
1019 *(char *)&cpi->cpi_vendorstr[12] = '\0';
1020 }
1021
1022 cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
1023 x86_vendor = cpi->cpi_vendor; /* for compatibility */
1024
1025 /*
1026 * Limit the range in case of weird hardware
1027 */
1028 if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
1029 cpi->cpi_maxeax = CPI_MAXEAX_MAX;
1030 if (cpi->cpi_maxeax < 1)
1031 goto pass1_done;
1032
1033 cp = &cpi->cpi_std[1];
1034 cp->cp_eax = 1;
1035 (void) __cpuid_insn(cp);
1036
1037 /*
1038 * Extract identifying constants for easy access.
1039 */
1040 cpi->cpi_model = CPI_MODEL(cpi);
1041 cpi->cpi_family = CPI_FAMILY(cpi);
1042
1043 if (cpi->cpi_family == 0xf)
1044 cpi->cpi_family += CPI_FAMILY_XTD(cpi);
1045
1046 /*
1047 * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
1048 * Intel, and presumably everyone else, uses model == 0xf, as
1049 * one would expect (max value means possible overflow). Sigh.
1050 */
1051
1052 switch (cpi->cpi_vendor) {
1053 case X86_VENDOR_Intel:
1054 if (IS_EXTENDED_MODEL_INTEL(cpi))
1055 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1056 break;
1057 case X86_VENDOR_AMD:
1058 if (CPI_FAMILY(cpi) == 0xf)
1059 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1060 break;
1061 default:
1062 if (cpi->cpi_model == 0xf)
1063 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1064 break;
1065 }
1066
1067 cpi->cpi_step = CPI_STEP(cpi);
1068 cpi->cpi_brandid = CPI_BRANDID(cpi);
1069
1070 /*
1071 * *default* assumptions:
1072 * - believe %edx feature word
1073 * - ignore %ecx feature word
1074 * - 32-bit virtual and physical addressing
1075 */
1076 mask_edx = 0xffffffff;
1077 mask_ecx = 0;
1078
1079 cpi->cpi_pabits = cpi->cpi_vabits = 32;
1080
1081 switch (cpi->cpi_vendor) {
1082 case X86_VENDOR_Intel:
1083 if (cpi->cpi_family == 5)
1084 x86_type = X86_TYPE_P5;
1085 else if (IS_LEGACY_P6(cpi)) {
1086 x86_type = X86_TYPE_P6;
1087 pentiumpro_bug4046376 = 1;
1088 /*
1089 * Clear the SEP bit when it was set erroneously
1090 */
1091 if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
1092 cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
1093 } else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
1094 x86_type = X86_TYPE_P4;
1095 /*
1096 * We don't currently depend on any of the %ecx
1097 * features until Prescott, so we'll only check
1098 * this from P4 onwards. We might want to revisit
1099 * that idea later.
1100 */
1101 mask_ecx = 0xffffffff;
1102 } else if (cpi->cpi_family > 0xf)
1103 mask_ecx = 0xffffffff;
1104 /*
1105 * We don't support MONITOR/MWAIT if leaf 5 is not available
1106 * to obtain the monitor linesize.
1107 */
1108 if (cpi->cpi_maxeax < 5)
1109 mask_ecx &= ~CPUID_INTC_ECX_MON;
1110 break;
1111 case X86_VENDOR_IntelClone:
1112 default:
1113 break;
1114 case X86_VENDOR_AMD:
1115 #if defined(OPTERON_ERRATUM_108)
1116 if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
1117 cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
1118 cpi->cpi_model = 0xc;
1119 } else
1120 #endif
1121 if (cpi->cpi_family == 5) {
1122 /*
1123 * AMD K5 and K6
1124 *
1125 * These CPUs have an incomplete implementation
1126 * of MCA/MCE which we mask away.
1127 */
1128 mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);
1129
1130 /*
1131 * Model 0 uses the wrong (APIC) bit
1132 * to indicate PGE. Fix it here.
1133 */
1134 if (cpi->cpi_model == 0) {
1135 if (cp->cp_edx & 0x200) {
1136 cp->cp_edx &= ~0x200;
1137 cp->cp_edx |= CPUID_INTC_EDX_PGE;
1138 }
1139 }
1140
1141 /*
1142 * Early models had problems w/ MMX; disable.
1143 */
1144 if (cpi->cpi_model < 6)
1145 mask_edx &= ~CPUID_INTC_EDX_MMX;
1146 }
1147
1148 /*
1149 * For newer families, SSE3 and CX16, at least, are valid;
1150 * enable all
1151 */
1152 if (cpi->cpi_family >= 0xf)
1153 mask_ecx = 0xffffffff;
1154 /*
1155 * We don't support MONITOR/MWAIT if leaf 5 is not available
1156 * to obtain the monitor linesize.
1157 */
1158 if (cpi->cpi_maxeax < 5)
1159 mask_ecx &= ~CPUID_INTC_ECX_MON;
1160
1161 /*
1162 * Do not use MONITOR/MWAIT to halt in the idle loop on any AMD
1163 * processors. AMD does not intend MWAIT to be used in the cpu
1164 * idle loop on current and future processors. 10h and future
1165 * AMD processors use more power in MWAIT than HLT.
1166 * Pre-family-10h Opterons do not have the MWAIT instruction.
1167 */
1168 idle_cpu_prefer_mwait = 0;
1169
1170 break;
1171 case X86_VENDOR_TM:
1172 /*
1173 * workaround the NT workaround in CMS 4.1
1174 */
1175 if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
1176 (cpi->cpi_step == 2 || cpi->cpi_step == 3))
1177 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1178 break;
1179 case X86_VENDOR_Centaur:
1180 /*
1181 * workaround the NT workarounds again
1182 */
1183 if (cpi->cpi_family == 6)
1184 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1185 break;
1186 case X86_VENDOR_Cyrix:
1187 /*
1188 * We rely heavily on the probing in locore
1189 * to actually figure out what parts, if any,
1190 * of the Cyrix cpuid instruction to believe.
1191 */
1192 switch (x86_type) {
1193 case X86_TYPE_CYRIX_486:
1194 mask_edx = 0;
1195 break;
1196 case X86_TYPE_CYRIX_6x86:
1197 mask_edx = 0;
1198 break;
1199 case X86_TYPE_CYRIX_6x86L:
1200 mask_edx =
1201 CPUID_INTC_EDX_DE |
1202 CPUID_INTC_EDX_CX8;
1203 break;
1204 case X86_TYPE_CYRIX_6x86MX:
1205 mask_edx =
1206 CPUID_INTC_EDX_DE |
1207 CPUID_INTC_EDX_MSR |
1208 CPUID_INTC_EDX_CX8 |
1209 CPUID_INTC_EDX_PGE |
1210 CPUID_INTC_EDX_CMOV |
1211 CPUID_INTC_EDX_MMX;
1212 break;
1213 case X86_TYPE_CYRIX_GXm:
1214 mask_edx =
1215 CPUID_INTC_EDX_MSR |
1216 CPUID_INTC_EDX_CX8 |
1217 CPUID_INTC_EDX_CMOV |
1218 CPUID_INTC_EDX_MMX;
1219 break;
1220 case X86_TYPE_CYRIX_MediaGX:
1221 break;
1222 case X86_TYPE_CYRIX_MII:
1223 case X86_TYPE_VIA_CYRIX_III:
1224 mask_edx =
1225 CPUID_INTC_EDX_DE |
1226 CPUID_INTC_EDX_TSC |
1227 CPUID_INTC_EDX_MSR |
1228 CPUID_INTC_EDX_CX8 |
1229 CPUID_INTC_EDX_PGE |
1230 CPUID_INTC_EDX_CMOV |
1231 CPUID_INTC_EDX_MMX;
1232 break;
1233 default:
1234 break;
1235 }
1236 break;
1237 }
1238
1239
1240 if (xsave_force_disable) {
1241 mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
1242 mask_ecx &= ~CPUID_INTC_ECX_AVX;
1243 mask_ecx &= ~CPUID_INTC_ECX_F16C;
1244 mask_ecx &= ~CPUID_INTC_ECX_FMA;
1245 }
1246
1247 /*
1248 * Now we've figured out the masks that determine
1249 * which bits we choose to believe, apply the masks
1250 * to the feature words, then map the kernel's view
1251 * of these feature words into its feature word.
1252 */
1253 cp->cp_edx &= mask_edx;
1254 cp->cp_ecx &= mask_ecx;
1255
1256 /*
1257 * apply any platform restrictions (we don't call this
1258 * immediately after __cpuid_insn here, because we need the
1259 * workarounds applied above first)
1260 */
1261 platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);
1262
1263 /*
1264 * In addition to ecx and edx, Intel and AMD are storing a bunch of
1265 * instruction set extensions in leaf 7's ebx, ecx, and edx.
1266 */
1267 if (cpi->cpi_maxeax >= 7) {
1268 struct cpuid_regs *ecp;
1269 ecp = &cpi->cpi_std[7];
1270 ecp->cp_eax = 7;
1271 ecp->cp_ecx = 0;
1272 (void) __cpuid_insn(ecp);
1273
1274 /*
1275 * If XSAVE has been disabled, just ignore all of the
1276 * extended-save-area dependent flags here.
1277 */
1278 if (xsave_force_disable) {
1279 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
1280 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
1281 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
1282 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_MPX;
1283 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_ALL_AVX512;
1284 ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_ALL_AVX512;
1285 ecp->cp_edx &= ~CPUID_INTC_EDX_7_0_ALL_AVX512;
1286 }
1287
1288 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMEP)
1289 add_x86_feature(featureset, X86FSET_SMEP);
1290
1291 /*
1292 * We check disable_smap here in addition to in startup_smap()
1293 * to ensure CPUs that aren't the boot CPU don't accidentally
1294 * include it in the feature set and thus generate a mismatched
1295 * x86 feature set across CPUs.
1296 */
1297 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMAP &&
1298 disable_smap == 0)
1299 add_x86_feature(featureset, X86FSET_SMAP);
1300
1301 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_RDSEED)
1302 add_x86_feature(featureset, X86FSET_RDSEED);
1303
1304 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_ADX)
1305 add_x86_feature(featureset, X86FSET_ADX);
1306
1307 if (cpi->cpi_vendor == X86_VENDOR_Intel) {
1308 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_INVPCID)
1309 add_x86_feature(featureset, X86FSET_INVPCID);
1310
1311 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_MPX)
1312 add_x86_feature(featureset, X86FSET_MPX);
1313 }
1314 }
1315
1316 /*
1317 * fold in overrides from the "eeprom" mechanism
1318 */
1319 cp->cp_edx |= cpuid_feature_edx_include;
1320 cp->cp_edx &= ~cpuid_feature_edx_exclude;
1321
1322 cp->cp_ecx |= cpuid_feature_ecx_include;
1323 cp->cp_ecx &= ~cpuid_feature_ecx_exclude;
1324
1325 if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
1326 add_x86_feature(featureset, X86FSET_LARGEPAGE);
1327 }
1328 if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
1329 add_x86_feature(featureset, X86FSET_TSC);
1330 }
1331 if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
1332 add_x86_feature(featureset, X86FSET_MSR);
1333 }
1334 if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
1335 add_x86_feature(featureset, X86FSET_MTRR);
1336 }
1337 if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
1338 add_x86_feature(featureset, X86FSET_PGE);
1339 }
1340 if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
1341 add_x86_feature(featureset, X86FSET_CMOV);
1342 }
1343 if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
1344 add_x86_feature(featureset, X86FSET_MMX);
1345 }
1346 if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
1347 (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
1348 add_x86_feature(featureset, X86FSET_MCA);
1349 }
1350 if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
1351 add_x86_feature(featureset, X86FSET_PAE);
1352 }
1353 if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
1354 add_x86_feature(featureset, X86FSET_CX8);
1355 }
1356 if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
1357 add_x86_feature(featureset, X86FSET_CX16);
1358 }
1359 if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
1360 add_x86_feature(featureset, X86FSET_PAT);
1361 }
1362 if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
1363 add_x86_feature(featureset, X86FSET_SEP);
1364 }
1365 if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
1366 /*
1367 * In our implementation, fxsave/fxrstor
1368 * are prerequisites before we'll even
1369 * try and do SSE things.
1370 */
1371 if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
1372 add_x86_feature(featureset, X86FSET_SSE);
1373 }
1374 if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
1375 add_x86_feature(featureset, X86FSET_SSE2);
1376 }
1377 if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
1378 add_x86_feature(featureset, X86FSET_SSE3);
1379 }
1380 if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
1381 add_x86_feature(featureset, X86FSET_SSSE3);
1382 }
1383 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
1384 add_x86_feature(featureset, X86FSET_SSE4_1);
1385 }
1386 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
1387 add_x86_feature(featureset, X86FSET_SSE4_2);
1388 }
1389 if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
1390 add_x86_feature(featureset, X86FSET_AES);
1391 }
1392 if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
1393 add_x86_feature(featureset, X86FSET_PCLMULQDQ);
1394 }
1395
1396 if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_SHA)
1397 add_x86_feature(featureset, X86FSET_SHA);
1398
1399 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_UMIP)
1400 add_x86_feature(featureset, X86FSET_UMIP);
1401 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_PKU)
1402 add_x86_feature(featureset, X86FSET_PKU);
1403 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_OSPKE)
1404 add_x86_feature(featureset, X86FSET_OSPKE);
1405
1406 if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
1407 add_x86_feature(featureset, X86FSET_XSAVE);
1408
1409 /* We only test AVX & AVX512 when there is XSAVE */
1410
1411 if (cp->cp_ecx & CPUID_INTC_ECX_AVX) {
1412 add_x86_feature(featureset,
1413 X86FSET_AVX);
1414
1415 /*
1416 * Intel says we can't check these without also
1417 * checking AVX.
1418 */
1419 if (cp->cp_ecx & CPUID_INTC_ECX_F16C)
1420 add_x86_feature(featureset,
1421 X86FSET_F16C);
1422
1423 if (cp->cp_ecx & CPUID_INTC_ECX_FMA)
1424 add_x86_feature(featureset,
1425 X86FSET_FMA);
1426
1427 if (cpi->cpi_std[7].cp_ebx &
1428 CPUID_INTC_EBX_7_0_BMI1)
1429 add_x86_feature(featureset,
1430 X86FSET_BMI1);
1431
1432 if (cpi->cpi_std[7].cp_ebx &
1433 CPUID_INTC_EBX_7_0_BMI2)
1434 add_x86_feature(featureset,
1435 X86FSET_BMI2);
1436
1437 if (cpi->cpi_std[7].cp_ebx &
1438 CPUID_INTC_EBX_7_0_AVX2)
1439 add_x86_feature(featureset,
1440 X86FSET_AVX2);
1441 }
1442
1443 if (cpi->cpi_vendor == X86_VENDOR_Intel &&
1444 (cpi->cpi_std[7].cp_ebx &
1445 CPUID_INTC_EBX_7_0_AVX512F) != 0) {
1446 add_x86_feature(featureset, X86FSET_AVX512F);
1447
1448 if (cpi->cpi_std[7].cp_ebx &
1449 CPUID_INTC_EBX_7_0_AVX512DQ)
1450 add_x86_feature(featureset,
1451 X86FSET_AVX512DQ);
1452 if (cpi->cpi_std[7].cp_ebx &
1453 CPUID_INTC_EBX_7_0_AVX512IFMA)
1454 add_x86_feature(featureset,
1455 X86FSET_AVX512FMA);
1456 if (cpi->cpi_std[7].cp_ebx &
1457 CPUID_INTC_EBX_7_0_AVX512PF)
1458 add_x86_feature(featureset,
1459 X86FSET_AVX512PF);
1460 if (cpi->cpi_std[7].cp_ebx &
1461 CPUID_INTC_EBX_7_0_AVX512ER)
1462 add_x86_feature(featureset,
1463 X86FSET_AVX512ER);
1464 if (cpi->cpi_std[7].cp_ebx &
1465 CPUID_INTC_EBX_7_0_AVX512CD)
1466 add_x86_feature(featureset,
1467 X86FSET_AVX512CD);
1468 if (cpi->cpi_std[7].cp_ebx &
1469 CPUID_INTC_EBX_7_0_AVX512BW)
1470 add_x86_feature(featureset,
1471 X86FSET_AVX512BW);
1472 if (cpi->cpi_std[7].cp_ebx &
1473 CPUID_INTC_EBX_7_0_AVX512VL)
1474 add_x86_feature(featureset,
1475 X86FSET_AVX512VL);
1476
1477 if (cpi->cpi_std[7].cp_ecx &
1478 CPUID_INTC_ECX_7_0_AVX512VBMI)
1479 add_x86_feature(featureset,
1480 X86FSET_AVX512VBMI);
1481 if (cpi->cpi_std[7].cp_ecx &
1482 CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
1483 add_x86_feature(featureset,
1484 X86FSET_AVX512VPOPCDQ);
1485
1486 if (cpi->cpi_std[7].cp_edx &
1487 CPUID_INTC_EDX_7_0_AVX5124NNIW)
1488 add_x86_feature(featureset,
1489 X86FSET_AVX512NNIW);
1490 if (cpi->cpi_std[7].cp_edx &
1491 CPUID_INTC_EDX_7_0_AVX5124FMAPS)
1492 add_x86_feature(featureset,
1493 X86FSET_AVX512FMAPS);
1494 }
1495 }
1496 }
1497
1498 if (cpi->cpi_vendor == X86_VENDOR_Intel) {
1499 if (cp->cp_ecx & CPUID_INTC_ECX_PCID) {
1500 add_x86_feature(featureset, X86FSET_PCID);
1501 }
1502 }
1503
1504 if (cp->cp_ecx & CPUID_INTC_ECX_X2APIC) {
1505 add_x86_feature(featureset, X86FSET_X2APIC);
1506 }
1507 if (cp->cp_edx & CPUID_INTC_EDX_DE) {
1508 add_x86_feature(featureset, X86FSET_DE);
1509 }
1510 if (cp->cp_ecx & CPUID_INTC_ECX_MON) {
1511
1512 /*
1513 * We require the CLFLUSH instruction for erratum workaround
1514 * to use MONITOR/MWAIT.
1515 */
1516 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1517 cpi->cpi_mwait.support |= MWAIT_SUPPORT;
1518 add_x86_feature(featureset, X86FSET_MWAIT);
1519 } else {
1520 extern int idle_cpu_assert_cflush_monitor;
1521
1522 /*
1523 * All processors we are aware of which have
1524 * MONITOR/MWAIT also have CLFLUSH.
1525 */
1526 if (idle_cpu_assert_cflush_monitor) {
1527 ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
1528 (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
1529 }
1530 }
1531 }
1532
1533 if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
1534 add_x86_feature(featureset, X86FSET_VMX);
1535 }
1536
1537 if (cp->cp_ecx & CPUID_INTC_ECX_RDRAND)
1538 add_x86_feature(featureset, X86FSET_RDRAND);
1539
1540 /*
1541 * Only need it first time, rest of the cpus would follow suit.
1542 * we only capture this for the bootcpu.
1543 */
1544 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1545 add_x86_feature(featureset, X86FSET_CLFSH);
1546 x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
1547 }
1548 if (is_x86_feature(featureset, X86FSET_PAE))
1549 cpi->cpi_pabits = 36;
1550
1551 /*
1552 * Hyperthreading configuration is slightly tricky on Intel
1553 * and pure clones, and even trickier on AMD.
1554 *
1555 * (AMD chose to set the HTT bit on their CMP processors,
1556 * even though they're not actually hyperthreaded. Thus it
1557 * takes a bit more work to figure out what's really going
1558 * on ... see the handling of the CMP_LGCY bit below)
1559 */
1560 if (cp->cp_edx & CPUID_INTC_EDX_HTT) {
1561 cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
1562 if (cpi->cpi_ncpu_per_chip > 1)
1563 add_x86_feature(featureset, X86FSET_HTT);
1564 } else {
1565 cpi->cpi_ncpu_per_chip = 1;
1566 }
1567
1568 if (cpi->cpi_maxeax >= 0xD && !xsave_force_disable) {
1569 struct cpuid_regs r, *ecp;
1570
1571 ecp = &r;
1572 ecp->cp_eax = 0xD;
1573 ecp->cp_ecx = 1;
1574 ecp->cp_edx = ecp->cp_ebx = 0;
1575 (void) __cpuid_insn(ecp);
1576
1577 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEOPT)
1578 add_x86_feature(featureset, X86FSET_XSAVEOPT);
1579 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEC)
1580 add_x86_feature(featureset, X86FSET_XSAVEC);
1581 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVES)
1582 add_x86_feature(featureset, X86FSET_XSAVES);
1583 }
1584
1585 /*
1586 * Work on the "extended" feature information, doing
1587 * some basic initialization for cpuid_pass2()
1588 */
1589 xcpuid = 0;
1590 switch (cpi->cpi_vendor) {
1591 case X86_VENDOR_Intel:
1592 /*
1593 * On KVM we know we will have proper support for extended
1594 * cpuid.
1595 */
1596 if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf ||
1597 (get_hwenv() == HW_KVM && cpi->cpi_family == 6 &&
1598 (cpi->cpi_model == 6 || cpi->cpi_model == 2)))
1599 xcpuid++;
1600 break;
1601 case X86_VENDOR_AMD:
1602 if (cpi->cpi_family > 5 ||
1603 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
1604 xcpuid++;
1605 break;
1606 case X86_VENDOR_Cyrix:
1607 /*
1608 * Only these Cyrix CPUs are -known- to support
1609 * extended cpuid operations.
1610 */
1611 if (x86_type == X86_TYPE_VIA_CYRIX_III ||
1612 x86_type == X86_TYPE_CYRIX_GXm)
1613 xcpuid++;
1614 break;
1615 case X86_VENDOR_Centaur:
1616 case X86_VENDOR_TM:
1617 default:
1618 xcpuid++;
1619 break;
1620 }
1621
1622 if (xcpuid) {
1623 cp = &cpi->cpi_extd[0];
1624 cp->cp_eax = 0x80000000;
1625 cpi->cpi_xmaxeax = __cpuid_insn(cp);
1626 }
1627
1628 if (cpi->cpi_xmaxeax & 0x80000000) {
1629
1630 if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
1631 cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;
1632
1633 switch (cpi->cpi_vendor) {
1634 case X86_VENDOR_Intel:
1635 case X86_VENDOR_AMD:
1636 if (cpi->cpi_xmaxeax < 0x80000001)
1637 break;
1638 cp = &cpi->cpi_extd[1];
1639 cp->cp_eax = 0x80000001;
1640 (void) __cpuid_insn(cp);
1641
1642 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1643 cpi->cpi_family == 5 &&
1644 cpi->cpi_model == 6 &&
1645 cpi->cpi_step == 6) {
1646 /*
1647 * K6 model 6 uses bit 10 to indicate SYSC
1648 * Later models use bit 11. Fix it here.
1649 */
1650 if (cp->cp_edx & 0x400) {
1651 cp->cp_edx &= ~0x400;
1652 cp->cp_edx |= CPUID_AMD_EDX_SYSC;
1653 }
1654 }
1655
1656 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);
1657
1658 /*
1659 * Compute the additions to the kernel's feature word.
1660 */
1661 if (cp->cp_edx & CPUID_AMD_EDX_NX) {
1662 add_x86_feature(featureset, X86FSET_NX);
1663 }
1664
1665 /*
1666 * Regardless whether or not we boot 64-bit,
1667 * we should have a way to identify whether
1668 * the CPU is capable of running 64-bit.
1669 */
1670 if (cp->cp_edx & CPUID_AMD_EDX_LM) {
1671 add_x86_feature(featureset, X86FSET_64);
1672 }
1673
1674 #if defined(__amd64)
1675 /* 1 GB large page - enable only for 64 bit kernel */
1676 if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
1677 add_x86_feature(featureset, X86FSET_1GPG);
1678 }
1679 #endif
1680
1681 if ((cpi->cpi_vendor == X86_VENDOR_AMD) &&
1682 (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
1683 (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
1684 add_x86_feature(featureset, X86FSET_SSE4A);
1685 }
1686
1687 /*
1688 * If both the HTT and CMP_LGCY bits are set,
1689 * then we're not actually HyperThreaded. Read
1690 * "AMD CPUID Specification" for more details.
1691 */
1692 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1693 is_x86_feature(featureset, X86FSET_HTT) &&
1694 (cp->cp_ecx & CPUID_AMD_ECX_CMP_LGCY)) {
1695 remove_x86_feature(featureset, X86FSET_HTT);
1696 add_x86_feature(featureset, X86FSET_CMP);
1697 }
1698 #if defined(__amd64)
1699 /*
1700 * It's really tricky to support syscall/sysret in
1701 * the i386 kernel; we rely on sysenter/sysexit
1702 * instead. In the amd64 kernel, things are -way-
1703 * better.
1704 */
1705 if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
1706 add_x86_feature(featureset, X86FSET_ASYSC);
1707 }
1708
1709 /*
1710 * While we're thinking about system calls, note
1711 * that AMD processors don't support sysenter
1712 * in long mode at all, so don't try to program them.
1713 */
1714 if (x86_vendor == X86_VENDOR_AMD) {
1715 remove_x86_feature(featureset, X86FSET_SEP);
1716 }
1717 #endif
1718 if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
1719 add_x86_feature(featureset, X86FSET_TSCP);
1720 }
1721
1722 if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
1723 add_x86_feature(featureset, X86FSET_SVM);
1724 }
1725
1726 if (cp->cp_ecx & CPUID_AMD_ECX_TOPOEXT) {
1727 add_x86_feature(featureset, X86FSET_TOPOEXT);
1728 }
1729 break;
1730 default:
1731 break;
1732 }
1733
1734 /*
1735 * Get CPUID data about processor cores and hyperthreads.
1736 */
1737 switch (cpi->cpi_vendor) {
1738 case X86_VENDOR_Intel:
1739 if (cpi->cpi_maxeax >= 4) {
1740 cp = &cpi->cpi_std[4];
1741 cp->cp_eax = 4;
1742 cp->cp_ecx = 0;
1743 (void) __cpuid_insn(cp);
1744 platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
1745 }
1746 /*FALLTHROUGH*/
1747 case X86_VENDOR_AMD:
1748 if (cpi->cpi_xmaxeax < 0x80000008)
1749 break;
1750 cp = &cpi->cpi_extd[8];
1751 cp->cp_eax = 0x80000008;
1752 (void) __cpuid_insn(cp);
1753 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, cp);
1754
1755 /*
1756 * Virtual and physical address limits from
1757 * cpuid override previously guessed values.
1758 */
1759 cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
1760 cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
1761 break;
1762 default:
1763 break;
1764 }
1765
1766 /*
1767 * Derive the number of cores per chip
1768 */
1769 switch (cpi->cpi_vendor) {
1770 case X86_VENDOR_Intel:
1771 if (cpi->cpi_maxeax < 4) {
1772 cpi->cpi_ncore_per_chip = 1;
1773 break;
1774 } else {
1775 cpi->cpi_ncore_per_chip =
1776 BITX((cpi)->cpi_std[4].cp_eax, 31, 26) + 1;
1777 }
1778 break;
1779 case X86_VENDOR_AMD:
1780 if (cpi->cpi_xmaxeax < 0x80000008) {
1781 cpi->cpi_ncore_per_chip = 1;
1782 break;
1783 } else {
1784 /*
1785 * On family 0xf cpuid fn 2 ECX[7:0] "NC" is
1786 * 1 less than the number of physical cores on
1787 * the chip. In family 0x10 this value can
1788 * be affected by "downcoring" - it reflects
1789 * 1 less than the number of cores actually
1790 * enabled on this node.
1791 */
1792 cpi->cpi_ncore_per_chip =
1793 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
1794 }
1795 break;
1796 default:
1797 cpi->cpi_ncore_per_chip = 1;
1798 break;
1799 }
1800
1801 /*
1802 * Get CPUID data about TSC Invariance in Deep C-State.
1803 */
1804 switch (cpi->cpi_vendor) {
1805 case X86_VENDOR_Intel:
1806 if (cpi->cpi_maxeax >= 7) {
1807 cp = &cpi->cpi_extd[7];
1808 cp->cp_eax = 0x80000007;
1809 cp->cp_ecx = 0;
1810 (void) __cpuid_insn(cp);
1811 }
1812 break;
1813 default:
1814 break;
1815 }
1816 } else {
1817 cpi->cpi_ncore_per_chip = 1;
1818 }
1819
1820 /*
1821 * If more than one core, then this processor is CMP.
1822 */
1823 if (cpi->cpi_ncore_per_chip > 1) {
1824 add_x86_feature(featureset, X86FSET_CMP);
1825 }
1826
1827 /*
1828 * If the number of cores is the same as the number
1829 * of CPUs, then we cannot have HyperThreading.
1830 */
1831 if (cpi->cpi_ncpu_per_chip == cpi->cpi_ncore_per_chip) {
1832 remove_x86_feature(featureset, X86FSET_HTT);
1833 }
1834
1835 cpi->cpi_apicid = CPI_APIC_ID(cpi);
1836 cpi->cpi_procnodes_per_pkg = 1;
1837 cpi->cpi_cores_per_compunit = 1;
1838 if (is_x86_feature(featureset, X86FSET_HTT) == B_FALSE &&
1839 is_x86_feature(featureset, X86FSET_CMP) == B_FALSE) {
1840 /*
1841 * Single-core single-threaded processors.
1842 */
1843 cpi->cpi_chipid = -1;
1844 cpi->cpi_clogid = 0;
1845 cpi->cpi_coreid = cpu->cpu_id;
1846 cpi->cpi_pkgcoreid = 0;
1847 if (cpi->cpi_vendor == X86_VENDOR_AMD)
1848 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
1849 else
1850 cpi->cpi_procnodeid = cpi->cpi_chipid;
1851 } else if (cpi->cpi_ncpu_per_chip > 1) {
1852 if (cpi->cpi_vendor == X86_VENDOR_Intel)
1853 cpuid_intel_getids(cpu, featureset);
1854 else if (cpi->cpi_vendor == X86_VENDOR_AMD)
1855 cpuid_amd_getids(cpu);
1856 else {
1857 /*
1858 * All other processors are currently
1859 * assumed to have single cores.
1860 */
1861 cpi->cpi_coreid = cpi->cpi_chipid;
1862 cpi->cpi_pkgcoreid = 0;
1863 cpi->cpi_procnodeid = cpi->cpi_chipid;
1864 cpi->cpi_compunitid = cpi->cpi_chipid;
1865 }
1866 }
1867
1868 /*
1869 * Synthesize chip "revision" and socket type
1870 */
1871 cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
1872 cpi->cpi_model, cpi->cpi_step);
1873 cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
1874 cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
1875 cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
1876 cpi->cpi_model, cpi->cpi_step);
1877
1878 /*
1879 * While we're here, check for the AMD "Error Pointer Zero/Restore"
1880 * feature. This can be used to setup the FP save handlers
1881 * appropriately.
1882 */
1883 if (cpi->cpi_vendor == X86_VENDOR_AMD) {
1884 if (cpi->cpi_xmaxeax >= 0x80000008 &&
1885 cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
1886 /* Special handling for AMD FP not necessary. */
1887 cpi->cpi_fp_amd_save = 0;
1888 } else {
1889 cpi->cpi_fp_amd_save = 1;
1890 }
1891 }
1892
1893 /*
1894 * Check the processor leaves that are used for security features.
1895 */
1896 cpuid_scan_security(cpu, featureset);
1897
1898 pass1_done:
1899 cpi->cpi_pass = 1;
1900 }
1901
1902 /*
1903 * Make copies of the cpuid table entries we depend on, in
1904 * part for ease of parsing now, in part so that we have only
1905 * one place to correct any of it, in part for ease of
1906 * later export to userland, and in part so we can look at
1907 * this stuff in a crash dump.
1908 */
1909
1910 /*ARGSUSED*/
1911 void
1912 cpuid_pass2(cpu_t *cpu)
1913 {
1914 uint_t n, nmax;
1915 int i;
1916 struct cpuid_regs *cp;
1917 uint8_t *dp;
1918 uint32_t *iptr;
1919 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
1920
1921 ASSERT(cpi->cpi_pass == 1);
1922
1923 if (cpi->cpi_maxeax < 1)
1924 goto pass2_done;
1925
1926 if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
1927 nmax = NMAX_CPI_STD;
1928 /*
1929 * (We already handled n == 0 and n == 1 in pass 1)
1930 */
1931 for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
1932 cp->cp_eax = n;
1933
1934 /*
1935 * n == 7 was handled in pass 1
1936 */
1937 if (n == 7)
1938 continue;
1939
1940 /*
1941 * CPUID function 4 expects %ecx to be initialized
1942 * with an index which indicates which cache to return
1943 * information about. The OS is expected to call function 4
1944 * with %ecx set to 0, 1, 2, ... until it returns with
1945 * EAX[4:0] set to 0, which indicates there are no more
1946 * caches.
1947 *
1948 * Here, populate cpi_std[4] with the information returned by
1949 * function 4 when %ecx == 0, and do the rest in cpuid_pass3()
1950 * when dynamic memory allocation becomes available.
1951 *
1952 * Note: we need to explicitly initialize %ecx here, since
1953 * function 4 may have been previously invoked.
1954 */
1955 if (n == 4)
1956 cp->cp_ecx = 0;
1957
1958 (void) __cpuid_insn(cp);
1959 platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
1960 switch (n) {
1961 case 2:
1962 /*
1963 * "the lower 8 bits of the %eax register
1964 * contain a value that identifies the number
1965 * of times the cpuid [instruction] has to be
1966 * executed to obtain a complete image of the
1967 * processor's caching systems."
1968 *
1969 * How *do* they make this stuff up?
1970 */
1971 cpi->cpi_ncache = sizeof (*cp) *
1972 BITX(cp->cp_eax, 7, 0);
1973 if (cpi->cpi_ncache == 0)
1974 break;
1975 cpi->cpi_ncache--; /* skip count byte */
1976
1977 /*
1978 * Well, for now, rather than attempt to implement
1979 * this slightly dubious algorithm, we just look
1980 * at the first 15 ..
1981 */
1982 if (cpi->cpi_ncache > (sizeof (*cp) - 1))
1983 cpi->cpi_ncache = sizeof (*cp) - 1;
1984
1985 dp = cpi->cpi_cacheinfo;
1986 if (BITX(cp->cp_eax, 31, 31) == 0) {
1987 uint8_t *p = (void *)&cp->cp_eax;
1988 for (i = 1; i < 4; i++)
1989 if (p[i] != 0)
1990 *dp++ = p[i];
1991 }
1992 if (BITX(cp->cp_ebx, 31, 31) == 0) {
1993 uint8_t *p = (void *)&cp->cp_ebx;
1994 for (i = 0; i < 4; i++)
1995 if (p[i] != 0)
1996 *dp++ = p[i];
1997 }
1998 if (BITX(cp->cp_ecx, 31, 31) == 0) {
1999 uint8_t *p = (void *)&cp->cp_ecx;
2000 for (i = 0; i < 4; i++)
2001 if (p[i] != 0)
2002 *dp++ = p[i];
2003 }
2004 if (BITX(cp->cp_edx, 31, 31) == 0) {
2005 uint8_t *p = (void *)&cp->cp_edx;
2006 for (i = 0; i < 4; i++)
2007 if (p[i] != 0)
2008 *dp++ = p[i];
2009 }
2010 break;
2011
2012 case 3: /* Processor serial number, if PSN supported */
2013 break;
2014
2015 case 4: /* Deterministic cache parameters */
2016 break;
2017
2018 case 5: /* Monitor/Mwait parameters */
2019 {
2020 size_t mwait_size;
2021
2022 /*
2023 * check cpi_mwait.support which was set in cpuid_pass1
2024 */
2025 if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
2026 break;
2027
2028 /*
2029 * Protect ourself from insane mwait line size.
2030 * Workaround for incomplete hardware emulator(s).
2031 */
2032 mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
2033 if (mwait_size < sizeof (uint32_t) ||
2034 !ISP2(mwait_size)) {
2035 #if DEBUG
2036 cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
2037 "size %ld", cpu->cpu_id, (long)mwait_size);
2038 #endif
2039 break;
2040 }
2041
2042 cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
2043 cpi->cpi_mwait.mon_max = mwait_size;
2044 if (MWAIT_EXTENSION(cpi)) {
2045 cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
2046 if (MWAIT_INT_ENABLE(cpi))
2047 cpi->cpi_mwait.support |=
2048 MWAIT_ECX_INT_ENABLE;
2049 }
2050 break;
2051 }
2052 default:
2053 break;
2054 }
2055 }
2056
2057 if (cpi->cpi_maxeax >= 0xB && cpi->cpi_vendor == X86_VENDOR_Intel) {
2058 struct cpuid_regs regs;
2059
2060 cp = &regs;
2061 cp->cp_eax = 0xB;
2062 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2063
2064 (void) __cpuid_insn(cp);
2065
2066 /*
2067 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
2068 * indicates that the extended topology enumeration leaf is
2069 * available.
2070 */
2071 if (cp->cp_ebx) {
2072 uint32_t x2apic_id;
2073 uint_t coreid_shift = 0;
2074 uint_t ncpu_per_core = 1;
2075 uint_t chipid_shift = 0;
2076 uint_t ncpu_per_chip = 1;
2077 uint_t i;
2078 uint_t level;
2079
2080 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
2081 cp->cp_eax = 0xB;
2082 cp->cp_ecx = i;
2083
2084 (void) __cpuid_insn(cp);
2085 level = CPI_CPU_LEVEL_TYPE(cp);
2086
2087 if (level == 1) {
2088 x2apic_id = cp->cp_edx;
2089 coreid_shift = BITX(cp->cp_eax, 4, 0);
2090 ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
2091 } else if (level == 2) {
2092 x2apic_id = cp->cp_edx;
2093 chipid_shift = BITX(cp->cp_eax, 4, 0);
2094 ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
2095 }
2096 }
2097
2098 cpi->cpi_apicid = x2apic_id;
2099 cpi->cpi_ncpu_per_chip = ncpu_per_chip;
2100 cpi->cpi_ncore_per_chip = ncpu_per_chip /
2101 ncpu_per_core;
2102 cpi->cpi_chipid = x2apic_id >> chipid_shift;
2103 cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
2104 cpi->cpi_coreid = x2apic_id >> coreid_shift;
2105 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
2106 }
2107
2108 /* Make cp NULL so that we don't stumble on others */
2109 cp = NULL;
2110 }
2111
2112 /*
2113 * XSAVE enumeration
2114 */
2115 if (cpi->cpi_maxeax >= 0xD) {
2116 struct cpuid_regs regs;
2117 boolean_t cpuid_d_valid = B_TRUE;
2118
2119 cp = &regs;
2120 cp->cp_eax = 0xD;
2121 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2122
2123 (void) __cpuid_insn(cp);
2124
2125 /*
2126 * Sanity checks for debug
2127 */
2128 if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
2129 (cp->cp_eax & XFEATURE_SSE) == 0) {
2130 cpuid_d_valid = B_FALSE;
2131 }
2132
2133 cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
2134 cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
2135 cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;
2136
2137 /*
2138 * If the hw supports AVX, get the size and offset in the save
2139 * area for the ymm state.
2140 */
2141 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
2142 cp->cp_eax = 0xD;
2143 cp->cp_ecx = 2;
2144 cp->cp_edx = cp->cp_ebx = 0;
2145
2146 (void) __cpuid_insn(cp);
2147
2148 if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
2149 cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
2150 cpuid_d_valid = B_FALSE;
2151 }
2152
2153 cpi->cpi_xsave.ymm_size = cp->cp_eax;
2154 cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
2155 }
2156
2157 /*
2158 * If the hw supports MPX, get the size and offset in the
2159 * save area for BNDREGS and BNDCSR.
2160 */
2161 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_MPX) {
2162 cp->cp_eax = 0xD;
2163 cp->cp_ecx = 3;
2164 cp->cp_edx = cp->cp_ebx = 0;
2165
2166 (void) __cpuid_insn(cp);
2167
2168 cpi->cpi_xsave.bndregs_size = cp->cp_eax;
2169 cpi->cpi_xsave.bndregs_offset = cp->cp_ebx;
2170
2171 cp->cp_eax = 0xD;
2172 cp->cp_ecx = 4;
2173 cp->cp_edx = cp->cp_ebx = 0;
2174
2175 (void) __cpuid_insn(cp);
2176
2177 cpi->cpi_xsave.bndcsr_size = cp->cp_eax;
2178 cpi->cpi_xsave.bndcsr_offset = cp->cp_ebx;
2179 }
2180
2181 /*
2182 * If the hw supports AVX512, get the size and offset in the
2183 * save area for the opmask registers and zmm state.
2184 */
2185 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX512) {
2186 cp->cp_eax = 0xD;
2187 cp->cp_ecx = 5;
2188 cp->cp_edx = cp->cp_ebx = 0;
2189
2190 (void) __cpuid_insn(cp);
2191
2192 cpi->cpi_xsave.opmask_size = cp->cp_eax;
2193 cpi->cpi_xsave.opmask_offset = cp->cp_ebx;
2194
2195 cp->cp_eax = 0xD;
2196 cp->cp_ecx = 6;
2197 cp->cp_edx = cp->cp_ebx = 0;
2198
2199 (void) __cpuid_insn(cp);
2200
2201 cpi->cpi_xsave.zmmlo_size = cp->cp_eax;
2202 cpi->cpi_xsave.zmmlo_offset = cp->cp_ebx;
2203
2204 cp->cp_eax = 0xD;
2205 cp->cp_ecx = 7;
2206 cp->cp_edx = cp->cp_ebx = 0;
2207
2208 (void) __cpuid_insn(cp);
2209
2210 cpi->cpi_xsave.zmmhi_size = cp->cp_eax;
2211 cpi->cpi_xsave.zmmhi_offset = cp->cp_ebx;
2212 }
2213
2214 if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
2215 xsave_state_size = 0;
2216 } else if (cpuid_d_valid) {
2217 xsave_state_size = cpi->cpi_xsave.xsav_max_size;
2218 } else {
2219 /* Broken CPUID 0xD, probably in HVM */
2220 cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
2221 "value: hw_low = %d, hw_high = %d, xsave_size = %d"
2222 ", ymm_size = %d, ymm_offset = %d\n",
2223 cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
2224 cpi->cpi_xsave.xsav_hw_features_high,
2225 (int)cpi->cpi_xsave.xsav_max_size,
2226 (int)cpi->cpi_xsave.ymm_size,
2227 (int)cpi->cpi_xsave.ymm_offset);
2228
2229 if (xsave_state_size != 0) {
2230 /*
2231 * This must be a non-boot CPU. We cannot
2232 * continue, because boot cpu has already
2233 * enabled XSAVE.
2234 */
2235 ASSERT(cpu->cpu_id != 0);
2236 cmn_err(CE_PANIC, "cpu%d: we have already "
2237 "enabled XSAVE on boot cpu, cannot "
2238 "continue.", cpu->cpu_id);
2239 } else {
2240 /*
2241 * If we reached here on the boot CPU, it's also
2242 * almost certain that we'll reach here on the
2243 * non-boot CPUs. When we're here on a boot CPU
2244 * we should disable the feature, on a non-boot
2245 * CPU we need to confirm that we have.
2246 */
2247 if (cpu->cpu_id == 0) {
2248 remove_x86_feature(x86_featureset,
2249 X86FSET_XSAVE);
2250 remove_x86_feature(x86_featureset,
2251 X86FSET_AVX);
2252 remove_x86_feature(x86_featureset,
2253 X86FSET_F16C);
2254 remove_x86_feature(x86_featureset,
2255 X86FSET_BMI1);
2256 remove_x86_feature(x86_featureset,
2257 X86FSET_BMI2);
2258 remove_x86_feature(x86_featureset,
2259 X86FSET_FMA);
2260 remove_x86_feature(x86_featureset,
2261 X86FSET_AVX2);
2262 remove_x86_feature(x86_featureset,
2263 X86FSET_MPX);
2264 remove_x86_feature(x86_featureset,
2265 X86FSET_AVX512F);
2266 remove_x86_feature(x86_featureset,
2267 X86FSET_AVX512DQ);
2268 remove_x86_feature(x86_featureset,
2269 X86FSET_AVX512PF);
2270 remove_x86_feature(x86_featureset,
2271 X86FSET_AVX512ER);
2272 remove_x86_feature(x86_featureset,
2273 X86FSET_AVX512CD);
2274 remove_x86_feature(x86_featureset,
2275 X86FSET_AVX512BW);
2276 remove_x86_feature(x86_featureset,
2277 X86FSET_AVX512VL);
2278 remove_x86_feature(x86_featureset,
2279 X86FSET_AVX512FMA);
2280 remove_x86_feature(x86_featureset,
2281 X86FSET_AVX512VBMI);
2282 remove_x86_feature(x86_featureset,
2283 X86FSET_AVX512VPOPCDQ);
2284 remove_x86_feature(x86_featureset,
2285 X86FSET_AVX512NNIW);
2286 remove_x86_feature(x86_featureset,
2287 X86FSET_AVX512FMAPS);
2288
2289 CPI_FEATURES_ECX(cpi) &=
2290 ~CPUID_INTC_ECX_XSAVE;
2291 CPI_FEATURES_ECX(cpi) &=
2292 ~CPUID_INTC_ECX_AVX;
2293 CPI_FEATURES_ECX(cpi) &=
2294 ~CPUID_INTC_ECX_F16C;
2295 CPI_FEATURES_ECX(cpi) &=
2296 ~CPUID_INTC_ECX_FMA;
2297 CPI_FEATURES_7_0_EBX(cpi) &=
2298 ~CPUID_INTC_EBX_7_0_BMI1;
2299 CPI_FEATURES_7_0_EBX(cpi) &=
2300 ~CPUID_INTC_EBX_7_0_BMI2;
2301 CPI_FEATURES_7_0_EBX(cpi) &=
2302 ~CPUID_INTC_EBX_7_0_AVX2;
2303 CPI_FEATURES_7_0_EBX(cpi) &=
2304 ~CPUID_INTC_EBX_7_0_MPX;
2305 CPI_FEATURES_7_0_EBX(cpi) &=
2306 ~CPUID_INTC_EBX_7_0_ALL_AVX512;
2307
2308 CPI_FEATURES_7_0_ECX(cpi) &=
2309 ~CPUID_INTC_ECX_7_0_ALL_AVX512;
2310
2311 CPI_FEATURES_7_0_EDX(cpi) &=
2312 ~CPUID_INTC_EDX_7_0_ALL_AVX512;
2313
2314 xsave_force_disable = B_TRUE;
2315 } else {
2316 VERIFY(is_x86_feature(x86_featureset,
2317 X86FSET_XSAVE) == B_FALSE);
2318 }
2319 }
2320 }
2321 }
2322
2323
2324 if ((cpi->cpi_xmaxeax & 0x80000000) == 0)
2325 goto pass2_done;
2326
2327 if ((nmax = cpi->cpi_xmaxeax - 0x80000000 + 1) > NMAX_CPI_EXTD)
2328 nmax = NMAX_CPI_EXTD;
2329 /*
2330 * Copy the extended properties, fixing them as we go.
2331 * (We already handled n == 0 and n == 1 in pass 1)
2332 */
2333 iptr = (void *)cpi->cpi_brandstr;
2334 for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
2335 cp->cp_eax = 0x80000000 + n;
2336 (void) __cpuid_insn(cp);
2337 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000000 + n, cp);
2338 switch (n) {
2339 case 2:
2340 case 3:
2341 case 4:
2342 /*
2343 * Extract the brand string
2344 */
2345 *iptr++ = cp->cp_eax;
2346 *iptr++ = cp->cp_ebx;
2347 *iptr++ = cp->cp_ecx;
2348 *iptr++ = cp->cp_edx;
2349 break;
2350 case 5:
2351 switch (cpi->cpi_vendor) {
2352 case X86_VENDOR_AMD:
2353 /*
2354 * The Athlon and Duron were the first
2355 * parts to report the sizes of the
2356 * TLB for large pages. Before then,
2357 * we don't trust the data.
2358 */
2359 if (cpi->cpi_family < 6 ||
2360 (cpi->cpi_family == 6 &&
2361 cpi->cpi_model < 1))
2362 cp->cp_eax = 0;
2363 break;
2364 default:
2365 break;
2366 }
2367 break;
2368 case 6:
2369 switch (cpi->cpi_vendor) {
2370 case X86_VENDOR_AMD:
2371 /*
2372 * The Athlon and Duron were the first
2373 * AMD parts with L2 TLB's.
2374 * Before then, don't trust the data.
2375 */
2376 if (cpi->cpi_family < 6 ||
2377 cpi->cpi_family == 6 &&
2378 cpi->cpi_model < 1)
2379 cp->cp_eax = cp->cp_ebx = 0;
2380 /*
2381 * AMD Duron rev A0 reports L2
2382 * cache size incorrectly as 1K
2383 * when it is really 64K
2384 */
2385 if (cpi->cpi_family == 6 &&
2386 cpi->cpi_model == 3 &&
2387 cpi->cpi_step == 0) {
2388 cp->cp_ecx &= 0xffff;
2389 cp->cp_ecx |= 0x400000;
2390 }
2391 break;
2392 case X86_VENDOR_Cyrix: /* VIA C3 */
2393 /*
2394 * VIA C3 processors are a bit messed
2395 * up w.r.t. encoding cache sizes in %ecx
2396 */
2397 if (cpi->cpi_family != 6)
2398 break;
2399 /*
2400 * model 7 and 8 were incorrectly encoded
2401 *
2402 * xxx is model 8 really broken?
2403 */
2404 if (cpi->cpi_model == 7 ||
2405 cpi->cpi_model == 8)
2406 cp->cp_ecx =
2407 BITX(cp->cp_ecx, 31, 24) << 16 |
2408 BITX(cp->cp_ecx, 23, 16) << 12 |
2409 BITX(cp->cp_ecx, 15, 8) << 8 |
2410 BITX(cp->cp_ecx, 7, 0);
2411 /*
2412 * model 9 stepping 1 has wrong associativity
2413 */
2414 if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
2415 cp->cp_ecx |= 8 << 12;
2416 break;
2417 case X86_VENDOR_Intel:
2418 /*
2419 * Extended L2 Cache features function.
2420 * First appeared on Prescott.
2421 */
2422 default:
2423 break;
2424 }
2425 break;
2426 default:
2427 break;
2428 }
2429 }
2430
2431 pass2_done:
2432 cpi->cpi_pass = 2;
2433 }
2434
2435 static const char *
2436 intel_cpubrand(const struct cpuid_info *cpi)
2437 {
2438 int i;
2439
2440 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2441 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2442 return ("i486");
2443
2444 switch (cpi->cpi_family) {
2445 case 5:
2446 return ("Intel Pentium(r)");
2447 case 6:
2448 switch (cpi->cpi_model) {
2449 uint_t celeron, xeon;
2450 const struct cpuid_regs *cp;
2451 case 0:
2452 case 1:
2453 case 2:
2454 return ("Intel Pentium(r) Pro");
2455 case 3:
2456 case 4:
2457 return ("Intel Pentium(r) II");
2458 case 6:
2459 return ("Intel Celeron(r)");
2460 case 5:
2461 case 7:
2462 celeron = xeon = 0;
2463 cp = &cpi->cpi_std[2]; /* cache info */
2464
2465 for (i = 1; i < 4; i++) {
2466 uint_t tmp;
2467
2468 tmp = (cp->cp_eax >> (8 * i)) & 0xff;
2469 if (tmp == 0x40)
2470 celeron++;
2471 if (tmp >= 0x44 && tmp <= 0x45)
2472 xeon++;
2473 }
2474
2475 for (i = 0; i < 2; i++) {
2476 uint_t tmp;
2477
2478 tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
2479 if (tmp == 0x40)
2480 celeron++;
2481 else if (tmp >= 0x44 && tmp <= 0x45)
2482 xeon++;
2483 }
2484
2485 for (i = 0; i < 4; i++) {
2486 uint_t tmp;
2487
2488 tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
2489 if (tmp == 0x40)
2490 celeron++;
2491 else if (tmp >= 0x44 && tmp <= 0x45)
2492 xeon++;
2493 }
2494
2495 for (i = 0; i < 4; i++) {
2496 uint_t tmp;
2497
2498 tmp = (cp->cp_edx >> (8 * i)) & 0xff;
2499 if (tmp == 0x40)
2500 celeron++;
2501 else if (tmp >= 0x44 && tmp <= 0x45)
2502 xeon++;
2503 }
2504
2505 if (celeron)
2506 return ("Intel Celeron(r)");
2507 if (xeon)
2508 return (cpi->cpi_model == 5 ?
2509 "Intel Pentium(r) II Xeon(tm)" :
2510 "Intel Pentium(r) III Xeon(tm)");
2511 return (cpi->cpi_model == 5 ?
2512 "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
2513 "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
2514 default:
2515 break;
2516 }
2517 default:
2518 break;
2519 }
2520
2521 /* BrandID is present if the field is nonzero */
2522 if (cpi->cpi_brandid != 0) {
2523 static const struct {
2524 uint_t bt_bid;
2525 const char *bt_str;
2526 } brand_tbl[] = {
2527 { 0x1, "Intel(r) Celeron(r)" },
2528 { 0x2, "Intel(r) Pentium(r) III" },
2529 { 0x3, "Intel(r) Pentium(r) III Xeon(tm)" },
2530 { 0x4, "Intel(r) Pentium(r) III" },
2531 { 0x6, "Mobile Intel(r) Pentium(r) III" },
2532 { 0x7, "Mobile Intel(r) Celeron(r)" },
2533 { 0x8, "Intel(r) Pentium(r) 4" },
2534 { 0x9, "Intel(r) Pentium(r) 4" },
2535 { 0xa, "Intel(r) Celeron(r)" },
2536 { 0xb, "Intel(r) Xeon(tm)" },
2537 { 0xc, "Intel(r) Xeon(tm) MP" },
2538 { 0xe, "Mobile Intel(r) Pentium(r) 4" },
2539 { 0xf, "Mobile Intel(r) Celeron(r)" },
2540 { 0x11, "Mobile Genuine Intel(r)" },
2541 { 0x12, "Intel(r) Celeron(r) M" },
2542 { 0x13, "Mobile Intel(r) Celeron(r)" },
2543 { 0x14, "Intel(r) Celeron(r)" },
2544 { 0x15, "Mobile Genuine Intel(r)" },
2545 { 0x16, "Intel(r) Pentium(r) M" },
2546 { 0x17, "Mobile Intel(r) Celeron(r)" }
2547 };
2548 uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
2549 uint_t sgn;
2550
2551 sgn = (cpi->cpi_family << 8) |
2552 (cpi->cpi_model << 4) | cpi->cpi_step;
2553
2554 for (i = 0; i < btblmax; i++)
2555 if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
2556 break;
2557 if (i < btblmax) {
2558 if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
2559 return ("Intel(r) Celeron(r)");
2560 if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
2561 return ("Intel(r) Xeon(tm) MP");
2562 if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
2563 return ("Intel(r) Xeon(tm)");
2564 return (brand_tbl[i].bt_str);
2565 }
2566 }
2567
2568 return (NULL);
2569 }
2570
2571 static const char *
2572 amd_cpubrand(const struct cpuid_info *cpi)
2573 {
2574 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2575 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2576 return ("i486 compatible");
2577
2578 switch (cpi->cpi_family) {
2579 case 5:
2580 switch (cpi->cpi_model) {
2581 case 0:
2582 case 1:
2583 case 2:
2584 case 3:
2585 case 4:
2586 case 5:
2587 return ("AMD-K5(r)");
2588 case 6:
2589 case 7:
2590 return ("AMD-K6(r)");
2591 case 8:
2592 return ("AMD-K6(r)-2");
2593 case 9:
2594 return ("AMD-K6(r)-III");
2595 default:
2596 return ("AMD (family 5)");
2597 }
2598 case 6:
2599 switch (cpi->cpi_model) {
2600 case 1:
2601 return ("AMD-K7(tm)");
2602 case 0:
2603 case 2:
2604 case 4:
2605 return ("AMD Athlon(tm)");
2606 case 3:
2607 case 7:
2608 return ("AMD Duron(tm)");
2609 case 6:
2610 case 8:
2611 case 10:
2612 /*
2613 * Use the L2 cache size to distinguish
2614 */
2615 return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
2616 "AMD Athlon(tm)" : "AMD Duron(tm)");
2617 default:
2618 return ("AMD (family 6)");
2619 }
2620 default:
2621 break;
2622 }
2623
2624 if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
2625 cpi->cpi_brandid != 0) {
2626 switch (BITX(cpi->cpi_brandid, 7, 5)) {
2627 case 3:
2628 return ("AMD Opteron(tm) UP 1xx");
2629 case 4:
2630 return ("AMD Opteron(tm) DP 2xx");
2631 case 5:
2632 return ("AMD Opteron(tm) MP 8xx");
2633 default:
2634 return ("AMD Opteron(tm)");
2635 }
2636 }
2637
2638 return (NULL);
2639 }
2640
2641 static const char *
2642 cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
2643 {
2644 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2645 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5 ||
2646 type == X86_TYPE_CYRIX_486)
2647 return ("i486 compatible");
2648
2649 switch (type) {
2650 case X86_TYPE_CYRIX_6x86:
2651 return ("Cyrix 6x86");
2652 case X86_TYPE_CYRIX_6x86L:
2653 return ("Cyrix 6x86L");
2654 case X86_TYPE_CYRIX_6x86MX:
2655 return ("Cyrix 6x86MX");
2656 case X86_TYPE_CYRIX_GXm:
2657 return ("Cyrix GXm");
2658 case X86_TYPE_CYRIX_MediaGX:
2659 return ("Cyrix MediaGX");
2660 case X86_TYPE_CYRIX_MII:
2661 return ("Cyrix M2");
2662 case X86_TYPE_VIA_CYRIX_III:
2663 return ("VIA Cyrix M3");
2664 default:
2665 /*
2666 * Have another wild guess ..
2667 */
2668 if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
2669 return ("Cyrix 5x86");
2670 else if (cpi->cpi_family == 5) {
2671 switch (cpi->cpi_model) {
2672 case 2:
2673 return ("Cyrix 6x86"); /* Cyrix M1 */
2674 case 4:
2675 return ("Cyrix MediaGX");
2676 default:
2677 break;
2678 }
2679 } else if (cpi->cpi_family == 6) {
2680 switch (cpi->cpi_model) {
2681 case 0:
2682 return ("Cyrix 6x86MX"); /* Cyrix M2? */
2683 case 5:
2684 case 6:
2685 case 7:
2686 case 8:
2687 case 9:
2688 return ("VIA C3");
2689 default:
2690 break;
2691 }
2692 }
2693 break;
2694 }
2695 return (NULL);
2696 }
2697
2698 /*
2699 * This only gets called in the case that the CPU extended
2700 * feature brand string (0x80000002, 0x80000003, 0x80000004)
2701 * aren't available, or contain null bytes for some reason.
2702 */
2703 static void
2704 fabricate_brandstr(struct cpuid_info *cpi)
2705 {
2706 const char *brand = NULL;
2707
2708 switch (cpi->cpi_vendor) {
2709 case X86_VENDOR_Intel:
2710 brand = intel_cpubrand(cpi);
2711 break;
2712 case X86_VENDOR_AMD:
2713 brand = amd_cpubrand(cpi);
2714 break;
2715 case X86_VENDOR_Cyrix:
2716 brand = cyrix_cpubrand(cpi, x86_type);
2717 break;
2718 case X86_VENDOR_NexGen:
2719 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2720 brand = "NexGen Nx586";
2721 break;
2722 case X86_VENDOR_Centaur:
2723 if (cpi->cpi_family == 5)
2724 switch (cpi->cpi_model) {
2725 case 4:
2726 brand = "Centaur C6";
2727 break;
2728 case 8:
2729 brand = "Centaur C2";
2730 break;
2731 case 9:
2732 brand = "Centaur C3";
2733 break;
2734 default:
2735 break;
2736 }
2737 break;
2738 case X86_VENDOR_Rise:
2739 if (cpi->cpi_family == 5 &&
2740 (cpi->cpi_model == 0 || cpi->cpi_model == 2))
2741 brand = "Rise mP6";
2742 break;
2743 case X86_VENDOR_SiS:
2744 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2745 brand = "SiS 55x";
2746 break;
2747 case X86_VENDOR_TM:
2748 if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
2749 brand = "Transmeta Crusoe TM3x00 or TM5x00";
2750 break;
2751 case X86_VENDOR_NSC:
2752 case X86_VENDOR_UMC:
2753 default:
2754 break;
2755 }
2756 if (brand) {
2757 (void) strcpy((char *)cpi->cpi_brandstr, brand);
2758 return;
2759 }
2760
2761 /*
2762 * If all else fails ...
2763 */
2764 (void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
2765 "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
2766 cpi->cpi_model, cpi->cpi_step);
2767 }
2768
2769 /*
2770 * This routine is called just after kernel memory allocation
2771 * becomes available on cpu0, and as part of mp_startup() on
2772 * the other cpus.
2773 *
2774 * Fixup the brand string, and collect any information from cpuid
2775 * that requires dynamically allocated storage to represent.
2776 */
2777 /*ARGSUSED*/
2778 void
2779 cpuid_pass3(cpu_t *cpu)
2780 {
2781 int i, max, shft, level, size;
2782 struct cpuid_regs regs;
2783 struct cpuid_regs *cp;
2784 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2785
2786 ASSERT(cpi->cpi_pass == 2);
2787
2788 /*
2789 * Function 4: Deterministic cache parameters
2790 *
2791 * Take this opportunity to detect the number of threads
2792 * sharing the last level cache, and construct a corresponding
2793 * cache id. The respective cpuid_info members are initialized
2794 * to the default case of "no last level cache sharing".
2795 */
2796 cpi->cpi_ncpu_shr_last_cache = 1;
2797 cpi->cpi_last_lvl_cacheid = cpu->cpu_id;
2798
2799 if (cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) {
2800
2801 /*
2802 * Find the # of elements (size) returned by fn 4, and along
2803 * the way detect last level cache sharing details.
2804 */
2805 bzero(&regs, sizeof (regs));
2806 cp = &regs;
2807 for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
2808 cp->cp_eax = 4;
2809 cp->cp_ecx = i;
2810
2811 (void) __cpuid_insn(cp);
2812
2813 if (CPI_CACHE_TYPE(cp) == 0)
2814 break;
2815 level = CPI_CACHE_LVL(cp);
2816 if (level > max) {
2817 max = level;
2818 cpi->cpi_ncpu_shr_last_cache =
2819 CPI_NTHR_SHR_CACHE(cp) + 1;
2820 }
2821 }
2822 cpi->cpi_std_4_size = size = i;
2823
2824 /*
2825 * Allocate the cpi_std_4 array. The first element
2826 * references the regs for fn 4, %ecx == 0, which
2827 * cpuid_pass2() stashed in cpi->cpi_std[4].
2828 */
2829 if (size > 0) {
2830 cpi->cpi_std_4 =
2831 kmem_alloc(size * sizeof (cp), KM_SLEEP);
2832 cpi->cpi_std_4[0] = &cpi->cpi_std[4];
2833
2834 /*
2835 * Allocate storage to hold the additional regs
2836 * for function 4, %ecx == 1 .. cpi_std_4_size.
2837 *
2838 * The regs for fn 4, %ecx == 0 has already
2839 * been allocated as indicated above.
2840 */
2841 for (i = 1; i < size; i++) {
2842 cp = cpi->cpi_std_4[i] =
2843 kmem_zalloc(sizeof (regs), KM_SLEEP);
2844 cp->cp_eax = 4;
2845 cp->cp_ecx = i;
2846
2847 (void) __cpuid_insn(cp);
2848 }
2849 }
2850 /*
2851 * Determine the number of bits needed to represent
2852 * the number of CPUs sharing the last level cache.
2853 *
2854 * Shift off that number of bits from the APIC id to
2855 * derive the cache id.
2856 */
2857 shft = 0;
2858 for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
2859 shft++;
2860 cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
2861 }
2862
2863 /*
2864 * Now fixup the brand string
2865 */
2866 if ((cpi->cpi_xmaxeax & 0x80000000) == 0) {
2867 fabricate_brandstr(cpi);
2868 } else {
2869
2870 /*
2871 * If we successfully extracted a brand string from the cpuid
2872 * instruction, clean it up by removing leading spaces and
2873 * similar junk.
2874 */
2875 if (cpi->cpi_brandstr[0]) {
2876 size_t maxlen = sizeof (cpi->cpi_brandstr);
2877 char *src, *dst;
2878
2879 dst = src = (char *)cpi->cpi_brandstr;
2880 src[maxlen - 1] = '\0';
2881 /*
2882 * strip leading spaces
2883 */
2884 while (*src == ' ')
2885 src++;
2886 /*
2887 * Remove any 'Genuine' or "Authentic" prefixes
2888 */
2889 if (strncmp(src, "Genuine ", 8) == 0)
2890 src += 8;
2891 if (strncmp(src, "Authentic ", 10) == 0)
2892 src += 10;
2893
2894 /*
2895 * Now do an in-place copy.
2896 * Map (R) to (r) and (TM) to (tm).
2897 * The era of teletypes is long gone, and there's
2898 * -really- no need to shout.
2899 */
2900 while (*src != '\0') {
2901 if (src[0] == '(') {
2902 if (strncmp(src + 1, "R)", 2) == 0) {
2903 (void) strncpy(dst, "(r)", 3);
2904 src += 3;
2905 dst += 3;
2906 continue;
2907 }
2908 if (strncmp(src + 1, "TM)", 3) == 0) {
2909 (void) strncpy(dst, "(tm)", 4);
2910 src += 4;
2911 dst += 4;
2912 continue;
2913 }
2914 }
2915 *dst++ = *src++;
2916 }
2917 *dst = '\0';
2918
2919 /*
2920 * Finally, remove any trailing spaces
2921 */
2922 while (--dst > cpi->cpi_brandstr)
2923 if (*dst == ' ')
2924 *dst = '\0';
2925 else
2926 break;
2927 } else
2928 fabricate_brandstr(cpi);
2929 }
2930 cpi->cpi_pass = 3;
2931 }
2932
2933 /*
2934 * This routine is called out of bind_hwcap() much later in the life
2935 * of the kernel (post_startup()). The job of this routine is to resolve
2936 * the hardware feature support and kernel support for those features into
2937 * what we're actually going to tell applications via the aux vector.
2938 */
2939 void
2940 cpuid_pass4(cpu_t *cpu, uint_t *hwcap_out)
2941 {
2942 struct cpuid_info *cpi;
2943 uint_t hwcap_flags = 0, hwcap_flags_2 = 0;
2944
2945 if (cpu == NULL)
2946 cpu = CPU;
2947 cpi = cpu->cpu_m.mcpu_cpi;
2948
2949 ASSERT(cpi->cpi_pass == 3);
2950
2951 if (cpi->cpi_maxeax >= 1) {
2952 uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
2953 uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];
2954 uint32_t *ebx = &cpi->cpi_support[STD_EBX_FEATURES];
2955
2956 *edx = CPI_FEATURES_EDX(cpi);
2957 *ecx = CPI_FEATURES_ECX(cpi);
2958 *ebx = CPI_FEATURES_7_0_EBX(cpi);
2959
2960 /*
2961 * [these require explicit kernel support]
2962 */
2963 if (!is_x86_feature(x86_featureset, X86FSET_SEP))
2964 *edx &= ~CPUID_INTC_EDX_SEP;
2965
2966 if (!is_x86_feature(x86_featureset, X86FSET_SSE))
2967 *edx &= ~(CPUID_INTC_EDX_FXSR|CPUID_INTC_EDX_SSE);
2968 if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
2969 *edx &= ~CPUID_INTC_EDX_SSE2;
2970
2971 if (!is_x86_feature(x86_featureset, X86FSET_HTT))
2972 *edx &= ~CPUID_INTC_EDX_HTT;
2973
2974 if (!is_x86_feature(x86_featureset, X86FSET_SSE3))
2975 *ecx &= ~CPUID_INTC_ECX_SSE3;
2976
2977 if (!is_x86_feature(x86_featureset, X86FSET_SSSE3))
2978 *ecx &= ~CPUID_INTC_ECX_SSSE3;
2979 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_1))
2980 *ecx &= ~CPUID_INTC_ECX_SSE4_1;
2981 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2))
2982 *ecx &= ~CPUID_INTC_ECX_SSE4_2;
2983 if (!is_x86_feature(x86_featureset, X86FSET_AES))
2984 *ecx &= ~CPUID_INTC_ECX_AES;
2985 if (!is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ))
2986 *ecx &= ~CPUID_INTC_ECX_PCLMULQDQ;
2987 if (!is_x86_feature(x86_featureset, X86FSET_XSAVE))
2988 *ecx &= ~(CPUID_INTC_ECX_XSAVE |
2989 CPUID_INTC_ECX_OSXSAVE);
2990 if (!is_x86_feature(x86_featureset, X86FSET_AVX))
2991 *ecx &= ~CPUID_INTC_ECX_AVX;
2992 if (!is_x86_feature(x86_featureset, X86FSET_F16C))
2993 *ecx &= ~CPUID_INTC_ECX_F16C;
2994 if (!is_x86_feature(x86_featureset, X86FSET_FMA))
2995 *ecx &= ~CPUID_INTC_ECX_FMA;
2996 if (!is_x86_feature(x86_featureset, X86FSET_BMI1))
2997 *ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
2998 if (!is_x86_feature(x86_featureset, X86FSET_BMI2))
2999 *ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
3000 if (!is_x86_feature(x86_featureset, X86FSET_AVX2))
3001 *ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
3002 if (!is_x86_feature(x86_featureset, X86FSET_RDSEED))
3003 *ebx &= ~CPUID_INTC_EBX_7_0_RDSEED;
3004 if (!is_x86_feature(x86_featureset, X86FSET_ADX))
3005 *ebx &= ~CPUID_INTC_EBX_7_0_ADX;
3006
3007 /*
3008 * [no explicit support required beyond x87 fp context]
3009 */
3010 if (!fpu_exists)
3011 *edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);
3012
3013 /*
3014 * Now map the supported feature vector to things that we
3015 * think userland will care about.
3016 */
3017 if (*edx & CPUID_INTC_EDX_SEP)
3018 hwcap_flags |= AV_386_SEP;
3019 if (*edx & CPUID_INTC_EDX_SSE)
3020 hwcap_flags |= AV_386_FXSR | AV_386_SSE;
3021 if (*edx & CPUID_INTC_EDX_SSE2)
3022 hwcap_flags |= AV_386_SSE2;
3023 if (*ecx & CPUID_INTC_ECX_SSE3)
3024 hwcap_flags |= AV_386_SSE3;
3025 if (*ecx & CPUID_INTC_ECX_SSSE3)
3026 hwcap_flags |= AV_386_SSSE3;
3027 if (*ecx & CPUID_INTC_ECX_SSE4_1)
3028 hwcap_flags |= AV_386_SSE4_1;
3029 if (*ecx & CPUID_INTC_ECX_SSE4_2)
3030 hwcap_flags |= AV_386_SSE4_2;
3031 if (*ecx & CPUID_INTC_ECX_MOVBE)
3032 hwcap_flags |= AV_386_MOVBE;
3033 if (*ecx & CPUID_INTC_ECX_AES)
3034 hwcap_flags |= AV_386_AES;
3035 if (*ecx & CPUID_INTC_ECX_PCLMULQDQ)
3036 hwcap_flags |= AV_386_PCLMULQDQ;
3037 if ((*ecx & CPUID_INTC_ECX_XSAVE) &&
3038 (*ecx & CPUID_INTC_ECX_OSXSAVE)) {
3039 hwcap_flags |= AV_386_XSAVE;
3040
3041 if (*ecx & CPUID_INTC_ECX_AVX) {
3042 uint32_t *ecx_7 = &CPI_FEATURES_7_0_ECX(cpi);
3043 uint32_t *edx_7 = &CPI_FEATURES_7_0_EDX(cpi);
3044
3045 hwcap_flags |= AV_386_AVX;
3046 if (*ecx & CPUID_INTC_ECX_F16C)
3047 hwcap_flags_2 |= AV_386_2_F16C;
3048 if (*ecx & CPUID_INTC_ECX_FMA)
3049 hwcap_flags_2 |= AV_386_2_FMA;
3050
3051 if (*ebx & CPUID_INTC_EBX_7_0_BMI1)
3052 hwcap_flags_2 |= AV_386_2_BMI1;
3053 if (*ebx & CPUID_INTC_EBX_7_0_BMI2)
3054 hwcap_flags_2 |= AV_386_2_BMI2;
3055 if (*ebx & CPUID_INTC_EBX_7_0_AVX2)
3056 hwcap_flags_2 |= AV_386_2_AVX2;
3057 if (*ebx & CPUID_INTC_EBX_7_0_AVX512F)
3058 hwcap_flags_2 |= AV_386_2_AVX512F;
3059 if (*ebx & CPUID_INTC_EBX_7_0_AVX512DQ)
3060 hwcap_flags_2 |= AV_386_2_AVX512DQ;
3061 if (*ebx & CPUID_INTC_EBX_7_0_AVX512IFMA)
3062 hwcap_flags_2 |= AV_386_2_AVX512IFMA;
3063 if (*ebx & CPUID_INTC_EBX_7_0_AVX512PF)
3064 hwcap_flags_2 |= AV_386_2_AVX512PF;
3065 if (*ebx & CPUID_INTC_EBX_7_0_AVX512ER)
3066 hwcap_flags_2 |= AV_386_2_AVX512ER;
3067 if (*ebx & CPUID_INTC_EBX_7_0_AVX512CD)
3068 hwcap_flags_2 |= AV_386_2_AVX512CD;
3069 if (*ebx & CPUID_INTC_EBX_7_0_AVX512BW)
3070 hwcap_flags_2 |= AV_386_2_AVX512BW;
3071 if (*ebx & CPUID_INTC_EBX_7_0_AVX512VL)
3072 hwcap_flags_2 |= AV_386_2_AVX512VL;
3073
3074 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VBMI)
3075 hwcap_flags_2 |= AV_386_2_AVX512VBMI;
3076 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
3077 hwcap_flags_2 |= AV_386_2_AVX512VPOPCDQ;
3078
3079 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124NNIW)
3080 hwcap_flags_2 |= AV_386_2_AVX512_4NNIW;
3081 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124FMAPS)
3082 hwcap_flags_2 |= AV_386_2_AVX512_4FMAPS;
3083 }
3084 }
3085 if (*ecx & CPUID_INTC_ECX_VMX)
3086 hwcap_flags |= AV_386_VMX;
3087 if (*ecx & CPUID_INTC_ECX_POPCNT)
3088 hwcap_flags |= AV_386_POPCNT;
3089 if (*edx & CPUID_INTC_EDX_FPU)
3090 hwcap_flags |= AV_386_FPU;
3091 if (*edx & CPUID_INTC_EDX_MMX)
3092 hwcap_flags |= AV_386_MMX;
3093
3094 if (*edx & CPUID_INTC_EDX_TSC)
3095 hwcap_flags |= AV_386_TSC;
3096 if (*edx & CPUID_INTC_EDX_CX8)
3097 hwcap_flags |= AV_386_CX8;
3098 if (*edx & CPUID_INTC_EDX_CMOV)
3099 hwcap_flags |= AV_386_CMOV;
3100 if (*ecx & CPUID_INTC_ECX_CX16)
3101 hwcap_flags |= AV_386_CX16;
3102
3103 if (*ecx & CPUID_INTC_ECX_RDRAND)
3104 hwcap_flags_2 |= AV_386_2_RDRAND;
3105 if (*ebx & CPUID_INTC_EBX_7_0_ADX)
3106 hwcap_flags_2 |= AV_386_2_ADX;
3107 if (*ebx & CPUID_INTC_EBX_7_0_RDSEED)
3108 hwcap_flags_2 |= AV_386_2_RDSEED;
3109 if (*ebx & CPUID_INTC_EBX_7_0_SHA)
3110 hwcap_flags_2 |= AV_386_2_SHA;
3111
3112 }
3113
3114 if (cpi->cpi_xmaxeax < 0x80000001)
3115 goto pass4_done;
3116
3117 switch (cpi->cpi_vendor) {
3118 struct cpuid_regs cp;
3119 uint32_t *edx, *ecx;
3120
3121 case X86_VENDOR_Intel:
3122 /*
3123 * Seems like Intel duplicated what we necessary
3124 * here to make the initial crop of 64-bit OS's work.
3125 * Hopefully, those are the only "extended" bits
3126 * they'll add.
3127 */
3128 /*FALLTHROUGH*/
3129
3130 case X86_VENDOR_AMD:
3131 edx = &cpi->cpi_support[AMD_EDX_FEATURES];
3132 ecx = &cpi->cpi_support[AMD_ECX_FEATURES];
3133
3134 *edx = CPI_FEATURES_XTD_EDX(cpi);
3135 *ecx = CPI_FEATURES_XTD_ECX(cpi);
3136
3137 /*
3138 * [these features require explicit kernel support]
3139 */
3140 switch (cpi->cpi_vendor) {
3141 case X86_VENDOR_Intel:
3142 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3143 *edx &= ~CPUID_AMD_EDX_TSCP;
3144 break;
3145
3146 case X86_VENDOR_AMD:
3147 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3148 *edx &= ~CPUID_AMD_EDX_TSCP;
3149 if (!is_x86_feature(x86_featureset, X86FSET_SSE4A))
3150 *ecx &= ~CPUID_AMD_ECX_SSE4A;
3151 break;
3152
3153 default:
3154 break;
3155 }
3156
3157 /*
3158 * [no explicit support required beyond
3159 * x87 fp context and exception handlers]
3160 */
3161 if (!fpu_exists)
3162 *edx &= ~(CPUID_AMD_EDX_MMXamd |
3163 CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);
3164
3165 if (!is_x86_feature(x86_featureset, X86FSET_NX))
3166 *edx &= ~CPUID_AMD_EDX_NX;
3167 #if !defined(__amd64)
3168 *edx &= ~CPUID_AMD_EDX_LM;
3169 #endif
3170 /*
3171 * Now map the supported feature vector to
3172 * things that we think userland will care about.
3173 */
3174 #if defined(__amd64)
3175 if (*edx & CPUID_AMD_EDX_SYSC)
3176 hwcap_flags |= AV_386_AMD_SYSC;
3177 #endif
3178 if (*edx & CPUID_AMD_EDX_MMXamd)
3179 hwcap_flags |= AV_386_AMD_MMX;
3180 if (*edx & CPUID_AMD_EDX_3DNow)
3181 hwcap_flags |= AV_386_AMD_3DNow;
3182 if (*edx & CPUID_AMD_EDX_3DNowx)
3183 hwcap_flags |= AV_386_AMD_3DNowx;
3184 if (*ecx & CPUID_AMD_ECX_SVM)
3185 hwcap_flags |= AV_386_AMD_SVM;
3186
3187 switch (cpi->cpi_vendor) {
3188 case X86_VENDOR_AMD:
3189 if (*edx & CPUID_AMD_EDX_TSCP)
3190 hwcap_flags |= AV_386_TSCP;
3191 if (*ecx & CPUID_AMD_ECX_AHF64)
3192 hwcap_flags |= AV_386_AHF;
3193 if (*ecx & CPUID_AMD_ECX_SSE4A)
3194 hwcap_flags |= AV_386_AMD_SSE4A;
3195 if (*ecx & CPUID_AMD_ECX_LZCNT)
3196 hwcap_flags |= AV_386_AMD_LZCNT;
3197 break;
3198
3199 case X86_VENDOR_Intel:
3200 if (*edx & CPUID_AMD_EDX_TSCP)
3201 hwcap_flags |= AV_386_TSCP;
3202 /*
3203 * Aarrgh.
3204 * Intel uses a different bit in the same word.
3205 */
3206 if (*ecx & CPUID_INTC_ECX_AHF64)
3207 hwcap_flags |= AV_386_AHF;
3208 break;
3209
3210 default:
3211 break;
3212 }
3213 break;
3214
3215 case X86_VENDOR_TM:
3216 cp.cp_eax = 0x80860001;
3217 (void) __cpuid_insn(&cp);
3218 cpi->cpi_support[TM_EDX_FEATURES] = cp.cp_edx;
3219 break;
3220
3221 default:
3222 break;
3223 }
3224
3225 pass4_done:
3226 cpi->cpi_pass = 4;
3227 if (hwcap_out != NULL) {
3228 hwcap_out[0] = hwcap_flags;
3229 hwcap_out[1] = hwcap_flags_2;
3230 }
3231 }
3232
3233
3234 /*
3235 * Simulate the cpuid instruction using the data we previously
3236 * captured about this CPU. We try our best to return the truth
3237 * about the hardware, independently of kernel support.
3238 */
3239 uint32_t
3240 cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
3241 {
3242 struct cpuid_info *cpi;
3243 struct cpuid_regs *xcp;
3244
3245 if (cpu == NULL)
3246 cpu = CPU;
3247 cpi = cpu->cpu_m.mcpu_cpi;
3248
3249 ASSERT(cpuid_checkpass(cpu, 3));
3250
3251 /*
3252 * CPUID data is cached in two separate places: cpi_std for standard
3253 * CPUID functions, and cpi_extd for extended CPUID functions.
3254 */
3255 if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD)
3256 xcp = &cpi->cpi_std[cp->cp_eax];
3257 else if (cp->cp_eax >= 0x80000000 && cp->cp_eax <= cpi->cpi_xmaxeax &&
3258 cp->cp_eax < 0x80000000 + NMAX_CPI_EXTD)
3259 xcp = &cpi->cpi_extd[cp->cp_eax - 0x80000000];
3260 else
3261 /*
3262 * The caller is asking for data from an input parameter which
3263 * the kernel has not cached. In this case we go fetch from
3264 * the hardware and return the data directly to the user.
3265 */
3266 return (__cpuid_insn(cp));
3267
3268 cp->cp_eax = xcp->cp_eax;
3269 cp->cp_ebx = xcp->cp_ebx;
3270 cp->cp_ecx = xcp->cp_ecx;
3271 cp->cp_edx = xcp->cp_edx;
3272 return (cp->cp_eax);
3273 }
3274
3275 int
3276 cpuid_checkpass(cpu_t *cpu, int pass)
3277 {
3278 return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
3279 cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
3280 }
3281
3282 int
3283 cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
3284 {
3285 ASSERT(cpuid_checkpass(cpu, 3));
3286
3287 return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
3288 }
3289
3290 int
3291 cpuid_is_cmt(cpu_t *cpu)
3292 {
3293 if (cpu == NULL)
3294 cpu = CPU;
3295
3296 ASSERT(cpuid_checkpass(cpu, 1));
3297
3298 return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
3299 }
3300
3301 /*
3302 * AMD and Intel both implement the 64-bit variant of the syscall
3303 * instruction (syscallq), so if there's -any- support for syscall,
3304 * cpuid currently says "yes, we support this".
3305 *
3306 * However, Intel decided to -not- implement the 32-bit variant of the
3307 * syscall instruction, so we provide a predicate to allow our caller
3308 * to test that subtlety here.
3309 *
3310 * XXPV Currently, 32-bit syscall instructions don't work via the hypervisor,
3311 * even in the case where the hardware would in fact support it.
3312 */
3313 /*ARGSUSED*/
3314 int
3315 cpuid_syscall32_insn(cpu_t *cpu)
3316 {
3317 ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), 1));
3318
3319 if (cpu == NULL)
3320 cpu = CPU;
3321
3322 /*CSTYLED*/
3323 {
3324 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3325
3326 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
3327 cpi->cpi_xmaxeax >= 0x80000001 &&
3328 (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
3329 return (1);
3330 }
3331 return (0);
3332 }
3333
3334 int
3335 cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
3336 {
3337 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3338
3339 static const char fmt[] =
3340 "x86 (%s %X family %d model %d step %d clock %d MHz)";
3341 static const char fmt_ht[] =
3342 "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";
3343
3344 ASSERT(cpuid_checkpass(cpu, 1));
3345
3346 if (cpuid_is_cmt(cpu))
3347 return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
3348 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3349 cpi->cpi_family, cpi->cpi_model,
3350 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3351 return (snprintf(s, n, fmt,
3352 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3353 cpi->cpi_family, cpi->cpi_model,
3354 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3355 }
3356
3357 const char *
3358 cpuid_getvendorstr(cpu_t *cpu)
3359 {
3360 ASSERT(cpuid_checkpass(cpu, 1));
3361 return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
3362 }
3363
3364 uint_t
3365 cpuid_getvendor(cpu_t *cpu)
3366 {
3367 ASSERT(cpuid_checkpass(cpu, 1));
3368 return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
3369 }
3370
3371 uint_t
3372 cpuid_getfamily(cpu_t *cpu)
3373 {
3374 ASSERT(cpuid_checkpass(cpu, 1));
3375 return (cpu->cpu_m.mcpu_cpi->cpi_family);
3376 }
3377
3378 uint_t
3379 cpuid_getmodel(cpu_t *cpu)
3380 {
3381 ASSERT(cpuid_checkpass(cpu, 1));
3382 return (cpu->cpu_m.mcpu_cpi->cpi_model);
3383 }
3384
3385 uint_t
3386 cpuid_get_ncpu_per_chip(cpu_t *cpu)
3387 {
3388 ASSERT(cpuid_checkpass(cpu, 1));
3389 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
3390 }
3391
3392 uint_t
3393 cpuid_get_ncore_per_chip(cpu_t *cpu)
3394 {
3395 ASSERT(cpuid_checkpass(cpu, 1));
3396 return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
3397 }
3398
3399 uint_t
3400 cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
3401 {
3402 ASSERT(cpuid_checkpass(cpu, 2));
3403 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
3404 }
3405
3406 id_t
3407 cpuid_get_last_lvl_cacheid(cpu_t *cpu)
3408 {
3409 ASSERT(cpuid_checkpass(cpu, 2));
3410 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3411 }
3412
3413 uint_t
3414 cpuid_getstep(cpu_t *cpu)
3415 {
3416 ASSERT(cpuid_checkpass(cpu, 1));
3417 return (cpu->cpu_m.mcpu_cpi->cpi_step);
3418 }
3419
3420 uint_t
3421 cpuid_getsig(struct cpu *cpu)
3422 {
3423 ASSERT(cpuid_checkpass(cpu, 1));
3424 return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
3425 }
3426
3427 uint32_t
3428 cpuid_getchiprev(struct cpu *cpu)
3429 {
3430 ASSERT(cpuid_checkpass(cpu, 1));
3431 return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
3432 }
3433
3434 const char *
3435 cpuid_getchiprevstr(struct cpu *cpu)
3436 {
3437 ASSERT(cpuid_checkpass(cpu, 1));
3438 return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
3439 }
3440
3441 uint32_t
3442 cpuid_getsockettype(struct cpu *cpu)
3443 {
3444 ASSERT(cpuid_checkpass(cpu, 1));
3445 return (cpu->cpu_m.mcpu_cpi->cpi_socket);
3446 }
3447
3448 const char *
3449 cpuid_getsocketstr(cpu_t *cpu)
3450 {
3451 static const char *socketstr = NULL;
3452 struct cpuid_info *cpi;
3453
3454 ASSERT(cpuid_checkpass(cpu, 1));
3455 cpi = cpu->cpu_m.mcpu_cpi;
3456
3457 /* Assume that socket types are the same across the system */
3458 if (socketstr == NULL)
3459 socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
3460 cpi->cpi_model, cpi->cpi_step);
3461
3462
3463 return (socketstr);
3464 }
3465
3466 int
3467 cpuid_get_chipid(cpu_t *cpu)
3468 {
3469 ASSERT(cpuid_checkpass(cpu, 1));
3470
3471 if (cpuid_is_cmt(cpu))
3472 return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
3473 return (cpu->cpu_id);
3474 }
3475
3476 id_t
3477 cpuid_get_coreid(cpu_t *cpu)
3478 {
3479 ASSERT(cpuid_checkpass(cpu, 1));
3480 return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
3481 }
3482
3483 int
3484 cpuid_get_pkgcoreid(cpu_t *cpu)
3485 {
3486 ASSERT(cpuid_checkpass(cpu, 1));
3487 return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
3488 }
3489
3490 int
3491 cpuid_get_clogid(cpu_t *cpu)
3492 {
3493 ASSERT(cpuid_checkpass(cpu, 1));
3494 return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
3495 }
3496
3497 int
3498 cpuid_get_cacheid(cpu_t *cpu)
3499 {
3500 ASSERT(cpuid_checkpass(cpu, 1));
3501 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3502 }
3503
3504 uint_t
3505 cpuid_get_procnodeid(cpu_t *cpu)
3506 {
3507 ASSERT(cpuid_checkpass(cpu, 1));
3508 return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
3509 }
3510
3511 uint_t
3512 cpuid_get_procnodes_per_pkg(cpu_t *cpu)
3513 {
3514 ASSERT(cpuid_checkpass(cpu, 1));
3515 return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
3516 }
3517
3518 uint_t
3519 cpuid_get_compunitid(cpu_t *cpu)
3520 {
3521 ASSERT(cpuid_checkpass(cpu, 1));
3522 return (cpu->cpu_m.mcpu_cpi->cpi_compunitid);
3523 }
3524
3525 uint_t
3526 cpuid_get_cores_per_compunit(cpu_t *cpu)
3527 {
3528 ASSERT(cpuid_checkpass(cpu, 1));
3529 return (cpu->cpu_m.mcpu_cpi->cpi_cores_per_compunit);
3530 }
3531
3532 /*ARGSUSED*/
3533 int
3534 cpuid_have_cr8access(cpu_t *cpu)
3535 {
3536 #if defined(__amd64)
3537 return (1);
3538 #else
3539 struct cpuid_info *cpi;
3540
3541 ASSERT(cpu != NULL);
3542 cpi = cpu->cpu_m.mcpu_cpi;
3543 if (cpi->cpi_vendor == X86_VENDOR_AMD && cpi->cpi_maxeax >= 1 &&
3544 (CPI_FEATURES_XTD_ECX(cpi) & CPUID_AMD_ECX_CR8D) != 0)
3545 return (1);
3546 return (0);
3547 #endif
3548 }
3549
3550 uint32_t
3551 cpuid_get_apicid(cpu_t *cpu)
3552 {
3553 ASSERT(cpuid_checkpass(cpu, 1));
3554 if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
3555 return (UINT32_MAX);
3556 } else {
3557 return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
3558 }
3559 }
3560
3561 void
3562 cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
3563 {
3564 struct cpuid_info *cpi;
3565
3566 if (cpu == NULL)
3567 cpu = CPU;
3568 cpi = cpu->cpu_m.mcpu_cpi;
3569
3570 ASSERT(cpuid_checkpass(cpu, 1));
3571
3572 if (pabits)
3573 *pabits = cpi->cpi_pabits;
3574 if (vabits)
3575 *vabits = cpi->cpi_vabits;
3576 }
3577
3578 size_t
3579 cpuid_get_xsave_size()
3580 {
3581 return (MAX(cpuid_info0.cpi_xsave.xsav_max_size,
3582 sizeof (struct xsave_state)));
3583 }
3584
3585 /*
3586 * Return true if the CPUs on this system require 'pointer clearing' for the
3587 * floating point error pointer exception handling. In the past, this has been
3588 * true for all AMD K7 & K8 CPUs, although newer AMD CPUs have been changed to
3589 * behave the same as Intel. This is checked via the CPUID_AMD_EBX_ERR_PTR_ZERO
3590 * feature bit and is reflected in the cpi_fp_amd_save member.
3591 */
3592 boolean_t
3593 cpuid_need_fp_excp_handling()
3594 {
3595 return (cpuid_info0.cpi_vendor == X86_VENDOR_AMD &&
3596 cpuid_info0.cpi_fp_amd_save != 0);
3597 }
3598
3599 /*
3600 * Returns the number of data TLB entries for a corresponding
3601 * pagesize. If it can't be computed, or isn't known, the
3602 * routine returns zero. If you ask about an architecturally
3603 * impossible pagesize, the routine will panic (so that the
3604 * hat implementor knows that things are inconsistent.)
3605 */
3606 uint_t
3607 cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
3608 {
3609 struct cpuid_info *cpi;
3610 uint_t dtlb_nent = 0;
3611
3612 if (cpu == NULL)
3613 cpu = CPU;
3614 cpi = cpu->cpu_m.mcpu_cpi;
3615
3616 ASSERT(cpuid_checkpass(cpu, 1));
3617
3618 /*
3619 * Check the L2 TLB info
3620 */
3621 if (cpi->cpi_xmaxeax >= 0x80000006) {
3622 struct cpuid_regs *cp = &cpi->cpi_extd[6];
3623
3624 switch (pagesize) {
3625
3626 case 4 * 1024:
3627 /*
3628 * All zero in the top 16 bits of the register
3629 * indicates a unified TLB. Size is in low 16 bits.
3630 */
3631 if ((cp->cp_ebx & 0xffff0000) == 0)
3632 dtlb_nent = cp->cp_ebx & 0x0000ffff;
3633 else
3634 dtlb_nent = BITX(cp->cp_ebx, 27, 16);
3635 break;
3636
3637 case 2 * 1024 * 1024:
3638 if ((cp->cp_eax & 0xffff0000) == 0)
3639 dtlb_nent = cp->cp_eax & 0x0000ffff;
3640 else
3641 dtlb_nent = BITX(cp->cp_eax, 27, 16);
3642 break;
3643
3644 default:
3645 panic("unknown L2 pagesize");
3646 /*NOTREACHED*/
3647 }
3648 }
3649
3650 if (dtlb_nent != 0)
3651 return (dtlb_nent);
3652
3653 /*
3654 * No L2 TLB support for this size, try L1.
3655 */
3656 if (cpi->cpi_xmaxeax >= 0x80000005) {
3657 struct cpuid_regs *cp = &cpi->cpi_extd[5];
3658
3659 switch (pagesize) {
3660 case 4 * 1024:
3661 dtlb_nent = BITX(cp->cp_ebx, 23, 16);
3662 break;
3663 case 2 * 1024 * 1024:
3664 dtlb_nent = BITX(cp->cp_eax, 23, 16);
3665 break;
3666 default:
3667 panic("unknown L1 d-TLB pagesize");
3668 /*NOTREACHED*/
3669 }
3670 }
3671
3672 return (dtlb_nent);
3673 }
3674
3675 /*
3676 * Return 0 if the erratum is not present or not applicable, positive
3677 * if it is, and negative if the status of the erratum is unknown.
3678 *
3679 * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
3680 * Processors" #25759, Rev 3.57, August 2005
3681 */
3682 int
3683 cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
3684 {
3685 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3686 uint_t eax;
3687
3688 /*
3689 * Bail out if this CPU isn't an AMD CPU, or if it's
3690 * a legacy (32-bit) AMD CPU.
3691 */
3692 if (cpi->cpi_vendor != X86_VENDOR_AMD ||
3693 cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
3694 cpi->cpi_family == 6)
3695
3696 return (0);
3697
3698 eax = cpi->cpi_std[1].cp_eax;
3699
3700 #define SH_B0(eax) (eax == 0xf40 || eax == 0xf50)
3701 #define SH_B3(eax) (eax == 0xf51)
3702 #define B(eax) (SH_B0(eax) || SH_B3(eax))
3703
3704 #define SH_C0(eax) (eax == 0xf48 || eax == 0xf58)
3705
3706 #define SH_CG(eax) (eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
3707 #define DH_CG(eax) (eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
3708 #define CH_CG(eax) (eax == 0xf82 || eax == 0xfb2)
3709 #define CG(eax) (SH_CG(eax) || DH_CG(eax) || CH_CG(eax))
3710
3711 #define SH_D0(eax) (eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
3712 #define DH_D0(eax) (eax == 0x10fc0 || eax == 0x10ff0)
3713 #define CH_D0(eax) (eax == 0x10f80 || eax == 0x10fb0)
3714 #define D0(eax) (SH_D0(eax) || DH_D0(eax) || CH_D0(eax))
3715
3716 #define SH_E0(eax) (eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
3717 #define JH_E1(eax) (eax == 0x20f10) /* JH8_E0 had 0x20f30 */
3718 #define DH_E3(eax) (eax == 0x20fc0 || eax == 0x20ff0)
3719 #define SH_E4(eax) (eax == 0x20f51 || eax == 0x20f71)
3720 #define BH_E4(eax) (eax == 0x20fb1)
3721 #define SH_E5(eax) (eax == 0x20f42)
3722 #define DH_E6(eax) (eax == 0x20ff2 || eax == 0x20fc2)
3723 #define JH_E6(eax) (eax == 0x20f12 || eax == 0x20f32)
3724 #define EX(eax) (SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
3725 SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
3726 DH_E6(eax) || JH_E6(eax))
3727
3728 #define DR_AX(eax) (eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
3729 #define DR_B0(eax) (eax == 0x100f20)
3730 #define DR_B1(eax) (eax == 0x100f21)
3731 #define DR_BA(eax) (eax == 0x100f2a)
3732 #define DR_B2(eax) (eax == 0x100f22)
3733 #define DR_B3(eax) (eax == 0x100f23)
3734 #define RB_C0(eax) (eax == 0x100f40)
3735
3736 switch (erratum) {
3737 case 1:
3738 return (cpi->cpi_family < 0x10);
3739 case 51: /* what does the asterisk mean? */
3740 return (B(eax) || SH_C0(eax) || CG(eax));
3741 case 52:
3742 return (B(eax));
3743 case 57:
3744 return (cpi->cpi_family <= 0x11);
3745 case 58:
3746 return (B(eax));
3747 case 60:
3748 return (cpi->cpi_family <= 0x11);
3749 case 61:
3750 case 62:
3751 case 63:
3752 case 64:
3753 case 65:
3754 case 66:
3755 case 68:
3756 case 69:
3757 case 70:
3758 case 71:
3759 return (B(eax));
3760 case 72:
3761 return (SH_B0(eax));
3762 case 74:
3763 return (B(eax));
3764 case 75:
3765 return (cpi->cpi_family < 0x10);
3766 case 76:
3767 return (B(eax));
3768 case 77:
3769 return (cpi->cpi_family <= 0x11);
3770 case 78:
3771 return (B(eax) || SH_C0(eax));
3772 case 79:
3773 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3774 case 80:
3775 case 81:
3776 case 82:
3777 return (B(eax));
3778 case 83:
3779 return (B(eax) || SH_C0(eax) || CG(eax));
3780 case 85:
3781 return (cpi->cpi_family < 0x10);
3782 case 86:
3783 return (SH_C0(eax) || CG(eax));
3784 case 88:
3785 #if !defined(__amd64)
3786 return (0);
3787 #else
3788 return (B(eax) || SH_C0(eax));
3789 #endif
3790 case 89:
3791 return (cpi->cpi_family < 0x10);
3792 case 90:
3793 return (B(eax) || SH_C0(eax) || CG(eax));
3794 case 91:
3795 case 92:
3796 return (B(eax) || SH_C0(eax));
3797 case 93:
3798 return (SH_C0(eax));
3799 case 94:
3800 return (B(eax) || SH_C0(eax) || CG(eax));
3801 case 95:
3802 #if !defined(__amd64)
3803 return (0);
3804 #else
3805 return (B(eax) || SH_C0(eax));
3806 #endif
3807 case 96:
3808 return (B(eax) || SH_C0(eax) || CG(eax));
3809 case 97:
3810 case 98:
3811 return (SH_C0(eax) || CG(eax));
3812 case 99:
3813 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3814 case 100:
3815 return (B(eax) || SH_C0(eax));
3816 case 101:
3817 case 103:
3818 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3819 case 104:
3820 return (SH_C0(eax) || CG(eax) || D0(eax));
3821 case 105:
3822 case 106:
3823 case 107:
3824 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3825 case 108:
3826 return (DH_CG(eax));
3827 case 109:
3828 return (SH_C0(eax) || CG(eax) || D0(eax));
3829 case 110:
3830 return (D0(eax) || EX(eax));
3831 case 111:
3832 return (CG(eax));
3833 case 112:
3834 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3835 case 113:
3836 return (eax == 0x20fc0);
3837 case 114:
3838 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3839 case 115:
3840 return (SH_E0(eax) || JH_E1(eax));
3841 case 116:
3842 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3843 case 117:
3844 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3845 case 118:
3846 return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
3847 JH_E6(eax));
3848 case 121:
3849 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3850 case 122:
3851 return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
3852 case 123:
3853 return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
3854 case 131:
3855 return (cpi->cpi_family < 0x10);
3856 case 6336786:
3857 /*
3858 * Test for AdvPowerMgmtInfo.TscPStateInvariant
3859 * if this is a K8 family or newer processor
3860 */
3861 if (CPI_FAMILY(cpi) == 0xf) {
3862 struct cpuid_regs regs;
3863 regs.cp_eax = 0x80000007;
3864 (void) __cpuid_insn(&regs);
3865 return (!(regs.cp_edx & 0x100));
3866 }
3867 return (0);
3868 case 6323525:
3869 return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
3870 (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);
3871
3872 case 6671130:
3873 /*
3874 * check for processors (pre-Shanghai) that do not provide
3875 * optimal management of 1gb ptes in its tlb.
3876 */
3877 return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);
3878
3879 case 298:
3880 return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
3881 DR_B2(eax) || RB_C0(eax));
3882
3883 case 721:
3884 #if defined(__amd64)
3885 return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);
3886 #else
3887 return (0);
3888 #endif
3889
3890 default:
3891 return (-1);
3892
3893 }
3894 }
3895
3896 /*
3897 * Determine if specified erratum is present via OSVW (OS Visible Workaround).
3898 * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
3899 */
3900 int
3901 osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
3902 {
3903 struct cpuid_info *cpi;
3904 uint_t osvwid;
3905 static int osvwfeature = -1;
3906 uint64_t osvwlength;
3907
3908
3909 cpi = cpu->cpu_m.mcpu_cpi;
3910
3911 /* confirm OSVW supported */
3912 if (osvwfeature == -1) {
3913 osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
3914 } else {
3915 /* assert that osvw feature setting is consistent on all cpus */
3916 ASSERT(osvwfeature ==
3917 (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
3918 }
3919 if (!osvwfeature)
3920 return (-1);
3921
3922 osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;
3923
3924 switch (erratum) {
3925 case 298: /* osvwid is 0 */
3926 osvwid = 0;
3927 if (osvwlength <= (uint64_t)osvwid) {
3928 /* osvwid 0 is unknown */
3929 return (-1);
3930 }
3931
3932 /*
3933 * Check the OSVW STATUS MSR to determine the state
3934 * of the erratum where:
3935 * 0 - fixed by HW
3936 * 1 - BIOS has applied the workaround when BIOS
3937 * workaround is available. (Or for other errata,
3938 * OS workaround is required.)
3939 * For a value of 1, caller will confirm that the
3940 * erratum 298 workaround has indeed been applied by BIOS.
3941 *
3942 * A 1 may be set in cpus that have a HW fix
3943 * in a mixed cpu system. Regarding erratum 298:
3944 * In a multiprocessor platform, the workaround above
3945 * should be applied to all processors regardless of
3946 * silicon revision when an affected processor is
3947 * present.
3948 */
3949
3950 return (rdmsr(MSR_AMD_OSVW_STATUS +
3951 (osvwid / OSVW_ID_CNT_PER_MSR)) &
3952 (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));
3953
3954 default:
3955 return (-1);
3956 }
3957 }
3958
3959 static const char assoc_str[] = "associativity";
3960 static const char line_str[] = "line-size";
3961 static const char size_str[] = "size";
3962
3963 static void
3964 add_cache_prop(dev_info_t *devi, const char *label, const char *type,
3965 uint32_t val)
3966 {
3967 char buf[128];
3968
3969 /*
3970 * ndi_prop_update_int() is used because it is desirable for
3971 * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
3972 */
3973 if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
3974 (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
3975 }
3976
3977 /*
3978 * Intel-style cache/tlb description
3979 *
3980 * Standard cpuid level 2 gives a randomly ordered
3981 * selection of tags that index into a table that describes
3982 * cache and tlb properties.
3983 */
3984
3985 static const char l1_icache_str[] = "l1-icache";
3986 static const char l1_dcache_str[] = "l1-dcache";
3987 static const char l2_cache_str[] = "l2-cache";
3988 static const char l3_cache_str[] = "l3-cache";
3989 static const char itlb4k_str[] = "itlb-4K";
3990 static const char dtlb4k_str[] = "dtlb-4K";
3991 static const char itlb2M_str[] = "itlb-2M";
3992 static const char itlb4M_str[] = "itlb-4M";
3993 static const char dtlb4M_str[] = "dtlb-4M";
3994 static const char dtlb24_str[] = "dtlb0-2M-4M";
3995 static const char itlb424_str[] = "itlb-4K-2M-4M";
3996 static const char itlb24_str[] = "itlb-2M-4M";
3997 static const char dtlb44_str[] = "dtlb-4K-4M";
3998 static const char sl1_dcache_str[] = "sectored-l1-dcache";
3999 static const char sl2_cache_str[] = "sectored-l2-cache";
4000 static const char itrace_str[] = "itrace-cache";
4001 static const char sl3_cache_str[] = "sectored-l3-cache";
4002 static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";
4003
4004 static const struct cachetab {
4005 uint8_t ct_code;
4006 uint8_t ct_assoc;
4007 uint16_t ct_line_size;
4008 size_t ct_size;
4009 const char *ct_label;
4010 } intel_ctab[] = {
4011 /*
4012 * maintain descending order!
4013 *
4014 * Codes ignored - Reason
4015 * ----------------------
4016 * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
4017 * f0H/f1H - Currently we do not interpret prefetch size by design
4018 */
4019 { 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
4020 { 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
4021 { 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
4022 { 0xde, 12, 64, 6*1024*1024, l3_cache_str},
4023 { 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
4024 { 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
4025 { 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
4026 { 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
4027 { 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
4028 { 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
4029 { 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
4030 { 0xd0, 4, 64, 512*1024, l3_cache_str},
4031 { 0xca, 4, 0, 512, sh_l2_tlb4k_str},
4032 { 0xc0, 4, 0, 8, dtlb44_str },
4033 { 0xba, 4, 0, 64, dtlb4k_str },
4034 { 0xb4, 4, 0, 256, dtlb4k_str },
4035 { 0xb3, 4, 0, 128, dtlb4k_str },
4036 { 0xb2, 4, 0, 64, itlb4k_str },
4037 { 0xb0, 4, 0, 128, itlb4k_str },
4038 { 0x87, 8, 64, 1024*1024, l2_cache_str},
4039 { 0x86, 4, 64, 512*1024, l2_cache_str},
4040 { 0x85, 8, 32, 2*1024*1024, l2_cache_str},
4041 { 0x84, 8, 32, 1024*1024, l2_cache_str},
4042 { 0x83, 8, 32, 512*1024, l2_cache_str},
4043 { 0x82, 8, 32, 256*1024, l2_cache_str},
4044 { 0x80, 8, 64, 512*1024, l2_cache_str},
4045 { 0x7f, 2, 64, 512*1024, l2_cache_str},
4046 { 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
4047 { 0x7c, 8, 64, 1024*1024, sl2_cache_str},
4048 { 0x7b, 8, 64, 512*1024, sl2_cache_str},
4049 { 0x7a, 8, 64, 256*1024, sl2_cache_str},
4050 { 0x79, 8, 64, 128*1024, sl2_cache_str},
4051 { 0x78, 8, 64, 1024*1024, l2_cache_str},
4052 { 0x73, 8, 0, 64*1024, itrace_str},
4053 { 0x72, 8, 0, 32*1024, itrace_str},
4054 { 0x71, 8, 0, 16*1024, itrace_str},
4055 { 0x70, 8, 0, 12*1024, itrace_str},
4056 { 0x68, 4, 64, 32*1024, sl1_dcache_str},
4057 { 0x67, 4, 64, 16*1024, sl1_dcache_str},
4058 { 0x66, 4, 64, 8*1024, sl1_dcache_str},
4059 { 0x60, 8, 64, 16*1024, sl1_dcache_str},
4060 { 0x5d, 0, 0, 256, dtlb44_str},
4061 { 0x5c, 0, 0, 128, dtlb44_str},
4062 { 0x5b, 0, 0, 64, dtlb44_str},
4063 { 0x5a, 4, 0, 32, dtlb24_str},
4064 { 0x59, 0, 0, 16, dtlb4k_str},
4065 { 0x57, 4, 0, 16, dtlb4k_str},
4066 { 0x56, 4, 0, 16, dtlb4M_str},
4067 { 0x55, 0, 0, 7, itlb24_str},
4068 { 0x52, 0, 0, 256, itlb424_str},
4069 { 0x51, 0, 0, 128, itlb424_str},
4070 { 0x50, 0, 0, 64, itlb424_str},
4071 { 0x4f, 0, 0, 32, itlb4k_str},
4072 { 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
4073 { 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
4074 { 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
4075 { 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
4076 { 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
4077 { 0x49, 16, 64, 4*1024*1024, l3_cache_str},
4078 { 0x48, 12, 64, 3*1024*1024, l2_cache_str},
4079 { 0x47, 8, 64, 8*1024*1024, l3_cache_str},
4080 { 0x46, 4, 64, 4*1024*1024, l3_cache_str},
4081 { 0x45, 4, 32, 2*1024*1024, l2_cache_str},
4082 { 0x44, 4, 32, 1024*1024, l2_cache_str},
4083 { 0x43, 4, 32, 512*1024, l2_cache_str},
4084 { 0x42, 4, 32, 256*1024, l2_cache_str},
4085 { 0x41, 4, 32, 128*1024, l2_cache_str},
4086 { 0x3e, 4, 64, 512*1024, sl2_cache_str},
4087 { 0x3d, 6, 64, 384*1024, sl2_cache_str},
4088 { 0x3c, 4, 64, 256*1024, sl2_cache_str},
4089 { 0x3b, 2, 64, 128*1024, sl2_cache_str},
4090 { 0x3a, 6, 64, 192*1024, sl2_cache_str},
4091 { 0x39, 4, 64, 128*1024, sl2_cache_str},
4092 { 0x30, 8, 64, 32*1024, l1_icache_str},
4093 { 0x2c, 8, 64, 32*1024, l1_dcache_str},
4094 { 0x29, 8, 64, 4096*1024, sl3_cache_str},
4095 { 0x25, 8, 64, 2048*1024, sl3_cache_str},
4096 { 0x23, 8, 64, 1024*1024, sl3_cache_str},
4097 { 0x22, 4, 64, 512*1024, sl3_cache_str},
4098 { 0x0e, 6, 64, 24*1024, l1_dcache_str},
4099 { 0x0d, 4, 32, 16*1024, l1_dcache_str},
4100 { 0x0c, 4, 32, 16*1024, l1_dcache_str},
4101 { 0x0b, 4, 0, 4, itlb4M_str},
4102 { 0x0a, 2, 32, 8*1024, l1_dcache_str},
4103 { 0x08, 4, 32, 16*1024, l1_icache_str},
4104 { 0x06, 4, 32, 8*1024, l1_icache_str},
4105 { 0x05, 4, 0, 32, dtlb4M_str},
4106 { 0x04, 4, 0, 8, dtlb4M_str},
4107 { 0x03, 4, 0, 64, dtlb4k_str},
4108 { 0x02, 4, 0, 2, itlb4M_str},
4109 { 0x01, 4, 0, 32, itlb4k_str},
4110 { 0 }
4111 };
4112
4113 static const struct cachetab cyrix_ctab[] = {
4114 { 0x70, 4, 0, 32, "tlb-4K" },
4115 { 0x80, 4, 16, 16*1024, "l1-cache" },
4116 { 0 }
4117 };
4118
4119 /*
4120 * Search a cache table for a matching entry
4121 */
4122 static const struct cachetab *
4123 find_cacheent(const struct cachetab *ct, uint_t code)
4124 {
4125 if (code != 0) {
4126 for (; ct->ct_code != 0; ct++)
4127 if (ct->ct_code <= code)
4128 break;
4129 if (ct->ct_code == code)
4130 return (ct);
4131 }
4132 return (NULL);
4133 }
4134
4135 /*
4136 * Populate cachetab entry with L2 or L3 cache-information using
4137 * cpuid function 4. This function is called from intel_walk_cacheinfo()
4138 * when descriptor 0x49 is encountered. It returns 0 if no such cache
4139 * information is found.
4140 */
4141 static int
4142 intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
4143 {
4144 uint32_t level, i;
4145 int ret = 0;
4146
4147 for (i = 0; i < cpi->cpi_std_4_size; i++) {
4148 level = CPI_CACHE_LVL(cpi->cpi_std_4[i]);
4149
4150 if (level == 2 || level == 3) {
4151 ct->ct_assoc = CPI_CACHE_WAYS(cpi->cpi_std_4[i]) + 1;
4152 ct->ct_line_size =
4153 CPI_CACHE_COH_LN_SZ(cpi->cpi_std_4[i]) + 1;
4154 ct->ct_size = ct->ct_assoc *
4155 (CPI_CACHE_PARTS(cpi->cpi_std_4[i]) + 1) *
4156 ct->ct_line_size *
4157 (cpi->cpi_std_4[i]->cp_ecx + 1);
4158
4159 if (level == 2) {
4160 ct->ct_label = l2_cache_str;
4161 } else if (level == 3) {
4162 ct->ct_label = l3_cache_str;
4163 }
4164 ret = 1;
4165 }
4166 }
4167
4168 return (ret);
4169 }
4170
4171 /*
4172 * Walk the cacheinfo descriptor, applying 'func' to every valid element
4173 * The walk is terminated if the walker returns non-zero.
4174 */
4175 static void
4176 intel_walk_cacheinfo(struct cpuid_info *cpi,
4177 void *arg, int (*func)(void *, const struct cachetab *))
4178 {
4179 const struct cachetab *ct;
4180 struct cachetab des_49_ct, des_b1_ct;
4181 uint8_t *dp;
4182 int i;
4183
4184 if ((dp = cpi->cpi_cacheinfo) == NULL)
4185 return;
4186 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4187 /*
4188 * For overloaded descriptor 0x49 we use cpuid function 4
4189 * if supported by the current processor, to create
4190 * cache information.
4191 * For overloaded descriptor 0xb1 we use X86_PAE flag
4192 * to disambiguate the cache information.
4193 */
4194 if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
4195 intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
4196 ct = &des_49_ct;
4197 } else if (*dp == 0xb1) {
4198 des_b1_ct.ct_code = 0xb1;
4199 des_b1_ct.ct_assoc = 4;
4200 des_b1_ct.ct_line_size = 0;
4201 if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
4202 des_b1_ct.ct_size = 8;
4203 des_b1_ct.ct_label = itlb2M_str;
4204 } else {
4205 des_b1_ct.ct_size = 4;
4206 des_b1_ct.ct_label = itlb4M_str;
4207 }
4208 ct = &des_b1_ct;
4209 } else {
4210 if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
4211 continue;
4212 }
4213 }
4214
4215 if (func(arg, ct) != 0) {
4216 break;
4217 }
4218 }
4219 }
4220
4221 /*
4222 * (Like the Intel one, except for Cyrix CPUs)
4223 */
4224 static void
4225 cyrix_walk_cacheinfo(struct cpuid_info *cpi,
4226 void *arg, int (*func)(void *, const struct cachetab *))
4227 {
4228 const struct cachetab *ct;
4229 uint8_t *dp;
4230 int i;
4231
4232 if ((dp = cpi->cpi_cacheinfo) == NULL)
4233 return;
4234 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4235 /*
4236 * Search Cyrix-specific descriptor table first ..
4237 */
4238 if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
4239 if (func(arg, ct) != 0)
4240 break;
4241 continue;
4242 }
4243 /*
4244 * .. else fall back to the Intel one
4245 */
4246 if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
4247 if (func(arg, ct) != 0)
4248 break;
4249 continue;
4250 }
4251 }
4252 }
4253
4254 /*
4255 * A cacheinfo walker that adds associativity, line-size, and size properties
4256 * to the devinfo node it is passed as an argument.
4257 */
4258 static int
4259 add_cacheent_props(void *arg, const struct cachetab *ct)
4260 {
4261 dev_info_t *devi = arg;
4262
4263 add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
4264 if (ct->ct_line_size != 0)
4265 add_cache_prop(devi, ct->ct_label, line_str,
4266 ct->ct_line_size);
4267 add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
4268 return (0);
4269 }
4270
4271
4272 static const char fully_assoc[] = "fully-associative?";
4273
4274 /*
4275 * AMD style cache/tlb description
4276 *
4277 * Extended functions 5 and 6 directly describe properties of
4278 * tlbs and various cache levels.
4279 */
4280 static void
4281 add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4282 {
4283 switch (assoc) {
4284 case 0: /* reserved; ignore */
4285 break;
4286 default:
4287 add_cache_prop(devi, label, assoc_str, assoc);
4288 break;
4289 case 0xff:
4290 add_cache_prop(devi, label, fully_assoc, 1);
4291 break;
4292 }
4293 }
4294
4295 static void
4296 add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4297 {
4298 if (size == 0)
4299 return;
4300 add_cache_prop(devi, label, size_str, size);
4301 add_amd_assoc(devi, label, assoc);
4302 }
4303
4304 static void
4305 add_amd_cache(dev_info_t *devi, const char *label,
4306 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4307 {
4308 if (size == 0 || line_size == 0)
4309 return;
4310 add_amd_assoc(devi, label, assoc);
4311 /*
4312 * Most AMD parts have a sectored cache. Multiple cache lines are
4313 * associated with each tag. A sector consists of all cache lines
4314 * associated with a tag. For example, the AMD K6-III has a sector
4315 * size of 2 cache lines per tag.
4316 */
4317 if (lines_per_tag != 0)
4318 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4319 add_cache_prop(devi, label, line_str, line_size);
4320 add_cache_prop(devi, label, size_str, size * 1024);
4321 }
4322
4323 static void
4324 add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4325 {
4326 switch (assoc) {
4327 case 0: /* off */
4328 break;
4329 case 1:
4330 case 2:
4331 case 4:
4332 add_cache_prop(devi, label, assoc_str, assoc);
4333 break;
4334 case 6:
4335 add_cache_prop(devi, label, assoc_str, 8);
4336 break;
4337 case 8:
4338 add_cache_prop(devi, label, assoc_str, 16);
4339 break;
4340 case 0xf:
4341 add_cache_prop(devi, label, fully_assoc, 1);
4342 break;
4343 default: /* reserved; ignore */
4344 break;
4345 }
4346 }
4347
4348 static void
4349 add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4350 {
4351 if (size == 0 || assoc == 0)
4352 return;
4353 add_amd_l2_assoc(devi, label, assoc);
4354 add_cache_prop(devi, label, size_str, size);
4355 }
4356
4357 static void
4358 add_amd_l2_cache(dev_info_t *devi, const char *label,
4359 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4360 {
4361 if (size == 0 || assoc == 0 || line_size == 0)
4362 return;
4363 add_amd_l2_assoc(devi, label, assoc);
4364 if (lines_per_tag != 0)
4365 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4366 add_cache_prop(devi, label, line_str, line_size);
4367 add_cache_prop(devi, label, size_str, size * 1024);
4368 }
4369
4370 static void
4371 amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
4372 {
4373 struct cpuid_regs *cp;
4374
4375 if (cpi->cpi_xmaxeax < 0x80000005)
4376 return;
4377 cp = &cpi->cpi_extd[5];
4378
4379 /*
4380 * 4M/2M L1 TLB configuration
4381 *
4382 * We report the size for 2M pages because AMD uses two
4383 * TLB entries for one 4M page.
4384 */
4385 add_amd_tlb(devi, "dtlb-2M",
4386 BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
4387 add_amd_tlb(devi, "itlb-2M",
4388 BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));
4389
4390 /*
4391 * 4K L1 TLB configuration
4392 */
4393
4394 switch (cpi->cpi_vendor) {
4395 uint_t nentries;
4396 case X86_VENDOR_TM:
4397 if (cpi->cpi_family >= 5) {
4398 /*
4399 * Crusoe processors have 256 TLB entries, but
4400 * cpuid data format constrains them to only
4401 * reporting 255 of them.
4402 */
4403 if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
4404 nentries = 256;
4405 /*
4406 * Crusoe processors also have a unified TLB
4407 */
4408 add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
4409 nentries);
4410 break;
4411 }
4412 /*FALLTHROUGH*/
4413 default:
4414 add_amd_tlb(devi, itlb4k_str,
4415 BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
4416 add_amd_tlb(devi, dtlb4k_str,
4417 BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
4418 break;
4419 }
4420
4421 /*
4422 * data L1 cache configuration
4423 */
4424
4425 add_amd_cache(devi, l1_dcache_str,
4426 BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
4427 BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));
4428
4429 /*
4430 * code L1 cache configuration
4431 */
4432
4433 add_amd_cache(devi, l1_icache_str,
4434 BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
4435 BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));
4436
4437 if (cpi->cpi_xmaxeax < 0x80000006)
4438 return;
4439 cp = &cpi->cpi_extd[6];
4440
4441 /* Check for a unified L2 TLB for large pages */
4442
4443 if (BITX(cp->cp_eax, 31, 16) == 0)
4444 add_amd_l2_tlb(devi, "l2-tlb-2M",
4445 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4446 else {
4447 add_amd_l2_tlb(devi, "l2-dtlb-2M",
4448 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4449 add_amd_l2_tlb(devi, "l2-itlb-2M",
4450 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4451 }
4452
4453 /* Check for a unified L2 TLB for 4K pages */
4454
4455 if (BITX(cp->cp_ebx, 31, 16) == 0) {
4456 add_amd_l2_tlb(devi, "l2-tlb-4K",
4457 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4458 } else {
4459 add_amd_l2_tlb(devi, "l2-dtlb-4K",
4460 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4461 add_amd_l2_tlb(devi, "l2-itlb-4K",
4462 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4463 }
4464
4465 add_amd_l2_cache(devi, l2_cache_str,
4466 BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
4467 BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
4468 }
4469
4470 /*
4471 * There are two basic ways that the x86 world describes it cache
4472 * and tlb architecture - Intel's way and AMD's way.
4473 *
4474 * Return which flavor of cache architecture we should use
4475 */
4476 static int
4477 x86_which_cacheinfo(struct cpuid_info *cpi)
4478 {
4479 switch (cpi->cpi_vendor) {
4480 case X86_VENDOR_Intel:
4481 if (cpi->cpi_maxeax >= 2)
4482 return (X86_VENDOR_Intel);
4483 break;
4484 case X86_VENDOR_AMD:
4485 /*
4486 * The K5 model 1 was the first part from AMD that reported
4487 * cache sizes via extended cpuid functions.
4488 */
4489 if (cpi->cpi_family > 5 ||
4490 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
4491 return (X86_VENDOR_AMD);
4492 break;
4493 case X86_VENDOR_TM:
4494 if (cpi->cpi_family >= 5)
4495 return (X86_VENDOR_AMD);
4496 /*FALLTHROUGH*/
4497 default:
4498 /*
4499 * If they have extended CPU data for 0x80000005
4500 * then we assume they have AMD-format cache
4501 * information.
4502 *
4503 * If not, and the vendor happens to be Cyrix,
4504 * then try our-Cyrix specific handler.
4505 *
4506 * If we're not Cyrix, then assume we're using Intel's
4507 * table-driven format instead.
4508 */
4509 if (cpi->cpi_xmaxeax >= 0x80000005)
4510 return (X86_VENDOR_AMD);
4511 else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
4512 return (X86_VENDOR_Cyrix);
4513 else if (cpi->cpi_maxeax >= 2)
4514 return (X86_VENDOR_Intel);
4515 break;
4516 }
4517 return (-1);
4518 }
4519
4520 void
4521 cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
4522 struct cpuid_info *cpi)
4523 {
4524 dev_info_t *cpu_devi;
4525 int create;
4526
4527 cpu_devi = (dev_info_t *)dip;
4528
4529 /* device_type */
4530 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4531 "device_type", "cpu");
4532
4533 /* reg */
4534 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4535 "reg", cpu_id);
4536
4537 /* cpu-mhz, and clock-frequency */
4538 if (cpu_freq > 0) {
4539 long long mul;
4540
4541 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4542 "cpu-mhz", cpu_freq);
4543 if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
4544 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4545 "clock-frequency", (int)mul);
4546 }
4547
4548 if (!is_x86_feature(x86_featureset, X86FSET_CPUID)) {
4549 return;
4550 }
4551
4552 /* vendor-id */
4553 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4554 "vendor-id", cpi->cpi_vendorstr);
4555
4556 if (cpi->cpi_maxeax == 0) {
4557 return;
4558 }
4559
4560 /*
4561 * family, model, and step
4562 */
4563 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4564 "family", CPI_FAMILY(cpi));
4565 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4566 "cpu-model", CPI_MODEL(cpi));
4567 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4568 "stepping-id", CPI_STEP(cpi));
4569
4570 /* type */
4571 switch (cpi->cpi_vendor) {
4572 case X86_VENDOR_Intel:
4573 create = 1;
4574 break;
4575 default:
4576 create = 0;
4577 break;
4578 }
4579 if (create)
4580 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4581 "type", CPI_TYPE(cpi));
4582
4583 /* ext-family */
4584 switch (cpi->cpi_vendor) {
4585 case X86_VENDOR_Intel:
4586 case X86_VENDOR_AMD:
4587 create = cpi->cpi_family >= 0xf;
4588 break;
4589 default:
4590 create = 0;
4591 break;
4592 }
4593 if (create)
4594 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4595 "ext-family", CPI_FAMILY_XTD(cpi));
4596
4597 /* ext-model */
4598 switch (cpi->cpi_vendor) {
4599 case X86_VENDOR_Intel:
4600 create = IS_EXTENDED_MODEL_INTEL(cpi);
4601 break;
4602 case X86_VENDOR_AMD:
4603 create = CPI_FAMILY(cpi) == 0xf;
4604 break;
4605 default:
4606 create = 0;
4607 break;
4608 }
4609 if (create)
4610 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4611 "ext-model", CPI_MODEL_XTD(cpi));
4612
4613 /* generation */
4614 switch (cpi->cpi_vendor) {
4615 case X86_VENDOR_AMD:
4616 /*
4617 * AMD K5 model 1 was the first part to support this
4618 */
4619 create = cpi->cpi_xmaxeax >= 0x80000001;
4620 break;
4621 default:
4622 create = 0;
4623 break;
4624 }
4625 if (create)
4626 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4627 "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));
4628
4629 /* brand-id */
4630 switch (cpi->cpi_vendor) {
4631 case X86_VENDOR_Intel:
4632 /*
4633 * brand id first appeared on Pentium III Xeon model 8,
4634 * and Celeron model 8 processors and Opteron
4635 */
4636 create = cpi->cpi_family > 6 ||
4637 (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
4638 break;
4639 case X86_VENDOR_AMD:
4640 create = cpi->cpi_family >= 0xf;
4641 break;
4642 default:
4643 create = 0;
4644 break;
4645 }
4646 if (create && cpi->cpi_brandid != 0) {
4647 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4648 "brand-id", cpi->cpi_brandid);
4649 }
4650
4651 /* chunks, and apic-id */
4652 switch (cpi->cpi_vendor) {
4653 /*
4654 * first available on Pentium IV and Opteron (K8)
4655 */
4656 case X86_VENDOR_Intel:
4657 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4658 break;
4659 case X86_VENDOR_AMD:
4660 create = cpi->cpi_family >= 0xf;
4661 break;
4662 default:
4663 create = 0;
4664 break;
4665 }
4666 if (create) {
4667 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4668 "chunks", CPI_CHUNKS(cpi));
4669 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4670 "apic-id", cpi->cpi_apicid);
4671 if (cpi->cpi_chipid >= 0) {
4672 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4673 "chip#", cpi->cpi_chipid);
4674 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4675 "clog#", cpi->cpi_clogid);
4676 }
4677 }
4678
4679 /* cpuid-features */
4680 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4681 "cpuid-features", CPI_FEATURES_EDX(cpi));
4682
4683
4684 /* cpuid-features-ecx */
4685 switch (cpi->cpi_vendor) {
4686 case X86_VENDOR_Intel:
4687 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4688 break;
4689 case X86_VENDOR_AMD:
4690 create = cpi->cpi_family >= 0xf;
4691 break;
4692 default:
4693 create = 0;
4694 break;
4695 }
4696 if (create)
4697 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4698 "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));
4699
4700 /* ext-cpuid-features */
4701 switch (cpi->cpi_vendor) {
4702 case X86_VENDOR_Intel:
4703 case X86_VENDOR_AMD:
4704 case X86_VENDOR_Cyrix:
4705 case X86_VENDOR_TM:
4706 case X86_VENDOR_Centaur:
4707 create = cpi->cpi_xmaxeax >= 0x80000001;
4708 break;
4709 default:
4710 create = 0;
4711 break;
4712 }
4713 if (create) {
4714 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4715 "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
4716 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4717 "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
4718 }
4719
4720 /*
4721 * Brand String first appeared in Intel Pentium IV, AMD K5
4722 * model 1, and Cyrix GXm. On earlier models we try and
4723 * simulate something similar .. so this string should always
4724 * same -something- about the processor, however lame.
4725 */
4726 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4727 "brand-string", cpi->cpi_brandstr);
4728
4729 /*
4730 * Finally, cache and tlb information
4731 */
4732 switch (x86_which_cacheinfo(cpi)) {
4733 case X86_VENDOR_Intel:
4734 intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4735 break;
4736 case X86_VENDOR_Cyrix:
4737 cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4738 break;
4739 case X86_VENDOR_AMD:
4740 amd_cache_info(cpi, cpu_devi);
4741 break;
4742 default:
4743 break;
4744 }
4745 }
4746
4747 struct l2info {
4748 int *l2i_csz;
4749 int *l2i_lsz;
4750 int *l2i_assoc;
4751 int l2i_ret;
4752 };
4753
4754 /*
4755 * A cacheinfo walker that fetches the size, line-size and associativity
4756 * of the L2 cache
4757 */
4758 static int
4759 intel_l2cinfo(void *arg, const struct cachetab *ct)
4760 {
4761 struct l2info *l2i = arg;
4762 int *ip;
4763
4764 if (ct->ct_label != l2_cache_str &&
4765 ct->ct_label != sl2_cache_str)
4766 return (0); /* not an L2 -- keep walking */
4767
4768 if ((ip = l2i->l2i_csz) != NULL)
4769 *ip = ct->ct_size;
4770 if ((ip = l2i->l2i_lsz) != NULL)
4771 *ip = ct->ct_line_size;
4772 if ((ip = l2i->l2i_assoc) != NULL)
4773 *ip = ct->ct_assoc;
4774 l2i->l2i_ret = ct->ct_size;
4775 return (1); /* was an L2 -- terminate walk */
4776 }
4777
4778 /*
4779 * AMD L2/L3 Cache and TLB Associativity Field Definition:
4780 *
4781 * Unlike the associativity for the L1 cache and tlb where the 8 bit
4782 * value is the associativity, the associativity for the L2 cache and
4783 * tlb is encoded in the following table. The 4 bit L2 value serves as
4784 * an index into the amd_afd[] array to determine the associativity.
4785 * -1 is undefined. 0 is fully associative.
4786 */
4787
4788 static int amd_afd[] =
4789 {-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};
4790
4791 static void
4792 amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
4793 {
4794 struct cpuid_regs *cp;
4795 uint_t size, assoc;
4796 int i;
4797 int *ip;
4798
4799 if (cpi->cpi_xmaxeax < 0x80000006)
4800 return;
4801 cp = &cpi->cpi_extd[6];
4802
4803 if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
4804 (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
4805 uint_t cachesz = size * 1024;
4806 assoc = amd_afd[i];
4807
4808 ASSERT(assoc != -1);
4809
4810 if ((ip = l2i->l2i_csz) != NULL)
4811 *ip = cachesz;
4812 if ((ip = l2i->l2i_lsz) != NULL)
4813 *ip = BITX(cp->cp_ecx, 7, 0);
4814 if ((ip = l2i->l2i_assoc) != NULL)
4815 *ip = assoc;
4816 l2i->l2i_ret = cachesz;
4817 }
4818 }
4819
4820 int
4821 getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
4822 {
4823 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
4824 struct l2info __l2info, *l2i = &__l2info;
4825
4826 l2i->l2i_csz = csz;
4827 l2i->l2i_lsz = lsz;
4828 l2i->l2i_assoc = assoc;
4829 l2i->l2i_ret = -1;
4830
4831 switch (x86_which_cacheinfo(cpi)) {
4832 case X86_VENDOR_Intel:
4833 intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4834 break;
4835 case X86_VENDOR_Cyrix:
4836 cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4837 break;
4838 case X86_VENDOR_AMD:
4839 amd_l2cacheinfo(cpi, l2i);
4840 break;
4841 default:
4842 break;
4843 }
4844 return (l2i->l2i_ret);
4845 }
4846
4847
4848 int
4849 cpuid_mwait_alloc(cpu_t *cpu)
4850 {
4851 uint32_t *buf;
4852 uint32_t *mwait;
4853 size_t size;
4854
4855 ASSERT(cpuid_checkpass(CPU, 2));
4856
4857 size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
4858 if (size == 0)
4859 return (EINVAL);
4860
4861 /*
4862 * kmem_alloc() returns cache line size aligned data for size
4863 * allocations. size is currently cache line sized. Neither of
4864 * these implementation details are guarantied to be true in the
4865 * future.
4866 *
4867 * First try allocating size as kmem_alloc() currently returns
4868 * correctly aligned memory. If kmem_alloc() does not return
4869 * size aligned memory, then use size ROUNDUP.
4870 *
4871 * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
4872 * decide to free this memory.
4873 */
4874 buf = kmem_zalloc(size, KM_SLEEP);
4875 if (buf == (uint32_t *)P2ROUNDUP((uintptr_t)buf, size)) {
4876 mwait = buf;
4877 } else {
4878 kmem_free(buf, size);
4879 buf = kmem_zalloc(size * 2, KM_SLEEP);
4880
4881 mwait = (uint32_t *)P2ROUNDUP((uintptr_t)buf, size);
4882 size *= 2;
4883 }
4884
4885 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = buf;
4886 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = size;
4887
4888 *mwait = MWAIT_RUNNING;
4889
4890 cpu->cpu_m.mcpu_mwait = mwait;
4891
4892 return (0);
4893 }
4894
4895 void
4896 cpuid_mwait_free(cpu_t *cpu)
4897 {
4898 if (cpu->cpu_m.mcpu_cpi == NULL) {
4899 return;
4900 }
4901
4902 if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
4903 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
4904 kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
4905 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
4906 }
4907
4908 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
4909 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
4910 }
4911
4912 void
4913 patch_tsc_read(int flag)
4914 {
4915 size_t cnt;
4916
4917 switch (flag) {
4918 case TSC_NONE:
4919 cnt = &_no_rdtsc_end - &_no_rdtsc_start;
4920 (void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
4921 break;
4922 case TSC_RDTSC_MFENCE:
4923 cnt = &_tsc_mfence_end - &_tsc_mfence_start;
4924 (void) memcpy((void *)tsc_read,
4925 (void *)&_tsc_mfence_start, cnt);
4926 break;
4927 case TSC_RDTSC_LFENCE:
4928 cnt = &_tsc_lfence_end - &_tsc_lfence_start;
4929 (void) memcpy((void *)tsc_read,
4930 (void *)&_tsc_lfence_start, cnt);
4931 break;
4932 case TSC_TSCP:
4933 cnt = &_tscp_end - &_tscp_start;
4934 (void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
4935 break;
4936 default:
4937 /* Bail for unexpected TSC types. (TSC_NONE covers 0) */
4938 cmn_err(CE_PANIC, "Unrecogized TSC type: %d", flag);
4939 break;
4940 }
4941 tsc_type = flag;
4942 }
4943
4944 int
4945 cpuid_deep_cstates_supported(void)
4946 {
4947 struct cpuid_info *cpi;
4948 struct cpuid_regs regs;
4949
4950 ASSERT(cpuid_checkpass(CPU, 1));
4951
4952 cpi = CPU->cpu_m.mcpu_cpi;
4953
4954 if (!is_x86_feature(x86_featureset, X86FSET_CPUID))
4955 return (0);
4956
4957 switch (cpi->cpi_vendor) {
4958 case X86_VENDOR_Intel:
4959 if (cpi->cpi_xmaxeax < 0x80000007)
4960 return (0);
4961
4962 /*
4963 * TSC run at a constant rate in all ACPI C-states?
4964 */
4965 regs.cp_eax = 0x80000007;
4966 (void) __cpuid_insn(&regs);
4967 return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);
4968
4969 default:
4970 return (0);
4971 }
4972 }
4973
4974
4975 void
4976 post_startup_cpu_fixups(void)
4977 {
4978 /*
4979 * Some AMD processors support C1E state. Entering this state will
4980 * cause the local APIC timer to stop, which we can't deal with at
4981 * this time.
4982 */
4983 if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
4984 on_trap_data_t otd;
4985 uint64_t reg;
4986
4987 if (!on_trap(&otd, OT_DATA_ACCESS)) {
4988 reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
4989 /* Disable C1E state if it is enabled by BIOS */
4990 if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
4991 AMD_ACTONCMPHALT_MASK) {
4992 reg &= ~(AMD_ACTONCMPHALT_MASK <<
4993 AMD_ACTONCMPHALT_SHIFT);
4994 wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
4995 }
4996 }
4997 no_trap();
4998 }
4999 }
5000
5001 void
5002 enable_pcid(void)
5003 {
5004 if (x86_use_pcid == -1)
5005 x86_use_pcid = is_x86_feature(x86_featureset, X86FSET_PCID);
5006
5007 if (x86_use_invpcid == -1) {
5008 x86_use_invpcid = is_x86_feature(x86_featureset,
5009 X86FSET_INVPCID);
5010 }
5011
5012 if (!x86_use_pcid)
5013 return;
5014
5015 /*
5016 * Intel say that on setting PCIDE, it immediately starts using the PCID
5017 * bits; better make sure there's nothing there.
5018 */
5019 ASSERT((getcr3() & MMU_PAGEOFFSET) == PCID_NONE);
5020
5021 setcr4(getcr4() | CR4_PCIDE);
5022 }
5023
5024 /*
5025 * Setup necessary registers to enable XSAVE feature on this processor.
5026 * This function needs to be called early enough, so that no xsave/xrstor
5027 * ops will execute on the processor before the MSRs are properly set up.
5028 *
5029 * Current implementation has the following assumption:
5030 * - cpuid_pass1() is done, so that X86 features are known.
5031 * - fpu_probe() is done, so that fp_save_mech is chosen.
5032 */
5033 void
5034 xsave_setup_msr(cpu_t *cpu)
5035 {
5036 ASSERT(fp_save_mech == FP_XSAVE);
5037 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
5038
5039 /* Enable OSXSAVE in CR4. */
5040 setcr4(getcr4() | CR4_OSXSAVE);
5041 /*
5042 * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
5043 * correct value.
5044 */
5045 cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
5046 setup_xfem();
5047 }
5048
5049 /*
5050 * Starting with the Westmere processor the local
5051 * APIC timer will continue running in all C-states,
5052 * including the deepest C-states.
5053 */
5054 int
5055 cpuid_arat_supported(void)
5056 {
5057 struct cpuid_info *cpi;
5058 struct cpuid_regs regs;
5059
5060 ASSERT(cpuid_checkpass(CPU, 1));
5061 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5062
5063 cpi = CPU->cpu_m.mcpu_cpi;
5064
5065 switch (cpi->cpi_vendor) {
5066 case X86_VENDOR_Intel:
5067 /*
5068 * Always-running Local APIC Timer is
5069 * indicated by CPUID.6.EAX[2].
5070 */
5071 if (cpi->cpi_maxeax >= 6) {
5072 regs.cp_eax = 6;
5073 (void) cpuid_insn(NULL, &regs);
5074 return (regs.cp_eax & CPUID_CSTATE_ARAT);
5075 } else {
5076 return (0);
5077 }
5078 default:
5079 return (0);
5080 }
5081 }
5082
5083 /*
5084 * Check support for Intel ENERGY_PERF_BIAS feature
5085 */
5086 int
5087 cpuid_iepb_supported(struct cpu *cp)
5088 {
5089 struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
5090 struct cpuid_regs regs;
5091
5092 ASSERT(cpuid_checkpass(cp, 1));
5093
5094 if (!(is_x86_feature(x86_featureset, X86FSET_CPUID)) ||
5095 !(is_x86_feature(x86_featureset, X86FSET_MSR))) {
5096 return (0);
5097 }
5098
5099 /*
5100 * Intel ENERGY_PERF_BIAS MSR is indicated by
5101 * capability bit CPUID.6.ECX.3
5102 */
5103 if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
5104 return (0);
5105
5106 regs.cp_eax = 0x6;
5107 (void) cpuid_insn(NULL, &regs);
5108 return (regs.cp_ecx & CPUID_EPB_SUPPORT);
5109 }
5110
5111 /*
5112 * Check support for TSC deadline timer
5113 *
5114 * TSC deadline timer provides a superior software programming
5115 * model over local APIC timer that eliminates "time drifts".
5116 * Instead of specifying a relative time, software specifies an
5117 * absolute time as the target at which the processor should
5118 * generate a timer event.
5119 */
5120 int
5121 cpuid_deadline_tsc_supported(void)
5122 {
5123 struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
5124 struct cpuid_regs regs;
5125
5126 ASSERT(cpuid_checkpass(CPU, 1));
5127 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5128
5129 switch (cpi->cpi_vendor) {
5130 case X86_VENDOR_Intel:
5131 if (cpi->cpi_maxeax >= 1) {
5132 regs.cp_eax = 1;
5133 (void) cpuid_insn(NULL, &regs);
5134 return (regs.cp_ecx & CPUID_DEADLINE_TSC);
5135 } else {
5136 return (0);
5137 }
5138 default:
5139 return (0);
5140 }
5141 }
5142
5143 #if defined(__amd64)
5144 /*
5145 * Patch in versions of bcopy for high performance Intel Nhm processors
5146 * and later...
5147 */
5148 void
5149 patch_memops(uint_t vendor)
5150 {
5151 size_t cnt, i;
5152 caddr_t to, from;
5153
5154 if ((vendor == X86_VENDOR_Intel) &&
5155 is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
5156 cnt = &bcopy_patch_end - &bcopy_patch_start;
5157 to = &bcopy_ck_size;
5158 from = &bcopy_patch_start;
5159 for (i = 0; i < cnt; i++) {
5160 *to++ = *from++;
5161 }
5162 }
5163 }
5164 #endif /* __amd64 */
5165
5166 /*
5167 * This function finds the number of bits to represent the number of cores per
5168 * chip and the number of strands per core for the Intel platforms.
5169 * It re-uses the x2APIC cpuid code of the cpuid_pass2().
5170 */
5171 void
5172 cpuid_get_ext_topo(uint_t vendor, uint_t *core_nbits, uint_t *strand_nbits)
5173 {
5174 struct cpuid_regs regs;
5175 struct cpuid_regs *cp = &regs;
5176
5177 if (vendor != X86_VENDOR_Intel) {
5178 return;
5179 }
5180
5181 /* if the cpuid level is 0xB, extended topo is available. */
5182 cp->cp_eax = 0;
5183 if (__cpuid_insn(cp) >= 0xB) {
5184
5185 cp->cp_eax = 0xB;
5186 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
5187 (void) __cpuid_insn(cp);
5188
5189 /*
5190 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
5191 * indicates that the extended topology enumeration leaf is
5192 * available.
5193 */
5194 if (cp->cp_ebx) {
5195 uint_t coreid_shift = 0;
5196 uint_t chipid_shift = 0;
5197 uint_t i;
5198 uint_t level;
5199
5200 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
5201 cp->cp_eax = 0xB;
5202 cp->cp_ecx = i;
5203
5204 (void) __cpuid_insn(cp);
5205 level = CPI_CPU_LEVEL_TYPE(cp);
5206
5207 if (level == 1) {
5208 /*
5209 * Thread level processor topology
5210 * Number of bits shift right APIC ID
5211 * to get the coreid.
5212 */
5213 coreid_shift = BITX(cp->cp_eax, 4, 0);
5214 } else if (level == 2) {
5215 /*
5216 * Core level processor topology
5217 * Number of bits shift right APIC ID
5218 * to get the chipid.
5219 */
5220 chipid_shift = BITX(cp->cp_eax, 4, 0);
5221 }
5222 }
5223
5224 if (coreid_shift > 0 && chipid_shift > coreid_shift) {
5225 *strand_nbits = coreid_shift;
5226 *core_nbits = chipid_shift - coreid_shift;
5227 }
5228 }
5229 }
5230 }
5231
5232 void
5233 cpuid_pass_ucode(cpu_t *cpu, uchar_t *fset)
5234 {
5235 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
5236 struct cpuid_regs cp;
5237
5238 /*
5239 * Reread the CPUID portions that we need for various security
5240 * information.
5241 */
5242 if (cpi->cpi_vendor == X86_VENDOR_Intel) {
5243 /*
5244 * Check if we now have leaf 7 available to us.
5245 */
5246 if (cpi->cpi_maxeax < 7) {
5247 bzero(&cp, sizeof (cp));
5248 cp.cp_eax = 0;
5249 cpi->cpi_maxeax = __cpuid_insn(&cp);
5250 if (cpi->cpi_maxeax < 7)
5251 return;
5252 }
5253
5254 bzero(&cp, sizeof (cp));
5255 cp.cp_eax = 7;
5256 cp.cp_ecx = 0;
5257 (void) __cpuid_insn(&cp);
5258 cpi->cpi_std[7] = cp;
5259 } else if (cpi->cpi_vendor == X86_VENDOR_AMD) {
5260 /* No xcpuid support */
5261 if (cpi->cpi_family < 5 ||
5262 (cpi->cpi_family == 5 && cpi->cpi_model < 1))
5263 return;
5264
5265 if (cpi->cpi_xmaxeax < 0x80000008) {
5266 bzero(&cp, sizeof (cp));
5267 cp.cp_eax = 0x80000000;
5268 cpi->cpi_xmaxeax = __cpuid_insn(&cp);
5269 if (cpi->cpi_xmaxeax < 0x80000008) {
5270 return;
5271 }
5272 }
5273
5274 bzero(&cp, sizeof (cp));
5275 cp.cp_eax = 0x80000008;
5276 (void) __cpuid_insn(&cp);
5277 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, &cp);
5278 cpi->cpi_extd[8] = cp;
5279 } else {
5280 /*
5281 * Nothing to do here. Return an empty set which has already
5282 * been zeroed for us.
5283 */
5284 return;
5285 }
5286 cpuid_scan_security(cpu, fset);
5287 }
5288
5289 /* ARGSUSED */
5290 static int
5291 cpuid_post_ucodeadm_xc(xc_arg_t arg0, xc_arg_t arg1, xc_arg_t arg2)
5292 {
5293 uchar_t *fset;
5294
5295 fset = (uchar_t *)(arg0 + sizeof (x86_featureset) * CPU->cpu_id);
5296 cpuid_pass_ucode(CPU, fset);
5297
5298 return (0);
5299 }
5300
5301 /*
5302 * After a microcode update where the version has changed, then we need to
5303 * rescan CPUID. To do this we check every CPU to make sure that they have the
5304 * same microcode. Then we perform a cross call to all such CPUs. It's the
5305 * caller's job to make sure that no one else can end up doing an update while
5306 * this is going on.
5307 *
5308 * We assume that the system is microcode capable if we're called.
5309 */
5310 void
5311 cpuid_post_ucodeadm(void)
5312 {
5313 uint32_t rev;
5314 int i;
5315 struct cpu *cpu;
5316 cpuset_t cpuset;
5317 void *argdata;
5318 uchar_t *f0;
5319
5320 argdata = kmem_zalloc(sizeof (x86_featureset) * NCPU, KM_SLEEP);
5321
5322 mutex_enter(&cpu_lock);
5323 cpu = cpu_get(0);
5324 rev = cpu->cpu_m.mcpu_ucode_info->cui_rev;
5325 CPUSET_ONLY(cpuset, 0);
5326 for (i = 1; i < max_ncpus; i++) {
5327 if ((cpu = cpu_get(i)) == NULL)
5328 continue;
5329
5330 if (cpu->cpu_m.mcpu_ucode_info->cui_rev != rev) {
5331 panic("post microcode update CPU %d has differing "
5332 "microcode revision (%u) from CPU 0 (%u)",
5333 i, cpu->cpu_m.mcpu_ucode_info->cui_rev, rev);
5334 }
5335 CPUSET_ADD(cpuset, i);
5336 }
5337
5338 kpreempt_disable();
5339 xc_sync((xc_arg_t)argdata, 0, 0, CPUSET2BV(cpuset),
5340 cpuid_post_ucodeadm_xc);
5341 kpreempt_enable();
5342
5343 /*
5344 * OK, now look at each CPU and see if their feature sets are equal.
5345 */
5346 f0 = argdata;
5347 for (i = 1; i < max_ncpus; i++) {
5348 uchar_t *fset;
5349 if (!CPU_IN_SET(cpuset, i))
5350 continue;
5351
5352 fset = (uchar_t *)((uintptr_t)argdata +
5353 sizeof (x86_featureset) * i);
5354
5355 if (!compare_x86_featureset(f0, fset)) {
5356 panic("Post microcode update CPU %d has "
5357 "differing security feature (%p) set from CPU 0 "
5358 "(%p), not appending to feature set", i,
5359 (void *)fset, (void *)f0);
5360 }
5361 }
5362
5363 mutex_exit(&cpu_lock);
5364
5365 for (i = 0; i < NUM_X86_FEATURES; i++) {
5366 cmn_err(CE_CONT, "?post-ucode x86_feature: %s\n",
5367 x86_feature_names[i]);
5368 if (is_x86_feature(f0, i)) {
5369 add_x86_feature(x86_featureset, i);
5370 }
5371 }
5372 kmem_free(argdata, sizeof (x86_featureset) * NCPU);
5373 }
Unleashed OS