1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
26 */
27 /*
28 * Copyright (c) 2010, Intel Corporation.
29 * All rights reserved.
30 */
31 /*
32 * Portions Copyright 2009 Advanced Micro Devices, Inc.
33 */
34 /*
35 * Copyright 2018 Joyent, Inc.
36 */
37 /*
38 * Various routines to handle identification
39 * and classification of x86 processors.
40 */
41
42 #include <sys/types.h>
43 #include <sys/archsystm.h>
44 #include <sys/x86_archext.h>
45 #include <sys/kmem.h>
46 #include <sys/systm.h>
47 #include <sys/cmn_err.h>
48 #include <sys/sunddi.h>
49 #include <sys/sunndi.h>
50 #include <sys/cpuvar.h>
51 #include <sys/processor.h>
52 #include <sys/sysmacros.h>
53 #include <sys/pg.h>
54 #include <sys/fp.h>
55 #include <sys/controlregs.h>
56 #include <sys/bitmap.h>
57 #include <sys/auxv_386.h>
58 #include <sys/memnode.h>
59 #include <sys/pci_cfgspace.h>
60 #include <sys/comm_page.h>
61 #include <sys/mach_mmu.h>
62 #include <sys/ucode.h>
63 #include <sys/tsc.h>
64
65 #ifdef __xpv
66 #include <sys/hypervisor.h>
67 #else
68 #include <sys/ontrap.h>
69 #endif
70
71 /*
72 * Pass 0 of cpuid feature analysis happens in locore. It contains special code
73 * to recognize Cyrix processors that are not cpuid-compliant, and to deal with
74 * them accordingly. For most modern processors, feature detection occurs here
75 * in pass 1.
76 *
77 * Pass 1 of cpuid feature analysis happens just at the beginning of mlsetup()
78 * for the boot CPU and does the basic analysis that the early kernel needs.
79 * x86_featureset is set based on the return value of cpuid_pass1() of the boot
80 * CPU.
81 *
82 * Pass 1 includes:
83 *
84 * o Determining vendor/model/family/stepping and setting x86_type and
85 * x86_vendor accordingly.
86 * o Processing the feature flags returned by the cpuid instruction while
87 * applying any workarounds or tricks for the specific processor.
88 * o Mapping the feature flags into illumos feature bits (X86_*).
89 * o Processing extended feature flags if supported by the processor,
90 * again while applying specific processor knowledge.
91 * o Determining the CMT characteristics of the system.
92 *
93 * Pass 1 is done on non-boot CPUs during their initialization and the results
94 * are used only as a meager attempt at ensuring that all processors within the
95 * system support the same features.
96 *
97 * Pass 2 of cpuid feature analysis happens just at the beginning
98 * of startup(). It just copies in and corrects the remainder
99 * of the cpuid data we depend on: standard cpuid functions that we didn't
100 * need for pass1 feature analysis, and extended cpuid functions beyond the
101 * simple feature processing done in pass1.
102 *
103 * Pass 3 of cpuid analysis is invoked after basic kernel services; in
104 * particular kernel memory allocation has been made available. It creates a
105 * readable brand string based on the data collected in the first two passes.
106 *
107 * Pass 4 of cpuid analysis is invoked after post_startup() when all
108 * the support infrastructure for various hardware features has been
109 * initialized. It determines which processor features will be reported
110 * to userland via the aux vector.
111 *
112 * All passes are executed on all CPUs, but only the boot CPU determines what
113 * features the kernel will use.
114 *
115 * Much of the worst junk in this file is for the support of processors
116 * that didn't really implement the cpuid instruction properly.
117 *
118 * NOTE: The accessor functions (cpuid_get*) are aware of, and ASSERT upon,
119 * the pass numbers. Accordingly, changes to the pass code may require changes
120 * to the accessor code.
121 */
122
123 uint_t x86_vendor = X86_VENDOR_IntelClone;
124 uint_t x86_type = X86_TYPE_OTHER;
125 uint_t x86_clflush_size = 0;
126
127 #if defined(__xpv)
128 int x86_use_pcid = 0;
129 int x86_use_invpcid = 0;
130 #else
131 int x86_use_pcid = -1;
132 int x86_use_invpcid = -1;
133 #endif
134
135 uint_t pentiumpro_bug4046376;
136
137 uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
138
139 static char *x86_feature_names[NUM_X86_FEATURES] = {
140 "lgpg",
141 "tsc",
142 "msr",
143 "mtrr",
144 "pge",
145 "de",
146 "cmov",
147 "mmx",
148 "mca",
149 "pae",
150 "cv8",
151 "pat",
152 "sep",
153 "sse",
154 "sse2",
155 "htt",
156 "asysc",
157 "nx",
158 "sse3",
159 "cx16",
160 "cmp",
161 "tscp",
162 "mwait",
163 "sse4a",
164 "cpuid",
165 "ssse3",
166 "sse4_1",
167 "sse4_2",
168 "1gpg",
169 "clfsh",
170 "64",
171 "aes",
172 "pclmulqdq",
173 "xsave",
174 "avx",
175 "vmx",
176 "svm",
177 "topoext",
178 "f16c",
179 "rdrand",
180 "x2apic",
181 "avx2",
182 "bmi1",
183 "bmi2",
184 "fma",
185 "smep",
186 "smap",
187 "adx",
188 "rdseed",
189 "mpx",
190 "avx512f",
191 "avx512dq",
192 "avx512pf",
193 "avx512er",
194 "avx512cd",
195 "avx512bw",
196 "avx512vl",
197 "avx512fma",
198 "avx512vbmi",
199 "avx512_vpopcntdq",
200 "avx512_4vnniw",
201 "avx512_4fmaps",
202 "xsaveopt",
203 "xsavec",
204 "xsaves",
205 "sha",
206 "umip",
207 "pku",
208 "ospke",
209 "pcid",
210 "invpcid",
211 "ibrs",
212 "ibpb",
213 "stibp",
214 "ssbd",
215 "ssbd_virt",
216 "rdcl_no",
217 "ibrs_all",
218 "rsba",
219 "ssb_no",
220 "stibp_all"
221 };
222
223 boolean_t
224 is_x86_feature(void *featureset, uint_t feature)
225 {
226 ASSERT(feature < NUM_X86_FEATURES);
227 return (BT_TEST((ulong_t *)featureset, feature));
228 }
229
230 void
231 add_x86_feature(void *featureset, uint_t feature)
232 {
233 ASSERT(feature < NUM_X86_FEATURES);
234 BT_SET((ulong_t *)featureset, feature);
235 }
236
237 void
238 remove_x86_feature(void *featureset, uint_t feature)
239 {
240 ASSERT(feature < NUM_X86_FEATURES);
241 BT_CLEAR((ulong_t *)featureset, feature);
242 }
243
244 boolean_t
245 compare_x86_featureset(void *setA, void *setB)
246 {
247 /*
248 * We assume that the unused bits of the bitmap are always zero.
249 */
250 if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
251 return (B_TRUE);
252 } else {
253 return (B_FALSE);
254 }
255 }
256
257 void
258 print_x86_featureset(void *featureset)
259 {
260 uint_t i;
261
262 for (i = 0; i < NUM_X86_FEATURES; i++) {
263 if (is_x86_feature(featureset, i)) {
264 cmn_err(CE_CONT, "?x86_feature: %s\n",
265 x86_feature_names[i]);
266 }
267 }
268 }
269
270 /* Note: This is the maximum size for the CPU, not the size of the structure. */
271 static size_t xsave_state_size = 0;
272 uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
273 boolean_t xsave_force_disable = B_FALSE;
274 extern int disable_smap;
275
276 /*
277 * This is set to platform type we are running on.
278 */
279 static int platform_type = -1;
280
281 #if !defined(__xpv)
282 /*
283 * Variable to patch if hypervisor platform detection needs to be
284 * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
285 */
286 int enable_platform_detection = 1;
287 #endif
288
289 /*
290 * monitor/mwait info.
291 *
292 * size_actual and buf_actual are the real address and size allocated to get
293 * proper mwait_buf alignement. buf_actual and size_actual should be passed
294 * to kmem_free(). Currently kmem_alloc() and mwait happen to both use
295 * processor cache-line alignment, but this is not guarantied in the furture.
296 */
297 struct mwait_info {
298 size_t mon_min; /* min size to avoid missed wakeups */
299 size_t mon_max; /* size to avoid false wakeups */
300 size_t size_actual; /* size actually allocated */
301 void *buf_actual; /* memory actually allocated */
302 uint32_t support; /* processor support of monitor/mwait */
303 };
304
305 /*
306 * xsave/xrestor info.
307 *
308 * This structure contains HW feature bits and the size of the xsave save area.
309 * Note: the kernel declares a fixed size (AVX_XSAVE_SIZE) structure
310 * (xsave_state) to describe the xsave layout. However, at runtime the
311 * per-lwp xsave area is dynamically allocated based on xsav_max_size. The
312 * xsave_state structure simply represents the legacy layout of the beginning
313 * of the xsave area.
314 */
315 struct xsave_info {
316 uint32_t xsav_hw_features_low; /* Supported HW features */
317 uint32_t xsav_hw_features_high; /* Supported HW features */
318 size_t xsav_max_size; /* max size save area for HW features */
319 size_t ymm_size; /* AVX: size of ymm save area */
320 size_t ymm_offset; /* AVX: offset for ymm save area */
321 size_t bndregs_size; /* MPX: size of bndregs save area */
322 size_t bndregs_offset; /* MPX: offset for bndregs save area */
323 size_t bndcsr_size; /* MPX: size of bndcsr save area */
324 size_t bndcsr_offset; /* MPX: offset for bndcsr save area */
325 size_t opmask_size; /* AVX512: size of opmask save */
326 size_t opmask_offset; /* AVX512: offset for opmask save */
327 size_t zmmlo_size; /* AVX512: size of zmm 256 save */
328 size_t zmmlo_offset; /* AVX512: offset for zmm 256 save */
329 size_t zmmhi_size; /* AVX512: size of zmm hi reg save */
330 size_t zmmhi_offset; /* AVX512: offset for zmm hi reg save */
331 };
332
333
334 /*
335 * These constants determine how many of the elements of the
336 * cpuid we cache in the cpuid_info data structure; the
337 * remaining elements are accessible via the cpuid instruction.
338 */
339
340 #define NMAX_CPI_STD 8 /* eax = 0 .. 7 */
341 #define NMAX_CPI_EXTD 0x1f /* eax = 0x80000000 .. 0x8000001e */
342
343 /*
344 * Some terminology needs to be explained:
345 * - Socket: Something that can be plugged into a motherboard.
346 * - Package: Same as socket
347 * - Chip: Same as socket. Note that AMD's documentation uses term "chip"
348 * differently: there, chip is the same as processor node (below)
349 * - Processor node: Some AMD processors have more than one
350 * "subprocessor" embedded in a package. These subprocessors (nodes)
351 * are fully-functional processors themselves with cores, caches,
352 * memory controllers, PCI configuration spaces. They are connected
353 * inside the package with Hypertransport links. On single-node
354 * processors, processor node is equivalent to chip/socket/package.
355 * - Compute Unit: Some AMD processors pair cores in "compute units" that
356 * share the FPU and the I$ and L2 caches.
357 */
358
359 struct cpuid_info {
360 uint_t cpi_pass; /* last pass completed */
361 /*
362 * standard function information
363 */
364 uint_t cpi_maxeax; /* fn 0: %eax */
365 char cpi_vendorstr[13]; /* fn 0: %ebx:%ecx:%edx */
366 uint_t cpi_vendor; /* enum of cpi_vendorstr */
367
368 uint_t cpi_family; /* fn 1: extended family */
369 uint_t cpi_model; /* fn 1: extended model */
370 uint_t cpi_step; /* fn 1: stepping */
371 chipid_t cpi_chipid; /* fn 1: %ebx: Intel: chip # */
372 /* AMD: package/socket # */
373 uint_t cpi_brandid; /* fn 1: %ebx: brand ID */
374 int cpi_clogid; /* fn 1: %ebx: thread # */
375 uint_t cpi_ncpu_per_chip; /* fn 1: %ebx: logical cpu count */
376 uint8_t cpi_cacheinfo[16]; /* fn 2: intel-style cache desc */
377 uint_t cpi_ncache; /* fn 2: number of elements */
378 uint_t cpi_ncpu_shr_last_cache; /* fn 4: %eax: ncpus sharing cache */
379 id_t cpi_last_lvl_cacheid; /* fn 4: %eax: derived cache id */
380 uint_t cpi_std_4_size; /* fn 4: number of fn 4 elements */
381 struct cpuid_regs **cpi_std_4; /* fn 4: %ecx == 0 .. fn4_size */
382 struct cpuid_regs cpi_std[NMAX_CPI_STD]; /* 0 .. 7 */
383 /*
384 * extended function information
385 */
386 uint_t cpi_xmaxeax; /* fn 0x80000000: %eax */
387 char cpi_brandstr[49]; /* fn 0x8000000[234] */
388 uint8_t cpi_pabits; /* fn 0x80000006: %eax */
389 uint8_t cpi_vabits; /* fn 0x80000006: %eax */
390 uint8_t cpi_fp_amd_save; /* AMD: FP error pointer save rqd. */
391 struct cpuid_regs cpi_extd[NMAX_CPI_EXTD]; /* 0x800000XX */
392
393 id_t cpi_coreid; /* same coreid => strands share core */
394 int cpi_pkgcoreid; /* core number within single package */
395 uint_t cpi_ncore_per_chip; /* AMD: fn 0x80000008: %ecx[7-0] */
396 /* Intel: fn 4: %eax[31-26] */
397 /*
398 * supported feature information
399 */
400 uint32_t cpi_support[6];
401 #define STD_EDX_FEATURES 0
402 #define AMD_EDX_FEATURES 1
403 #define TM_EDX_FEATURES 2
404 #define STD_ECX_FEATURES 3
405 #define AMD_ECX_FEATURES 4
406 #define STD_EBX_FEATURES 5
407 /*
408 * Synthesized information, where known.
409 */
410 uint32_t cpi_chiprev; /* See X86_CHIPREV_* in x86_archext.h */
411 const char *cpi_chiprevstr; /* May be NULL if chiprev unknown */
412 uint32_t cpi_socket; /* Chip package/socket type */
413
414 struct mwait_info cpi_mwait; /* fn 5: monitor/mwait info */
415 uint32_t cpi_apicid;
416 uint_t cpi_procnodeid; /* AMD: nodeID on HT, Intel: chipid */
417 uint_t cpi_procnodes_per_pkg; /* AMD: # of nodes in the package */
418 /* Intel: 1 */
419 uint_t cpi_compunitid; /* AMD: ComputeUnit ID, Intel: coreid */
420 uint_t cpi_cores_per_compunit; /* AMD: # of cores in the ComputeUnit */
421
422 struct xsave_info cpi_xsave; /* fn D: xsave/xrestor info */
423 };
424
425
426 static struct cpuid_info cpuid_info0;
427
428 /*
429 * These bit fields are defined by the Intel Application Note AP-485
430 * "Intel Processor Identification and the CPUID Instruction"
431 */
432 #define CPI_FAMILY_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
433 #define CPI_MODEL_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
434 #define CPI_TYPE(cpi) BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
435 #define CPI_FAMILY(cpi) BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
436 #define CPI_STEP(cpi) BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
437 #define CPI_MODEL(cpi) BITX((cpi)->cpi_std[1].cp_eax, 7, 4)
438
439 #define CPI_FEATURES_EDX(cpi) ((cpi)->cpi_std[1].cp_edx)
440 #define CPI_FEATURES_ECX(cpi) ((cpi)->cpi_std[1].cp_ecx)
441 #define CPI_FEATURES_XTD_EDX(cpi) ((cpi)->cpi_extd[1].cp_edx)
442 #define CPI_FEATURES_XTD_ECX(cpi) ((cpi)->cpi_extd[1].cp_ecx)
443 #define CPI_FEATURES_7_0_EBX(cpi) ((cpi)->cpi_std[7].cp_ebx)
444 #define CPI_FEATURES_7_0_ECX(cpi) ((cpi)->cpi_std[7].cp_ecx)
445 #define CPI_FEATURES_7_0_EDX(cpi) ((cpi)->cpi_std[7].cp_edx)
446
447 #define CPI_BRANDID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
448 #define CPI_CHUNKS(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
449 #define CPI_CPU_COUNT(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
450 #define CPI_APIC_ID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)
451
452 #define CPI_MAXEAX_MAX 0x100 /* sanity control */
453 #define CPI_XMAXEAX_MAX 0x80000100
454 #define CPI_FN4_ECX_MAX 0x20 /* sanity: max fn 4 levels */
455 #define CPI_FNB_ECX_MAX 0x20 /* sanity: max fn B levels */
456
457 /*
458 * Function 4 (Deterministic Cache Parameters) macros
459 * Defined by Intel Application Note AP-485
460 */
461 #define CPI_NUM_CORES(regs) BITX((regs)->cp_eax, 31, 26)
462 #define CPI_NTHR_SHR_CACHE(regs) BITX((regs)->cp_eax, 25, 14)
463 #define CPI_FULL_ASSOC_CACHE(regs) BITX((regs)->cp_eax, 9, 9)
464 #define CPI_SELF_INIT_CACHE(regs) BITX((regs)->cp_eax, 8, 8)
465 #define CPI_CACHE_LVL(regs) BITX((regs)->cp_eax, 7, 5)
466 #define CPI_CACHE_TYPE(regs) BITX((regs)->cp_eax, 4, 0)
467 #define CPI_CPU_LEVEL_TYPE(regs) BITX((regs)->cp_ecx, 15, 8)
468
469 #define CPI_CACHE_WAYS(regs) BITX((regs)->cp_ebx, 31, 22)
470 #define CPI_CACHE_PARTS(regs) BITX((regs)->cp_ebx, 21, 12)
471 #define CPI_CACHE_COH_LN_SZ(regs) BITX((regs)->cp_ebx, 11, 0)
472
473 #define CPI_CACHE_SETS(regs) BITX((regs)->cp_ecx, 31, 0)
474
475 #define CPI_PREFCH_STRIDE(regs) BITX((regs)->cp_edx, 9, 0)
476
477
478 /*
479 * A couple of shorthand macros to identify "later" P6-family chips
480 * like the Pentium M and Core. First, the "older" P6-based stuff
481 * (loosely defined as "pre-Pentium-4"):
482 * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
483 */
484 #define IS_LEGACY_P6(cpi) ( \
485 cpi->cpi_family == 6 && \
486 (cpi->cpi_model == 1 || \
487 cpi->cpi_model == 3 || \
488 cpi->cpi_model == 5 || \
489 cpi->cpi_model == 6 || \
490 cpi->cpi_model == 7 || \
491 cpi->cpi_model == 8 || \
492 cpi->cpi_model == 0xA || \
493 cpi->cpi_model == 0xB) \
494 )
495
496 /* A "new F6" is everything with family 6 that's not the above */
497 #define IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))
498
499 /* Extended family/model support */
500 #define IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
501 cpi->cpi_family >= 0xf)
502
503 /*
504 * Info for monitor/mwait idle loop.
505 *
506 * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
507 * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
508 * 2006.
509 * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
510 * Documentation Updates" #33633, Rev 2.05, December 2006.
511 */
512 #define MWAIT_SUPPORT (0x00000001) /* mwait supported */
513 #define MWAIT_EXTENSIONS (0x00000002) /* extenstion supported */
514 #define MWAIT_ECX_INT_ENABLE (0x00000004) /* ecx 1 extension supported */
515 #define MWAIT_SUPPORTED(cpi) ((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
516 #define MWAIT_INT_ENABLE(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x2)
517 #define MWAIT_EXTENSION(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x1)
518 #define MWAIT_SIZE_MIN(cpi) BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
519 #define MWAIT_SIZE_MAX(cpi) BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
520 /*
521 * Number of sub-cstates for a given c-state.
522 */
523 #define MWAIT_NUM_SUBC_STATES(cpi, c_state) \
524 BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)
525
526 /*
527 * XSAVE leaf 0xD enumeration
528 */
529 #define CPUID_LEAFD_2_YMM_OFFSET 576
530 #define CPUID_LEAFD_2_YMM_SIZE 256
531
532 /*
533 * Functions we consune from cpuid_subr.c; don't publish these in a header
534 * file to try and keep people using the expected cpuid_* interfaces.
535 */
536 extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
537 extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
538 extern uint32_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
539 extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
540 extern uint_t _cpuid_vendorstr_to_vendorcode(char *);
541
542 /*
543 * Apply up various platform-dependent restrictions where the
544 * underlying platform restrictions mean the CPU can be marked
545 * as less capable than its cpuid instruction would imply.
546 */
547 #if defined(__xpv)
548 static void
549 platform_cpuid_mangle(uint_t vendor, uint32_t eax, struct cpuid_regs *cp)
550 {
551 switch (eax) {
552 case 1: {
553 uint32_t mcamask = DOMAIN_IS_INITDOMAIN(xen_info) ?
554 0 : CPUID_INTC_EDX_MCA;
555 cp->cp_edx &=
556 ~(mcamask |
557 CPUID_INTC_EDX_PSE |
558 CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
559 CPUID_INTC_EDX_SEP | CPUID_INTC_EDX_MTRR |
560 CPUID_INTC_EDX_PGE | CPUID_INTC_EDX_PAT |
561 CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
562 CPUID_INTC_EDX_PSE36 | CPUID_INTC_EDX_HTT);
563 break;
564 }
565
566 case 0x80000001:
567 cp->cp_edx &=
568 ~(CPUID_AMD_EDX_PSE |
569 CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
570 CPUID_AMD_EDX_MTRR | CPUID_AMD_EDX_PGE |
571 CPUID_AMD_EDX_PAT | CPUID_AMD_EDX_PSE36 |
572 CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
573 CPUID_AMD_EDX_TSCP);
574 cp->cp_ecx &= ~CPUID_AMD_ECX_CMP_LGCY;
575 break;
576 default:
577 break;
578 }
579
580 switch (vendor) {
581 case X86_VENDOR_Intel:
582 switch (eax) {
583 case 4:
584 /*
585 * Zero out the (ncores-per-chip - 1) field
586 */
587 cp->cp_eax &= 0x03fffffff;
588 break;
589 default:
590 break;
591 }
592 break;
593 case X86_VENDOR_AMD:
594 switch (eax) {
595
596 case 0x80000001:
597 cp->cp_ecx &= ~CPUID_AMD_ECX_CR8D;
598 break;
599
600 case 0x80000008:
601 /*
602 * Zero out the (ncores-per-chip - 1) field
603 */
604 cp->cp_ecx &= 0xffffff00;
605 break;
606 default:
607 break;
608 }
609 break;
610 default:
611 break;
612 }
613 }
614 #else
615 #define platform_cpuid_mangle(vendor, eax, cp) /* nothing */
616 #endif
617
618 /*
619 * Some undocumented ways of patching the results of the cpuid
620 * instruction to permit running Solaris 10 on future cpus that
621 * we don't currently support. Could be set to non-zero values
622 * via settings in eeprom.
623 */
624
625 uint32_t cpuid_feature_ecx_include;
626 uint32_t cpuid_feature_ecx_exclude;
627 uint32_t cpuid_feature_edx_include;
628 uint32_t cpuid_feature_edx_exclude;
629
630 /*
631 * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
632 */
633 void
634 cpuid_alloc_space(cpu_t *cpu)
635 {
636 /*
637 * By convention, cpu0 is the boot cpu, which is set up
638 * before memory allocation is available. All other cpus get
639 * their cpuid_info struct allocated here.
640 */
641 ASSERT(cpu->cpu_id != 0);
642 ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
643 cpu->cpu_m.mcpu_cpi =
644 kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
645 }
646
647 void
648 cpuid_free_space(cpu_t *cpu)
649 {
650 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
651 int i;
652
653 ASSERT(cpi != NULL);
654 ASSERT(cpi != &cpuid_info0);
655
656 /*
657 * Free up any function 4 related dynamic storage
658 */
659 for (i = 1; i < cpi->cpi_std_4_size; i++)
660 kmem_free(cpi->cpi_std_4[i], sizeof (struct cpuid_regs));
661 if (cpi->cpi_std_4_size > 0)
662 kmem_free(cpi->cpi_std_4,
663 cpi->cpi_std_4_size * sizeof (struct cpuid_regs *));
664
665 kmem_free(cpi, sizeof (*cpi));
666 cpu->cpu_m.mcpu_cpi = NULL;
667 }
668
669 #if !defined(__xpv)
670 /*
671 * Determine the type of the underlying platform. This is used to customize
672 * initialization of various subsystems (e.g. TSC). determine_platform() must
673 * only ever be called once to prevent two processors from seeing different
674 * values of platform_type. Must be called before cpuid_pass1(), the earliest
675 * consumer to execute (uses _cpuid_chiprev --> synth_amd_info --> get_hwenv).
676 */
677 void
678 determine_platform(void)
679 {
680 struct cpuid_regs cp;
681 uint32_t base;
682 uint32_t regs[4];
683 char *hvstr = (char *)regs;
684
685 ASSERT(platform_type == -1);
686
687 platform_type = HW_NATIVE;
688
689 if (!enable_platform_detection)
690 return;
691
692 /*
693 * If Hypervisor CPUID bit is set, try to determine hypervisor
694 * vendor signature, and set platform type accordingly.
695 *
696 * References:
697 * http://lkml.org/lkml/2008/10/1/246
698 * http://kb.vmware.com/kb/1009458
699 */
700 cp.cp_eax = 0x1;
701 (void) __cpuid_insn(&cp);
702 if ((cp.cp_ecx & CPUID_INTC_ECX_HV) != 0) {
703 cp.cp_eax = 0x40000000;
704 (void) __cpuid_insn(&cp);
705 regs[0] = cp.cp_ebx;
706 regs[1] = cp.cp_ecx;
707 regs[2] = cp.cp_edx;
708 regs[3] = 0;
709 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0) {
710 platform_type = HW_XEN_HVM;
711 return;
712 }
713 if (strcmp(hvstr, HVSIG_VMWARE) == 0) {
714 platform_type = HW_VMWARE;
715 return;
716 }
717 if (strcmp(hvstr, HVSIG_KVM) == 0) {
718 platform_type = HW_KVM;
719 return;
720 }
721 if (strcmp(hvstr, HVSIG_BHYVE) == 0) {
722 platform_type = HW_BHYVE;
723 return;
724 }
725 if (strcmp(hvstr, HVSIG_MICROSOFT) == 0)
726 platform_type = HW_MICROSOFT;
727 } else {
728 /*
729 * Check older VMware hardware versions. VMware hypervisor is
730 * detected by performing an IN operation to VMware hypervisor
731 * port and checking that value returned in %ebx is VMware
732 * hypervisor magic value.
733 *
734 * References: http://kb.vmware.com/kb/1009458
735 */
736 vmware_port(VMWARE_HVCMD_GETVERSION, regs);
737 if (regs[1] == VMWARE_HVMAGIC) {
738 platform_type = HW_VMWARE;
739 return;
740 }
741 }
742
743 /*
744 * Check Xen hypervisor. In a fully virtualized domain,
745 * Xen's pseudo-cpuid function returns a string representing the
746 * Xen signature in %ebx, %ecx, and %edx. %eax contains the maximum
747 * supported cpuid function. We need at least a (base + 2) leaf value
748 * to do what we want to do. Try different base values, since the
749 * hypervisor might use a different one depending on whether Hyper-V
750 * emulation is switched on by default or not.
751 */
752 for (base = 0x40000000; base < 0x40010000; base += 0x100) {
753 cp.cp_eax = base;
754 (void) __cpuid_insn(&cp);
755 regs[0] = cp.cp_ebx;
756 regs[1] = cp.cp_ecx;
757 regs[2] = cp.cp_edx;
758 regs[3] = 0;
759 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0 &&
760 cp.cp_eax >= (base + 2)) {
761 platform_type &= ~HW_NATIVE;
762 platform_type |= HW_XEN_HVM;
763 return;
764 }
765 }
766 }
767
768 int
769 get_hwenv(void)
770 {
771 ASSERT(platform_type != -1);
772 return (platform_type);
773 }
774
775 int
776 is_controldom(void)
777 {
778 return (0);
779 }
780
781 #else
782
783 int
784 get_hwenv(void)
785 {
786 return (HW_XEN_PV);
787 }
788
789 int
790 is_controldom(void)
791 {
792 return (DOMAIN_IS_INITDOMAIN(xen_info));
793 }
794
795 #endif /* __xpv */
796
797 static void
798 cpuid_intel_getids(cpu_t *cpu, void *feature)
799 {
800 uint_t i;
801 uint_t chipid_shift = 0;
802 uint_t coreid_shift = 0;
803 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
804
805 for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
806 chipid_shift++;
807
808 cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
809 cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);
810
811 if (is_x86_feature(feature, X86FSET_CMP)) {
812 /*
813 * Multi-core (and possibly multi-threaded)
814 * processors.
815 */
816 uint_t ncpu_per_core;
817 if (cpi->cpi_ncore_per_chip == 1)
818 ncpu_per_core = cpi->cpi_ncpu_per_chip;
819 else if (cpi->cpi_ncore_per_chip > 1)
820 ncpu_per_core = cpi->cpi_ncpu_per_chip /
821 cpi->cpi_ncore_per_chip;
822 /*
823 * 8bit APIC IDs on dual core Pentiums
824 * look like this:
825 *
826 * +-----------------------+------+------+
827 * | Physical Package ID | MC | HT |
828 * +-----------------------+------+------+
829 * <------- chipid -------->
830 * <------- coreid --------------->
831 * <--- clogid -->
832 * <------>
833 * pkgcoreid
834 *
835 * Where the number of bits necessary to
836 * represent MC and HT fields together equals
837 * to the minimum number of bits necessary to
838 * store the value of cpi->cpi_ncpu_per_chip.
839 * Of those bits, the MC part uses the number
840 * of bits necessary to store the value of
841 * cpi->cpi_ncore_per_chip.
842 */
843 for (i = 1; i < ncpu_per_core; i <<= 1)
844 coreid_shift++;
845 cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
846 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
847 } else if (is_x86_feature(feature, X86FSET_HTT)) {
848 /*
849 * Single-core multi-threaded processors.
850 */
851 cpi->cpi_coreid = cpi->cpi_chipid;
852 cpi->cpi_pkgcoreid = 0;
853 }
854 cpi->cpi_procnodeid = cpi->cpi_chipid;
855 cpi->cpi_compunitid = cpi->cpi_coreid;
856 }
857
858 static void
859 cpuid_amd_getids(cpu_t *cpu)
860 {
861 int i, first_half, coreidsz;
862 uint32_t nb_caps_reg;
863 uint_t node2_1;
864 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
865 struct cpuid_regs *cp;
866
867 /*
868 * AMD CMP chips currently have a single thread per core.
869 *
870 * Since no two cpus share a core we must assign a distinct coreid
871 * per cpu, and we do this by using the cpu_id. This scheme does not,
872 * however, guarantee that sibling cores of a chip will have sequential
873 * coreids starting at a multiple of the number of cores per chip -
874 * that is usually the case, but if the ACPI MADT table is presented
875 * in a different order then we need to perform a few more gymnastics
876 * for the pkgcoreid.
877 *
878 * All processors in the system have the same number of enabled
879 * cores. Cores within a processor are always numbered sequentially
880 * from 0 regardless of how many or which are disabled, and there
881 * is no way for operating system to discover the real core id when some
882 * are disabled.
883 *
884 * In family 0x15, the cores come in pairs called compute units. They
885 * share I$ and L2 caches and the FPU. Enumeration of this feature is
886 * simplified by the new topology extensions CPUID leaf, indicated by
887 * the X86 feature X86FSET_TOPOEXT.
888 */
889
890 cpi->cpi_coreid = cpu->cpu_id;
891 cpi->cpi_compunitid = cpu->cpu_id;
892
893 if (cpi->cpi_xmaxeax >= 0x80000008) {
894
895 coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);
896
897 /*
898 * In AMD parlance chip is really a node while Solaris
899 * sees chip as equivalent to socket/package.
900 */
901 cpi->cpi_ncore_per_chip =
902 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
903 if (coreidsz == 0) {
904 /* Use legacy method */
905 for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
906 coreidsz++;
907 if (coreidsz == 0)
908 coreidsz = 1;
909 }
910 } else {
911 /* Assume single-core part */
912 cpi->cpi_ncore_per_chip = 1;
913 coreidsz = 1;
914 }
915
916 cpi->cpi_clogid = cpi->cpi_pkgcoreid =
917 cpi->cpi_apicid & ((1<<coreidsz) - 1);
918 cpi->cpi_ncpu_per_chip = cpi->cpi_ncore_per_chip;
919
920 /* Get node ID, compute unit ID */
921 if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
922 cpi->cpi_xmaxeax >= 0x8000001e) {
923 cp = &cpi->cpi_extd[0x1e];
924 cp->cp_eax = 0x8000001e;
925 (void) __cpuid_insn(cp);
926
927 cpi->cpi_procnodes_per_pkg = BITX(cp->cp_ecx, 10, 8) + 1;
928 cpi->cpi_procnodeid = BITX(cp->cp_ecx, 7, 0);
929 cpi->cpi_cores_per_compunit = BITX(cp->cp_ebx, 15, 8) + 1;
930 cpi->cpi_compunitid = BITX(cp->cp_ebx, 7, 0)
931 + (cpi->cpi_ncore_per_chip / cpi->cpi_cores_per_compunit)
932 * (cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg);
933 } else if (cpi->cpi_family == 0xf || cpi->cpi_family >= 0x11) {
934 cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
935 } else if (cpi->cpi_family == 0x10) {
936 /*
937 * See if we are a multi-node processor.
938 * All processors in the system have the same number of nodes
939 */
940 nb_caps_reg = pci_getl_func(0, 24, 3, 0xe8);
941 if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
942 /* Single-node */
943 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
944 coreidsz);
945 } else {
946
947 /*
948 * Multi-node revision D (2 nodes per package
949 * are supported)
950 */
951 cpi->cpi_procnodes_per_pkg = 2;
952
953 first_half = (cpi->cpi_pkgcoreid <=
954 (cpi->cpi_ncore_per_chip/2 - 1));
955
956 if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
957 /* We are BSP */
958 cpi->cpi_procnodeid = (first_half ? 0 : 1);
959 } else {
960
961 /* We are AP */
962 /* NodeId[2:1] bits to use for reading F3xe8 */
963 node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;
964
965 nb_caps_reg =
966 pci_getl_func(0, 24 + node2_1, 3, 0xe8);
967
968 /*
969 * Check IntNodeNum bit (31:30, but bit 31 is
970 * always 0 on dual-node processors)
971 */
972 if (BITX(nb_caps_reg, 30, 30) == 0)
973 cpi->cpi_procnodeid = node2_1 +
974 !first_half;
975 else
976 cpi->cpi_procnodeid = node2_1 +
977 first_half;
978 }
979 }
980 } else {
981 cpi->cpi_procnodeid = 0;
982 }
983
984 cpi->cpi_chipid =
985 cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg;
986 }
987
988 static void
989 cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
990 {
991 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
992
993 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
994 cpi->cpi_xmaxeax >= 0x80000008) {
995 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBPB)
996 add_x86_feature(featureset, X86FSET_IBPB);
997 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBRS)
998 add_x86_feature(featureset, X86FSET_IBRS);
999 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP)
1000 add_x86_feature(featureset, X86FSET_STIBP);
1001 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBRS_ALL)
1002 add_x86_feature(featureset, X86FSET_IBRS_ALL);
1003 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP_ALL)
1004 add_x86_feature(featureset, X86FSET_STIBP_ALL);
1005 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_PREFER_IBRS)
1006 add_x86_feature(featureset, X86FSET_RSBA);
1007 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSBD)
1008 add_x86_feature(featureset, X86FSET_SSBD);
1009 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_VIRT_SSBD)
1010 add_x86_feature(featureset, X86FSET_SSBD_VIRT);
1011 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSB_NO)
1012 add_x86_feature(featureset, X86FSET_SSB_NO);
1013 } else if (cpi->cpi_vendor == X86_VENDOR_Intel &&
1014 cpi->cpi_maxeax >= 7) {
1015 struct cpuid_regs *ecp;
1016 ecp = &cpi->cpi_std[7];
1017
1018 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SPEC_CTRL) {
1019 add_x86_feature(featureset, X86FSET_IBRS);
1020 add_x86_feature(featureset, X86FSET_IBPB);
1021 }
1022
1023 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_STIBP) {
1024 add_x86_feature(featureset, X86FSET_STIBP);
1025 }
1026
1027 /*
1028 * Don't read the arch caps MSR on xpv where we lack the
1029 * on_trap().
1030 */
1031 #ifndef __xpv
1032 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_ARCH_CAPS) {
1033 on_trap_data_t otd;
1034
1035 /*
1036 * Be paranoid and assume we'll get a #GP.
1037 */
1038 if (!on_trap(&otd, OT_DATA_ACCESS)) {
1039 uint64_t reg;
1040
1041 reg = rdmsr(MSR_IA32_ARCH_CAPABILITIES);
1042 if (reg & IA32_ARCH_CAP_RDCL_NO) {
1043 add_x86_feature(featureset,
1044 X86FSET_RDCL_NO);
1045 }
1046 if (reg & IA32_ARCH_CAP_IBRS_ALL) {
1047 add_x86_feature(featureset,
1048 X86FSET_IBRS_ALL);
1049 }
1050 if (reg & IA32_ARCH_CAP_RSBA) {
1051 add_x86_feature(featureset,
1052 X86FSET_RSBA);
1053 }
1054 if (reg & IA32_ARCH_CAP_SSB_NO) {
1055 add_x86_feature(featureset,
1056 X86FSET_SSB_NO);
1057 }
1058 }
1059 no_trap();
1060 }
1061 #endif /* !__xpv */
1062
1063 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SSBD)
1064 add_x86_feature(featureset, X86FSET_SSBD);
1065 }
1066 }
1067
1068 /*
1069 * Setup XFeature_Enabled_Mask register. Required by xsave feature.
1070 */
1071 void
1072 setup_xfem(void)
1073 {
1074 uint64_t flags = XFEATURE_LEGACY_FP;
1075
1076 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
1077
1078 if (is_x86_feature(x86_featureset, X86FSET_SSE))
1079 flags |= XFEATURE_SSE;
1080
1081 if (is_x86_feature(x86_featureset, X86FSET_AVX))
1082 flags |= XFEATURE_AVX;
1083
1084 if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
1085 flags |= XFEATURE_AVX512;
1086
1087 set_xcr(XFEATURE_ENABLED_MASK, flags);
1088
1089 xsave_bv_all = flags;
1090 }
1091
1092 void
1093 cpuid_pass1(cpu_t *cpu, uchar_t *featureset)
1094 {
1095 uint32_t mask_ecx, mask_edx;
1096 struct cpuid_info *cpi;
1097 struct cpuid_regs *cp;
1098 int xcpuid;
1099 #if !defined(__xpv)
1100 extern int idle_cpu_prefer_mwait;
1101 #endif
1102
1103 /*
1104 * Space statically allocated for BSP, ensure pointer is set
1105 */
1106 if (cpu->cpu_id == 0) {
1107 if (cpu->cpu_m.mcpu_cpi == NULL)
1108 cpu->cpu_m.mcpu_cpi = &cpuid_info0;
1109 }
1110
1111 add_x86_feature(featureset, X86FSET_CPUID);
1112
1113 cpi = cpu->cpu_m.mcpu_cpi;
1114 ASSERT(cpi != NULL);
1115 cp = &cpi->cpi_std[0];
1116 cp->cp_eax = 0;
1117 cpi->cpi_maxeax = __cpuid_insn(cp);
1118 {
1119 uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
1120 *iptr++ = cp->cp_ebx;
1121 *iptr++ = cp->cp_edx;
1122 *iptr++ = cp->cp_ecx;
1123 *(char *)&cpi->cpi_vendorstr[12] = '\0';
1124 }
1125
1126 cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
1127 x86_vendor = cpi->cpi_vendor; /* for compatibility */
1128
1129 /*
1130 * Limit the range in case of weird hardware
1131 */
1132 if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
1133 cpi->cpi_maxeax = CPI_MAXEAX_MAX;
1134 if (cpi->cpi_maxeax < 1)
1135 goto pass1_done;
1136
1137 cp = &cpi->cpi_std[1];
1138 cp->cp_eax = 1;
1139 (void) __cpuid_insn(cp);
1140
1141 /*
1142 * Extract identifying constants for easy access.
1143 */
1144 cpi->cpi_model = CPI_MODEL(cpi);
1145 cpi->cpi_family = CPI_FAMILY(cpi);
1146
1147 if (cpi->cpi_family == 0xf)
1148 cpi->cpi_family += CPI_FAMILY_XTD(cpi);
1149
1150 /*
1151 * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
1152 * Intel, and presumably everyone else, uses model == 0xf, as
1153 * one would expect (max value means possible overflow). Sigh.
1154 */
1155
1156 switch (cpi->cpi_vendor) {
1157 case X86_VENDOR_Intel:
1158 if (IS_EXTENDED_MODEL_INTEL(cpi))
1159 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1160 break;
1161 case X86_VENDOR_AMD:
1162 if (CPI_FAMILY(cpi) == 0xf)
1163 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1164 break;
1165 default:
1166 if (cpi->cpi_model == 0xf)
1167 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1168 break;
1169 }
1170
1171 cpi->cpi_step = CPI_STEP(cpi);
1172 cpi->cpi_brandid = CPI_BRANDID(cpi);
1173
1174 /*
1175 * *default* assumptions:
1176 * - believe %edx feature word
1177 * - ignore %ecx feature word
1178 * - 32-bit virtual and physical addressing
1179 */
1180 mask_edx = 0xffffffff;
1181 mask_ecx = 0;
1182
1183 cpi->cpi_pabits = cpi->cpi_vabits = 32;
1184
1185 switch (cpi->cpi_vendor) {
1186 case X86_VENDOR_Intel:
1187 if (cpi->cpi_family == 5)
1188 x86_type = X86_TYPE_P5;
1189 else if (IS_LEGACY_P6(cpi)) {
1190 x86_type = X86_TYPE_P6;
1191 pentiumpro_bug4046376 = 1;
1192 /*
1193 * Clear the SEP bit when it was set erroneously
1194 */
1195 if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
1196 cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
1197 } else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
1198 x86_type = X86_TYPE_P4;
1199 /*
1200 * We don't currently depend on any of the %ecx
1201 * features until Prescott, so we'll only check
1202 * this from P4 onwards. We might want to revisit
1203 * that idea later.
1204 */
1205 mask_ecx = 0xffffffff;
1206 } else if (cpi->cpi_family > 0xf)
1207 mask_ecx = 0xffffffff;
1208 /*
1209 * We don't support MONITOR/MWAIT if leaf 5 is not available
1210 * to obtain the monitor linesize.
1211 */
1212 if (cpi->cpi_maxeax < 5)
1213 mask_ecx &= ~CPUID_INTC_ECX_MON;
1214 break;
1215 case X86_VENDOR_IntelClone:
1216 default:
1217 break;
1218 case X86_VENDOR_AMD:
1219 #if defined(OPTERON_ERRATUM_108)
1220 if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
1221 cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
1222 cpi->cpi_model = 0xc;
1223 } else
1224 #endif
1225 if (cpi->cpi_family == 5) {
1226 /*
1227 * AMD K5 and K6
1228 *
1229 * These CPUs have an incomplete implementation
1230 * of MCA/MCE which we mask away.
1231 */
1232 mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);
1233
1234 /*
1235 * Model 0 uses the wrong (APIC) bit
1236 * to indicate PGE. Fix it here.
1237 */
1238 if (cpi->cpi_model == 0) {
1239 if (cp->cp_edx & 0x200) {
1240 cp->cp_edx &= ~0x200;
1241 cp->cp_edx |= CPUID_INTC_EDX_PGE;
1242 }
1243 }
1244
1245 /*
1246 * Early models had problems w/ MMX; disable.
1247 */
1248 if (cpi->cpi_model < 6)
1249 mask_edx &= ~CPUID_INTC_EDX_MMX;
1250 }
1251
1252 /*
1253 * For newer families, SSE3 and CX16, at least, are valid;
1254 * enable all
1255 */
1256 if (cpi->cpi_family >= 0xf)
1257 mask_ecx = 0xffffffff;
1258 /*
1259 * We don't support MONITOR/MWAIT if leaf 5 is not available
1260 * to obtain the monitor linesize.
1261 */
1262 if (cpi->cpi_maxeax < 5)
1263 mask_ecx &= ~CPUID_INTC_ECX_MON;
1264
1265 #if !defined(__xpv)
1266 /*
1267 * Do not use MONITOR/MWAIT to halt in the idle loop on any AMD
1268 * processors. AMD does not intend MWAIT to be used in the cpu
1269 * idle loop on current and future processors. 10h and future
1270 * AMD processors use more power in MWAIT than HLT.
1271 * Pre-family-10h Opterons do not have the MWAIT instruction.
1272 */
1273 idle_cpu_prefer_mwait = 0;
1274 #endif
1275
1276 break;
1277 case X86_VENDOR_TM:
1278 /*
1279 * workaround the NT workaround in CMS 4.1
1280 */
1281 if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
1282 (cpi->cpi_step == 2 || cpi->cpi_step == 3))
1283 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1284 break;
1285 case X86_VENDOR_Centaur:
1286 /*
1287 * workaround the NT workarounds again
1288 */
1289 if (cpi->cpi_family == 6)
1290 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1291 break;
1292 case X86_VENDOR_Cyrix:
1293 /*
1294 * We rely heavily on the probing in locore
1295 * to actually figure out what parts, if any,
1296 * of the Cyrix cpuid instruction to believe.
1297 */
1298 switch (x86_type) {
1299 case X86_TYPE_CYRIX_486:
1300 mask_edx = 0;
1301 break;
1302 case X86_TYPE_CYRIX_6x86:
1303 mask_edx = 0;
1304 break;
1305 case X86_TYPE_CYRIX_6x86L:
1306 mask_edx =
1307 CPUID_INTC_EDX_DE |
1308 CPUID_INTC_EDX_CX8;
1309 break;
1310 case X86_TYPE_CYRIX_6x86MX:
1311 mask_edx =
1312 CPUID_INTC_EDX_DE |
1313 CPUID_INTC_EDX_MSR |
1314 CPUID_INTC_EDX_CX8 |
1315 CPUID_INTC_EDX_PGE |
1316 CPUID_INTC_EDX_CMOV |
1317 CPUID_INTC_EDX_MMX;
1318 break;
1319 case X86_TYPE_CYRIX_GXm:
1320 mask_edx =
1321 CPUID_INTC_EDX_MSR |
1322 CPUID_INTC_EDX_CX8 |
1323 CPUID_INTC_EDX_CMOV |
1324 CPUID_INTC_EDX_MMX;
1325 break;
1326 case X86_TYPE_CYRIX_MediaGX:
1327 break;
1328 case X86_TYPE_CYRIX_MII:
1329 case X86_TYPE_VIA_CYRIX_III:
1330 mask_edx =
1331 CPUID_INTC_EDX_DE |
1332 CPUID_INTC_EDX_TSC |
1333 CPUID_INTC_EDX_MSR |
1334 CPUID_INTC_EDX_CX8 |
1335 CPUID_INTC_EDX_PGE |
1336 CPUID_INTC_EDX_CMOV |
1337 CPUID_INTC_EDX_MMX;
1338 break;
1339 default:
1340 break;
1341 }
1342 break;
1343 }
1344
1345 #if defined(__xpv)
1346 /*
1347 * Do not support MONITOR/MWAIT under a hypervisor
1348 */
1349 mask_ecx &= ~CPUID_INTC_ECX_MON;
1350 /*
1351 * Do not support XSAVE under a hypervisor for now
1352 */
1353 xsave_force_disable = B_TRUE;
1354
1355 #endif /* __xpv */
1356
1357 if (xsave_force_disable) {
1358 mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
1359 mask_ecx &= ~CPUID_INTC_ECX_AVX;
1360 mask_ecx &= ~CPUID_INTC_ECX_F16C;
1361 mask_ecx &= ~CPUID_INTC_ECX_FMA;
1362 }
1363
1364 /*
1365 * Now we've figured out the masks that determine
1366 * which bits we choose to believe, apply the masks
1367 * to the feature words, then map the kernel's view
1368 * of these feature words into its feature word.
1369 */
1370 cp->cp_edx &= mask_edx;
1371 cp->cp_ecx &= mask_ecx;
1372
1373 /*
1374 * apply any platform restrictions (we don't call this
1375 * immediately after __cpuid_insn here, because we need the
1376 * workarounds applied above first)
1377 */
1378 platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);
1379
1380 /*
1381 * In addition to ecx and edx, Intel and AMD are storing a bunch of
1382 * instruction set extensions in leaf 7's ebx, ecx, and edx.
1383 */
1384 if (cpi->cpi_maxeax >= 7) {
1385 struct cpuid_regs *ecp;
1386 ecp = &cpi->cpi_std[7];
1387 ecp->cp_eax = 7;
1388 ecp->cp_ecx = 0;
1389 (void) __cpuid_insn(ecp);
1390
1391 /*
1392 * If XSAVE has been disabled, just ignore all of the
1393 * extended-save-area dependent flags here.
1394 */
1395 if (xsave_force_disable) {
1396 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
1397 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
1398 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
1399 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_MPX;
1400 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_ALL_AVX512;
1401 ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_ALL_AVX512;
1402 ecp->cp_edx &= ~CPUID_INTC_EDX_7_0_ALL_AVX512;
1403 }
1404
1405 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMEP)
1406 add_x86_feature(featureset, X86FSET_SMEP);
1407
1408 /*
1409 * We check disable_smap here in addition to in startup_smap()
1410 * to ensure CPUs that aren't the boot CPU don't accidentally
1411 * include it in the feature set and thus generate a mismatched
1412 * x86 feature set across CPUs.
1413 */
1414 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMAP &&
1415 disable_smap == 0)
1416 add_x86_feature(featureset, X86FSET_SMAP);
1417
1418 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_RDSEED)
1419 add_x86_feature(featureset, X86FSET_RDSEED);
1420
1421 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_ADX)
1422 add_x86_feature(featureset, X86FSET_ADX);
1423
1424 if (cpi->cpi_vendor == X86_VENDOR_Intel) {
1425 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_INVPCID)
1426 add_x86_feature(featureset, X86FSET_INVPCID);
1427
1428 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_MPX)
1429 add_x86_feature(featureset, X86FSET_MPX);
1430 }
1431 }
1432
1433 /*
1434 * fold in overrides from the "eeprom" mechanism
1435 */
1436 cp->cp_edx |= cpuid_feature_edx_include;
1437 cp->cp_edx &= ~cpuid_feature_edx_exclude;
1438
1439 cp->cp_ecx |= cpuid_feature_ecx_include;
1440 cp->cp_ecx &= ~cpuid_feature_ecx_exclude;
1441
1442 if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
1443 add_x86_feature(featureset, X86FSET_LARGEPAGE);
1444 }
1445 if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
1446 add_x86_feature(featureset, X86FSET_TSC);
1447 }
1448 if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
1449 add_x86_feature(featureset, X86FSET_MSR);
1450 }
1451 if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
1452 add_x86_feature(featureset, X86FSET_MTRR);
1453 }
1454 if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
1455 add_x86_feature(featureset, X86FSET_PGE);
1456 }
1457 if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
1458 add_x86_feature(featureset, X86FSET_CMOV);
1459 }
1460 if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
1461 add_x86_feature(featureset, X86FSET_MMX);
1462 }
1463 if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
1464 (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
1465 add_x86_feature(featureset, X86FSET_MCA);
1466 }
1467 if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
1468 add_x86_feature(featureset, X86FSET_PAE);
1469 }
1470 if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
1471 add_x86_feature(featureset, X86FSET_CX8);
1472 }
1473 if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
1474 add_x86_feature(featureset, X86FSET_CX16);
1475 }
1476 if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
1477 add_x86_feature(featureset, X86FSET_PAT);
1478 }
1479 if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
1480 add_x86_feature(featureset, X86FSET_SEP);
1481 }
1482 if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
1483 /*
1484 * In our implementation, fxsave/fxrstor
1485 * are prerequisites before we'll even
1486 * try and do SSE things.
1487 */
1488 if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
1489 add_x86_feature(featureset, X86FSET_SSE);
1490 }
1491 if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
1492 add_x86_feature(featureset, X86FSET_SSE2);
1493 }
1494 if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
1495 add_x86_feature(featureset, X86FSET_SSE3);
1496 }
1497 if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
1498 add_x86_feature(featureset, X86FSET_SSSE3);
1499 }
1500 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
1501 add_x86_feature(featureset, X86FSET_SSE4_1);
1502 }
1503 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
1504 add_x86_feature(featureset, X86FSET_SSE4_2);
1505 }
1506 if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
1507 add_x86_feature(featureset, X86FSET_AES);
1508 }
1509 if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
1510 add_x86_feature(featureset, X86FSET_PCLMULQDQ);
1511 }
1512
1513 if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_SHA)
1514 add_x86_feature(featureset, X86FSET_SHA);
1515
1516 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_UMIP)
1517 add_x86_feature(featureset, X86FSET_UMIP);
1518 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_PKU)
1519 add_x86_feature(featureset, X86FSET_PKU);
1520 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_OSPKE)
1521 add_x86_feature(featureset, X86FSET_OSPKE);
1522
1523 if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
1524 add_x86_feature(featureset, X86FSET_XSAVE);
1525
1526 /* We only test AVX & AVX512 when there is XSAVE */
1527
1528 if (cp->cp_ecx & CPUID_INTC_ECX_AVX) {
1529 add_x86_feature(featureset,
1530 X86FSET_AVX);
1531
1532 /*
1533 * Intel says we can't check these without also
1534 * checking AVX.
1535 */
1536 if (cp->cp_ecx & CPUID_INTC_ECX_F16C)
1537 add_x86_feature(featureset,
1538 X86FSET_F16C);
1539
1540 if (cp->cp_ecx & CPUID_INTC_ECX_FMA)
1541 add_x86_feature(featureset,
1542 X86FSET_FMA);
1543
1544 if (cpi->cpi_std[7].cp_ebx &
1545 CPUID_INTC_EBX_7_0_BMI1)
1546 add_x86_feature(featureset,
1547 X86FSET_BMI1);
1548
1549 if (cpi->cpi_std[7].cp_ebx &
1550 CPUID_INTC_EBX_7_0_BMI2)
1551 add_x86_feature(featureset,
1552 X86FSET_BMI2);
1553
1554 if (cpi->cpi_std[7].cp_ebx &
1555 CPUID_INTC_EBX_7_0_AVX2)
1556 add_x86_feature(featureset,
1557 X86FSET_AVX2);
1558 }
1559
1560 if (cpi->cpi_vendor == X86_VENDOR_Intel &&
1561 (cpi->cpi_std[7].cp_ebx &
1562 CPUID_INTC_EBX_7_0_AVX512F) != 0) {
1563 add_x86_feature(featureset, X86FSET_AVX512F);
1564
1565 if (cpi->cpi_std[7].cp_ebx &
1566 CPUID_INTC_EBX_7_0_AVX512DQ)
1567 add_x86_feature(featureset,
1568 X86FSET_AVX512DQ);
1569 if (cpi->cpi_std[7].cp_ebx &
1570 CPUID_INTC_EBX_7_0_AVX512IFMA)
1571 add_x86_feature(featureset,
1572 X86FSET_AVX512FMA);
1573 if (cpi->cpi_std[7].cp_ebx &
1574 CPUID_INTC_EBX_7_0_AVX512PF)
1575 add_x86_feature(featureset,
1576 X86FSET_AVX512PF);
1577 if (cpi->cpi_std[7].cp_ebx &
1578 CPUID_INTC_EBX_7_0_AVX512ER)
1579 add_x86_feature(featureset,
1580 X86FSET_AVX512ER);
1581 if (cpi->cpi_std[7].cp_ebx &
1582 CPUID_INTC_EBX_7_0_AVX512CD)
1583 add_x86_feature(featureset,
1584 X86FSET_AVX512CD);
1585 if (cpi->cpi_std[7].cp_ebx &
1586 CPUID_INTC_EBX_7_0_AVX512BW)
1587 add_x86_feature(featureset,
1588 X86FSET_AVX512BW);
1589 if (cpi->cpi_std[7].cp_ebx &
1590 CPUID_INTC_EBX_7_0_AVX512VL)
1591 add_x86_feature(featureset,
1592 X86FSET_AVX512VL);
1593
1594 if (cpi->cpi_std[7].cp_ecx &
1595 CPUID_INTC_ECX_7_0_AVX512VBMI)
1596 add_x86_feature(featureset,
1597 X86FSET_AVX512VBMI);
1598 if (cpi->cpi_std[7].cp_ecx &
1599 CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
1600 add_x86_feature(featureset,
1601 X86FSET_AVX512VPOPCDQ);
1602
1603 if (cpi->cpi_std[7].cp_edx &
1604 CPUID_INTC_EDX_7_0_AVX5124NNIW)
1605 add_x86_feature(featureset,
1606 X86FSET_AVX512NNIW);
1607 if (cpi->cpi_std[7].cp_edx &
1608 CPUID_INTC_EDX_7_0_AVX5124FMAPS)
1609 add_x86_feature(featureset,
1610 X86FSET_AVX512FMAPS);
1611 }
1612 }
1613 }
1614
1615 if (cpi->cpi_vendor == X86_VENDOR_Intel) {
1616 if (cp->cp_ecx & CPUID_INTC_ECX_PCID) {
1617 add_x86_feature(featureset, X86FSET_PCID);
1618 }
1619 }
1620
1621 if (cp->cp_ecx & CPUID_INTC_ECX_X2APIC) {
1622 add_x86_feature(featureset, X86FSET_X2APIC);
1623 }
1624 if (cp->cp_edx & CPUID_INTC_EDX_DE) {
1625 add_x86_feature(featureset, X86FSET_DE);
1626 }
1627 #if !defined(__xpv)
1628 if (cp->cp_ecx & CPUID_INTC_ECX_MON) {
1629
1630 /*
1631 * We require the CLFLUSH instruction for erratum workaround
1632 * to use MONITOR/MWAIT.
1633 */
1634 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1635 cpi->cpi_mwait.support |= MWAIT_SUPPORT;
1636 add_x86_feature(featureset, X86FSET_MWAIT);
1637 } else {
1638 extern int idle_cpu_assert_cflush_monitor;
1639
1640 /*
1641 * All processors we are aware of which have
1642 * MONITOR/MWAIT also have CLFLUSH.
1643 */
1644 if (idle_cpu_assert_cflush_monitor) {
1645 ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
1646 (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
1647 }
1648 }
1649 }
1650 #endif /* __xpv */
1651
1652 if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
1653 add_x86_feature(featureset, X86FSET_VMX);
1654 }
1655
1656 if (cp->cp_ecx & CPUID_INTC_ECX_RDRAND)
1657 add_x86_feature(featureset, X86FSET_RDRAND);
1658
1659 /*
1660 * Only need it first time, rest of the cpus would follow suit.
1661 * we only capture this for the bootcpu.
1662 */
1663 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1664 add_x86_feature(featureset, X86FSET_CLFSH);
1665 x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
1666 }
1667 if (is_x86_feature(featureset, X86FSET_PAE))
1668 cpi->cpi_pabits = 36;
1669
1670 /*
1671 * Hyperthreading configuration is slightly tricky on Intel
1672 * and pure clones, and even trickier on AMD.
1673 *
1674 * (AMD chose to set the HTT bit on their CMP processors,
1675 * even though they're not actually hyperthreaded. Thus it
1676 * takes a bit more work to figure out what's really going
1677 * on ... see the handling of the CMP_LGCY bit below)
1678 */
1679 if (cp->cp_edx & CPUID_INTC_EDX_HTT) {
1680 cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
1681 if (cpi->cpi_ncpu_per_chip > 1)
1682 add_x86_feature(featureset, X86FSET_HTT);
1683 } else {
1684 cpi->cpi_ncpu_per_chip = 1;
1685 }
1686
1687 if (cpi->cpi_maxeax >= 0xD && !xsave_force_disable) {
1688 struct cpuid_regs r, *ecp;
1689
1690 ecp = &r;
1691 ecp->cp_eax = 0xD;
1692 ecp->cp_ecx = 1;
1693 ecp->cp_edx = ecp->cp_ebx = 0;
1694 (void) __cpuid_insn(ecp);
1695
1696 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEOPT)
1697 add_x86_feature(featureset, X86FSET_XSAVEOPT);
1698 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEC)
1699 add_x86_feature(featureset, X86FSET_XSAVEC);
1700 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVES)
1701 add_x86_feature(featureset, X86FSET_XSAVES);
1702 }
1703
1704 /*
1705 * Work on the "extended" feature information, doing
1706 * some basic initialization for cpuid_pass2()
1707 */
1708 xcpuid = 0;
1709 switch (cpi->cpi_vendor) {
1710 case X86_VENDOR_Intel:
1711 /*
1712 * On KVM we know we will have proper support for extended
1713 * cpuid.
1714 */
1715 if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf ||
1716 (get_hwenv() == HW_KVM && cpi->cpi_family == 6 &&
1717 (cpi->cpi_model == 6 || cpi->cpi_model == 2)))
1718 xcpuid++;
1719 break;
1720 case X86_VENDOR_AMD:
1721 if (cpi->cpi_family > 5 ||
1722 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
1723 xcpuid++;
1724 break;
1725 case X86_VENDOR_Cyrix:
1726 /*
1727 * Only these Cyrix CPUs are -known- to support
1728 * extended cpuid operations.
1729 */
1730 if (x86_type == X86_TYPE_VIA_CYRIX_III ||
1731 x86_type == X86_TYPE_CYRIX_GXm)
1732 xcpuid++;
1733 break;
1734 case X86_VENDOR_Centaur:
1735 case X86_VENDOR_TM:
1736 default:
1737 xcpuid++;
1738 break;
1739 }
1740
1741 if (xcpuid) {
1742 cp = &cpi->cpi_extd[0];
1743 cp->cp_eax = 0x80000000;
1744 cpi->cpi_xmaxeax = __cpuid_insn(cp);
1745 }
1746
1747 if (cpi->cpi_xmaxeax & 0x80000000) {
1748
1749 if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
1750 cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;
1751
1752 switch (cpi->cpi_vendor) {
1753 case X86_VENDOR_Intel:
1754 case X86_VENDOR_AMD:
1755 if (cpi->cpi_xmaxeax < 0x80000001)
1756 break;
1757 cp = &cpi->cpi_extd[1];
1758 cp->cp_eax = 0x80000001;
1759 (void) __cpuid_insn(cp);
1760
1761 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1762 cpi->cpi_family == 5 &&
1763 cpi->cpi_model == 6 &&
1764 cpi->cpi_step == 6) {
1765 /*
1766 * K6 model 6 uses bit 10 to indicate SYSC
1767 * Later models use bit 11. Fix it here.
1768 */
1769 if (cp->cp_edx & 0x400) {
1770 cp->cp_edx &= ~0x400;
1771 cp->cp_edx |= CPUID_AMD_EDX_SYSC;
1772 }
1773 }
1774
1775 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);
1776
1777 /*
1778 * Compute the additions to the kernel's feature word.
1779 */
1780 if (cp->cp_edx & CPUID_AMD_EDX_NX) {
1781 add_x86_feature(featureset, X86FSET_NX);
1782 }
1783
1784 /*
1785 * Regardless whether or not we boot 64-bit,
1786 * we should have a way to identify whether
1787 * the CPU is capable of running 64-bit.
1788 */
1789 if (cp->cp_edx & CPUID_AMD_EDX_LM) {
1790 add_x86_feature(featureset, X86FSET_64);
1791 }
1792
1793 #if defined(__amd64)
1794 /* 1 GB large page - enable only for 64 bit kernel */
1795 if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
1796 add_x86_feature(featureset, X86FSET_1GPG);
1797 }
1798 #endif
1799
1800 if ((cpi->cpi_vendor == X86_VENDOR_AMD) &&
1801 (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
1802 (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
1803 add_x86_feature(featureset, X86FSET_SSE4A);
1804 }
1805
1806 /*
1807 * If both the HTT and CMP_LGCY bits are set,
1808 * then we're not actually HyperThreaded. Read
1809 * "AMD CPUID Specification" for more details.
1810 */
1811 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1812 is_x86_feature(featureset, X86FSET_HTT) &&
1813 (cp->cp_ecx & CPUID_AMD_ECX_CMP_LGCY)) {
1814 remove_x86_feature(featureset, X86FSET_HTT);
1815 add_x86_feature(featureset, X86FSET_CMP);
1816 }
1817 #if defined(__amd64)
1818 /*
1819 * It's really tricky to support syscall/sysret in
1820 * the i386 kernel; we rely on sysenter/sysexit
1821 * instead. In the amd64 kernel, things are -way-
1822 * better.
1823 */
1824 if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
1825 add_x86_feature(featureset, X86FSET_ASYSC);
1826 }
1827
1828 /*
1829 * While we're thinking about system calls, note
1830 * that AMD processors don't support sysenter
1831 * in long mode at all, so don't try to program them.
1832 */
1833 if (x86_vendor == X86_VENDOR_AMD) {
1834 remove_x86_feature(featureset, X86FSET_SEP);
1835 }
1836 #endif
1837 if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
1838 add_x86_feature(featureset, X86FSET_TSCP);
1839 }
1840
1841 if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
1842 add_x86_feature(featureset, X86FSET_SVM);
1843 }
1844
1845 if (cp->cp_ecx & CPUID_AMD_ECX_TOPOEXT) {
1846 add_x86_feature(featureset, X86FSET_TOPOEXT);
1847 }
1848 break;
1849 default:
1850 break;
1851 }
1852
1853 /*
1854 * Get CPUID data about processor cores and hyperthreads.
1855 */
1856 switch (cpi->cpi_vendor) {
1857 case X86_VENDOR_Intel:
1858 if (cpi->cpi_maxeax >= 4) {
1859 cp = &cpi->cpi_std[4];
1860 cp->cp_eax = 4;
1861 cp->cp_ecx = 0;
1862 (void) __cpuid_insn(cp);
1863 platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
1864 }
1865 /*FALLTHROUGH*/
1866 case X86_VENDOR_AMD:
1867 if (cpi->cpi_xmaxeax < 0x80000008)
1868 break;
1869 cp = &cpi->cpi_extd[8];
1870 cp->cp_eax = 0x80000008;
1871 (void) __cpuid_insn(cp);
1872 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, cp);
1873
1874 /*
1875 * Virtual and physical address limits from
1876 * cpuid override previously guessed values.
1877 */
1878 cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
1879 cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
1880 break;
1881 default:
1882 break;
1883 }
1884
1885 /*
1886 * Derive the number of cores per chip
1887 */
1888 switch (cpi->cpi_vendor) {
1889 case X86_VENDOR_Intel:
1890 if (cpi->cpi_maxeax < 4) {
1891 cpi->cpi_ncore_per_chip = 1;
1892 break;
1893 } else {
1894 cpi->cpi_ncore_per_chip =
1895 BITX((cpi)->cpi_std[4].cp_eax, 31, 26) + 1;
1896 }
1897 break;
1898 case X86_VENDOR_AMD:
1899 if (cpi->cpi_xmaxeax < 0x80000008) {
1900 cpi->cpi_ncore_per_chip = 1;
1901 break;
1902 } else {
1903 /*
1904 * On family 0xf cpuid fn 2 ECX[7:0] "NC" is
1905 * 1 less than the number of physical cores on
1906 * the chip. In family 0x10 this value can
1907 * be affected by "downcoring" - it reflects
1908 * 1 less than the number of cores actually
1909 * enabled on this node.
1910 */
1911 cpi->cpi_ncore_per_chip =
1912 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
1913 }
1914 break;
1915 default:
1916 cpi->cpi_ncore_per_chip = 1;
1917 break;
1918 }
1919
1920 /*
1921 * Get CPUID data about TSC Invariance in Deep C-State.
1922 */
1923 switch (cpi->cpi_vendor) {
1924 case X86_VENDOR_Intel:
1925 if (cpi->cpi_maxeax >= 7) {
1926 cp = &cpi->cpi_extd[7];
1927 cp->cp_eax = 0x80000007;
1928 cp->cp_ecx = 0;
1929 (void) __cpuid_insn(cp);
1930 }
1931 break;
1932 default:
1933 break;
1934 }
1935 } else {
1936 cpi->cpi_ncore_per_chip = 1;
1937 }
1938
1939 /*
1940 * If more than one core, then this processor is CMP.
1941 */
1942 if (cpi->cpi_ncore_per_chip > 1) {
1943 add_x86_feature(featureset, X86FSET_CMP);
1944 }
1945
1946 /*
1947 * If the number of cores is the same as the number
1948 * of CPUs, then we cannot have HyperThreading.
1949 */
1950 if (cpi->cpi_ncpu_per_chip == cpi->cpi_ncore_per_chip) {
1951 remove_x86_feature(featureset, X86FSET_HTT);
1952 }
1953
1954 cpi->cpi_apicid = CPI_APIC_ID(cpi);
1955 cpi->cpi_procnodes_per_pkg = 1;
1956 cpi->cpi_cores_per_compunit = 1;
1957 if (is_x86_feature(featureset, X86FSET_HTT) == B_FALSE &&
1958 is_x86_feature(featureset, X86FSET_CMP) == B_FALSE) {
1959 /*
1960 * Single-core single-threaded processors.
1961 */
1962 cpi->cpi_chipid = -1;
1963 cpi->cpi_clogid = 0;
1964 cpi->cpi_coreid = cpu->cpu_id;
1965 cpi->cpi_pkgcoreid = 0;
1966 if (cpi->cpi_vendor == X86_VENDOR_AMD)
1967 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
1968 else
1969 cpi->cpi_procnodeid = cpi->cpi_chipid;
1970 } else if (cpi->cpi_ncpu_per_chip > 1) {
1971 if (cpi->cpi_vendor == X86_VENDOR_Intel)
1972 cpuid_intel_getids(cpu, featureset);
1973 else if (cpi->cpi_vendor == X86_VENDOR_AMD)
1974 cpuid_amd_getids(cpu);
1975 else {
1976 /*
1977 * All other processors are currently
1978 * assumed to have single cores.
1979 */
1980 cpi->cpi_coreid = cpi->cpi_chipid;
1981 cpi->cpi_pkgcoreid = 0;
1982 cpi->cpi_procnodeid = cpi->cpi_chipid;
1983 cpi->cpi_compunitid = cpi->cpi_chipid;
1984 }
1985 }
1986
1987 /*
1988 * Synthesize chip "revision" and socket type
1989 */
1990 cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
1991 cpi->cpi_model, cpi->cpi_step);
1992 cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
1993 cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
1994 cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
1995 cpi->cpi_model, cpi->cpi_step);
1996
1997 /*
1998 * While we're here, check for the AMD "Error Pointer Zero/Restore"
1999 * feature. This can be used to setup the FP save handlers
2000 * appropriately.
2001 */
2002 if (cpi->cpi_vendor == X86_VENDOR_AMD) {
2003 if (cpi->cpi_xmaxeax >= 0x80000008 &&
2004 cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
2005 /* Special handling for AMD FP not necessary. */
2006 cpi->cpi_fp_amd_save = 0;
2007 } else {
2008 cpi->cpi_fp_amd_save = 1;
2009 }
2010 }
2011
2012 /*
2013 * Check the processor leaves that are used for security features.
2014 */
2015 cpuid_scan_security(cpu, featureset);
2016
2017 pass1_done:
2018 cpi->cpi_pass = 1;
2019 }
2020
2021 /*
2022 * Make copies of the cpuid table entries we depend on, in
2023 * part for ease of parsing now, in part so that we have only
2024 * one place to correct any of it, in part for ease of
2025 * later export to userland, and in part so we can look at
2026 * this stuff in a crash dump.
2027 */
2028
2029 /*ARGSUSED*/
2030 void
2031 cpuid_pass2(cpu_t *cpu)
2032 {
2033 uint_t n, nmax;
2034 int i;
2035 struct cpuid_regs *cp;
2036 uint8_t *dp;
2037 uint32_t *iptr;
2038 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2039
2040 ASSERT(cpi->cpi_pass == 1);
2041
2042 if (cpi->cpi_maxeax < 1)
2043 goto pass2_done;
2044
2045 if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
2046 nmax = NMAX_CPI_STD;
2047 /*
2048 * (We already handled n == 0 and n == 1 in pass 1)
2049 */
2050 for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
2051 cp->cp_eax = n;
2052
2053 /*
2054 * n == 7 was handled in pass 1
2055 */
2056 if (n == 7)
2057 continue;
2058
2059 /*
2060 * CPUID function 4 expects %ecx to be initialized
2061 * with an index which indicates which cache to return
2062 * information about. The OS is expected to call function 4
2063 * with %ecx set to 0, 1, 2, ... until it returns with
2064 * EAX[4:0] set to 0, which indicates there are no more
2065 * caches.
2066 *
2067 * Here, populate cpi_std[4] with the information returned by
2068 * function 4 when %ecx == 0, and do the rest in cpuid_pass3()
2069 * when dynamic memory allocation becomes available.
2070 *
2071 * Note: we need to explicitly initialize %ecx here, since
2072 * function 4 may have been previously invoked.
2073 */
2074 if (n == 4)
2075 cp->cp_ecx = 0;
2076
2077 (void) __cpuid_insn(cp);
2078 platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
2079 switch (n) {
2080 case 2:
2081 /*
2082 * "the lower 8 bits of the %eax register
2083 * contain a value that identifies the number
2084 * of times the cpuid [instruction] has to be
2085 * executed to obtain a complete image of the
2086 * processor's caching systems."
2087 *
2088 * How *do* they make this stuff up?
2089 */
2090 cpi->cpi_ncache = sizeof (*cp) *
2091 BITX(cp->cp_eax, 7, 0);
2092 if (cpi->cpi_ncache == 0)
2093 break;
2094 cpi->cpi_ncache--; /* skip count byte */
2095
2096 /*
2097 * Well, for now, rather than attempt to implement
2098 * this slightly dubious algorithm, we just look
2099 * at the first 15 ..
2100 */
2101 if (cpi->cpi_ncache > (sizeof (*cp) - 1))
2102 cpi->cpi_ncache = sizeof (*cp) - 1;
2103
2104 dp = cpi->cpi_cacheinfo;
2105 if (BITX(cp->cp_eax, 31, 31) == 0) {
2106 uint8_t *p = (void *)&cp->cp_eax;
2107 for (i = 1; i < 4; i++)
2108 if (p[i] != 0)
2109 *dp++ = p[i];
2110 }
2111 if (BITX(cp->cp_ebx, 31, 31) == 0) {
2112 uint8_t *p = (void *)&cp->cp_ebx;
2113 for (i = 0; i < 4; i++)
2114 if (p[i] != 0)
2115 *dp++ = p[i];
2116 }
2117 if (BITX(cp->cp_ecx, 31, 31) == 0) {
2118 uint8_t *p = (void *)&cp->cp_ecx;
2119 for (i = 0; i < 4; i++)
2120 if (p[i] != 0)
2121 *dp++ = p[i];
2122 }
2123 if (BITX(cp->cp_edx, 31, 31) == 0) {
2124 uint8_t *p = (void *)&cp->cp_edx;
2125 for (i = 0; i < 4; i++)
2126 if (p[i] != 0)
2127 *dp++ = p[i];
2128 }
2129 break;
2130
2131 case 3: /* Processor serial number, if PSN supported */
2132 break;
2133
2134 case 4: /* Deterministic cache parameters */
2135 break;
2136
2137 case 5: /* Monitor/Mwait parameters */
2138 {
2139 size_t mwait_size;
2140
2141 /*
2142 * check cpi_mwait.support which was set in cpuid_pass1
2143 */
2144 if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
2145 break;
2146
2147 /*
2148 * Protect ourself from insane mwait line size.
2149 * Workaround for incomplete hardware emulator(s).
2150 */
2151 mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
2152 if (mwait_size < sizeof (uint32_t) ||
2153 !ISP2(mwait_size)) {
2154 #if DEBUG
2155 cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
2156 "size %ld", cpu->cpu_id, (long)mwait_size);
2157 #endif
2158 break;
2159 }
2160
2161 cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
2162 cpi->cpi_mwait.mon_max = mwait_size;
2163 if (MWAIT_EXTENSION(cpi)) {
2164 cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
2165 if (MWAIT_INT_ENABLE(cpi))
2166 cpi->cpi_mwait.support |=
2167 MWAIT_ECX_INT_ENABLE;
2168 }
2169 break;
2170 }
2171 default:
2172 break;
2173 }
2174 }
2175
2176 if (cpi->cpi_maxeax >= 0xB && cpi->cpi_vendor == X86_VENDOR_Intel) {
2177 struct cpuid_regs regs;
2178
2179 cp = ®s;
2180 cp->cp_eax = 0xB;
2181 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2182
2183 (void) __cpuid_insn(cp);
2184
2185 /*
2186 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
2187 * indicates that the extended topology enumeration leaf is
2188 * available.
2189 */
2190 if (cp->cp_ebx) {
2191 uint32_t x2apic_id;
2192 uint_t coreid_shift = 0;
2193 uint_t ncpu_per_core = 1;
2194 uint_t chipid_shift = 0;
2195 uint_t ncpu_per_chip = 1;
2196 uint_t i;
2197 uint_t level;
2198
2199 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
2200 cp->cp_eax = 0xB;
2201 cp->cp_ecx = i;
2202
2203 (void) __cpuid_insn(cp);
2204 level = CPI_CPU_LEVEL_TYPE(cp);
2205
2206 if (level == 1) {
2207 x2apic_id = cp->cp_edx;
2208 coreid_shift = BITX(cp->cp_eax, 4, 0);
2209 ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
2210 } else if (level == 2) {
2211 x2apic_id = cp->cp_edx;
2212 chipid_shift = BITX(cp->cp_eax, 4, 0);
2213 ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
2214 }
2215 }
2216
2217 cpi->cpi_apicid = x2apic_id;
2218 cpi->cpi_ncpu_per_chip = ncpu_per_chip;
2219 cpi->cpi_ncore_per_chip = ncpu_per_chip /
2220 ncpu_per_core;
2221 cpi->cpi_chipid = x2apic_id >> chipid_shift;
2222 cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
2223 cpi->cpi_coreid = x2apic_id >> coreid_shift;
2224 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
2225 }
2226
2227 /* Make cp NULL so that we don't stumble on others */
2228 cp = NULL;
2229 }
2230
2231 /*
2232 * XSAVE enumeration
2233 */
2234 if (cpi->cpi_maxeax >= 0xD) {
2235 struct cpuid_regs regs;
2236 boolean_t cpuid_d_valid = B_TRUE;
2237
2238 cp = ®s;
2239 cp->cp_eax = 0xD;
2240 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2241
2242 (void) __cpuid_insn(cp);
2243
2244 /*
2245 * Sanity checks for debug
2246 */
2247 if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
2248 (cp->cp_eax & XFEATURE_SSE) == 0) {
2249 cpuid_d_valid = B_FALSE;
2250 }
2251
2252 cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
2253 cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
2254 cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;
2255
2256 /*
2257 * If the hw supports AVX, get the size and offset in the save
2258 * area for the ymm state.
2259 */
2260 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
2261 cp->cp_eax = 0xD;
2262 cp->cp_ecx = 2;
2263 cp->cp_edx = cp->cp_ebx = 0;
2264
2265 (void) __cpuid_insn(cp);
2266
2267 if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
2268 cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
2269 cpuid_d_valid = B_FALSE;
2270 }
2271
2272 cpi->cpi_xsave.ymm_size = cp->cp_eax;
2273 cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
2274 }
2275
2276 /*
2277 * If the hw supports MPX, get the size and offset in the
2278 * save area for BNDREGS and BNDCSR.
2279 */
2280 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_MPX) {
2281 cp->cp_eax = 0xD;
2282 cp->cp_ecx = 3;
2283 cp->cp_edx = cp->cp_ebx = 0;
2284
2285 (void) __cpuid_insn(cp);
2286
2287 cpi->cpi_xsave.bndregs_size = cp->cp_eax;
2288 cpi->cpi_xsave.bndregs_offset = cp->cp_ebx;
2289
2290 cp->cp_eax = 0xD;
2291 cp->cp_ecx = 4;
2292 cp->cp_edx = cp->cp_ebx = 0;
2293
2294 (void) __cpuid_insn(cp);
2295
2296 cpi->cpi_xsave.bndcsr_size = cp->cp_eax;
2297 cpi->cpi_xsave.bndcsr_offset = cp->cp_ebx;
2298 }
2299
2300 /*
2301 * If the hw supports AVX512, get the size and offset in the
2302 * save area for the opmask registers and zmm state.
2303 */
2304 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX512) {
2305 cp->cp_eax = 0xD;
2306 cp->cp_ecx = 5;
2307 cp->cp_edx = cp->cp_ebx = 0;
2308
2309 (void) __cpuid_insn(cp);
2310
2311 cpi->cpi_xsave.opmask_size = cp->cp_eax;
2312 cpi->cpi_xsave.opmask_offset = cp->cp_ebx;
2313
2314 cp->cp_eax = 0xD;
2315 cp->cp_ecx = 6;
2316 cp->cp_edx = cp->cp_ebx = 0;
2317
2318 (void) __cpuid_insn(cp);
2319
2320 cpi->cpi_xsave.zmmlo_size = cp->cp_eax;
2321 cpi->cpi_xsave.zmmlo_offset = cp->cp_ebx;
2322
2323 cp->cp_eax = 0xD;
2324 cp->cp_ecx = 7;
2325 cp->cp_edx = cp->cp_ebx = 0;
2326
2327 (void) __cpuid_insn(cp);
2328
2329 cpi->cpi_xsave.zmmhi_size = cp->cp_eax;
2330 cpi->cpi_xsave.zmmhi_offset = cp->cp_ebx;
2331 }
2332
2333 if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
2334 xsave_state_size = 0;
2335 } else if (cpuid_d_valid) {
2336 xsave_state_size = cpi->cpi_xsave.xsav_max_size;
2337 } else {
2338 /* Broken CPUID 0xD, probably in HVM */
2339 cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
2340 "value: hw_low = %d, hw_high = %d, xsave_size = %d"
2341 ", ymm_size = %d, ymm_offset = %d\n",
2342 cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
2343 cpi->cpi_xsave.xsav_hw_features_high,
2344 (int)cpi->cpi_xsave.xsav_max_size,
2345 (int)cpi->cpi_xsave.ymm_size,
2346 (int)cpi->cpi_xsave.ymm_offset);
2347
2348 if (xsave_state_size != 0) {
2349 /*
2350 * This must be a non-boot CPU. We cannot
2351 * continue, because boot cpu has already
2352 * enabled XSAVE.
2353 */
2354 ASSERT(cpu->cpu_id != 0);
2355 cmn_err(CE_PANIC, "cpu%d: we have already "
2356 "enabled XSAVE on boot cpu, cannot "
2357 "continue.", cpu->cpu_id);
2358 } else {
2359 /*
2360 * If we reached here on the boot CPU, it's also
2361 * almost certain that we'll reach here on the
2362 * non-boot CPUs. When we're here on a boot CPU
2363 * we should disable the feature, on a non-boot
2364 * CPU we need to confirm that we have.
2365 */
2366 if (cpu->cpu_id == 0) {
2367 remove_x86_feature(x86_featureset,
2368 X86FSET_XSAVE);
2369 remove_x86_feature(x86_featureset,
2370 X86FSET_AVX);
2371 remove_x86_feature(x86_featureset,
2372 X86FSET_F16C);
2373 remove_x86_feature(x86_featureset,
2374 X86FSET_BMI1);
2375 remove_x86_feature(x86_featureset,
2376 X86FSET_BMI2);
2377 remove_x86_feature(x86_featureset,
2378 X86FSET_FMA);
2379 remove_x86_feature(x86_featureset,
2380 X86FSET_AVX2);
2381 remove_x86_feature(x86_featureset,
2382 X86FSET_MPX);
2383 remove_x86_feature(x86_featureset,
2384 X86FSET_AVX512F);
2385 remove_x86_feature(x86_featureset,
2386 X86FSET_AVX512DQ);
2387 remove_x86_feature(x86_featureset,
2388 X86FSET_AVX512PF);
2389 remove_x86_feature(x86_featureset,
2390 X86FSET_AVX512ER);
2391 remove_x86_feature(x86_featureset,
2392 X86FSET_AVX512CD);
2393 remove_x86_feature(x86_featureset,
2394 X86FSET_AVX512BW);
2395 remove_x86_feature(x86_featureset,
2396 X86FSET_AVX512VL);
2397 remove_x86_feature(x86_featureset,
2398 X86FSET_AVX512FMA);
2399 remove_x86_feature(x86_featureset,
2400 X86FSET_AVX512VBMI);
2401 remove_x86_feature(x86_featureset,
2402 X86FSET_AVX512VPOPCDQ);
2403 remove_x86_feature(x86_featureset,
2404 X86FSET_AVX512NNIW);
2405 remove_x86_feature(x86_featureset,
2406 X86FSET_AVX512FMAPS);
2407
2408 CPI_FEATURES_ECX(cpi) &=
2409 ~CPUID_INTC_ECX_XSAVE;
2410 CPI_FEATURES_ECX(cpi) &=
2411 ~CPUID_INTC_ECX_AVX;
2412 CPI_FEATURES_ECX(cpi) &=
2413 ~CPUID_INTC_ECX_F16C;
2414 CPI_FEATURES_ECX(cpi) &=
2415 ~CPUID_INTC_ECX_FMA;
2416 CPI_FEATURES_7_0_EBX(cpi) &=
2417 ~CPUID_INTC_EBX_7_0_BMI1;
2418 CPI_FEATURES_7_0_EBX(cpi) &=
2419 ~CPUID_INTC_EBX_7_0_BMI2;
2420 CPI_FEATURES_7_0_EBX(cpi) &=
2421 ~CPUID_INTC_EBX_7_0_AVX2;
2422 CPI_FEATURES_7_0_EBX(cpi) &=
2423 ~CPUID_INTC_EBX_7_0_MPX;
2424 CPI_FEATURES_7_0_EBX(cpi) &=
2425 ~CPUID_INTC_EBX_7_0_ALL_AVX512;
2426
2427 CPI_FEATURES_7_0_ECX(cpi) &=
2428 ~CPUID_INTC_ECX_7_0_ALL_AVX512;
2429
2430 CPI_FEATURES_7_0_EDX(cpi) &=
2431 ~CPUID_INTC_EDX_7_0_ALL_AVX512;
2432
2433 xsave_force_disable = B_TRUE;
2434 } else {
2435 VERIFY(is_x86_feature(x86_featureset,
2436 X86FSET_XSAVE) == B_FALSE);
2437 }
2438 }
2439 }
2440 }
2441
2442
2443 if ((cpi->cpi_xmaxeax & 0x80000000) == 0)
2444 goto pass2_done;
2445
2446 if ((nmax = cpi->cpi_xmaxeax - 0x80000000 + 1) > NMAX_CPI_EXTD)
2447 nmax = NMAX_CPI_EXTD;
2448 /*
2449 * Copy the extended properties, fixing them as we go.
2450 * (We already handled n == 0 and n == 1 in pass 1)
2451 */
2452 iptr = (void *)cpi->cpi_brandstr;
2453 for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
2454 cp->cp_eax = 0x80000000 + n;
2455 (void) __cpuid_insn(cp);
2456 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000000 + n, cp);
2457 switch (n) {
2458 case 2:
2459 case 3:
2460 case 4:
2461 /*
2462 * Extract the brand string
2463 */
2464 *iptr++ = cp->cp_eax;
2465 *iptr++ = cp->cp_ebx;
2466 *iptr++ = cp->cp_ecx;
2467 *iptr++ = cp->cp_edx;
2468 break;
2469 case 5:
2470 switch (cpi->cpi_vendor) {
2471 case X86_VENDOR_AMD:
2472 /*
2473 * The Athlon and Duron were the first
2474 * parts to report the sizes of the
2475 * TLB for large pages. Before then,
2476 * we don't trust the data.
2477 */
2478 if (cpi->cpi_family < 6 ||
2479 (cpi->cpi_family == 6 &&
2480 cpi->cpi_model < 1))
2481 cp->cp_eax = 0;
2482 break;
2483 default:
2484 break;
2485 }
2486 break;
2487 case 6:
2488 switch (cpi->cpi_vendor) {
2489 case X86_VENDOR_AMD:
2490 /*
2491 * The Athlon and Duron were the first
2492 * AMD parts with L2 TLB's.
2493 * Before then, don't trust the data.
2494 */
2495 if (cpi->cpi_family < 6 ||
2496 cpi->cpi_family == 6 &&
2497 cpi->cpi_model < 1)
2498 cp->cp_eax = cp->cp_ebx = 0;
2499 /*
2500 * AMD Duron rev A0 reports L2
2501 * cache size incorrectly as 1K
2502 * when it is really 64K
2503 */
2504 if (cpi->cpi_family == 6 &&
2505 cpi->cpi_model == 3 &&
2506 cpi->cpi_step == 0) {
2507 cp->cp_ecx &= 0xffff;
2508 cp->cp_ecx |= 0x400000;
2509 }
2510 break;
2511 case X86_VENDOR_Cyrix: /* VIA C3 */
2512 /*
2513 * VIA C3 processors are a bit messed
2514 * up w.r.t. encoding cache sizes in %ecx
2515 */
2516 if (cpi->cpi_family != 6)
2517 break;
2518 /*
2519 * model 7 and 8 were incorrectly encoded
2520 *
2521 * xxx is model 8 really broken?
2522 */
2523 if (cpi->cpi_model == 7 ||
2524 cpi->cpi_model == 8)
2525 cp->cp_ecx =
2526 BITX(cp->cp_ecx, 31, 24) << 16 |
2527 BITX(cp->cp_ecx, 23, 16) << 12 |
2528 BITX(cp->cp_ecx, 15, 8) << 8 |
2529 BITX(cp->cp_ecx, 7, 0);
2530 /*
2531 * model 9 stepping 1 has wrong associativity
2532 */
2533 if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
2534 cp->cp_ecx |= 8 << 12;
2535 break;
2536 case X86_VENDOR_Intel:
2537 /*
2538 * Extended L2 Cache features function.
2539 * First appeared on Prescott.
2540 */
2541 default:
2542 break;
2543 }
2544 break;
2545 default:
2546 break;
2547 }
2548 }
2549
2550 pass2_done:
2551 cpi->cpi_pass = 2;
2552 }
2553
2554 static const char *
2555 intel_cpubrand(const struct cpuid_info *cpi)
2556 {
2557 int i;
2558
2559 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2560 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2561 return ("i486");
2562
2563 switch (cpi->cpi_family) {
2564 case 5:
2565 return ("Intel Pentium(r)");
2566 case 6:
2567 switch (cpi->cpi_model) {
2568 uint_t celeron, xeon;
2569 const struct cpuid_regs *cp;
2570 case 0:
2571 case 1:
2572 case 2:
2573 return ("Intel Pentium(r) Pro");
2574 case 3:
2575 case 4:
2576 return ("Intel Pentium(r) II");
2577 case 6:
2578 return ("Intel Celeron(r)");
2579 case 5:
2580 case 7:
2581 celeron = xeon = 0;
2582 cp = &cpi->cpi_std[2]; /* cache info */
2583
2584 for (i = 1; i < 4; i++) {
2585 uint_t tmp;
2586
2587 tmp = (cp->cp_eax >> (8 * i)) & 0xff;
2588 if (tmp == 0x40)
2589 celeron++;
2590 if (tmp >= 0x44 && tmp <= 0x45)
2591 xeon++;
2592 }
2593
2594 for (i = 0; i < 2; i++) {
2595 uint_t tmp;
2596
2597 tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
2598 if (tmp == 0x40)
2599 celeron++;
2600 else if (tmp >= 0x44 && tmp <= 0x45)
2601 xeon++;
2602 }
2603
2604 for (i = 0; i < 4; i++) {
2605 uint_t tmp;
2606
2607 tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
2608 if (tmp == 0x40)
2609 celeron++;
2610 else if (tmp >= 0x44 && tmp <= 0x45)
2611 xeon++;
2612 }
2613
2614 for (i = 0; i < 4; i++) {
2615 uint_t tmp;
2616
2617 tmp = (cp->cp_edx >> (8 * i)) & 0xff;
2618 if (tmp == 0x40)
2619 celeron++;
2620 else if (tmp >= 0x44 && tmp <= 0x45)
2621 xeon++;
2622 }
2623
2624 if (celeron)
2625 return ("Intel Celeron(r)");
2626 if (xeon)
2627 return (cpi->cpi_model == 5 ?
2628 "Intel Pentium(r) II Xeon(tm)" :
2629 "Intel Pentium(r) III Xeon(tm)");
2630 return (cpi->cpi_model == 5 ?
2631 "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
2632 "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
2633 default:
2634 break;
2635 }
2636 default:
2637 break;
2638 }
2639
2640 /* BrandID is present if the field is nonzero */
2641 if (cpi->cpi_brandid != 0) {
2642 static const struct {
2643 uint_t bt_bid;
2644 const char *bt_str;
2645 } brand_tbl[] = {
2646 { 0x1, "Intel(r) Celeron(r)" },
2647 { 0x2, "Intel(r) Pentium(r) III" },
2648 { 0x3, "Intel(r) Pentium(r) III Xeon(tm)" },
2649 { 0x4, "Intel(r) Pentium(r) III" },
2650 { 0x6, "Mobile Intel(r) Pentium(r) III" },
2651 { 0x7, "Mobile Intel(r) Celeron(r)" },
2652 { 0x8, "Intel(r) Pentium(r) 4" },
2653 { 0x9, "Intel(r) Pentium(r) 4" },
2654 { 0xa, "Intel(r) Celeron(r)" },
2655 { 0xb, "Intel(r) Xeon(tm)" },
2656 { 0xc, "Intel(r) Xeon(tm) MP" },
2657 { 0xe, "Mobile Intel(r) Pentium(r) 4" },
2658 { 0xf, "Mobile Intel(r) Celeron(r)" },
2659 { 0x11, "Mobile Genuine Intel(r)" },
2660 { 0x12, "Intel(r) Celeron(r) M" },
2661 { 0x13, "Mobile Intel(r) Celeron(r)" },
2662 { 0x14, "Intel(r) Celeron(r)" },
2663 { 0x15, "Mobile Genuine Intel(r)" },
2664 { 0x16, "Intel(r) Pentium(r) M" },
2665 { 0x17, "Mobile Intel(r) Celeron(r)" }
2666 };
2667 uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
2668 uint_t sgn;
2669
2670 sgn = (cpi->cpi_family << 8) |
2671 (cpi->cpi_model << 4) | cpi->cpi_step;
2672
2673 for (i = 0; i < btblmax; i++)
2674 if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
2675 break;
2676 if (i < btblmax) {
2677 if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
2678 return ("Intel(r) Celeron(r)");
2679 if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
2680 return ("Intel(r) Xeon(tm) MP");
2681 if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
2682 return ("Intel(r) Xeon(tm)");
2683 return (brand_tbl[i].bt_str);
2684 }
2685 }
2686
2687 return (NULL);
2688 }
2689
2690 static const char *
2691 amd_cpubrand(const struct cpuid_info *cpi)
2692 {
2693 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2694 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2695 return ("i486 compatible");
2696
2697 switch (cpi->cpi_family) {
2698 case 5:
2699 switch (cpi->cpi_model) {
2700 case 0:
2701 case 1:
2702 case 2:
2703 case 3:
2704 case 4:
2705 case 5:
2706 return ("AMD-K5(r)");
2707 case 6:
2708 case 7:
2709 return ("AMD-K6(r)");
2710 case 8:
2711 return ("AMD-K6(r)-2");
2712 case 9:
2713 return ("AMD-K6(r)-III");
2714 default:
2715 return ("AMD (family 5)");
2716 }
2717 case 6:
2718 switch (cpi->cpi_model) {
2719 case 1:
2720 return ("AMD-K7(tm)");
2721 case 0:
2722 case 2:
2723 case 4:
2724 return ("AMD Athlon(tm)");
2725 case 3:
2726 case 7:
2727 return ("AMD Duron(tm)");
2728 case 6:
2729 case 8:
2730 case 10:
2731 /*
2732 * Use the L2 cache size to distinguish
2733 */
2734 return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
2735 "AMD Athlon(tm)" : "AMD Duron(tm)");
2736 default:
2737 return ("AMD (family 6)");
2738 }
2739 default:
2740 break;
2741 }
2742
2743 if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
2744 cpi->cpi_brandid != 0) {
2745 switch (BITX(cpi->cpi_brandid, 7, 5)) {
2746 case 3:
2747 return ("AMD Opteron(tm) UP 1xx");
2748 case 4:
2749 return ("AMD Opteron(tm) DP 2xx");
2750 case 5:
2751 return ("AMD Opteron(tm) MP 8xx");
2752 default:
2753 return ("AMD Opteron(tm)");
2754 }
2755 }
2756
2757 return (NULL);
2758 }
2759
2760 static const char *
2761 cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
2762 {
2763 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2764 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5 ||
2765 type == X86_TYPE_CYRIX_486)
2766 return ("i486 compatible");
2767
2768 switch (type) {
2769 case X86_TYPE_CYRIX_6x86:
2770 return ("Cyrix 6x86");
2771 case X86_TYPE_CYRIX_6x86L:
2772 return ("Cyrix 6x86L");
2773 case X86_TYPE_CYRIX_6x86MX:
2774 return ("Cyrix 6x86MX");
2775 case X86_TYPE_CYRIX_GXm:
2776 return ("Cyrix GXm");
2777 case X86_TYPE_CYRIX_MediaGX:
2778 return ("Cyrix MediaGX");
2779 case X86_TYPE_CYRIX_MII:
2780 return ("Cyrix M2");
2781 case X86_TYPE_VIA_CYRIX_III:
2782 return ("VIA Cyrix M3");
2783 default:
2784 /*
2785 * Have another wild guess ..
2786 */
2787 if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
2788 return ("Cyrix 5x86");
2789 else if (cpi->cpi_family == 5) {
2790 switch (cpi->cpi_model) {
2791 case 2:
2792 return ("Cyrix 6x86"); /* Cyrix M1 */
2793 case 4:
2794 return ("Cyrix MediaGX");
2795 default:
2796 break;
2797 }
2798 } else if (cpi->cpi_family == 6) {
2799 switch (cpi->cpi_model) {
2800 case 0:
2801 return ("Cyrix 6x86MX"); /* Cyrix M2? */
2802 case 5:
2803 case 6:
2804 case 7:
2805 case 8:
2806 case 9:
2807 return ("VIA C3");
2808 default:
2809 break;
2810 }
2811 }
2812 break;
2813 }
2814 return (NULL);
2815 }
2816
2817 /*
2818 * This only gets called in the case that the CPU extended
2819 * feature brand string (0x80000002, 0x80000003, 0x80000004)
2820 * aren't available, or contain null bytes for some reason.
2821 */
2822 static void
2823 fabricate_brandstr(struct cpuid_info *cpi)
2824 {
2825 const char *brand = NULL;
2826
2827 switch (cpi->cpi_vendor) {
2828 case X86_VENDOR_Intel:
2829 brand = intel_cpubrand(cpi);
2830 break;
2831 case X86_VENDOR_AMD:
2832 brand = amd_cpubrand(cpi);
2833 break;
2834 case X86_VENDOR_Cyrix:
2835 brand = cyrix_cpubrand(cpi, x86_type);
2836 break;
2837 case X86_VENDOR_NexGen:
2838 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2839 brand = "NexGen Nx586";
2840 break;
2841 case X86_VENDOR_Centaur:
2842 if (cpi->cpi_family == 5)
2843 switch (cpi->cpi_model) {
2844 case 4:
2845 brand = "Centaur C6";
2846 break;
2847 case 8:
2848 brand = "Centaur C2";
2849 break;
2850 case 9:
2851 brand = "Centaur C3";
2852 break;
2853 default:
2854 break;
2855 }
2856 break;
2857 case X86_VENDOR_Rise:
2858 if (cpi->cpi_family == 5 &&
2859 (cpi->cpi_model == 0 || cpi->cpi_model == 2))
2860 brand = "Rise mP6";
2861 break;
2862 case X86_VENDOR_SiS:
2863 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2864 brand = "SiS 55x";
2865 break;
2866 case X86_VENDOR_TM:
2867 if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
2868 brand = "Transmeta Crusoe TM3x00 or TM5x00";
2869 break;
2870 case X86_VENDOR_NSC:
2871 case X86_VENDOR_UMC:
2872 default:
2873 break;
2874 }
2875 if (brand) {
2876 (void) strcpy((char *)cpi->cpi_brandstr, brand);
2877 return;
2878 }
2879
2880 /*
2881 * If all else fails ...
2882 */
2883 (void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
2884 "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
2885 cpi->cpi_model, cpi->cpi_step);
2886 }
2887
2888 /*
2889 * This routine is called just after kernel memory allocation
2890 * becomes available on cpu0, and as part of mp_startup() on
2891 * the other cpus.
2892 *
2893 * Fixup the brand string, and collect any information from cpuid
2894 * that requires dynamically allocated storage to represent.
2895 */
2896 /*ARGSUSED*/
2897 void
2898 cpuid_pass3(cpu_t *cpu)
2899 {
2900 int i, max, shft, level, size;
2901 struct cpuid_regs regs;
2902 struct cpuid_regs *cp;
2903 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2904
2905 ASSERT(cpi->cpi_pass == 2);
2906
2907 /*
2908 * Function 4: Deterministic cache parameters
2909 *
2910 * Take this opportunity to detect the number of threads
2911 * sharing the last level cache, and construct a corresponding
2912 * cache id. The respective cpuid_info members are initialized
2913 * to the default case of "no last level cache sharing".
2914 */
2915 cpi->cpi_ncpu_shr_last_cache = 1;
2916 cpi->cpi_last_lvl_cacheid = cpu->cpu_id;
2917
2918 if (cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) {
2919
2920 /*
2921 * Find the # of elements (size) returned by fn 4, and along
2922 * the way detect last level cache sharing details.
2923 */
2924 bzero(®s, sizeof (regs));
2925 cp = ®s;
2926 for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
2927 cp->cp_eax = 4;
2928 cp->cp_ecx = i;
2929
2930 (void) __cpuid_insn(cp);
2931
2932 if (CPI_CACHE_TYPE(cp) == 0)
2933 break;
2934 level = CPI_CACHE_LVL(cp);
2935 if (level > max) {
2936 max = level;
2937 cpi->cpi_ncpu_shr_last_cache =
2938 CPI_NTHR_SHR_CACHE(cp) + 1;
2939 }
2940 }
2941 cpi->cpi_std_4_size = size = i;
2942
2943 /*
2944 * Allocate the cpi_std_4 array. The first element
2945 * references the regs for fn 4, %ecx == 0, which
2946 * cpuid_pass2() stashed in cpi->cpi_std[4].
2947 */
2948 if (size > 0) {
2949 cpi->cpi_std_4 =
2950 kmem_alloc(size * sizeof (cp), KM_SLEEP);
2951 cpi->cpi_std_4[0] = &cpi->cpi_std[4];
2952
2953 /*
2954 * Allocate storage to hold the additional regs
2955 * for function 4, %ecx == 1 .. cpi_std_4_size.
2956 *
2957 * The regs for fn 4, %ecx == 0 has already
2958 * been allocated as indicated above.
2959 */
2960 for (i = 1; i < size; i++) {
2961 cp = cpi->cpi_std_4[i] =
2962 kmem_zalloc(sizeof (regs), KM_SLEEP);
2963 cp->cp_eax = 4;
2964 cp->cp_ecx = i;
2965
2966 (void) __cpuid_insn(cp);
2967 }
2968 }
2969 /*
2970 * Determine the number of bits needed to represent
2971 * the number of CPUs sharing the last level cache.
2972 *
2973 * Shift off that number of bits from the APIC id to
2974 * derive the cache id.
2975 */
2976 shft = 0;
2977 for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
2978 shft++;
2979 cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
2980 }
2981
2982 /*
2983 * Now fixup the brand string
2984 */
2985 if ((cpi->cpi_xmaxeax & 0x80000000) == 0) {
2986 fabricate_brandstr(cpi);
2987 } else {
2988
2989 /*
2990 * If we successfully extracted a brand string from the cpuid
2991 * instruction, clean it up by removing leading spaces and
2992 * similar junk.
2993 */
2994 if (cpi->cpi_brandstr[0]) {
2995 size_t maxlen = sizeof (cpi->cpi_brandstr);
2996 char *src, *dst;
2997
2998 dst = src = (char *)cpi->cpi_brandstr;
2999 src[maxlen - 1] = '\0';
3000 /*
3001 * strip leading spaces
3002 */
3003 while (*src == ' ')
3004 src++;
3005 /*
3006 * Remove any 'Genuine' or "Authentic" prefixes
3007 */
3008 if (strncmp(src, "Genuine ", 8) == 0)
3009 src += 8;
3010 if (strncmp(src, "Authentic ", 10) == 0)
3011 src += 10;
3012
3013 /*
3014 * Now do an in-place copy.
3015 * Map (R) to (r) and (TM) to (tm).
3016 * The era of teletypes is long gone, and there's
3017 * -really- no need to shout.
3018 */
3019 while (*src != '\0') {
3020 if (src[0] == '(') {
3021 if (strncmp(src + 1, "R)", 2) == 0) {
3022 (void) strncpy(dst, "(r)", 3);
3023 src += 3;
3024 dst += 3;
3025 continue;
3026 }
3027 if (strncmp(src + 1, "TM)", 3) == 0) {
3028 (void) strncpy(dst, "(tm)", 4);
3029 src += 4;
3030 dst += 4;
3031 continue;
3032 }
3033 }
3034 *dst++ = *src++;
3035 }
3036 *dst = '\0';
3037
3038 /*
3039 * Finally, remove any trailing spaces
3040 */
3041 while (--dst > cpi->cpi_brandstr)
3042 if (*dst == ' ')
3043 *dst = '\0';
3044 else
3045 break;
3046 } else
3047 fabricate_brandstr(cpi);
3048 }
3049 cpi->cpi_pass = 3;
3050 }
3051
3052 /*
3053 * This routine is called out of bind_hwcap() much later in the life
3054 * of the kernel (post_startup()). The job of this routine is to resolve
3055 * the hardware feature support and kernel support for those features into
3056 * what we're actually going to tell applications via the aux vector.
3057 */
3058 void
3059 cpuid_pass4(cpu_t *cpu, uint_t *hwcap_out)
3060 {
3061 struct cpuid_info *cpi;
3062 uint_t hwcap_flags = 0, hwcap_flags_2 = 0;
3063
3064 if (cpu == NULL)
3065 cpu = CPU;
3066 cpi = cpu->cpu_m.mcpu_cpi;
3067
3068 ASSERT(cpi->cpi_pass == 3);
3069
3070 if (cpi->cpi_maxeax >= 1) {
3071 uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
3072 uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];
3073 uint32_t *ebx = &cpi->cpi_support[STD_EBX_FEATURES];
3074
3075 *edx = CPI_FEATURES_EDX(cpi);
3076 *ecx = CPI_FEATURES_ECX(cpi);
3077 *ebx = CPI_FEATURES_7_0_EBX(cpi);
3078
3079 /*
3080 * [these require explicit kernel support]
3081 */
3082 if (!is_x86_feature(x86_featureset, X86FSET_SEP))
3083 *edx &= ~CPUID_INTC_EDX_SEP;
3084
3085 if (!is_x86_feature(x86_featureset, X86FSET_SSE))
3086 *edx &= ~(CPUID_INTC_EDX_FXSR|CPUID_INTC_EDX_SSE);
3087 if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
3088 *edx &= ~CPUID_INTC_EDX_SSE2;
3089
3090 if (!is_x86_feature(x86_featureset, X86FSET_HTT))
3091 *edx &= ~CPUID_INTC_EDX_HTT;
3092
3093 if (!is_x86_feature(x86_featureset, X86FSET_SSE3))
3094 *ecx &= ~CPUID_INTC_ECX_SSE3;
3095
3096 if (!is_x86_feature(x86_featureset, X86FSET_SSSE3))
3097 *ecx &= ~CPUID_INTC_ECX_SSSE3;
3098 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_1))
3099 *ecx &= ~CPUID_INTC_ECX_SSE4_1;
3100 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2))
3101 *ecx &= ~CPUID_INTC_ECX_SSE4_2;
3102 if (!is_x86_feature(x86_featureset, X86FSET_AES))
3103 *ecx &= ~CPUID_INTC_ECX_AES;
3104 if (!is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ))
3105 *ecx &= ~CPUID_INTC_ECX_PCLMULQDQ;
3106 if (!is_x86_feature(x86_featureset, X86FSET_XSAVE))
3107 *ecx &= ~(CPUID_INTC_ECX_XSAVE |
3108 CPUID_INTC_ECX_OSXSAVE);
3109 if (!is_x86_feature(x86_featureset, X86FSET_AVX))
3110 *ecx &= ~CPUID_INTC_ECX_AVX;
3111 if (!is_x86_feature(x86_featureset, X86FSET_F16C))
3112 *ecx &= ~CPUID_INTC_ECX_F16C;
3113 if (!is_x86_feature(x86_featureset, X86FSET_FMA))
3114 *ecx &= ~CPUID_INTC_ECX_FMA;
3115 if (!is_x86_feature(x86_featureset, X86FSET_BMI1))
3116 *ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
3117 if (!is_x86_feature(x86_featureset, X86FSET_BMI2))
3118 *ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
3119 if (!is_x86_feature(x86_featureset, X86FSET_AVX2))
3120 *ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
3121 if (!is_x86_feature(x86_featureset, X86FSET_RDSEED))
3122 *ebx &= ~CPUID_INTC_EBX_7_0_RDSEED;
3123 if (!is_x86_feature(x86_featureset, X86FSET_ADX))
3124 *ebx &= ~CPUID_INTC_EBX_7_0_ADX;
3125
3126 /*
3127 * [no explicit support required beyond x87 fp context]
3128 */
3129 if (!fpu_exists)
3130 *edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);
3131
3132 /*
3133 * Now map the supported feature vector to things that we
3134 * think userland will care about.
3135 */
3136 if (*edx & CPUID_INTC_EDX_SEP)
3137 hwcap_flags |= AV_386_SEP;
3138 if (*edx & CPUID_INTC_EDX_SSE)
3139 hwcap_flags |= AV_386_FXSR | AV_386_SSE;
3140 if (*edx & CPUID_INTC_EDX_SSE2)
3141 hwcap_flags |= AV_386_SSE2;
3142 if (*ecx & CPUID_INTC_ECX_SSE3)
3143 hwcap_flags |= AV_386_SSE3;
3144 if (*ecx & CPUID_INTC_ECX_SSSE3)
3145 hwcap_flags |= AV_386_SSSE3;
3146 if (*ecx & CPUID_INTC_ECX_SSE4_1)
3147 hwcap_flags |= AV_386_SSE4_1;
3148 if (*ecx & CPUID_INTC_ECX_SSE4_2)
3149 hwcap_flags |= AV_386_SSE4_2;
3150 if (*ecx & CPUID_INTC_ECX_MOVBE)
3151 hwcap_flags |= AV_386_MOVBE;
3152 if (*ecx & CPUID_INTC_ECX_AES)
3153 hwcap_flags |= AV_386_AES;
3154 if (*ecx & CPUID_INTC_ECX_PCLMULQDQ)
3155 hwcap_flags |= AV_386_PCLMULQDQ;
3156 if ((*ecx & CPUID_INTC_ECX_XSAVE) &&
3157 (*ecx & CPUID_INTC_ECX_OSXSAVE)) {
3158 hwcap_flags |= AV_386_XSAVE;
3159
3160 if (*ecx & CPUID_INTC_ECX_AVX) {
3161 uint32_t *ecx_7 = &CPI_FEATURES_7_0_ECX(cpi);
3162 uint32_t *edx_7 = &CPI_FEATURES_7_0_EDX(cpi);
3163
3164 hwcap_flags |= AV_386_AVX;
3165 if (*ecx & CPUID_INTC_ECX_F16C)
3166 hwcap_flags_2 |= AV_386_2_F16C;
3167 if (*ecx & CPUID_INTC_ECX_FMA)
3168 hwcap_flags_2 |= AV_386_2_FMA;
3169
3170 if (*ebx & CPUID_INTC_EBX_7_0_BMI1)
3171 hwcap_flags_2 |= AV_386_2_BMI1;
3172 if (*ebx & CPUID_INTC_EBX_7_0_BMI2)
3173 hwcap_flags_2 |= AV_386_2_BMI2;
3174 if (*ebx & CPUID_INTC_EBX_7_0_AVX2)
3175 hwcap_flags_2 |= AV_386_2_AVX2;
3176 if (*ebx & CPUID_INTC_EBX_7_0_AVX512F)
3177 hwcap_flags_2 |= AV_386_2_AVX512F;
3178 if (*ebx & CPUID_INTC_EBX_7_0_AVX512DQ)
3179 hwcap_flags_2 |= AV_386_2_AVX512DQ;
3180 if (*ebx & CPUID_INTC_EBX_7_0_AVX512IFMA)
3181 hwcap_flags_2 |= AV_386_2_AVX512IFMA;
3182 if (*ebx & CPUID_INTC_EBX_7_0_AVX512PF)
3183 hwcap_flags_2 |= AV_386_2_AVX512PF;
3184 if (*ebx & CPUID_INTC_EBX_7_0_AVX512ER)
3185 hwcap_flags_2 |= AV_386_2_AVX512ER;
3186 if (*ebx & CPUID_INTC_EBX_7_0_AVX512CD)
3187 hwcap_flags_2 |= AV_386_2_AVX512CD;
3188 if (*ebx & CPUID_INTC_EBX_7_0_AVX512BW)
3189 hwcap_flags_2 |= AV_386_2_AVX512BW;
3190 if (*ebx & CPUID_INTC_EBX_7_0_AVX512VL)
3191 hwcap_flags_2 |= AV_386_2_AVX512VL;
3192
3193 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VBMI)
3194 hwcap_flags_2 |= AV_386_2_AVX512VBMI;
3195 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
3196 hwcap_flags_2 |= AV_386_2_AVX512VPOPCDQ;
3197
3198 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124NNIW)
3199 hwcap_flags_2 |= AV_386_2_AVX512_4NNIW;
3200 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124FMAPS)
3201 hwcap_flags_2 |= AV_386_2_AVX512_4FMAPS;
3202 }
3203 }
3204 if (*ecx & CPUID_INTC_ECX_VMX)
3205 hwcap_flags |= AV_386_VMX;
3206 if (*ecx & CPUID_INTC_ECX_POPCNT)
3207 hwcap_flags |= AV_386_POPCNT;
3208 if (*edx & CPUID_INTC_EDX_FPU)
3209 hwcap_flags |= AV_386_FPU;
3210 if (*edx & CPUID_INTC_EDX_MMX)
3211 hwcap_flags |= AV_386_MMX;
3212
3213 if (*edx & CPUID_INTC_EDX_TSC)
3214 hwcap_flags |= AV_386_TSC;
3215 if (*edx & CPUID_INTC_EDX_CX8)
3216 hwcap_flags |= AV_386_CX8;
3217 if (*edx & CPUID_INTC_EDX_CMOV)
3218 hwcap_flags |= AV_386_CMOV;
3219 if (*ecx & CPUID_INTC_ECX_CX16)
3220 hwcap_flags |= AV_386_CX16;
3221
3222 if (*ecx & CPUID_INTC_ECX_RDRAND)
3223 hwcap_flags_2 |= AV_386_2_RDRAND;
3224 if (*ebx & CPUID_INTC_EBX_7_0_ADX)
3225 hwcap_flags_2 |= AV_386_2_ADX;
3226 if (*ebx & CPUID_INTC_EBX_7_0_RDSEED)
3227 hwcap_flags_2 |= AV_386_2_RDSEED;
3228 if (*ebx & CPUID_INTC_EBX_7_0_SHA)
3229 hwcap_flags_2 |= AV_386_2_SHA;
3230
3231 }
3232
3233 if (cpi->cpi_xmaxeax < 0x80000001)
3234 goto pass4_done;
3235
3236 switch (cpi->cpi_vendor) {
3237 struct cpuid_regs cp;
3238 uint32_t *edx, *ecx;
3239
3240 case X86_VENDOR_Intel:
3241 /*
3242 * Seems like Intel duplicated what we necessary
3243 * here to make the initial crop of 64-bit OS's work.
3244 * Hopefully, those are the only "extended" bits
3245 * they'll add.
3246 */
3247 /*FALLTHROUGH*/
3248
3249 case X86_VENDOR_AMD:
3250 edx = &cpi->cpi_support[AMD_EDX_FEATURES];
3251 ecx = &cpi->cpi_support[AMD_ECX_FEATURES];
3252
3253 *edx = CPI_FEATURES_XTD_EDX(cpi);
3254 *ecx = CPI_FEATURES_XTD_ECX(cpi);
3255
3256 /*
3257 * [these features require explicit kernel support]
3258 */
3259 switch (cpi->cpi_vendor) {
3260 case X86_VENDOR_Intel:
3261 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3262 *edx &= ~CPUID_AMD_EDX_TSCP;
3263 break;
3264
3265 case X86_VENDOR_AMD:
3266 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3267 *edx &= ~CPUID_AMD_EDX_TSCP;
3268 if (!is_x86_feature(x86_featureset, X86FSET_SSE4A))
3269 *ecx &= ~CPUID_AMD_ECX_SSE4A;
3270 break;
3271
3272 default:
3273 break;
3274 }
3275
3276 /*
3277 * [no explicit support required beyond
3278 * x87 fp context and exception handlers]
3279 */
3280 if (!fpu_exists)
3281 *edx &= ~(CPUID_AMD_EDX_MMXamd |
3282 CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);
3283
3284 if (!is_x86_feature(x86_featureset, X86FSET_NX))
3285 *edx &= ~CPUID_AMD_EDX_NX;
3286 #if !defined(__amd64)
3287 *edx &= ~CPUID_AMD_EDX_LM;
3288 #endif
3289 /*
3290 * Now map the supported feature vector to
3291 * things that we think userland will care about.
3292 */
3293 #if defined(__amd64)
3294 if (*edx & CPUID_AMD_EDX_SYSC)
3295 hwcap_flags |= AV_386_AMD_SYSC;
3296 #endif
3297 if (*edx & CPUID_AMD_EDX_MMXamd)
3298 hwcap_flags |= AV_386_AMD_MMX;
3299 if (*edx & CPUID_AMD_EDX_3DNow)
3300 hwcap_flags |= AV_386_AMD_3DNow;
3301 if (*edx & CPUID_AMD_EDX_3DNowx)
3302 hwcap_flags |= AV_386_AMD_3DNowx;
3303 if (*ecx & CPUID_AMD_ECX_SVM)
3304 hwcap_flags |= AV_386_AMD_SVM;
3305
3306 switch (cpi->cpi_vendor) {
3307 case X86_VENDOR_AMD:
3308 if (*edx & CPUID_AMD_EDX_TSCP)
3309 hwcap_flags |= AV_386_TSCP;
3310 if (*ecx & CPUID_AMD_ECX_AHF64)
3311 hwcap_flags |= AV_386_AHF;
3312 if (*ecx & CPUID_AMD_ECX_SSE4A)
3313 hwcap_flags |= AV_386_AMD_SSE4A;
3314 if (*ecx & CPUID_AMD_ECX_LZCNT)
3315 hwcap_flags |= AV_386_AMD_LZCNT;
3316 break;
3317
3318 case X86_VENDOR_Intel:
3319 if (*edx & CPUID_AMD_EDX_TSCP)
3320 hwcap_flags |= AV_386_TSCP;
3321 /*
3322 * Aarrgh.
3323 * Intel uses a different bit in the same word.
3324 */
3325 if (*ecx & CPUID_INTC_ECX_AHF64)
3326 hwcap_flags |= AV_386_AHF;
3327 break;
3328
3329 default:
3330 break;
3331 }
3332 break;
3333
3334 case X86_VENDOR_TM:
3335 cp.cp_eax = 0x80860001;
3336 (void) __cpuid_insn(&cp);
3337 cpi->cpi_support[TM_EDX_FEATURES] = cp.cp_edx;
3338 break;
3339
3340 default:
3341 break;
3342 }
3343
3344 pass4_done:
3345 cpi->cpi_pass = 4;
3346 if (hwcap_out != NULL) {
3347 hwcap_out[0] = hwcap_flags;
3348 hwcap_out[1] = hwcap_flags_2;
3349 }
3350 }
3351
3352
3353 /*
3354 * Simulate the cpuid instruction using the data we previously
3355 * captured about this CPU. We try our best to return the truth
3356 * about the hardware, independently of kernel support.
3357 */
3358 uint32_t
3359 cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
3360 {
3361 struct cpuid_info *cpi;
3362 struct cpuid_regs *xcp;
3363
3364 if (cpu == NULL)
3365 cpu = CPU;
3366 cpi = cpu->cpu_m.mcpu_cpi;
3367
3368 ASSERT(cpuid_checkpass(cpu, 3));
3369
3370 /*
3371 * CPUID data is cached in two separate places: cpi_std for standard
3372 * CPUID functions, and cpi_extd for extended CPUID functions.
3373 */
3374 if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD)
3375 xcp = &cpi->cpi_std[cp->cp_eax];
3376 else if (cp->cp_eax >= 0x80000000 && cp->cp_eax <= cpi->cpi_xmaxeax &&
3377 cp->cp_eax < 0x80000000 + NMAX_CPI_EXTD)
3378 xcp = &cpi->cpi_extd[cp->cp_eax - 0x80000000];
3379 else
3380 /*
3381 * The caller is asking for data from an input parameter which
3382 * the kernel has not cached. In this case we go fetch from
3383 * the hardware and return the data directly to the user.
3384 */
3385 return (__cpuid_insn(cp));
3386
3387 cp->cp_eax = xcp->cp_eax;
3388 cp->cp_ebx = xcp->cp_ebx;
3389 cp->cp_ecx = xcp->cp_ecx;
3390 cp->cp_edx = xcp->cp_edx;
3391 return (cp->cp_eax);
3392 }
3393
3394 int
3395 cpuid_checkpass(cpu_t *cpu, int pass)
3396 {
3397 return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
3398 cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
3399 }
3400
3401 int
3402 cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
3403 {
3404 ASSERT(cpuid_checkpass(cpu, 3));
3405
3406 return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
3407 }
3408
3409 int
3410 cpuid_is_cmt(cpu_t *cpu)
3411 {
3412 if (cpu == NULL)
3413 cpu = CPU;
3414
3415 ASSERT(cpuid_checkpass(cpu, 1));
3416
3417 return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
3418 }
3419
3420 /*
3421 * AMD and Intel both implement the 64-bit variant of the syscall
3422 * instruction (syscallq), so if there's -any- support for syscall,
3423 * cpuid currently says "yes, we support this".
3424 *
3425 * However, Intel decided to -not- implement the 32-bit variant of the
3426 * syscall instruction, so we provide a predicate to allow our caller
3427 * to test that subtlety here.
3428 *
3429 * XXPV Currently, 32-bit syscall instructions don't work via the hypervisor,
3430 * even in the case where the hardware would in fact support it.
3431 */
3432 /*ARGSUSED*/
3433 int
3434 cpuid_syscall32_insn(cpu_t *cpu)
3435 {
3436 ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), 1));
3437
3438 #if !defined(__xpv)
3439 if (cpu == NULL)
3440 cpu = CPU;
3441
3442 /*CSTYLED*/
3443 {
3444 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3445
3446 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
3447 cpi->cpi_xmaxeax >= 0x80000001 &&
3448 (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
3449 return (1);
3450 }
3451 #endif
3452 return (0);
3453 }
3454
3455 int
3456 cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
3457 {
3458 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3459
3460 static const char fmt[] =
3461 "x86 (%s %X family %d model %d step %d clock %d MHz)";
3462 static const char fmt_ht[] =
3463 "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";
3464
3465 ASSERT(cpuid_checkpass(cpu, 1));
3466
3467 if (cpuid_is_cmt(cpu))
3468 return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
3469 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3470 cpi->cpi_family, cpi->cpi_model,
3471 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3472 return (snprintf(s, n, fmt,
3473 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3474 cpi->cpi_family, cpi->cpi_model,
3475 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3476 }
3477
3478 const char *
3479 cpuid_getvendorstr(cpu_t *cpu)
3480 {
3481 ASSERT(cpuid_checkpass(cpu, 1));
3482 return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
3483 }
3484
3485 uint_t
3486 cpuid_getvendor(cpu_t *cpu)
3487 {
3488 ASSERT(cpuid_checkpass(cpu, 1));
3489 return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
3490 }
3491
3492 uint_t
3493 cpuid_getfamily(cpu_t *cpu)
3494 {
3495 ASSERT(cpuid_checkpass(cpu, 1));
3496 return (cpu->cpu_m.mcpu_cpi->cpi_family);
3497 }
3498
3499 uint_t
3500 cpuid_getmodel(cpu_t *cpu)
3501 {
3502 ASSERT(cpuid_checkpass(cpu, 1));
3503 return (cpu->cpu_m.mcpu_cpi->cpi_model);
3504 }
3505
3506 uint_t
3507 cpuid_get_ncpu_per_chip(cpu_t *cpu)
3508 {
3509 ASSERT(cpuid_checkpass(cpu, 1));
3510 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
3511 }
3512
3513 uint_t
3514 cpuid_get_ncore_per_chip(cpu_t *cpu)
3515 {
3516 ASSERT(cpuid_checkpass(cpu, 1));
3517 return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
3518 }
3519
3520 uint_t
3521 cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
3522 {
3523 ASSERT(cpuid_checkpass(cpu, 2));
3524 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
3525 }
3526
3527 id_t
3528 cpuid_get_last_lvl_cacheid(cpu_t *cpu)
3529 {
3530 ASSERT(cpuid_checkpass(cpu, 2));
3531 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3532 }
3533
3534 uint_t
3535 cpuid_getstep(cpu_t *cpu)
3536 {
3537 ASSERT(cpuid_checkpass(cpu, 1));
3538 return (cpu->cpu_m.mcpu_cpi->cpi_step);
3539 }
3540
3541 uint_t
3542 cpuid_getsig(struct cpu *cpu)
3543 {
3544 ASSERT(cpuid_checkpass(cpu, 1));
3545 return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
3546 }
3547
3548 uint32_t
3549 cpuid_getchiprev(struct cpu *cpu)
3550 {
3551 ASSERT(cpuid_checkpass(cpu, 1));
3552 return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
3553 }
3554
3555 const char *
3556 cpuid_getchiprevstr(struct cpu *cpu)
3557 {
3558 ASSERT(cpuid_checkpass(cpu, 1));
3559 return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
3560 }
3561
3562 uint32_t
3563 cpuid_getsockettype(struct cpu *cpu)
3564 {
3565 ASSERT(cpuid_checkpass(cpu, 1));
3566 return (cpu->cpu_m.mcpu_cpi->cpi_socket);
3567 }
3568
3569 const char *
3570 cpuid_getsocketstr(cpu_t *cpu)
3571 {
3572 static const char *socketstr = NULL;
3573 struct cpuid_info *cpi;
3574
3575 ASSERT(cpuid_checkpass(cpu, 1));
3576 cpi = cpu->cpu_m.mcpu_cpi;
3577
3578 /* Assume that socket types are the same across the system */
3579 if (socketstr == NULL)
3580 socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
3581 cpi->cpi_model, cpi->cpi_step);
3582
3583
3584 return (socketstr);
3585 }
3586
3587 int
3588 cpuid_get_chipid(cpu_t *cpu)
3589 {
3590 ASSERT(cpuid_checkpass(cpu, 1));
3591
3592 if (cpuid_is_cmt(cpu))
3593 return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
3594 return (cpu->cpu_id);
3595 }
3596
3597 id_t
3598 cpuid_get_coreid(cpu_t *cpu)
3599 {
3600 ASSERT(cpuid_checkpass(cpu, 1));
3601 return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
3602 }
3603
3604 int
3605 cpuid_get_pkgcoreid(cpu_t *cpu)
3606 {
3607 ASSERT(cpuid_checkpass(cpu, 1));
3608 return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
3609 }
3610
3611 int
3612 cpuid_get_clogid(cpu_t *cpu)
3613 {
3614 ASSERT(cpuid_checkpass(cpu, 1));
3615 return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
3616 }
3617
3618 int
3619 cpuid_get_cacheid(cpu_t *cpu)
3620 {
3621 ASSERT(cpuid_checkpass(cpu, 1));
3622 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3623 }
3624
3625 uint_t
3626 cpuid_get_procnodeid(cpu_t *cpu)
3627 {
3628 ASSERT(cpuid_checkpass(cpu, 1));
3629 return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
3630 }
3631
3632 uint_t
3633 cpuid_get_procnodes_per_pkg(cpu_t *cpu)
3634 {
3635 ASSERT(cpuid_checkpass(cpu, 1));
3636 return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
3637 }
3638
3639 uint_t
3640 cpuid_get_compunitid(cpu_t *cpu)
3641 {
3642 ASSERT(cpuid_checkpass(cpu, 1));
3643 return (cpu->cpu_m.mcpu_cpi->cpi_compunitid);
3644 }
3645
3646 uint_t
3647 cpuid_get_cores_per_compunit(cpu_t *cpu)
3648 {
3649 ASSERT(cpuid_checkpass(cpu, 1));
3650 return (cpu->cpu_m.mcpu_cpi->cpi_cores_per_compunit);
3651 }
3652
3653 /*ARGSUSED*/
3654 int
3655 cpuid_have_cr8access(cpu_t *cpu)
3656 {
3657 #if defined(__amd64)
3658 return (1);
3659 #else
3660 struct cpuid_info *cpi;
3661
3662 ASSERT(cpu != NULL);
3663 cpi = cpu->cpu_m.mcpu_cpi;
3664 if (cpi->cpi_vendor == X86_VENDOR_AMD && cpi->cpi_maxeax >= 1 &&
3665 (CPI_FEATURES_XTD_ECX(cpi) & CPUID_AMD_ECX_CR8D) != 0)
3666 return (1);
3667 return (0);
3668 #endif
3669 }
3670
3671 uint32_t
3672 cpuid_get_apicid(cpu_t *cpu)
3673 {
3674 ASSERT(cpuid_checkpass(cpu, 1));
3675 if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
3676 return (UINT32_MAX);
3677 } else {
3678 return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
3679 }
3680 }
3681
3682 void
3683 cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
3684 {
3685 struct cpuid_info *cpi;
3686
3687 if (cpu == NULL)
3688 cpu = CPU;
3689 cpi = cpu->cpu_m.mcpu_cpi;
3690
3691 ASSERT(cpuid_checkpass(cpu, 1));
3692
3693 if (pabits)
3694 *pabits = cpi->cpi_pabits;
3695 if (vabits)
3696 *vabits = cpi->cpi_vabits;
3697 }
3698
3699 size_t
3700 cpuid_get_xsave_size()
3701 {
3702 return (MAX(cpuid_info0.cpi_xsave.xsav_max_size,
3703 sizeof (struct xsave_state)));
3704 }
3705
3706 /*
3707 * Return true if the CPUs on this system require 'pointer clearing' for the
3708 * floating point error pointer exception handling. In the past, this has been
3709 * true for all AMD K7 & K8 CPUs, although newer AMD CPUs have been changed to
3710 * behave the same as Intel. This is checked via the CPUID_AMD_EBX_ERR_PTR_ZERO
3711 * feature bit and is reflected in the cpi_fp_amd_save member.
3712 */
3713 boolean_t
3714 cpuid_need_fp_excp_handling()
3715 {
3716 return (cpuid_info0.cpi_vendor == X86_VENDOR_AMD &&
3717 cpuid_info0.cpi_fp_amd_save != 0);
3718 }
3719
3720 /*
3721 * Returns the number of data TLB entries for a corresponding
3722 * pagesize. If it can't be computed, or isn't known, the
3723 * routine returns zero. If you ask about an architecturally
3724 * impossible pagesize, the routine will panic (so that the
3725 * hat implementor knows that things are inconsistent.)
3726 */
3727 uint_t
3728 cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
3729 {
3730 struct cpuid_info *cpi;
3731 uint_t dtlb_nent = 0;
3732
3733 if (cpu == NULL)
3734 cpu = CPU;
3735 cpi = cpu->cpu_m.mcpu_cpi;
3736
3737 ASSERT(cpuid_checkpass(cpu, 1));
3738
3739 /*
3740 * Check the L2 TLB info
3741 */
3742 if (cpi->cpi_xmaxeax >= 0x80000006) {
3743 struct cpuid_regs *cp = &cpi->cpi_extd[6];
3744
3745 switch (pagesize) {
3746
3747 case 4 * 1024:
3748 /*
3749 * All zero in the top 16 bits of the register
3750 * indicates a unified TLB. Size is in low 16 bits.
3751 */
3752 if ((cp->cp_ebx & 0xffff0000) == 0)
3753 dtlb_nent = cp->cp_ebx & 0x0000ffff;
3754 else
3755 dtlb_nent = BITX(cp->cp_ebx, 27, 16);
3756 break;
3757
3758 case 2 * 1024 * 1024:
3759 if ((cp->cp_eax & 0xffff0000) == 0)
3760 dtlb_nent = cp->cp_eax & 0x0000ffff;
3761 else
3762 dtlb_nent = BITX(cp->cp_eax, 27, 16);
3763 break;
3764
3765 default:
3766 panic("unknown L2 pagesize");
3767 /*NOTREACHED*/
3768 }
3769 }
3770
3771 if (dtlb_nent != 0)
3772 return (dtlb_nent);
3773
3774 /*
3775 * No L2 TLB support for this size, try L1.
3776 */
3777 if (cpi->cpi_xmaxeax >= 0x80000005) {
3778 struct cpuid_regs *cp = &cpi->cpi_extd[5];
3779
3780 switch (pagesize) {
3781 case 4 * 1024:
3782 dtlb_nent = BITX(cp->cp_ebx, 23, 16);
3783 break;
3784 case 2 * 1024 * 1024:
3785 dtlb_nent = BITX(cp->cp_eax, 23, 16);
3786 break;
3787 default:
3788 panic("unknown L1 d-TLB pagesize");
3789 /*NOTREACHED*/
3790 }
3791 }
3792
3793 return (dtlb_nent);
3794 }
3795
3796 /*
3797 * Return 0 if the erratum is not present or not applicable, positive
3798 * if it is, and negative if the status of the erratum is unknown.
3799 *
3800 * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
3801 * Processors" #25759, Rev 3.57, August 2005
3802 */
3803 int
3804 cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
3805 {
3806 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3807 uint_t eax;
3808
3809 /*
3810 * Bail out if this CPU isn't an AMD CPU, or if it's
3811 * a legacy (32-bit) AMD CPU.
3812 */
3813 if (cpi->cpi_vendor != X86_VENDOR_AMD ||
3814 cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
3815 cpi->cpi_family == 6)
3816
3817 return (0);
3818
3819 eax = cpi->cpi_std[1].cp_eax;
3820
3821 #define SH_B0(eax) (eax == 0xf40 || eax == 0xf50)
3822 #define SH_B3(eax) (eax == 0xf51)
3823 #define B(eax) (SH_B0(eax) || SH_B3(eax))
3824
3825 #define SH_C0(eax) (eax == 0xf48 || eax == 0xf58)
3826
3827 #define SH_CG(eax) (eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
3828 #define DH_CG(eax) (eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
3829 #define CH_CG(eax) (eax == 0xf82 || eax == 0xfb2)
3830 #define CG(eax) (SH_CG(eax) || DH_CG(eax) || CH_CG(eax))
3831
3832 #define SH_D0(eax) (eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
3833 #define DH_D0(eax) (eax == 0x10fc0 || eax == 0x10ff0)
3834 #define CH_D0(eax) (eax == 0x10f80 || eax == 0x10fb0)
3835 #define D0(eax) (SH_D0(eax) || DH_D0(eax) || CH_D0(eax))
3836
3837 #define SH_E0(eax) (eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
3838 #define JH_E1(eax) (eax == 0x20f10) /* JH8_E0 had 0x20f30 */
3839 #define DH_E3(eax) (eax == 0x20fc0 || eax == 0x20ff0)
3840 #define SH_E4(eax) (eax == 0x20f51 || eax == 0x20f71)
3841 #define BH_E4(eax) (eax == 0x20fb1)
3842 #define SH_E5(eax) (eax == 0x20f42)
3843 #define DH_E6(eax) (eax == 0x20ff2 || eax == 0x20fc2)
3844 #define JH_E6(eax) (eax == 0x20f12 || eax == 0x20f32)
3845 #define EX(eax) (SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
3846 SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
3847 DH_E6(eax) || JH_E6(eax))
3848
3849 #define DR_AX(eax) (eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
3850 #define DR_B0(eax) (eax == 0x100f20)
3851 #define DR_B1(eax) (eax == 0x100f21)
3852 #define DR_BA(eax) (eax == 0x100f2a)
3853 #define DR_B2(eax) (eax == 0x100f22)
3854 #define DR_B3(eax) (eax == 0x100f23)
3855 #define RB_C0(eax) (eax == 0x100f40)
3856
3857 switch (erratum) {
3858 case 1:
3859 return (cpi->cpi_family < 0x10);
3860 case 51: /* what does the asterisk mean? */
3861 return (B(eax) || SH_C0(eax) || CG(eax));
3862 case 52:
3863 return (B(eax));
3864 case 57:
3865 return (cpi->cpi_family <= 0x11);
3866 case 58:
3867 return (B(eax));
3868 case 60:
3869 return (cpi->cpi_family <= 0x11);
3870 case 61:
3871 case 62:
3872 case 63:
3873 case 64:
3874 case 65:
3875 case 66:
3876 case 68:
3877 case 69:
3878 case 70:
3879 case 71:
3880 return (B(eax));
3881 case 72:
3882 return (SH_B0(eax));
3883 case 74:
3884 return (B(eax));
3885 case 75:
3886 return (cpi->cpi_family < 0x10);
3887 case 76:
3888 return (B(eax));
3889 case 77:
3890 return (cpi->cpi_family <= 0x11);
3891 case 78:
3892 return (B(eax) || SH_C0(eax));
3893 case 79:
3894 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3895 case 80:
3896 case 81:
3897 case 82:
3898 return (B(eax));
3899 case 83:
3900 return (B(eax) || SH_C0(eax) || CG(eax));
3901 case 85:
3902 return (cpi->cpi_family < 0x10);
3903 case 86:
3904 return (SH_C0(eax) || CG(eax));
3905 case 88:
3906 #if !defined(__amd64)
3907 return (0);
3908 #else
3909 return (B(eax) || SH_C0(eax));
3910 #endif
3911 case 89:
3912 return (cpi->cpi_family < 0x10);
3913 case 90:
3914 return (B(eax) || SH_C0(eax) || CG(eax));
3915 case 91:
3916 case 92:
3917 return (B(eax) || SH_C0(eax));
3918 case 93:
3919 return (SH_C0(eax));
3920 case 94:
3921 return (B(eax) || SH_C0(eax) || CG(eax));
3922 case 95:
3923 #if !defined(__amd64)
3924 return (0);
3925 #else
3926 return (B(eax) || SH_C0(eax));
3927 #endif
3928 case 96:
3929 return (B(eax) || SH_C0(eax) || CG(eax));
3930 case 97:
3931 case 98:
3932 return (SH_C0(eax) || CG(eax));
3933 case 99:
3934 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3935 case 100:
3936 return (B(eax) || SH_C0(eax));
3937 case 101:
3938 case 103:
3939 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3940 case 104:
3941 return (SH_C0(eax) || CG(eax) || D0(eax));
3942 case 105:
3943 case 106:
3944 case 107:
3945 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3946 case 108:
3947 return (DH_CG(eax));
3948 case 109:
3949 return (SH_C0(eax) || CG(eax) || D0(eax));
3950 case 110:
3951 return (D0(eax) || EX(eax));
3952 case 111:
3953 return (CG(eax));
3954 case 112:
3955 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3956 case 113:
3957 return (eax == 0x20fc0);
3958 case 114:
3959 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3960 case 115:
3961 return (SH_E0(eax) || JH_E1(eax));
3962 case 116:
3963 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3964 case 117:
3965 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3966 case 118:
3967 return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
3968 JH_E6(eax));
3969 case 121:
3970 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3971 case 122:
3972 return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
3973 case 123:
3974 return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
3975 case 131:
3976 return (cpi->cpi_family < 0x10);
3977 case 6336786:
3978 /*
3979 * Test for AdvPowerMgmtInfo.TscPStateInvariant
3980 * if this is a K8 family or newer processor
3981 */
3982 if (CPI_FAMILY(cpi) == 0xf) {
3983 struct cpuid_regs regs;
3984 regs.cp_eax = 0x80000007;
3985 (void) __cpuid_insn(®s);
3986 return (!(regs.cp_edx & 0x100));
3987 }
3988 return (0);
3989 case 6323525:
3990 return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
3991 (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);
3992
3993 case 6671130:
3994 /*
3995 * check for processors (pre-Shanghai) that do not provide
3996 * optimal management of 1gb ptes in its tlb.
3997 */
3998 return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);
3999
4000 case 298:
4001 return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
4002 DR_B2(eax) || RB_C0(eax));
4003
4004 case 721:
4005 #if defined(__amd64)
4006 return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);
4007 #else
4008 return (0);
4009 #endif
4010
4011 default:
4012 return (-1);
4013
4014 }
4015 }
4016
4017 /*
4018 * Determine if specified erratum is present via OSVW (OS Visible Workaround).
4019 * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
4020 */
4021 int
4022 osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
4023 {
4024 struct cpuid_info *cpi;
4025 uint_t osvwid;
4026 static int osvwfeature = -1;
4027 uint64_t osvwlength;
4028
4029
4030 cpi = cpu->cpu_m.mcpu_cpi;
4031
4032 /* confirm OSVW supported */
4033 if (osvwfeature == -1) {
4034 osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
4035 } else {
4036 /* assert that osvw feature setting is consistent on all cpus */
4037 ASSERT(osvwfeature ==
4038 (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
4039 }
4040 if (!osvwfeature)
4041 return (-1);
4042
4043 osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;
4044
4045 switch (erratum) {
4046 case 298: /* osvwid is 0 */
4047 osvwid = 0;
4048 if (osvwlength <= (uint64_t)osvwid) {
4049 /* osvwid 0 is unknown */
4050 return (-1);
4051 }
4052
4053 /*
4054 * Check the OSVW STATUS MSR to determine the state
4055 * of the erratum where:
4056 * 0 - fixed by HW
4057 * 1 - BIOS has applied the workaround when BIOS
4058 * workaround is available. (Or for other errata,
4059 * OS workaround is required.)
4060 * For a value of 1, caller will confirm that the
4061 * erratum 298 workaround has indeed been applied by BIOS.
4062 *
4063 * A 1 may be set in cpus that have a HW fix
4064 * in a mixed cpu system. Regarding erratum 298:
4065 * In a multiprocessor platform, the workaround above
4066 * should be applied to all processors regardless of
4067 * silicon revision when an affected processor is
4068 * present.
4069 */
4070
4071 return (rdmsr(MSR_AMD_OSVW_STATUS +
4072 (osvwid / OSVW_ID_CNT_PER_MSR)) &
4073 (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));
4074
4075 default:
4076 return (-1);
4077 }
4078 }
4079
4080 static const char assoc_str[] = "associativity";
4081 static const char line_str[] = "line-size";
4082 static const char size_str[] = "size";
4083
4084 static void
4085 add_cache_prop(dev_info_t *devi, const char *label, const char *type,
4086 uint32_t val)
4087 {
4088 char buf[128];
4089
4090 /*
4091 * ndi_prop_update_int() is used because it is desirable for
4092 * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
4093 */
4094 if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
4095 (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
4096 }
4097
4098 /*
4099 * Intel-style cache/tlb description
4100 *
4101 * Standard cpuid level 2 gives a randomly ordered
4102 * selection of tags that index into a table that describes
4103 * cache and tlb properties.
4104 */
4105
4106 static const char l1_icache_str[] = "l1-icache";
4107 static const char l1_dcache_str[] = "l1-dcache";
4108 static const char l2_cache_str[] = "l2-cache";
4109 static const char l3_cache_str[] = "l3-cache";
4110 static const char itlb4k_str[] = "itlb-4K";
4111 static const char dtlb4k_str[] = "dtlb-4K";
4112 static const char itlb2M_str[] = "itlb-2M";
4113 static const char itlb4M_str[] = "itlb-4M";
4114 static const char dtlb4M_str[] = "dtlb-4M";
4115 static const char dtlb24_str[] = "dtlb0-2M-4M";
4116 static const char itlb424_str[] = "itlb-4K-2M-4M";
4117 static const char itlb24_str[] = "itlb-2M-4M";
4118 static const char dtlb44_str[] = "dtlb-4K-4M";
4119 static const char sl1_dcache_str[] = "sectored-l1-dcache";
4120 static const char sl2_cache_str[] = "sectored-l2-cache";
4121 static const char itrace_str[] = "itrace-cache";
4122 static const char sl3_cache_str[] = "sectored-l3-cache";
4123 static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";
4124
4125 static const struct cachetab {
4126 uint8_t ct_code;
4127 uint8_t ct_assoc;
4128 uint16_t ct_line_size;
4129 size_t ct_size;
4130 const char *ct_label;
4131 } intel_ctab[] = {
4132 /*
4133 * maintain descending order!
4134 *
4135 * Codes ignored - Reason
4136 * ----------------------
4137 * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
4138 * f0H/f1H - Currently we do not interpret prefetch size by design
4139 */
4140 { 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
4141 { 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
4142 { 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
4143 { 0xde, 12, 64, 6*1024*1024, l3_cache_str},
4144 { 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
4145 { 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
4146 { 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
4147 { 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
4148 { 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
4149 { 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
4150 { 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
4151 { 0xd0, 4, 64, 512*1024, l3_cache_str},
4152 { 0xca, 4, 0, 512, sh_l2_tlb4k_str},
4153 { 0xc0, 4, 0, 8, dtlb44_str },
4154 { 0xba, 4, 0, 64, dtlb4k_str },
4155 { 0xb4, 4, 0, 256, dtlb4k_str },
4156 { 0xb3, 4, 0, 128, dtlb4k_str },
4157 { 0xb2, 4, 0, 64, itlb4k_str },
4158 { 0xb0, 4, 0, 128, itlb4k_str },
4159 { 0x87, 8, 64, 1024*1024, l2_cache_str},
4160 { 0x86, 4, 64, 512*1024, l2_cache_str},
4161 { 0x85, 8, 32, 2*1024*1024, l2_cache_str},
4162 { 0x84, 8, 32, 1024*1024, l2_cache_str},
4163 { 0x83, 8, 32, 512*1024, l2_cache_str},
4164 { 0x82, 8, 32, 256*1024, l2_cache_str},
4165 { 0x80, 8, 64, 512*1024, l2_cache_str},
4166 { 0x7f, 2, 64, 512*1024, l2_cache_str},
4167 { 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
4168 { 0x7c, 8, 64, 1024*1024, sl2_cache_str},
4169 { 0x7b, 8, 64, 512*1024, sl2_cache_str},
4170 { 0x7a, 8, 64, 256*1024, sl2_cache_str},
4171 { 0x79, 8, 64, 128*1024, sl2_cache_str},
4172 { 0x78, 8, 64, 1024*1024, l2_cache_str},
4173 { 0x73, 8, 0, 64*1024, itrace_str},
4174 { 0x72, 8, 0, 32*1024, itrace_str},
4175 { 0x71, 8, 0, 16*1024, itrace_str},
4176 { 0x70, 8, 0, 12*1024, itrace_str},
4177 { 0x68, 4, 64, 32*1024, sl1_dcache_str},
4178 { 0x67, 4, 64, 16*1024, sl1_dcache_str},
4179 { 0x66, 4, 64, 8*1024, sl1_dcache_str},
4180 { 0x60, 8, 64, 16*1024, sl1_dcache_str},
4181 { 0x5d, 0, 0, 256, dtlb44_str},
4182 { 0x5c, 0, 0, 128, dtlb44_str},
4183 { 0x5b, 0, 0, 64, dtlb44_str},
4184 { 0x5a, 4, 0, 32, dtlb24_str},
4185 { 0x59, 0, 0, 16, dtlb4k_str},
4186 { 0x57, 4, 0, 16, dtlb4k_str},
4187 { 0x56, 4, 0, 16, dtlb4M_str},
4188 { 0x55, 0, 0, 7, itlb24_str},
4189 { 0x52, 0, 0, 256, itlb424_str},
4190 { 0x51, 0, 0, 128, itlb424_str},
4191 { 0x50, 0, 0, 64, itlb424_str},
4192 { 0x4f, 0, 0, 32, itlb4k_str},
4193 { 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
4194 { 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
4195 { 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
4196 { 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
4197 { 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
4198 { 0x49, 16, 64, 4*1024*1024, l3_cache_str},
4199 { 0x48, 12, 64, 3*1024*1024, l2_cache_str},
4200 { 0x47, 8, 64, 8*1024*1024, l3_cache_str},
4201 { 0x46, 4, 64, 4*1024*1024, l3_cache_str},
4202 { 0x45, 4, 32, 2*1024*1024, l2_cache_str},
4203 { 0x44, 4, 32, 1024*1024, l2_cache_str},
4204 { 0x43, 4, 32, 512*1024, l2_cache_str},
4205 { 0x42, 4, 32, 256*1024, l2_cache_str},
4206 { 0x41, 4, 32, 128*1024, l2_cache_str},
4207 { 0x3e, 4, 64, 512*1024, sl2_cache_str},
4208 { 0x3d, 6, 64, 384*1024, sl2_cache_str},
4209 { 0x3c, 4, 64, 256*1024, sl2_cache_str},
4210 { 0x3b, 2, 64, 128*1024, sl2_cache_str},
4211 { 0x3a, 6, 64, 192*1024, sl2_cache_str},
4212 { 0x39, 4, 64, 128*1024, sl2_cache_str},
4213 { 0x30, 8, 64, 32*1024, l1_icache_str},
4214 { 0x2c, 8, 64, 32*1024, l1_dcache_str},
4215 { 0x29, 8, 64, 4096*1024, sl3_cache_str},
4216 { 0x25, 8, 64, 2048*1024, sl3_cache_str},
4217 { 0x23, 8, 64, 1024*1024, sl3_cache_str},
4218 { 0x22, 4, 64, 512*1024, sl3_cache_str},
4219 { 0x0e, 6, 64, 24*1024, l1_dcache_str},
4220 { 0x0d, 4, 32, 16*1024, l1_dcache_str},
4221 { 0x0c, 4, 32, 16*1024, l1_dcache_str},
4222 { 0x0b, 4, 0, 4, itlb4M_str},
4223 { 0x0a, 2, 32, 8*1024, l1_dcache_str},
4224 { 0x08, 4, 32, 16*1024, l1_icache_str},
4225 { 0x06, 4, 32, 8*1024, l1_icache_str},
4226 { 0x05, 4, 0, 32, dtlb4M_str},
4227 { 0x04, 4, 0, 8, dtlb4M_str},
4228 { 0x03, 4, 0, 64, dtlb4k_str},
4229 { 0x02, 4, 0, 2, itlb4M_str},
4230 { 0x01, 4, 0, 32, itlb4k_str},
4231 { 0 }
4232 };
4233
4234 static const struct cachetab cyrix_ctab[] = {
4235 { 0x70, 4, 0, 32, "tlb-4K" },
4236 { 0x80, 4, 16, 16*1024, "l1-cache" },
4237 { 0 }
4238 };
4239
4240 /*
4241 * Search a cache table for a matching entry
4242 */
4243 static const struct cachetab *
4244 find_cacheent(const struct cachetab *ct, uint_t code)
4245 {
4246 if (code != 0) {
4247 for (; ct->ct_code != 0; ct++)
4248 if (ct->ct_code <= code)
4249 break;
4250 if (ct->ct_code == code)
4251 return (ct);
4252 }
4253 return (NULL);
4254 }
4255
4256 /*
4257 * Populate cachetab entry with L2 or L3 cache-information using
4258 * cpuid function 4. This function is called from intel_walk_cacheinfo()
4259 * when descriptor 0x49 is encountered. It returns 0 if no such cache
4260 * information is found.
4261 */
4262 static int
4263 intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
4264 {
4265 uint32_t level, i;
4266 int ret = 0;
4267
4268 for (i = 0; i < cpi->cpi_std_4_size; i++) {
4269 level = CPI_CACHE_LVL(cpi->cpi_std_4[i]);
4270
4271 if (level == 2 || level == 3) {
4272 ct->ct_assoc = CPI_CACHE_WAYS(cpi->cpi_std_4[i]) + 1;
4273 ct->ct_line_size =
4274 CPI_CACHE_COH_LN_SZ(cpi->cpi_std_4[i]) + 1;
4275 ct->ct_size = ct->ct_assoc *
4276 (CPI_CACHE_PARTS(cpi->cpi_std_4[i]) + 1) *
4277 ct->ct_line_size *
4278 (cpi->cpi_std_4[i]->cp_ecx + 1);
4279
4280 if (level == 2) {
4281 ct->ct_label = l2_cache_str;
4282 } else if (level == 3) {
4283 ct->ct_label = l3_cache_str;
4284 }
4285 ret = 1;
4286 }
4287 }
4288
4289 return (ret);
4290 }
4291
4292 /*
4293 * Walk the cacheinfo descriptor, applying 'func' to every valid element
4294 * The walk is terminated if the walker returns non-zero.
4295 */
4296 static void
4297 intel_walk_cacheinfo(struct cpuid_info *cpi,
4298 void *arg, int (*func)(void *, const struct cachetab *))
4299 {
4300 const struct cachetab *ct;
4301 struct cachetab des_49_ct, des_b1_ct;
4302 uint8_t *dp;
4303 int i;
4304
4305 if ((dp = cpi->cpi_cacheinfo) == NULL)
4306 return;
4307 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4308 /*
4309 * For overloaded descriptor 0x49 we use cpuid function 4
4310 * if supported by the current processor, to create
4311 * cache information.
4312 * For overloaded descriptor 0xb1 we use X86_PAE flag
4313 * to disambiguate the cache information.
4314 */
4315 if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
4316 intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
4317 ct = &des_49_ct;
4318 } else if (*dp == 0xb1) {
4319 des_b1_ct.ct_code = 0xb1;
4320 des_b1_ct.ct_assoc = 4;
4321 des_b1_ct.ct_line_size = 0;
4322 if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
4323 des_b1_ct.ct_size = 8;
4324 des_b1_ct.ct_label = itlb2M_str;
4325 } else {
4326 des_b1_ct.ct_size = 4;
4327 des_b1_ct.ct_label = itlb4M_str;
4328 }
4329 ct = &des_b1_ct;
4330 } else {
4331 if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
4332 continue;
4333 }
4334 }
4335
4336 if (func(arg, ct) != 0) {
4337 break;
4338 }
4339 }
4340 }
4341
4342 /*
4343 * (Like the Intel one, except for Cyrix CPUs)
4344 */
4345 static void
4346 cyrix_walk_cacheinfo(struct cpuid_info *cpi,
4347 void *arg, int (*func)(void *, const struct cachetab *))
4348 {
4349 const struct cachetab *ct;
4350 uint8_t *dp;
4351 int i;
4352
4353 if ((dp = cpi->cpi_cacheinfo) == NULL)
4354 return;
4355 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4356 /*
4357 * Search Cyrix-specific descriptor table first ..
4358 */
4359 if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
4360 if (func(arg, ct) != 0)
4361 break;
4362 continue;
4363 }
4364 /*
4365 * .. else fall back to the Intel one
4366 */
4367 if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
4368 if (func(arg, ct) != 0)
4369 break;
4370 continue;
4371 }
4372 }
4373 }
4374
4375 /*
4376 * A cacheinfo walker that adds associativity, line-size, and size properties
4377 * to the devinfo node it is passed as an argument.
4378 */
4379 static int
4380 add_cacheent_props(void *arg, const struct cachetab *ct)
4381 {
4382 dev_info_t *devi = arg;
4383
4384 add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
4385 if (ct->ct_line_size != 0)
4386 add_cache_prop(devi, ct->ct_label, line_str,
4387 ct->ct_line_size);
4388 add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
4389 return (0);
4390 }
4391
4392
4393 static const char fully_assoc[] = "fully-associative?";
4394
4395 /*
4396 * AMD style cache/tlb description
4397 *
4398 * Extended functions 5 and 6 directly describe properties of
4399 * tlbs and various cache levels.
4400 */
4401 static void
4402 add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4403 {
4404 switch (assoc) {
4405 case 0: /* reserved; ignore */
4406 break;
4407 default:
4408 add_cache_prop(devi, label, assoc_str, assoc);
4409 break;
4410 case 0xff:
4411 add_cache_prop(devi, label, fully_assoc, 1);
4412 break;
4413 }
4414 }
4415
4416 static void
4417 add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4418 {
4419 if (size == 0)
4420 return;
4421 add_cache_prop(devi, label, size_str, size);
4422 add_amd_assoc(devi, label, assoc);
4423 }
4424
4425 static void
4426 add_amd_cache(dev_info_t *devi, const char *label,
4427 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4428 {
4429 if (size == 0 || line_size == 0)
4430 return;
4431 add_amd_assoc(devi, label, assoc);
4432 /*
4433 * Most AMD parts have a sectored cache. Multiple cache lines are
4434 * associated with each tag. A sector consists of all cache lines
4435 * associated with a tag. For example, the AMD K6-III has a sector
4436 * size of 2 cache lines per tag.
4437 */
4438 if (lines_per_tag != 0)
4439 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4440 add_cache_prop(devi, label, line_str, line_size);
4441 add_cache_prop(devi, label, size_str, size * 1024);
4442 }
4443
4444 static void
4445 add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4446 {
4447 switch (assoc) {
4448 case 0: /* off */
4449 break;
4450 case 1:
4451 case 2:
4452 case 4:
4453 add_cache_prop(devi, label, assoc_str, assoc);
4454 break;
4455 case 6:
4456 add_cache_prop(devi, label, assoc_str, 8);
4457 break;
4458 case 8:
4459 add_cache_prop(devi, label, assoc_str, 16);
4460 break;
4461 case 0xf:
4462 add_cache_prop(devi, label, fully_assoc, 1);
4463 break;
4464 default: /* reserved; ignore */
4465 break;
4466 }
4467 }
4468
4469 static void
4470 add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4471 {
4472 if (size == 0 || assoc == 0)
4473 return;
4474 add_amd_l2_assoc(devi, label, assoc);
4475 add_cache_prop(devi, label, size_str, size);
4476 }
4477
4478 static void
4479 add_amd_l2_cache(dev_info_t *devi, const char *label,
4480 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4481 {
4482 if (size == 0 || assoc == 0 || line_size == 0)
4483 return;
4484 add_amd_l2_assoc(devi, label, assoc);
4485 if (lines_per_tag != 0)
4486 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4487 add_cache_prop(devi, label, line_str, line_size);
4488 add_cache_prop(devi, label, size_str, size * 1024);
4489 }
4490
4491 static void
4492 amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
4493 {
4494 struct cpuid_regs *cp;
4495
4496 if (cpi->cpi_xmaxeax < 0x80000005)
4497 return;
4498 cp = &cpi->cpi_extd[5];
4499
4500 /*
4501 * 4M/2M L1 TLB configuration
4502 *
4503 * We report the size for 2M pages because AMD uses two
4504 * TLB entries for one 4M page.
4505 */
4506 add_amd_tlb(devi, "dtlb-2M",
4507 BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
4508 add_amd_tlb(devi, "itlb-2M",
4509 BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));
4510
4511 /*
4512 * 4K L1 TLB configuration
4513 */
4514
4515 switch (cpi->cpi_vendor) {
4516 uint_t nentries;
4517 case X86_VENDOR_TM:
4518 if (cpi->cpi_family >= 5) {
4519 /*
4520 * Crusoe processors have 256 TLB entries, but
4521 * cpuid data format constrains them to only
4522 * reporting 255 of them.
4523 */
4524 if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
4525 nentries = 256;
4526 /*
4527 * Crusoe processors also have a unified TLB
4528 */
4529 add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
4530 nentries);
4531 break;
4532 }
4533 /*FALLTHROUGH*/
4534 default:
4535 add_amd_tlb(devi, itlb4k_str,
4536 BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
4537 add_amd_tlb(devi, dtlb4k_str,
4538 BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
4539 break;
4540 }
4541
4542 /*
4543 * data L1 cache configuration
4544 */
4545
4546 add_amd_cache(devi, l1_dcache_str,
4547 BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
4548 BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));
4549
4550 /*
4551 * code L1 cache configuration
4552 */
4553
4554 add_amd_cache(devi, l1_icache_str,
4555 BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
4556 BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));
4557
4558 if (cpi->cpi_xmaxeax < 0x80000006)
4559 return;
4560 cp = &cpi->cpi_extd[6];
4561
4562 /* Check for a unified L2 TLB for large pages */
4563
4564 if (BITX(cp->cp_eax, 31, 16) == 0)
4565 add_amd_l2_tlb(devi, "l2-tlb-2M",
4566 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4567 else {
4568 add_amd_l2_tlb(devi, "l2-dtlb-2M",
4569 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4570 add_amd_l2_tlb(devi, "l2-itlb-2M",
4571 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4572 }
4573
4574 /* Check for a unified L2 TLB for 4K pages */
4575
4576 if (BITX(cp->cp_ebx, 31, 16) == 0) {
4577 add_amd_l2_tlb(devi, "l2-tlb-4K",
4578 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4579 } else {
4580 add_amd_l2_tlb(devi, "l2-dtlb-4K",
4581 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4582 add_amd_l2_tlb(devi, "l2-itlb-4K",
4583 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4584 }
4585
4586 add_amd_l2_cache(devi, l2_cache_str,
4587 BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
4588 BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
4589 }
4590
4591 /*
4592 * There are two basic ways that the x86 world describes it cache
4593 * and tlb architecture - Intel's way and AMD's way.
4594 *
4595 * Return which flavor of cache architecture we should use
4596 */
4597 static int
4598 x86_which_cacheinfo(struct cpuid_info *cpi)
4599 {
4600 switch (cpi->cpi_vendor) {
4601 case X86_VENDOR_Intel:
4602 if (cpi->cpi_maxeax >= 2)
4603 return (X86_VENDOR_Intel);
4604 break;
4605 case X86_VENDOR_AMD:
4606 /*
4607 * The K5 model 1 was the first part from AMD that reported
4608 * cache sizes via extended cpuid functions.
4609 */
4610 if (cpi->cpi_family > 5 ||
4611 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
4612 return (X86_VENDOR_AMD);
4613 break;
4614 case X86_VENDOR_TM:
4615 if (cpi->cpi_family >= 5)
4616 return (X86_VENDOR_AMD);
4617 /*FALLTHROUGH*/
4618 default:
4619 /*
4620 * If they have extended CPU data for 0x80000005
4621 * then we assume they have AMD-format cache
4622 * information.
4623 *
4624 * If not, and the vendor happens to be Cyrix,
4625 * then try our-Cyrix specific handler.
4626 *
4627 * If we're not Cyrix, then assume we're using Intel's
4628 * table-driven format instead.
4629 */
4630 if (cpi->cpi_xmaxeax >= 0x80000005)
4631 return (X86_VENDOR_AMD);
4632 else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
4633 return (X86_VENDOR_Cyrix);
4634 else if (cpi->cpi_maxeax >= 2)
4635 return (X86_VENDOR_Intel);
4636 break;
4637 }
4638 return (-1);
4639 }
4640
4641 void
4642 cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
4643 struct cpuid_info *cpi)
4644 {
4645 dev_info_t *cpu_devi;
4646 int create;
4647
4648 cpu_devi = (dev_info_t *)dip;
4649
4650 /* device_type */
4651 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4652 "device_type", "cpu");
4653
4654 /* reg */
4655 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4656 "reg", cpu_id);
4657
4658 /* cpu-mhz, and clock-frequency */
4659 if (cpu_freq > 0) {
4660 long long mul;
4661
4662 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4663 "cpu-mhz", cpu_freq);
4664 if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
4665 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4666 "clock-frequency", (int)mul);
4667 }
4668
4669 if (!is_x86_feature(x86_featureset, X86FSET_CPUID)) {
4670 return;
4671 }
4672
4673 /* vendor-id */
4674 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4675 "vendor-id", cpi->cpi_vendorstr);
4676
4677 if (cpi->cpi_maxeax == 0) {
4678 return;
4679 }
4680
4681 /*
4682 * family, model, and step
4683 */
4684 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4685 "family", CPI_FAMILY(cpi));
4686 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4687 "cpu-model", CPI_MODEL(cpi));
4688 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4689 "stepping-id", CPI_STEP(cpi));
4690
4691 /* type */
4692 switch (cpi->cpi_vendor) {
4693 case X86_VENDOR_Intel:
4694 create = 1;
4695 break;
4696 default:
4697 create = 0;
4698 break;
4699 }
4700 if (create)
4701 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4702 "type", CPI_TYPE(cpi));
4703
4704 /* ext-family */
4705 switch (cpi->cpi_vendor) {
4706 case X86_VENDOR_Intel:
4707 case X86_VENDOR_AMD:
4708 create = cpi->cpi_family >= 0xf;
4709 break;
4710 default:
4711 create = 0;
4712 break;
4713 }
4714 if (create)
4715 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4716 "ext-family", CPI_FAMILY_XTD(cpi));
4717
4718 /* ext-model */
4719 switch (cpi->cpi_vendor) {
4720 case X86_VENDOR_Intel:
4721 create = IS_EXTENDED_MODEL_INTEL(cpi);
4722 break;
4723 case X86_VENDOR_AMD:
4724 create = CPI_FAMILY(cpi) == 0xf;
4725 break;
4726 default:
4727 create = 0;
4728 break;
4729 }
4730 if (create)
4731 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4732 "ext-model", CPI_MODEL_XTD(cpi));
4733
4734 /* generation */
4735 switch (cpi->cpi_vendor) {
4736 case X86_VENDOR_AMD:
4737 /*
4738 * AMD K5 model 1 was the first part to support this
4739 */
4740 create = cpi->cpi_xmaxeax >= 0x80000001;
4741 break;
4742 default:
4743 create = 0;
4744 break;
4745 }
4746 if (create)
4747 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4748 "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));
4749
4750 /* brand-id */
4751 switch (cpi->cpi_vendor) {
4752 case X86_VENDOR_Intel:
4753 /*
4754 * brand id first appeared on Pentium III Xeon model 8,
4755 * and Celeron model 8 processors and Opteron
4756 */
4757 create = cpi->cpi_family > 6 ||
4758 (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
4759 break;
4760 case X86_VENDOR_AMD:
4761 create = cpi->cpi_family >= 0xf;
4762 break;
4763 default:
4764 create = 0;
4765 break;
4766 }
4767 if (create && cpi->cpi_brandid != 0) {
4768 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4769 "brand-id", cpi->cpi_brandid);
4770 }
4771
4772 /* chunks, and apic-id */
4773 switch (cpi->cpi_vendor) {
4774 /*
4775 * first available on Pentium IV and Opteron (K8)
4776 */
4777 case X86_VENDOR_Intel:
4778 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4779 break;
4780 case X86_VENDOR_AMD:
4781 create = cpi->cpi_family >= 0xf;
4782 break;
4783 default:
4784 create = 0;
4785 break;
4786 }
4787 if (create) {
4788 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4789 "chunks", CPI_CHUNKS(cpi));
4790 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4791 "apic-id", cpi->cpi_apicid);
4792 if (cpi->cpi_chipid >= 0) {
4793 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4794 "chip#", cpi->cpi_chipid);
4795 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4796 "clog#", cpi->cpi_clogid);
4797 }
4798 }
4799
4800 /* cpuid-features */
4801 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4802 "cpuid-features", CPI_FEATURES_EDX(cpi));
4803
4804
4805 /* cpuid-features-ecx */
4806 switch (cpi->cpi_vendor) {
4807 case X86_VENDOR_Intel:
4808 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4809 break;
4810 case X86_VENDOR_AMD:
4811 create = cpi->cpi_family >= 0xf;
4812 break;
4813 default:
4814 create = 0;
4815 break;
4816 }
4817 if (create)
4818 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4819 "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));
4820
4821 /* ext-cpuid-features */
4822 switch (cpi->cpi_vendor) {
4823 case X86_VENDOR_Intel:
4824 case X86_VENDOR_AMD:
4825 case X86_VENDOR_Cyrix:
4826 case X86_VENDOR_TM:
4827 case X86_VENDOR_Centaur:
4828 create = cpi->cpi_xmaxeax >= 0x80000001;
4829 break;
4830 default:
4831 create = 0;
4832 break;
4833 }
4834 if (create) {
4835 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4836 "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
4837 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4838 "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
4839 }
4840
4841 /*
4842 * Brand String first appeared in Intel Pentium IV, AMD K5
4843 * model 1, and Cyrix GXm. On earlier models we try and
4844 * simulate something similar .. so this string should always
4845 * same -something- about the processor, however lame.
4846 */
4847 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4848 "brand-string", cpi->cpi_brandstr);
4849
4850 /*
4851 * Finally, cache and tlb information
4852 */
4853 switch (x86_which_cacheinfo(cpi)) {
4854 case X86_VENDOR_Intel:
4855 intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4856 break;
4857 case X86_VENDOR_Cyrix:
4858 cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4859 break;
4860 case X86_VENDOR_AMD:
4861 amd_cache_info(cpi, cpu_devi);
4862 break;
4863 default:
4864 break;
4865 }
4866 }
4867
4868 struct l2info {
4869 int *l2i_csz;
4870 int *l2i_lsz;
4871 int *l2i_assoc;
4872 int l2i_ret;
4873 };
4874
4875 /*
4876 * A cacheinfo walker that fetches the size, line-size and associativity
4877 * of the L2 cache
4878 */
4879 static int
4880 intel_l2cinfo(void *arg, const struct cachetab *ct)
4881 {
4882 struct l2info *l2i = arg;
4883 int *ip;
4884
4885 if (ct->ct_label != l2_cache_str &&
4886 ct->ct_label != sl2_cache_str)
4887 return (0); /* not an L2 -- keep walking */
4888
4889 if ((ip = l2i->l2i_csz) != NULL)
4890 *ip = ct->ct_size;
4891 if ((ip = l2i->l2i_lsz) != NULL)
4892 *ip = ct->ct_line_size;
4893 if ((ip = l2i->l2i_assoc) != NULL)
4894 *ip = ct->ct_assoc;
4895 l2i->l2i_ret = ct->ct_size;
4896 return (1); /* was an L2 -- terminate walk */
4897 }
4898
4899 /*
4900 * AMD L2/L3 Cache and TLB Associativity Field Definition:
4901 *
4902 * Unlike the associativity for the L1 cache and tlb where the 8 bit
4903 * value is the associativity, the associativity for the L2 cache and
4904 * tlb is encoded in the following table. The 4 bit L2 value serves as
4905 * an index into the amd_afd[] array to determine the associativity.
4906 * -1 is undefined. 0 is fully associative.
4907 */
4908
4909 static int amd_afd[] =
4910 {-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};
4911
4912 static void
4913 amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
4914 {
4915 struct cpuid_regs *cp;
4916 uint_t size, assoc;
4917 int i;
4918 int *ip;
4919
4920 if (cpi->cpi_xmaxeax < 0x80000006)
4921 return;
4922 cp = &cpi->cpi_extd[6];
4923
4924 if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
4925 (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
4926 uint_t cachesz = size * 1024;
4927 assoc = amd_afd[i];
4928
4929 ASSERT(assoc != -1);
4930
4931 if ((ip = l2i->l2i_csz) != NULL)
4932 *ip = cachesz;
4933 if ((ip = l2i->l2i_lsz) != NULL)
4934 *ip = BITX(cp->cp_ecx, 7, 0);
4935 if ((ip = l2i->l2i_assoc) != NULL)
4936 *ip = assoc;
4937 l2i->l2i_ret = cachesz;
4938 }
4939 }
4940
4941 int
4942 getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
4943 {
4944 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
4945 struct l2info __l2info, *l2i = &__l2info;
4946
4947 l2i->l2i_csz = csz;
4948 l2i->l2i_lsz = lsz;
4949 l2i->l2i_assoc = assoc;
4950 l2i->l2i_ret = -1;
4951
4952 switch (x86_which_cacheinfo(cpi)) {
4953 case X86_VENDOR_Intel:
4954 intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4955 break;
4956 case X86_VENDOR_Cyrix:
4957 cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4958 break;
4959 case X86_VENDOR_AMD:
4960 amd_l2cacheinfo(cpi, l2i);
4961 break;
4962 default:
4963 break;
4964 }
4965 return (l2i->l2i_ret);
4966 }
4967
4968 #if !defined(__xpv)
4969
4970 uint32_t *
4971 cpuid_mwait_alloc(cpu_t *cpu)
4972 {
4973 uint32_t *ret;
4974 size_t mwait_size;
4975
4976 ASSERT(cpuid_checkpass(CPU, 2));
4977
4978 mwait_size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
4979 if (mwait_size == 0)
4980 return (NULL);
4981
4982 /*
4983 * kmem_alloc() returns cache line size aligned data for mwait_size
4984 * allocations. mwait_size is currently cache line sized. Neither
4985 * of these implementation details are guarantied to be true in the
4986 * future.
4987 *
4988 * First try allocating mwait_size as kmem_alloc() currently returns
4989 * correctly aligned memory. If kmem_alloc() does not return
4990 * mwait_size aligned memory, then use mwait_size ROUNDUP.
4991 *
4992 * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
4993 * decide to free this memory.
4994 */
4995 ret = kmem_zalloc(mwait_size, KM_SLEEP);
4996 if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) {
4997 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4998 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size;
4999 *ret = MWAIT_RUNNING;
5000 return (ret);
5001 } else {
5002 kmem_free(ret, mwait_size);
5003 ret = kmem_zalloc(mwait_size * 2, KM_SLEEP);
5004 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
5005 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size * 2;
5006 ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size);
5007 *ret = MWAIT_RUNNING;
5008 return (ret);
5009 }
5010 }
5011
5012 void
5013 cpuid_mwait_free(cpu_t *cpu)
5014 {
5015 if (cpu->cpu_m.mcpu_cpi == NULL) {
5016 return;
5017 }
5018
5019 if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
5020 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
5021 kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
5022 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
5023 }
5024
5025 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
5026 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
5027 }
5028
5029 void
5030 patch_tsc_read(int flag)
5031 {
5032 size_t cnt;
5033
5034 switch (flag) {
5035 case TSC_NONE:
5036 cnt = &_no_rdtsc_end - &_no_rdtsc_start;
5037 (void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
5038 break;
5039 case TSC_RDTSC_MFENCE:
5040 cnt = &_tsc_mfence_end - &_tsc_mfence_start;
5041 (void) memcpy((void *)tsc_read,
5042 (void *)&_tsc_mfence_start, cnt);
5043 break;
5044 case TSC_RDTSC_LFENCE:
5045 cnt = &_tsc_lfence_end - &_tsc_lfence_start;
5046 (void) memcpy((void *)tsc_read,
5047 (void *)&_tsc_lfence_start, cnt);
5048 break;
5049 case TSC_TSCP:
5050 cnt = &_tscp_end - &_tscp_start;
5051 (void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
5052 break;
5053 default:
5054 /* Bail for unexpected TSC types. (TSC_NONE covers 0) */
5055 cmn_err(CE_PANIC, "Unrecogized TSC type: %d", flag);
5056 break;
5057 }
5058 tsc_type = flag;
5059 }
5060
5061 int
5062 cpuid_deep_cstates_supported(void)
5063 {
5064 struct cpuid_info *cpi;
5065 struct cpuid_regs regs;
5066
5067 ASSERT(cpuid_checkpass(CPU, 1));
5068
5069 cpi = CPU->cpu_m.mcpu_cpi;
5070
5071 if (!is_x86_feature(x86_featureset, X86FSET_CPUID))
5072 return (0);
5073
5074 switch (cpi->cpi_vendor) {
5075 case X86_VENDOR_Intel:
5076 if (cpi->cpi_xmaxeax < 0x80000007)
5077 return (0);
5078
5079 /*
5080 * TSC run at a constant rate in all ACPI C-states?
5081 */
5082 regs.cp_eax = 0x80000007;
5083 (void) __cpuid_insn(®s);
5084 return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);
5085
5086 default:
5087 return (0);
5088 }
5089 }
5090
5091 #endif /* !__xpv */
5092
5093 void
5094 post_startup_cpu_fixups(void)
5095 {
5096 #ifndef __xpv
5097 /*
5098 * Some AMD processors support C1E state. Entering this state will
5099 * cause the local APIC timer to stop, which we can't deal with at
5100 * this time.
5101 */
5102 if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
5103 on_trap_data_t otd;
5104 uint64_t reg;
5105
5106 if (!on_trap(&otd, OT_DATA_ACCESS)) {
5107 reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
5108 /* Disable C1E state if it is enabled by BIOS */
5109 if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
5110 AMD_ACTONCMPHALT_MASK) {
5111 reg &= ~(AMD_ACTONCMPHALT_MASK <<
5112 AMD_ACTONCMPHALT_SHIFT);
5113 wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
5114 }
5115 }
5116 no_trap();
5117 }
5118 #endif /* !__xpv */
5119 }
5120
5121 void
5122 enable_pcid(void)
5123 {
5124 if (x86_use_pcid == -1)
5125 x86_use_pcid = is_x86_feature(x86_featureset, X86FSET_PCID);
5126
5127 if (x86_use_invpcid == -1) {
5128 x86_use_invpcid = is_x86_feature(x86_featureset,
5129 X86FSET_INVPCID);
5130 }
5131
5132 if (!x86_use_pcid)
5133 return;
5134
5135 /*
5136 * Intel say that on setting PCIDE, it immediately starts using the PCID
5137 * bits; better make sure there's nothing there.
5138 */
5139 ASSERT((getcr3() & MMU_PAGEOFFSET) == PCID_NONE);
5140
5141 setcr4(getcr4() | CR4_PCIDE);
5142 }
5143
5144 /*
5145 * Setup necessary registers to enable XSAVE feature on this processor.
5146 * This function needs to be called early enough, so that no xsave/xrstor
5147 * ops will execute on the processor before the MSRs are properly set up.
5148 *
5149 * Current implementation has the following assumption:
5150 * - cpuid_pass1() is done, so that X86 features are known.
5151 * - fpu_probe() is done, so that fp_save_mech is chosen.
5152 */
5153 void
5154 xsave_setup_msr(cpu_t *cpu)
5155 {
5156 ASSERT(fp_save_mech == FP_XSAVE);
5157 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
5158
5159 /* Enable OSXSAVE in CR4. */
5160 setcr4(getcr4() | CR4_OSXSAVE);
5161 /*
5162 * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
5163 * correct value.
5164 */
5165 cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
5166 setup_xfem();
5167 }
5168
5169 /*
5170 * Starting with the Westmere processor the local
5171 * APIC timer will continue running in all C-states,
5172 * including the deepest C-states.
5173 */
5174 int
5175 cpuid_arat_supported(void)
5176 {
5177 struct cpuid_info *cpi;
5178 struct cpuid_regs regs;
5179
5180 ASSERT(cpuid_checkpass(CPU, 1));
5181 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5182
5183 cpi = CPU->cpu_m.mcpu_cpi;
5184
5185 switch (cpi->cpi_vendor) {
5186 case X86_VENDOR_Intel:
5187 /*
5188 * Always-running Local APIC Timer is
5189 * indicated by CPUID.6.EAX[2].
5190 */
5191 if (cpi->cpi_maxeax >= 6) {
5192 regs.cp_eax = 6;
5193 (void) cpuid_insn(NULL, ®s);
5194 return (regs.cp_eax & CPUID_CSTATE_ARAT);
5195 } else {
5196 return (0);
5197 }
5198 default:
5199 return (0);
5200 }
5201 }
5202
5203 /*
5204 * Check support for Intel ENERGY_PERF_BIAS feature
5205 */
5206 int
5207 cpuid_iepb_supported(struct cpu *cp)
5208 {
5209 struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
5210 struct cpuid_regs regs;
5211
5212 ASSERT(cpuid_checkpass(cp, 1));
5213
5214 if (!(is_x86_feature(x86_featureset, X86FSET_CPUID)) ||
5215 !(is_x86_feature(x86_featureset, X86FSET_MSR))) {
5216 return (0);
5217 }
5218
5219 /*
5220 * Intel ENERGY_PERF_BIAS MSR is indicated by
5221 * capability bit CPUID.6.ECX.3
5222 */
5223 if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
5224 return (0);
5225
5226 regs.cp_eax = 0x6;
5227 (void) cpuid_insn(NULL, ®s);
5228 return (regs.cp_ecx & CPUID_EPB_SUPPORT);
5229 }
5230
5231 /*
5232 * Check support for TSC deadline timer
5233 *
5234 * TSC deadline timer provides a superior software programming
5235 * model over local APIC timer that eliminates "time drifts".
5236 * Instead of specifying a relative time, software specifies an
5237 * absolute time as the target at which the processor should
5238 * generate a timer event.
5239 */
5240 int
5241 cpuid_deadline_tsc_supported(void)
5242 {
5243 struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
5244 struct cpuid_regs regs;
5245
5246 ASSERT(cpuid_checkpass(CPU, 1));
5247 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5248
5249 switch (cpi->cpi_vendor) {
5250 case X86_VENDOR_Intel:
5251 if (cpi->cpi_maxeax >= 1) {
5252 regs.cp_eax = 1;
5253 (void) cpuid_insn(NULL, ®s);
5254 return (regs.cp_ecx & CPUID_DEADLINE_TSC);
5255 } else {
5256 return (0);
5257 }
5258 default:
5259 return (0);
5260 }
5261 }
5262
5263 #if defined(__amd64) && !defined(__xpv)
5264 /*
5265 * Patch in versions of bcopy for high performance Intel Nhm processors
5266 * and later...
5267 */
5268 void
5269 patch_memops(uint_t vendor)
5270 {
5271 size_t cnt, i;
5272 caddr_t to, from;
5273
5274 if ((vendor == X86_VENDOR_Intel) &&
5275 is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
5276 cnt = &bcopy_patch_end - &bcopy_patch_start;
5277 to = &bcopy_ck_size;
5278 from = &bcopy_patch_start;
5279 for (i = 0; i < cnt; i++) {
5280 *to++ = *from++;
5281 }
5282 }
5283 }
5284 #endif /* __amd64 && !__xpv */
5285
5286 /*
5287 * This function finds the number of bits to represent the number of cores per
5288 * chip and the number of strands per core for the Intel platforms.
5289 * It re-uses the x2APIC cpuid code of the cpuid_pass2().
5290 */
5291 void
5292 cpuid_get_ext_topo(uint_t vendor, uint_t *core_nbits, uint_t *strand_nbits)
5293 {
5294 struct cpuid_regs regs;
5295 struct cpuid_regs *cp = ®s;
5296
5297 if (vendor != X86_VENDOR_Intel) {
5298 return;
5299 }
5300
5301 /* if the cpuid level is 0xB, extended topo is available. */
5302 cp->cp_eax = 0;
5303 if (__cpuid_insn(cp) >= 0xB) {
5304
5305 cp->cp_eax = 0xB;
5306 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
5307 (void) __cpuid_insn(cp);
5308
5309 /*
5310 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
5311 * indicates that the extended topology enumeration leaf is
5312 * available.
5313 */
5314 if (cp->cp_ebx) {
5315 uint_t coreid_shift = 0;
5316 uint_t chipid_shift = 0;
5317 uint_t i;
5318 uint_t level;
5319
5320 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
5321 cp->cp_eax = 0xB;
5322 cp->cp_ecx = i;
5323
5324 (void) __cpuid_insn(cp);
5325 level = CPI_CPU_LEVEL_TYPE(cp);
5326
5327 if (level == 1) {
5328 /*
5329 * Thread level processor topology
5330 * Number of bits shift right APIC ID
5331 * to get the coreid.
5332 */
5333 coreid_shift = BITX(cp->cp_eax, 4, 0);
5334 } else if (level == 2) {
5335 /*
5336 * Core level processor topology
5337 * Number of bits shift right APIC ID
5338 * to get the chipid.
5339 */
5340 chipid_shift = BITX(cp->cp_eax, 4, 0);
5341 }
5342 }
5343
5344 if (coreid_shift > 0 && chipid_shift > coreid_shift) {
5345 *strand_nbits = coreid_shift;
5346 *core_nbits = chipid_shift - coreid_shift;
5347 }
5348 }
5349 }
5350 }
5351
5352 void
5353 cpuid_pass_ucode(cpu_t *cpu, uchar_t *fset)
5354 {
5355 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
5356 struct cpuid_regs cp;
5357
5358 /*
5359 * Reread the CPUID portions that we need for various security
5360 * information.
5361 */
5362 if (cpi->cpi_vendor == X86_VENDOR_Intel) {
5363 /*
5364 * Check if we now have leaf 7 available to us.
5365 */
5366 if (cpi->cpi_maxeax < 7) {
5367 bzero(&cp, sizeof (cp));
5368 cp.cp_eax = 0;
5369 cpi->cpi_maxeax = __cpuid_insn(&cp);
5370 if (cpi->cpi_maxeax < 7)
5371 return;
5372 }
5373
5374 bzero(&cp, sizeof (cp));
5375 cp.cp_eax = 7;
5376 cp.cp_ecx = 0;
5377 (void) __cpuid_insn(&cp);
5378 cpi->cpi_std[7] = cp;
5379 } else if (cpi->cpi_vendor == X86_VENDOR_AMD) {
5380 /* No xcpuid support */
5381 if (cpi->cpi_family < 5 ||
5382 (cpi->cpi_family == 5 && cpi->cpi_model < 1))
5383 return;
5384
5385 if (cpi->cpi_xmaxeax < 0x80000008) {
5386 bzero(&cp, sizeof (cp));
5387 cp.cp_eax = 0x80000000;
5388 cpi->cpi_xmaxeax = __cpuid_insn(&cp);
5389 if (cpi->cpi_xmaxeax < 0x80000008) {
5390 return;
5391 }
5392 }
5393
5394 bzero(&cp, sizeof (cp));
5395 cp.cp_eax = 0x80000008;
5396 (void) __cpuid_insn(&cp);
5397 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, &cp);
5398 cpi->cpi_extd[8] = cp;
5399 } else {
5400 /*
5401 * Nothing to do here. Return an empty set which has already
5402 * been zeroed for us.
5403 */
5404 return;
5405 }
5406 cpuid_scan_security(cpu, fset);
5407 }
5408
5409 /* ARGSUSED */
5410 static int
5411 cpuid_post_ucodeadm_xc(xc_arg_t arg0, xc_arg_t arg1, xc_arg_t arg2)
5412 {
5413 uchar_t *fset;
5414
5415 fset = (uchar_t *)(arg0 + sizeof (x86_featureset) * CPU->cpu_id);
5416 cpuid_pass_ucode(CPU, fset);
5417
5418 return (0);
5419 }
5420
5421 /*
5422 * After a microcode update where the version has changed, then we need to
5423 * rescan CPUID. To do this we check every CPU to make sure that they have the
5424 * same microcode. Then we perform a cross call to all such CPUs. It's the
5425 * caller's job to make sure that no one else can end up doing an update while
5426 * this is going on.
5427 *
5428 * We assume that the system is microcode capable if we're called.
5429 */
5430 void
5431 cpuid_post_ucodeadm(void)
5432 {
5433 uint32_t rev;
5434 int i;
5435 struct cpu *cpu;
5436 cpuset_t cpuset;
5437 void *argdata;
5438 uchar_t *f0;
5439
5440 argdata = kmem_zalloc(sizeof (x86_featureset) * NCPU, KM_SLEEP);
5441
5442 mutex_enter(&cpu_lock);
5443 cpu = cpu_get(0);
5444 rev = cpu->cpu_m.mcpu_ucode_info->cui_rev;
5445 CPUSET_ONLY(cpuset, 0);
5446 for (i = 1; i < max_ncpus; i++) {
5447 if ((cpu = cpu_get(i)) == NULL)
5448 continue;
5449
5450 if (cpu->cpu_m.mcpu_ucode_info->cui_rev != rev) {
5451 panic("post microcode update CPU %d has differing "
5452 "microcode revision (%u) from CPU 0 (%u)",
5453 i, cpu->cpu_m.mcpu_ucode_info->cui_rev, rev);
5454 }
5455 CPUSET_ADD(cpuset, i);
5456 }
5457
5458 kpreempt_disable();
5459 xc_sync((xc_arg_t)argdata, 0, 0, CPUSET2BV(cpuset),
5460 cpuid_post_ucodeadm_xc);
5461 kpreempt_enable();
5462
5463 /*
5464 * OK, now look at each CPU and see if their feature sets are equal.
5465 */
5466 f0 = argdata;
5467 for (i = 1; i < max_ncpus; i++) {
5468 uchar_t *fset;
5469 if (!CPU_IN_SET(cpuset, i))
5470 continue;
5471
5472 fset = (uchar_t *)((uintptr_t)argdata +
5473 sizeof (x86_featureset) * i);
5474
5475 if (!compare_x86_featureset(f0, fset)) {
5476 panic("Post microcode update CPU %d has "
5477 "differing security feature (%p) set from CPU 0 "
5478 "(%p), not appending to feature set", i,
5479 (void *)fset, (void *)f0);
5480 }
5481 }
5482
5483 mutex_exit(&cpu_lock);
5484
5485 for (i = 0; i < NUM_X86_FEATURES; i++) {
5486 cmn_err(CE_CONT, "?post-ucode x86_feature: %s\n",
5487 x86_feature_names[i]);
5488 if (is_x86_feature(f0, i)) {
5489 add_x86_feature(x86_featureset, i);
5490 }
5491 }
5492 kmem_free(argdata, sizeof (x86_featureset) * NCPU);
5493 }