1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
26 */
27 /*
28 * Copyright (c) 2010, Intel Corporation.
29 * All rights reserved.
30 */
31 /*
32 * Portions Copyright 2009 Advanced Micro Devices, Inc.
33 */
34 /*
35 * Copyright (c) 2019, Joyent, Inc.
36 */
37 /*
38 * Various routines to handle identification
39 * and classification of x86 processors.
40 */
41
42 #include <sys/types.h>
43 #include <sys/archsystm.h>
44 #include <sys/x86_archext.h>
45 #include <sys/kmem.h>
46 #include <sys/systm.h>
47 #include <sys/cmn_err.h>
48 #include <sys/sunddi.h>
49 #include <sys/sunndi.h>
50 #include <sys/cpuvar.h>
51 #include <sys/processor.h>
52 #include <sys/sysmacros.h>
53 #include <sys/pg.h>
54 #include <sys/fp.h>
55 #include <sys/controlregs.h>
56 #include <sys/bitmap.h>
57 #include <sys/auxv_386.h>
58 #include <sys/memnode.h>
59 #include <sys/pci_cfgspace.h>
60 #include <sys/comm_page.h>
61 #include <sys/mach_mmu.h>
62 #include <sys/ucode.h>
63 #include <sys/tsc.h>
64
65 #ifdef __xpv
66 #include <sys/hypervisor.h>
67 #else
68 #include <sys/ontrap.h>
69 #endif
70
71 /*
72 * Pass 0 of cpuid feature analysis happens in locore. It contains special code
73 * to recognize Cyrix processors that are not cpuid-compliant, and to deal with
74 * them accordingly. For most modern processors, feature detection occurs here
75 * in pass 1.
76 *
77 * Pass 1 of cpuid feature analysis happens just at the beginning of mlsetup()
78 * for the boot CPU and does the basic analysis that the early kernel needs.
79 * x86_featureset is set based on the return value of cpuid_pass1() of the boot
80 * CPU.
81 *
82 * Pass 1 includes:
83 *
84 * o Determining vendor/model/family/stepping and setting x86_type and
85 * x86_vendor accordingly.
86 * o Processing the feature flags returned by the cpuid instruction while
87 * applying any workarounds or tricks for the specific processor.
88 * o Mapping the feature flags into illumos feature bits (X86_*).
89 * o Processing extended feature flags if supported by the processor,
90 * again while applying specific processor knowledge.
91 * o Determining the CMT characteristics of the system.
92 *
93 * Pass 1 is done on non-boot CPUs during their initialization and the results
94 * are used only as a meager attempt at ensuring that all processors within the
95 * system support the same features.
96 *
97 * Pass 2 of cpuid feature analysis happens just at the beginning
98 * of startup(). It just copies in and corrects the remainder
99 * of the cpuid data we depend on: standard cpuid functions that we didn't
100 * need for pass1 feature analysis, and extended cpuid functions beyond the
101 * simple feature processing done in pass1.
102 *
103 * Pass 3 of cpuid analysis is invoked after basic kernel services; in
104 * particular kernel memory allocation has been made available. It creates a
105 * readable brand string based on the data collected in the first two passes.
106 *
107 * Pass 4 of cpuid analysis is invoked after post_startup() when all
108 * the support infrastructure for various hardware features has been
109 * initialized. It determines which processor features will be reported
110 * to userland via the aux vector.
111 *
112 * All passes are executed on all CPUs, but only the boot CPU determines what
113 * features the kernel will use.
114 *
115 * Much of the worst junk in this file is for the support of processors
116 * that didn't really implement the cpuid instruction properly.
117 *
118 * NOTE: The accessor functions (cpuid_get*) are aware of, and ASSERT upon,
119 * the pass numbers. Accordingly, changes to the pass code may require changes
120 * to the accessor code.
121 */
122
123 uint_t x86_vendor = X86_VENDOR_IntelClone;
124 uint_t x86_type = X86_TYPE_OTHER;
125 uint_t x86_clflush_size = 0;
126
127 #if defined(__xpv)
128 int x86_use_pcid = 0;
129 int x86_use_invpcid = 0;
130 #else
131 int x86_use_pcid = -1;
132 int x86_use_invpcid = -1;
133 #endif
134
135 uint_t pentiumpro_bug4046376;
136
137 uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
138
139 static char *x86_feature_names[NUM_X86_FEATURES] = {
140 "lgpg",
141 "tsc",
142 "msr",
143 "mtrr",
144 "pge",
145 "de",
146 "cmov",
147 "mmx",
148 "mca",
149 "pae",
150 "cv8",
151 "pat",
152 "sep",
153 "sse",
154 "sse2",
155 "htt",
156 "asysc",
157 "nx",
158 "sse3",
159 "cx16",
160 "cmp",
161 "tscp",
162 "mwait",
163 "sse4a",
164 "cpuid",
165 "ssse3",
166 "sse4_1",
167 "sse4_2",
168 "1gpg",
169 "clfsh",
170 "64",
171 "aes",
172 "pclmulqdq",
173 "xsave",
174 "avx",
175 "vmx",
176 "svm",
177 "topoext",
178 "f16c",
179 "rdrand",
180 "x2apic",
181 "avx2",
182 "bmi1",
183 "bmi2",
184 "fma",
185 "smep",
186 "smap",
187 "adx",
188 "rdseed",
189 "mpx",
190 "avx512f",
191 "avx512dq",
192 "avx512pf",
193 "avx512er",
194 "avx512cd",
195 "avx512bw",
196 "avx512vl",
197 "avx512fma",
198 "avx512vbmi",
199 "avx512_vpopcntdq",
200 "avx512_4vnniw",
201 "avx512_4fmaps",
202 "xsaveopt",
203 "xsavec",
204 "xsaves",
205 "sha",
206 "umip",
207 "pku",
208 "ospke",
209 "pcid",
210 "invpcid",
211 "ibrs",
212 "ibpb",
213 "stibp",
214 "ssbd",
215 "ssbd_virt",
216 "rdcl_no",
217 "ibrs_all",
218 "rsba",
219 "ssb_no",
220 "stibp_all",
221 "flush_cmd",
222 "l1d_vmentry_no"
223 };
224
225 boolean_t
226 is_x86_feature(void *featureset, uint_t feature)
227 {
228 ASSERT(feature < NUM_X86_FEATURES);
229 return (BT_TEST((ulong_t *)featureset, feature));
230 }
231
232 void
233 add_x86_feature(void *featureset, uint_t feature)
234 {
235 ASSERT(feature < NUM_X86_FEATURES);
236 BT_SET((ulong_t *)featureset, feature);
237 }
238
239 void
240 remove_x86_feature(void *featureset, uint_t feature)
241 {
242 ASSERT(feature < NUM_X86_FEATURES);
243 BT_CLEAR((ulong_t *)featureset, feature);
244 }
245
246 boolean_t
247 compare_x86_featureset(void *setA, void *setB)
248 {
249 /*
250 * We assume that the unused bits of the bitmap are always zero.
251 */
252 if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
253 return (B_TRUE);
254 } else {
255 return (B_FALSE);
256 }
257 }
258
259 void
260 print_x86_featureset(void *featureset)
261 {
262 uint_t i;
263
264 for (i = 0; i < NUM_X86_FEATURES; i++) {
265 if (is_x86_feature(featureset, i)) {
266 cmn_err(CE_CONT, "?x86_feature: %s\n",
267 x86_feature_names[i]);
268 }
269 }
270 }
271
272 /* Note: This is the maximum size for the CPU, not the size of the structure. */
273 static size_t xsave_state_size = 0;
274 uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
275 boolean_t xsave_force_disable = B_FALSE;
276 extern int disable_smap;
277
278 /*
279 * This is set to platform type we are running on.
280 */
281 static int platform_type = -1;
282
283 #if !defined(__xpv)
284 /*
285 * Variable to patch if hypervisor platform detection needs to be
286 * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
287 */
288 int enable_platform_detection = 1;
289 #endif
290
291 /*
292 * monitor/mwait info.
293 *
294 * size_actual and buf_actual are the real address and size allocated to get
295 * proper mwait_buf alignement. buf_actual and size_actual should be passed
296 * to kmem_free(). Currently kmem_alloc() and mwait happen to both use
297 * processor cache-line alignment, but this is not guarantied in the furture.
298 */
299 struct mwait_info {
300 size_t mon_min; /* min size to avoid missed wakeups */
301 size_t mon_max; /* size to avoid false wakeups */
302 size_t size_actual; /* size actually allocated */
303 void *buf_actual; /* memory actually allocated */
304 uint32_t support; /* processor support of monitor/mwait */
305 };
306
307 /*
308 * xsave/xrestor info.
309 *
310 * This structure contains HW feature bits and the size of the xsave save area.
311 * Note: the kernel declares a fixed size (AVX_XSAVE_SIZE) structure
312 * (xsave_state) to describe the xsave layout. However, at runtime the
313 * per-lwp xsave area is dynamically allocated based on xsav_max_size. The
314 * xsave_state structure simply represents the legacy layout of the beginning
315 * of the xsave area.
316 */
317 struct xsave_info {
318 uint32_t xsav_hw_features_low; /* Supported HW features */
319 uint32_t xsav_hw_features_high; /* Supported HW features */
320 size_t xsav_max_size; /* max size save area for HW features */
321 size_t ymm_size; /* AVX: size of ymm save area */
322 size_t ymm_offset; /* AVX: offset for ymm save area */
323 size_t bndregs_size; /* MPX: size of bndregs save area */
324 size_t bndregs_offset; /* MPX: offset for bndregs save area */
325 size_t bndcsr_size; /* MPX: size of bndcsr save area */
326 size_t bndcsr_offset; /* MPX: offset for bndcsr save area */
327 size_t opmask_size; /* AVX512: size of opmask save */
328 size_t opmask_offset; /* AVX512: offset for opmask save */
329 size_t zmmlo_size; /* AVX512: size of zmm 256 save */
330 size_t zmmlo_offset; /* AVX512: offset for zmm 256 save */
331 size_t zmmhi_size; /* AVX512: size of zmm hi reg save */
332 size_t zmmhi_offset; /* AVX512: offset for zmm hi reg save */
333 };
334
335
336 /*
337 * These constants determine how many of the elements of the
338 * cpuid we cache in the cpuid_info data structure; the
339 * remaining elements are accessible via the cpuid instruction.
340 */
341
342 #define NMAX_CPI_STD 8 /* eax = 0 .. 7 */
343 #define NMAX_CPI_EXTD 0x1f /* eax = 0x80000000 .. 0x8000001e */
344
345 /*
346 * Some terminology needs to be explained:
347 * - Socket: Something that can be plugged into a motherboard.
348 * - Package: Same as socket
349 * - Chip: Same as socket. Note that AMD's documentation uses term "chip"
350 * differently: there, chip is the same as processor node (below)
351 * - Processor node: Some AMD processors have more than one
352 * "subprocessor" embedded in a package. These subprocessors (nodes)
353 * are fully-functional processors themselves with cores, caches,
354 * memory controllers, PCI configuration spaces. They are connected
355 * inside the package with Hypertransport links. On single-node
356 * processors, processor node is equivalent to chip/socket/package.
357 * - Compute Unit: Some AMD processors pair cores in "compute units" that
358 * share the FPU and the I$ and L2 caches.
359 */
360
361 struct cpuid_info {
362 uint_t cpi_pass; /* last pass completed */
363 /*
364 * standard function information
365 */
366 uint_t cpi_maxeax; /* fn 0: %eax */
367 char cpi_vendorstr[13]; /* fn 0: %ebx:%ecx:%edx */
368 uint_t cpi_vendor; /* enum of cpi_vendorstr */
369
370 uint_t cpi_family; /* fn 1: extended family */
371 uint_t cpi_model; /* fn 1: extended model */
372 uint_t cpi_step; /* fn 1: stepping */
373 chipid_t cpi_chipid; /* fn 1: %ebx: Intel: chip # */
374 /* AMD: package/socket # */
375 uint_t cpi_brandid; /* fn 1: %ebx: brand ID */
376 int cpi_clogid; /* fn 1: %ebx: thread # */
377 uint_t cpi_ncpu_per_chip; /* fn 1: %ebx: logical cpu count */
378 uint8_t cpi_cacheinfo[16]; /* fn 2: intel-style cache desc */
379 uint_t cpi_ncache; /* fn 2: number of elements */
380 uint_t cpi_ncpu_shr_last_cache; /* fn 4: %eax: ncpus sharing cache */
381 id_t cpi_last_lvl_cacheid; /* fn 4: %eax: derived cache id */
382 uint_t cpi_std_4_size; /* fn 4: number of fn 4 elements */
383 struct cpuid_regs **cpi_std_4; /* fn 4: %ecx == 0 .. fn4_size */
384 struct cpuid_regs cpi_std[NMAX_CPI_STD]; /* 0 .. 7 */
385 /*
386 * extended function information
387 */
388 uint_t cpi_xmaxeax; /* fn 0x80000000: %eax */
389 char cpi_brandstr[49]; /* fn 0x8000000[234] */
390 uint8_t cpi_pabits; /* fn 0x80000006: %eax */
391 uint8_t cpi_vabits; /* fn 0x80000006: %eax */
392 uint8_t cpi_fp_amd_save; /* AMD: FP error pointer save rqd. */
393 struct cpuid_regs cpi_extd[NMAX_CPI_EXTD]; /* 0x800000XX */
394
395 id_t cpi_coreid; /* same coreid => strands share core */
396 int cpi_pkgcoreid; /* core number within single package */
397 uint_t cpi_ncore_per_chip; /* AMD: fn 0x80000008: %ecx[7-0] */
398 /* Intel: fn 4: %eax[31-26] */
399 /*
400 * supported feature information
401 */
402 uint32_t cpi_support[6];
403 #define STD_EDX_FEATURES 0
404 #define AMD_EDX_FEATURES 1
405 #define TM_EDX_FEATURES 2
406 #define STD_ECX_FEATURES 3
407 #define AMD_ECX_FEATURES 4
408 #define STD_EBX_FEATURES 5
409 /*
410 * Synthesized information, where known.
411 */
412 uint32_t cpi_chiprev; /* See X86_CHIPREV_* in x86_archext.h */
413 const char *cpi_chiprevstr; /* May be NULL if chiprev unknown */
414 uint32_t cpi_socket; /* Chip package/socket type */
415
416 struct mwait_info cpi_mwait; /* fn 5: monitor/mwait info */
417 uint32_t cpi_apicid;
418 uint_t cpi_procnodeid; /* AMD: nodeID on HT, Intel: chipid */
419 uint_t cpi_procnodes_per_pkg; /* AMD: # of nodes in the package */
420 /* Intel: 1 */
421 uint_t cpi_compunitid; /* AMD: ComputeUnit ID, Intel: coreid */
422 uint_t cpi_cores_per_compunit; /* AMD: # of cores in the ComputeUnit */
423
424 struct xsave_info cpi_xsave; /* fn D: xsave/xrestor info */
425 };
426
427
428 static struct cpuid_info cpuid_info0;
429
430 /*
431 * These bit fields are defined by the Intel Application Note AP-485
432 * "Intel Processor Identification and the CPUID Instruction"
433 */
434 #define CPI_FAMILY_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
435 #define CPI_MODEL_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
436 #define CPI_TYPE(cpi) BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
437 #define CPI_FAMILY(cpi) BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
438 #define CPI_STEP(cpi) BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
439 #define CPI_MODEL(cpi) BITX((cpi)->cpi_std[1].cp_eax, 7, 4)
440
441 #define CPI_FEATURES_EDX(cpi) ((cpi)->cpi_std[1].cp_edx)
442 #define CPI_FEATURES_ECX(cpi) ((cpi)->cpi_std[1].cp_ecx)
443 #define CPI_FEATURES_XTD_EDX(cpi) ((cpi)->cpi_extd[1].cp_edx)
444 #define CPI_FEATURES_XTD_ECX(cpi) ((cpi)->cpi_extd[1].cp_ecx)
445 #define CPI_FEATURES_7_0_EBX(cpi) ((cpi)->cpi_std[7].cp_ebx)
446 #define CPI_FEATURES_7_0_ECX(cpi) ((cpi)->cpi_std[7].cp_ecx)
447 #define CPI_FEATURES_7_0_EDX(cpi) ((cpi)->cpi_std[7].cp_edx)
448
449 #define CPI_BRANDID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
450 #define CPI_CHUNKS(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
451 #define CPI_CPU_COUNT(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
452 #define CPI_APIC_ID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)
453
454 #define CPI_MAXEAX_MAX 0x100 /* sanity control */
455 #define CPI_XMAXEAX_MAX 0x80000100
456 #define CPI_FN4_ECX_MAX 0x20 /* sanity: max fn 4 levels */
457 #define CPI_FNB_ECX_MAX 0x20 /* sanity: max fn B levels */
458
459 /*
460 * Function 4 (Deterministic Cache Parameters) macros
461 * Defined by Intel Application Note AP-485
462 */
463 #define CPI_NUM_CORES(regs) BITX((regs)->cp_eax, 31, 26)
464 #define CPI_NTHR_SHR_CACHE(regs) BITX((regs)->cp_eax, 25, 14)
465 #define CPI_FULL_ASSOC_CACHE(regs) BITX((regs)->cp_eax, 9, 9)
466 #define CPI_SELF_INIT_CACHE(regs) BITX((regs)->cp_eax, 8, 8)
467 #define CPI_CACHE_LVL(regs) BITX((regs)->cp_eax, 7, 5)
468 #define CPI_CACHE_TYPE(regs) BITX((regs)->cp_eax, 4, 0)
469 #define CPI_CPU_LEVEL_TYPE(regs) BITX((regs)->cp_ecx, 15, 8)
470
471 #define CPI_CACHE_WAYS(regs) BITX((regs)->cp_ebx, 31, 22)
472 #define CPI_CACHE_PARTS(regs) BITX((regs)->cp_ebx, 21, 12)
473 #define CPI_CACHE_COH_LN_SZ(regs) BITX((regs)->cp_ebx, 11, 0)
474
475 #define CPI_CACHE_SETS(regs) BITX((regs)->cp_ecx, 31, 0)
476
477 #define CPI_PREFCH_STRIDE(regs) BITX((regs)->cp_edx, 9, 0)
478
479
480 /*
481 * A couple of shorthand macros to identify "later" P6-family chips
482 * like the Pentium M and Core. First, the "older" P6-based stuff
483 * (loosely defined as "pre-Pentium-4"):
484 * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
485 */
486 #define IS_LEGACY_P6(cpi) ( \
487 cpi->cpi_family == 6 && \
488 (cpi->cpi_model == 1 || \
489 cpi->cpi_model == 3 || \
490 cpi->cpi_model == 5 || \
491 cpi->cpi_model == 6 || \
492 cpi->cpi_model == 7 || \
493 cpi->cpi_model == 8 || \
494 cpi->cpi_model == 0xA || \
495 cpi->cpi_model == 0xB) \
496 )
497
498 /* A "new F6" is everything with family 6 that's not the above */
499 #define IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))
500
501 /* Extended family/model support */
502 #define IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
503 cpi->cpi_family >= 0xf)
504
505 /*
506 * Info for monitor/mwait idle loop.
507 *
508 * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
509 * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
510 * 2006.
511 * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
512 * Documentation Updates" #33633, Rev 2.05, December 2006.
513 */
514 #define MWAIT_SUPPORT (0x00000001) /* mwait supported */
515 #define MWAIT_EXTENSIONS (0x00000002) /* extenstion supported */
516 #define MWAIT_ECX_INT_ENABLE (0x00000004) /* ecx 1 extension supported */
517 #define MWAIT_SUPPORTED(cpi) ((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
518 #define MWAIT_INT_ENABLE(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x2)
519 #define MWAIT_EXTENSION(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x1)
520 #define MWAIT_SIZE_MIN(cpi) BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
521 #define MWAIT_SIZE_MAX(cpi) BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
522 /*
523 * Number of sub-cstates for a given c-state.
524 */
525 #define MWAIT_NUM_SUBC_STATES(cpi, c_state) \
526 BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)
527
528 /*
529 * XSAVE leaf 0xD enumeration
530 */
531 #define CPUID_LEAFD_2_YMM_OFFSET 576
532 #define CPUID_LEAFD_2_YMM_SIZE 256
533
534 /*
535 * Functions we consune from cpuid_subr.c; don't publish these in a header
536 * file to try and keep people using the expected cpuid_* interfaces.
537 */
538 extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
539 extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
540 extern uint32_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
541 extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
542 extern uint_t _cpuid_vendorstr_to_vendorcode(char *);
543
544 /*
545 * Apply up various platform-dependent restrictions where the
546 * underlying platform restrictions mean the CPU can be marked
547 * as less capable than its cpuid instruction would imply.
548 */
549 #if defined(__xpv)
550 static void
551 platform_cpuid_mangle(uint_t vendor, uint32_t eax, struct cpuid_regs *cp)
552 {
553 switch (eax) {
554 case 1: {
555 uint32_t mcamask = DOMAIN_IS_INITDOMAIN(xen_info) ?
556 0 : CPUID_INTC_EDX_MCA;
557 cp->cp_edx &=
558 ~(mcamask |
559 CPUID_INTC_EDX_PSE |
560 CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
561 CPUID_INTC_EDX_SEP | CPUID_INTC_EDX_MTRR |
562 CPUID_INTC_EDX_PGE | CPUID_INTC_EDX_PAT |
563 CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
564 CPUID_INTC_EDX_PSE36 | CPUID_INTC_EDX_HTT);
565 break;
566 }
567
568 case 0x80000001:
569 cp->cp_edx &=
570 ~(CPUID_AMD_EDX_PSE |
571 CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
572 CPUID_AMD_EDX_MTRR | CPUID_AMD_EDX_PGE |
573 CPUID_AMD_EDX_PAT | CPUID_AMD_EDX_PSE36 |
574 CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
575 CPUID_AMD_EDX_TSCP);
576 cp->cp_ecx &= ~CPUID_AMD_ECX_CMP_LGCY;
577 break;
578 default:
579 break;
580 }
581
582 switch (vendor) {
583 case X86_VENDOR_Intel:
584 switch (eax) {
585 case 4:
586 /*
587 * Zero out the (ncores-per-chip - 1) field
588 */
589 cp->cp_eax &= 0x03fffffff;
590 break;
591 default:
592 break;
593 }
594 break;
595 case X86_VENDOR_AMD:
596 switch (eax) {
597
598 case 0x80000001:
599 cp->cp_ecx &= ~CPUID_AMD_ECX_CR8D;
600 break;
601
602 case 0x80000008:
603 /*
604 * Zero out the (ncores-per-chip - 1) field
605 */
606 cp->cp_ecx &= 0xffffff00;
607 break;
608 default:
609 break;
610 }
611 break;
612 default:
613 break;
614 }
615 }
616 #else
617 #define platform_cpuid_mangle(vendor, eax, cp) /* nothing */
618 #endif
619
620 /*
621 * Some undocumented ways of patching the results of the cpuid
622 * instruction to permit running Solaris 10 on future cpus that
623 * we don't currently support. Could be set to non-zero values
624 * via settings in eeprom.
625 */
626
627 uint32_t cpuid_feature_ecx_include;
628 uint32_t cpuid_feature_ecx_exclude;
629 uint32_t cpuid_feature_edx_include;
630 uint32_t cpuid_feature_edx_exclude;
631
632 /*
633 * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
634 */
635 void
636 cpuid_alloc_space(cpu_t *cpu)
637 {
638 /*
639 * By convention, cpu0 is the boot cpu, which is set up
640 * before memory allocation is available. All other cpus get
641 * their cpuid_info struct allocated here.
642 */
643 ASSERT(cpu->cpu_id != 0);
644 ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
645 cpu->cpu_m.mcpu_cpi =
646 kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
647 }
648
649 void
650 cpuid_free_space(cpu_t *cpu)
651 {
652 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
653 int i;
654
655 ASSERT(cpi != NULL);
656 ASSERT(cpi != &cpuid_info0);
657
658 /*
659 * Free up any function 4 related dynamic storage
660 */
661 for (i = 1; i < cpi->cpi_std_4_size; i++)
662 kmem_free(cpi->cpi_std_4[i], sizeof (struct cpuid_regs));
663 if (cpi->cpi_std_4_size > 0)
664 kmem_free(cpi->cpi_std_4,
665 cpi->cpi_std_4_size * sizeof (struct cpuid_regs *));
666
667 kmem_free(cpi, sizeof (*cpi));
668 cpu->cpu_m.mcpu_cpi = NULL;
669 }
670
671 #if !defined(__xpv)
672 /*
673 * Determine the type of the underlying platform. This is used to customize
674 * initialization of various subsystems (e.g. TSC). determine_platform() must
675 * only ever be called once to prevent two processors from seeing different
676 * values of platform_type. Must be called before cpuid_pass1(), the earliest
677 * consumer to execute (uses _cpuid_chiprev --> synth_amd_info --> get_hwenv).
678 */
679 void
680 determine_platform(void)
681 {
682 struct cpuid_regs cp;
683 uint32_t base;
684 uint32_t regs[4];
685 char *hvstr = (char *)regs;
686
687 ASSERT(platform_type == -1);
688
689 platform_type = HW_NATIVE;
690
691 if (!enable_platform_detection)
692 return;
693
694 /*
695 * If Hypervisor CPUID bit is set, try to determine hypervisor
696 * vendor signature, and set platform type accordingly.
697 *
698 * References:
699 * http://lkml.org/lkml/2008/10/1/246
700 * http://kb.vmware.com/kb/1009458
701 */
702 cp.cp_eax = 0x1;
703 (void) __cpuid_insn(&cp);
704 if ((cp.cp_ecx & CPUID_INTC_ECX_HV) != 0) {
705 cp.cp_eax = 0x40000000;
706 (void) __cpuid_insn(&cp);
707 regs[0] = cp.cp_ebx;
708 regs[1] = cp.cp_ecx;
709 regs[2] = cp.cp_edx;
710 regs[3] = 0;
711 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0) {
712 platform_type = HW_XEN_HVM;
713 return;
714 }
715 if (strcmp(hvstr, HVSIG_VMWARE) == 0) {
716 platform_type = HW_VMWARE;
717 return;
718 }
719 if (strcmp(hvstr, HVSIG_KVM) == 0) {
720 platform_type = HW_KVM;
721 return;
722 }
723 if (strcmp(hvstr, HVSIG_BHYVE) == 0) {
724 platform_type = HW_BHYVE;
725 return;
726 }
727 if (strcmp(hvstr, HVSIG_MICROSOFT) == 0)
728 platform_type = HW_MICROSOFT;
729 } else {
730 /*
731 * Check older VMware hardware versions. VMware hypervisor is
732 * detected by performing an IN operation to VMware hypervisor
733 * port and checking that value returned in %ebx is VMware
734 * hypervisor magic value.
735 *
736 * References: http://kb.vmware.com/kb/1009458
737 */
738 vmware_port(VMWARE_HVCMD_GETVERSION, regs);
739 if (regs[1] == VMWARE_HVMAGIC) {
740 platform_type = HW_VMWARE;
741 return;
742 }
743 }
744
745 /*
746 * Check Xen hypervisor. In a fully virtualized domain,
747 * Xen's pseudo-cpuid function returns a string representing the
748 * Xen signature in %ebx, %ecx, and %edx. %eax contains the maximum
749 * supported cpuid function. We need at least a (base + 2) leaf value
750 * to do what we want to do. Try different base values, since the
751 * hypervisor might use a different one depending on whether Hyper-V
752 * emulation is switched on by default or not.
753 */
754 for (base = 0x40000000; base < 0x40010000; base += 0x100) {
755 cp.cp_eax = base;
756 (void) __cpuid_insn(&cp);
757 regs[0] = cp.cp_ebx;
758 regs[1] = cp.cp_ecx;
759 regs[2] = cp.cp_edx;
760 regs[3] = 0;
761 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0 &&
762 cp.cp_eax >= (base + 2)) {
763 platform_type &= ~HW_NATIVE;
764 platform_type |= HW_XEN_HVM;
765 return;
766 }
767 }
768 }
769
770 int
771 get_hwenv(void)
772 {
773 ASSERT(platform_type != -1);
774 return (platform_type);
775 }
776
777 int
778 is_controldom(void)
779 {
780 return (0);
781 }
782
783 #else
784
785 int
786 get_hwenv(void)
787 {
788 return (HW_XEN_PV);
789 }
790
791 int
792 is_controldom(void)
793 {
794 return (DOMAIN_IS_INITDOMAIN(xen_info));
795 }
796
797 #endif /* __xpv */
798
799 static void
800 cpuid_intel_getids(cpu_t *cpu, void *feature)
801 {
802 uint_t i;
803 uint_t chipid_shift = 0;
804 uint_t coreid_shift = 0;
805 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
806
807 for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
808 chipid_shift++;
809
810 cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
811 cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);
812
813 if (is_x86_feature(feature, X86FSET_CMP)) {
814 /*
815 * Multi-core (and possibly multi-threaded)
816 * processors.
817 */
818 uint_t ncpu_per_core;
819 if (cpi->cpi_ncore_per_chip == 1)
820 ncpu_per_core = cpi->cpi_ncpu_per_chip;
821 else if (cpi->cpi_ncore_per_chip > 1)
822 ncpu_per_core = cpi->cpi_ncpu_per_chip /
823 cpi->cpi_ncore_per_chip;
824 /*
825 * 8bit APIC IDs on dual core Pentiums
826 * look like this:
827 *
828 * +-----------------------+------+------+
829 * | Physical Package ID | MC | HT |
830 * +-----------------------+------+------+
831 * <------- chipid -------->
832 * <------- coreid --------------->
833 * <--- clogid -->
834 * <------>
835 * pkgcoreid
836 *
837 * Where the number of bits necessary to
838 * represent MC and HT fields together equals
839 * to the minimum number of bits necessary to
840 * store the value of cpi->cpi_ncpu_per_chip.
841 * Of those bits, the MC part uses the number
842 * of bits necessary to store the value of
843 * cpi->cpi_ncore_per_chip.
844 */
845 for (i = 1; i < ncpu_per_core; i <<= 1)
846 coreid_shift++;
847 cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
848 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
849 } else if (is_x86_feature(feature, X86FSET_HTT)) {
850 /*
851 * Single-core multi-threaded processors.
852 */
853 cpi->cpi_coreid = cpi->cpi_chipid;
854 cpi->cpi_pkgcoreid = 0;
855 }
856 cpi->cpi_procnodeid = cpi->cpi_chipid;
857 cpi->cpi_compunitid = cpi->cpi_coreid;
858 }
859
860 static void
861 cpuid_amd_getids(cpu_t *cpu)
862 {
863 int i, first_half, coreidsz;
864 uint32_t nb_caps_reg;
865 uint_t node2_1;
866 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
867 struct cpuid_regs *cp;
868
869 /*
870 * AMD CMP chips currently have a single thread per core.
871 *
872 * Since no two cpus share a core we must assign a distinct coreid
873 * per cpu, and we do this by using the cpu_id. This scheme does not,
874 * however, guarantee that sibling cores of a chip will have sequential
875 * coreids starting at a multiple of the number of cores per chip -
876 * that is usually the case, but if the ACPI MADT table is presented
877 * in a different order then we need to perform a few more gymnastics
878 * for the pkgcoreid.
879 *
880 * All processors in the system have the same number of enabled
881 * cores. Cores within a processor are always numbered sequentially
882 * from 0 regardless of how many or which are disabled, and there
883 * is no way for operating system to discover the real core id when some
884 * are disabled.
885 *
886 * In family 0x15, the cores come in pairs called compute units. They
887 * share I$ and L2 caches and the FPU. Enumeration of this feature is
888 * simplified by the new topology extensions CPUID leaf, indicated by
889 * the X86 feature X86FSET_TOPOEXT.
890 */
891
892 cpi->cpi_coreid = cpu->cpu_id;
893 cpi->cpi_compunitid = cpu->cpu_id;
894
895 if (cpi->cpi_xmaxeax >= 0x80000008) {
896
897 coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);
898
899 /*
900 * In AMD parlance chip is really a node while Solaris
901 * sees chip as equivalent to socket/package.
902 */
903 cpi->cpi_ncore_per_chip =
904 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
905 if (coreidsz == 0) {
906 /* Use legacy method */
907 for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
908 coreidsz++;
909 if (coreidsz == 0)
910 coreidsz = 1;
911 }
912 } else {
913 /* Assume single-core part */
914 cpi->cpi_ncore_per_chip = 1;
915 coreidsz = 1;
916 }
917
918 cpi->cpi_clogid = cpi->cpi_pkgcoreid =
919 cpi->cpi_apicid & ((1<<coreidsz) - 1);
920 cpi->cpi_ncpu_per_chip = cpi->cpi_ncore_per_chip;
921
922 /* Get node ID, compute unit ID */
923 if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
924 cpi->cpi_xmaxeax >= 0x8000001e) {
925 cp = &cpi->cpi_extd[0x1e];
926 cp->cp_eax = 0x8000001e;
927 (void) __cpuid_insn(cp);
928
929 cpi->cpi_procnodes_per_pkg = BITX(cp->cp_ecx, 10, 8) + 1;
930 cpi->cpi_procnodeid = BITX(cp->cp_ecx, 7, 0);
931 cpi->cpi_cores_per_compunit = BITX(cp->cp_ebx, 15, 8) + 1;
932 cpi->cpi_compunitid = BITX(cp->cp_ebx, 7, 0)
933 + (cpi->cpi_ncore_per_chip / cpi->cpi_cores_per_compunit)
934 * (cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg);
935 } else if (cpi->cpi_family == 0xf || cpi->cpi_family >= 0x11) {
936 cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
937 } else if (cpi->cpi_family == 0x10) {
938 /*
939 * See if we are a multi-node processor.
940 * All processors in the system have the same number of nodes
941 */
942 nb_caps_reg = pci_getl_func(0, 24, 3, 0xe8);
943 if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
944 /* Single-node */
945 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
946 coreidsz);
947 } else {
948
949 /*
950 * Multi-node revision D (2 nodes per package
951 * are supported)
952 */
953 cpi->cpi_procnodes_per_pkg = 2;
954
955 first_half = (cpi->cpi_pkgcoreid <=
956 (cpi->cpi_ncore_per_chip/2 - 1));
957
958 if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
959 /* We are BSP */
960 cpi->cpi_procnodeid = (first_half ? 0 : 1);
961 } else {
962
963 /* We are AP */
964 /* NodeId[2:1] bits to use for reading F3xe8 */
965 node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;
966
967 nb_caps_reg =
968 pci_getl_func(0, 24 + node2_1, 3, 0xe8);
969
970 /*
971 * Check IntNodeNum bit (31:30, but bit 31 is
972 * always 0 on dual-node processors)
973 */
974 if (BITX(nb_caps_reg, 30, 30) == 0)
975 cpi->cpi_procnodeid = node2_1 +
976 !first_half;
977 else
978 cpi->cpi_procnodeid = node2_1 +
979 first_half;
980 }
981 }
982 } else {
983 cpi->cpi_procnodeid = 0;
984 }
985
986 cpi->cpi_chipid =
987 cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg;
988 }
989
990 static void
991 cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
992 {
993 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
994
995 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
996 cpi->cpi_xmaxeax >= 0x80000008) {
997 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBPB)
998 add_x86_feature(featureset, X86FSET_IBPB);
999 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBRS)
1000 add_x86_feature(featureset, X86FSET_IBRS);
1001 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP)
1002 add_x86_feature(featureset, X86FSET_STIBP);
1003 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBRS_ALL)
1004 add_x86_feature(featureset, X86FSET_IBRS_ALL);
1005 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP_ALL)
1006 add_x86_feature(featureset, X86FSET_STIBP_ALL);
1007 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_PREFER_IBRS)
1008 add_x86_feature(featureset, X86FSET_RSBA);
1009 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSBD)
1010 add_x86_feature(featureset, X86FSET_SSBD);
1011 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_VIRT_SSBD)
1012 add_x86_feature(featureset, X86FSET_SSBD_VIRT);
1013 if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSB_NO)
1014 add_x86_feature(featureset, X86FSET_SSB_NO);
1015 } else if (cpi->cpi_vendor == X86_VENDOR_Intel &&
1016 cpi->cpi_maxeax >= 7) {
1017 struct cpuid_regs *ecp;
1018 ecp = &cpi->cpi_std[7];
1019
1020 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SPEC_CTRL) {
1021 add_x86_feature(featureset, X86FSET_IBRS);
1022 add_x86_feature(featureset, X86FSET_IBPB);
1023 }
1024
1025 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_STIBP) {
1026 add_x86_feature(featureset, X86FSET_STIBP);
1027 }
1028
1029 /*
1030 * Don't read the arch caps MSR on xpv where we lack the
1031 * on_trap().
1032 */
1033 #ifndef __xpv
1034 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_ARCH_CAPS) {
1035 on_trap_data_t otd;
1036
1037 /*
1038 * Be paranoid and assume we'll get a #GP.
1039 */
1040 if (!on_trap(&otd, OT_DATA_ACCESS)) {
1041 uint64_t reg;
1042
1043 reg = rdmsr(MSR_IA32_ARCH_CAPABILITIES);
1044 if (reg & IA32_ARCH_CAP_RDCL_NO) {
1045 add_x86_feature(featureset,
1046 X86FSET_RDCL_NO);
1047 }
1048 if (reg & IA32_ARCH_CAP_IBRS_ALL) {
1049 add_x86_feature(featureset,
1050 X86FSET_IBRS_ALL);
1051 }
1052 if (reg & IA32_ARCH_CAP_RSBA) {
1053 add_x86_feature(featureset,
1054 X86FSET_RSBA);
1055 }
1056 if (reg & IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) {
1057 add_x86_feature(featureset,
1058 X86FSET_L1D_VM_NO);
1059 }
1060 if (reg & IA32_ARCH_CAP_SSB_NO) {
1061 add_x86_feature(featureset,
1062 X86FSET_SSB_NO);
1063 }
1064 }
1065 no_trap();
1066 }
1067 #endif /* !__xpv */
1068
1069 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SSBD)
1070 add_x86_feature(featureset, X86FSET_SSBD);
1071
1072 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_FLUSH_CMD)
1073 add_x86_feature(featureset, X86FSET_FLUSH_CMD);
1074 }
1075 }
1076
1077 /*
1078 * Setup XFeature_Enabled_Mask register. Required by xsave feature.
1079 */
1080 void
1081 setup_xfem(void)
1082 {
1083 uint64_t flags = XFEATURE_LEGACY_FP;
1084
1085 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
1086
1087 if (is_x86_feature(x86_featureset, X86FSET_SSE))
1088 flags |= XFEATURE_SSE;
1089
1090 if (is_x86_feature(x86_featureset, X86FSET_AVX))
1091 flags |= XFEATURE_AVX;
1092
1093 if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
1094 flags |= XFEATURE_AVX512;
1095
1096 set_xcr(XFEATURE_ENABLED_MASK, flags);
1097
1098 xsave_bv_all = flags;
1099 }
1100
1101 void
1102 cpuid_pass1(cpu_t *cpu, uchar_t *featureset)
1103 {
1104 uint32_t mask_ecx, mask_edx;
1105 struct cpuid_info *cpi;
1106 struct cpuid_regs *cp;
1107 int xcpuid;
1108 #if !defined(__xpv)
1109 extern int idle_cpu_prefer_mwait;
1110 #endif
1111
1112 /*
1113 * Space statically allocated for BSP, ensure pointer is set
1114 */
1115 if (cpu->cpu_id == 0) {
1116 if (cpu->cpu_m.mcpu_cpi == NULL)
1117 cpu->cpu_m.mcpu_cpi = &cpuid_info0;
1118 }
1119
1120 add_x86_feature(featureset, X86FSET_CPUID);
1121
1122 cpi = cpu->cpu_m.mcpu_cpi;
1123 ASSERT(cpi != NULL);
1124 cp = &cpi->cpi_std[0];
1125 cp->cp_eax = 0;
1126 cpi->cpi_maxeax = __cpuid_insn(cp);
1127 {
1128 uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
1129 *iptr++ = cp->cp_ebx;
1130 *iptr++ = cp->cp_edx;
1131 *iptr++ = cp->cp_ecx;
1132 *(char *)&cpi->cpi_vendorstr[12] = '\0';
1133 }
1134
1135 cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
1136 x86_vendor = cpi->cpi_vendor; /* for compatibility */
1137
1138 /*
1139 * Limit the range in case of weird hardware
1140 */
1141 if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
1142 cpi->cpi_maxeax = CPI_MAXEAX_MAX;
1143 if (cpi->cpi_maxeax < 1)
1144 goto pass1_done;
1145
1146 cp = &cpi->cpi_std[1];
1147 cp->cp_eax = 1;
1148 (void) __cpuid_insn(cp);
1149
1150 /*
1151 * Extract identifying constants for easy access.
1152 */
1153 cpi->cpi_model = CPI_MODEL(cpi);
1154 cpi->cpi_family = CPI_FAMILY(cpi);
1155
1156 if (cpi->cpi_family == 0xf)
1157 cpi->cpi_family += CPI_FAMILY_XTD(cpi);
1158
1159 /*
1160 * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
1161 * Intel, and presumably everyone else, uses model == 0xf, as
1162 * one would expect (max value means possible overflow). Sigh.
1163 */
1164
1165 switch (cpi->cpi_vendor) {
1166 case X86_VENDOR_Intel:
1167 if (IS_EXTENDED_MODEL_INTEL(cpi))
1168 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1169 break;
1170 case X86_VENDOR_AMD:
1171 if (CPI_FAMILY(cpi) == 0xf)
1172 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1173 break;
1174 default:
1175 if (cpi->cpi_model == 0xf)
1176 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1177 break;
1178 }
1179
1180 cpi->cpi_step = CPI_STEP(cpi);
1181 cpi->cpi_brandid = CPI_BRANDID(cpi);
1182
1183 /*
1184 * *default* assumptions:
1185 * - believe %edx feature word
1186 * - ignore %ecx feature word
1187 * - 32-bit virtual and physical addressing
1188 */
1189 mask_edx = 0xffffffff;
1190 mask_ecx = 0;
1191
1192 cpi->cpi_pabits = cpi->cpi_vabits = 32;
1193
1194 switch (cpi->cpi_vendor) {
1195 case X86_VENDOR_Intel:
1196 if (cpi->cpi_family == 5)
1197 x86_type = X86_TYPE_P5;
1198 else if (IS_LEGACY_P6(cpi)) {
1199 x86_type = X86_TYPE_P6;
1200 pentiumpro_bug4046376 = 1;
1201 /*
1202 * Clear the SEP bit when it was set erroneously
1203 */
1204 if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
1205 cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
1206 } else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
1207 x86_type = X86_TYPE_P4;
1208 /*
1209 * We don't currently depend on any of the %ecx
1210 * features until Prescott, so we'll only check
1211 * this from P4 onwards. We might want to revisit
1212 * that idea later.
1213 */
1214 mask_ecx = 0xffffffff;
1215 } else if (cpi->cpi_family > 0xf)
1216 mask_ecx = 0xffffffff;
1217 /*
1218 * We don't support MONITOR/MWAIT if leaf 5 is not available
1219 * to obtain the monitor linesize.
1220 */
1221 if (cpi->cpi_maxeax < 5)
1222 mask_ecx &= ~CPUID_INTC_ECX_MON;
1223 break;
1224 case X86_VENDOR_IntelClone:
1225 default:
1226 break;
1227 case X86_VENDOR_AMD:
1228 #if defined(OPTERON_ERRATUM_108)
1229 if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
1230 cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
1231 cpi->cpi_model = 0xc;
1232 } else
1233 #endif
1234 if (cpi->cpi_family == 5) {
1235 /*
1236 * AMD K5 and K6
1237 *
1238 * These CPUs have an incomplete implementation
1239 * of MCA/MCE which we mask away.
1240 */
1241 mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);
1242
1243 /*
1244 * Model 0 uses the wrong (APIC) bit
1245 * to indicate PGE. Fix it here.
1246 */
1247 if (cpi->cpi_model == 0) {
1248 if (cp->cp_edx & 0x200) {
1249 cp->cp_edx &= ~0x200;
1250 cp->cp_edx |= CPUID_INTC_EDX_PGE;
1251 }
1252 }
1253
1254 /*
1255 * Early models had problems w/ MMX; disable.
1256 */
1257 if (cpi->cpi_model < 6)
1258 mask_edx &= ~CPUID_INTC_EDX_MMX;
1259 }
1260
1261 /*
1262 * For newer families, SSE3 and CX16, at least, are valid;
1263 * enable all
1264 */
1265 if (cpi->cpi_family >= 0xf)
1266 mask_ecx = 0xffffffff;
1267 /*
1268 * We don't support MONITOR/MWAIT if leaf 5 is not available
1269 * to obtain the monitor linesize.
1270 */
1271 if (cpi->cpi_maxeax < 5)
1272 mask_ecx &= ~CPUID_INTC_ECX_MON;
1273
1274 #if !defined(__xpv)
1275 /*
1276 * Do not use MONITOR/MWAIT to halt in the idle loop on any AMD
1277 * processors. AMD does not intend MWAIT to be used in the cpu
1278 * idle loop on current and future processors. 10h and future
1279 * AMD processors use more power in MWAIT than HLT.
1280 * Pre-family-10h Opterons do not have the MWAIT instruction.
1281 */
1282 idle_cpu_prefer_mwait = 0;
1283 #endif
1284
1285 break;
1286 case X86_VENDOR_TM:
1287 /*
1288 * workaround the NT workaround in CMS 4.1
1289 */
1290 if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
1291 (cpi->cpi_step == 2 || cpi->cpi_step == 3))
1292 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1293 break;
1294 case X86_VENDOR_Centaur:
1295 /*
1296 * workaround the NT workarounds again
1297 */
1298 if (cpi->cpi_family == 6)
1299 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1300 break;
1301 case X86_VENDOR_Cyrix:
1302 /*
1303 * We rely heavily on the probing in locore
1304 * to actually figure out what parts, if any,
1305 * of the Cyrix cpuid instruction to believe.
1306 */
1307 switch (x86_type) {
1308 case X86_TYPE_CYRIX_486:
1309 mask_edx = 0;
1310 break;
1311 case X86_TYPE_CYRIX_6x86:
1312 mask_edx = 0;
1313 break;
1314 case X86_TYPE_CYRIX_6x86L:
1315 mask_edx =
1316 CPUID_INTC_EDX_DE |
1317 CPUID_INTC_EDX_CX8;
1318 break;
1319 case X86_TYPE_CYRIX_6x86MX:
1320 mask_edx =
1321 CPUID_INTC_EDX_DE |
1322 CPUID_INTC_EDX_MSR |
1323 CPUID_INTC_EDX_CX8 |
1324 CPUID_INTC_EDX_PGE |
1325 CPUID_INTC_EDX_CMOV |
1326 CPUID_INTC_EDX_MMX;
1327 break;
1328 case X86_TYPE_CYRIX_GXm:
1329 mask_edx =
1330 CPUID_INTC_EDX_MSR |
1331 CPUID_INTC_EDX_CX8 |
1332 CPUID_INTC_EDX_CMOV |
1333 CPUID_INTC_EDX_MMX;
1334 break;
1335 case X86_TYPE_CYRIX_MediaGX:
1336 break;
1337 case X86_TYPE_CYRIX_MII:
1338 case X86_TYPE_VIA_CYRIX_III:
1339 mask_edx =
1340 CPUID_INTC_EDX_DE |
1341 CPUID_INTC_EDX_TSC |
1342 CPUID_INTC_EDX_MSR |
1343 CPUID_INTC_EDX_CX8 |
1344 CPUID_INTC_EDX_PGE |
1345 CPUID_INTC_EDX_CMOV |
1346 CPUID_INTC_EDX_MMX;
1347 break;
1348 default:
1349 break;
1350 }
1351 break;
1352 }
1353
1354 #if defined(__xpv)
1355 /*
1356 * Do not support MONITOR/MWAIT under a hypervisor
1357 */
1358 mask_ecx &= ~CPUID_INTC_ECX_MON;
1359 /*
1360 * Do not support XSAVE under a hypervisor for now
1361 */
1362 xsave_force_disable = B_TRUE;
1363
1364 #endif /* __xpv */
1365
1366 if (xsave_force_disable) {
1367 mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
1368 mask_ecx &= ~CPUID_INTC_ECX_AVX;
1369 mask_ecx &= ~CPUID_INTC_ECX_F16C;
1370 mask_ecx &= ~CPUID_INTC_ECX_FMA;
1371 }
1372
1373 /*
1374 * Now we've figured out the masks that determine
1375 * which bits we choose to believe, apply the masks
1376 * to the feature words, then map the kernel's view
1377 * of these feature words into its feature word.
1378 */
1379 cp->cp_edx &= mask_edx;
1380 cp->cp_ecx &= mask_ecx;
1381
1382 /*
1383 * apply any platform restrictions (we don't call this
1384 * immediately after __cpuid_insn here, because we need the
1385 * workarounds applied above first)
1386 */
1387 platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);
1388
1389 /*
1390 * In addition to ecx and edx, Intel and AMD are storing a bunch of
1391 * instruction set extensions in leaf 7's ebx, ecx, and edx.
1392 */
1393 if (cpi->cpi_maxeax >= 7) {
1394 struct cpuid_regs *ecp;
1395 ecp = &cpi->cpi_std[7];
1396 ecp->cp_eax = 7;
1397 ecp->cp_ecx = 0;
1398 (void) __cpuid_insn(ecp);
1399
1400 /*
1401 * If XSAVE has been disabled, just ignore all of the
1402 * extended-save-area dependent flags here.
1403 */
1404 if (xsave_force_disable) {
1405 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
1406 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
1407 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
1408 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_MPX;
1409 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_ALL_AVX512;
1410 ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_ALL_AVX512;
1411 ecp->cp_edx &= ~CPUID_INTC_EDX_7_0_ALL_AVX512;
1412 }
1413
1414 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMEP)
1415 add_x86_feature(featureset, X86FSET_SMEP);
1416
1417 /*
1418 * We check disable_smap here in addition to in startup_smap()
1419 * to ensure CPUs that aren't the boot CPU don't accidentally
1420 * include it in the feature set and thus generate a mismatched
1421 * x86 feature set across CPUs.
1422 */
1423 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMAP &&
1424 disable_smap == 0)
1425 add_x86_feature(featureset, X86FSET_SMAP);
1426
1427 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_RDSEED)
1428 add_x86_feature(featureset, X86FSET_RDSEED);
1429
1430 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_ADX)
1431 add_x86_feature(featureset, X86FSET_ADX);
1432
1433 if (cpi->cpi_vendor == X86_VENDOR_Intel) {
1434 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_INVPCID)
1435 add_x86_feature(featureset, X86FSET_INVPCID);
1436
1437 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_MPX)
1438 add_x86_feature(featureset, X86FSET_MPX);
1439 }
1440 }
1441
1442 /*
1443 * fold in overrides from the "eeprom" mechanism
1444 */
1445 cp->cp_edx |= cpuid_feature_edx_include;
1446 cp->cp_edx &= ~cpuid_feature_edx_exclude;
1447
1448 cp->cp_ecx |= cpuid_feature_ecx_include;
1449 cp->cp_ecx &= ~cpuid_feature_ecx_exclude;
1450
1451 if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
1452 add_x86_feature(featureset, X86FSET_LARGEPAGE);
1453 }
1454 if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
1455 add_x86_feature(featureset, X86FSET_TSC);
1456 }
1457 if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
1458 add_x86_feature(featureset, X86FSET_MSR);
1459 }
1460 if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
1461 add_x86_feature(featureset, X86FSET_MTRR);
1462 }
1463 if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
1464 add_x86_feature(featureset, X86FSET_PGE);
1465 }
1466 if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
1467 add_x86_feature(featureset, X86FSET_CMOV);
1468 }
1469 if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
1470 add_x86_feature(featureset, X86FSET_MMX);
1471 }
1472 if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
1473 (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
1474 add_x86_feature(featureset, X86FSET_MCA);
1475 }
1476 if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
1477 add_x86_feature(featureset, X86FSET_PAE);
1478 }
1479 if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
1480 add_x86_feature(featureset, X86FSET_CX8);
1481 }
1482 if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
1483 add_x86_feature(featureset, X86FSET_CX16);
1484 }
1485 if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
1486 add_x86_feature(featureset, X86FSET_PAT);
1487 }
1488 if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
1489 add_x86_feature(featureset, X86FSET_SEP);
1490 }
1491 if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
1492 /*
1493 * In our implementation, fxsave/fxrstor
1494 * are prerequisites before we'll even
1495 * try and do SSE things.
1496 */
1497 if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
1498 add_x86_feature(featureset, X86FSET_SSE);
1499 }
1500 if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
1501 add_x86_feature(featureset, X86FSET_SSE2);
1502 }
1503 if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
1504 add_x86_feature(featureset, X86FSET_SSE3);
1505 }
1506 if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
1507 add_x86_feature(featureset, X86FSET_SSSE3);
1508 }
1509 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
1510 add_x86_feature(featureset, X86FSET_SSE4_1);
1511 }
1512 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
1513 add_x86_feature(featureset, X86FSET_SSE4_2);
1514 }
1515 if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
1516 add_x86_feature(featureset, X86FSET_AES);
1517 }
1518 if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
1519 add_x86_feature(featureset, X86FSET_PCLMULQDQ);
1520 }
1521
1522 if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_SHA)
1523 add_x86_feature(featureset, X86FSET_SHA);
1524
1525 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_UMIP)
1526 add_x86_feature(featureset, X86FSET_UMIP);
1527 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_PKU)
1528 add_x86_feature(featureset, X86FSET_PKU);
1529 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_OSPKE)
1530 add_x86_feature(featureset, X86FSET_OSPKE);
1531
1532 if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
1533 add_x86_feature(featureset, X86FSET_XSAVE);
1534
1535 /* We only test AVX & AVX512 when there is XSAVE */
1536
1537 if (cp->cp_ecx & CPUID_INTC_ECX_AVX) {
1538 add_x86_feature(featureset,
1539 X86FSET_AVX);
1540
1541 /*
1542 * Intel says we can't check these without also
1543 * checking AVX.
1544 */
1545 if (cp->cp_ecx & CPUID_INTC_ECX_F16C)
1546 add_x86_feature(featureset,
1547 X86FSET_F16C);
1548
1549 if (cp->cp_ecx & CPUID_INTC_ECX_FMA)
1550 add_x86_feature(featureset,
1551 X86FSET_FMA);
1552
1553 if (cpi->cpi_std[7].cp_ebx &
1554 CPUID_INTC_EBX_7_0_BMI1)
1555 add_x86_feature(featureset,
1556 X86FSET_BMI1);
1557
1558 if (cpi->cpi_std[7].cp_ebx &
1559 CPUID_INTC_EBX_7_0_BMI2)
1560 add_x86_feature(featureset,
1561 X86FSET_BMI2);
1562
1563 if (cpi->cpi_std[7].cp_ebx &
1564 CPUID_INTC_EBX_7_0_AVX2)
1565 add_x86_feature(featureset,
1566 X86FSET_AVX2);
1567 }
1568
1569 if (cpi->cpi_vendor == X86_VENDOR_Intel &&
1570 (cpi->cpi_std[7].cp_ebx &
1571 CPUID_INTC_EBX_7_0_AVX512F) != 0) {
1572 add_x86_feature(featureset, X86FSET_AVX512F);
1573
1574 if (cpi->cpi_std[7].cp_ebx &
1575 CPUID_INTC_EBX_7_0_AVX512DQ)
1576 add_x86_feature(featureset,
1577 X86FSET_AVX512DQ);
1578 if (cpi->cpi_std[7].cp_ebx &
1579 CPUID_INTC_EBX_7_0_AVX512IFMA)
1580 add_x86_feature(featureset,
1581 X86FSET_AVX512FMA);
1582 if (cpi->cpi_std[7].cp_ebx &
1583 CPUID_INTC_EBX_7_0_AVX512PF)
1584 add_x86_feature(featureset,
1585 X86FSET_AVX512PF);
1586 if (cpi->cpi_std[7].cp_ebx &
1587 CPUID_INTC_EBX_7_0_AVX512ER)
1588 add_x86_feature(featureset,
1589 X86FSET_AVX512ER);
1590 if (cpi->cpi_std[7].cp_ebx &
1591 CPUID_INTC_EBX_7_0_AVX512CD)
1592 add_x86_feature(featureset,
1593 X86FSET_AVX512CD);
1594 if (cpi->cpi_std[7].cp_ebx &
1595 CPUID_INTC_EBX_7_0_AVX512BW)
1596 add_x86_feature(featureset,
1597 X86FSET_AVX512BW);
1598 if (cpi->cpi_std[7].cp_ebx &
1599 CPUID_INTC_EBX_7_0_AVX512VL)
1600 add_x86_feature(featureset,
1601 X86FSET_AVX512VL);
1602
1603 if (cpi->cpi_std[7].cp_ecx &
1604 CPUID_INTC_ECX_7_0_AVX512VBMI)
1605 add_x86_feature(featureset,
1606 X86FSET_AVX512VBMI);
1607 if (cpi->cpi_std[7].cp_ecx &
1608 CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
1609 add_x86_feature(featureset,
1610 X86FSET_AVX512VPOPCDQ);
1611
1612 if (cpi->cpi_std[7].cp_edx &
1613 CPUID_INTC_EDX_7_0_AVX5124NNIW)
1614 add_x86_feature(featureset,
1615 X86FSET_AVX512NNIW);
1616 if (cpi->cpi_std[7].cp_edx &
1617 CPUID_INTC_EDX_7_0_AVX5124FMAPS)
1618 add_x86_feature(featureset,
1619 X86FSET_AVX512FMAPS);
1620 }
1621 }
1622 }
1623
1624 if (cpi->cpi_vendor == X86_VENDOR_Intel) {
1625 if (cp->cp_ecx & CPUID_INTC_ECX_PCID) {
1626 add_x86_feature(featureset, X86FSET_PCID);
1627 }
1628 }
1629
1630 if (cp->cp_ecx & CPUID_INTC_ECX_X2APIC) {
1631 add_x86_feature(featureset, X86FSET_X2APIC);
1632 }
1633 if (cp->cp_edx & CPUID_INTC_EDX_DE) {
1634 add_x86_feature(featureset, X86FSET_DE);
1635 }
1636 #if !defined(__xpv)
1637 if (cp->cp_ecx & CPUID_INTC_ECX_MON) {
1638
1639 /*
1640 * We require the CLFLUSH instruction for erratum workaround
1641 * to use MONITOR/MWAIT.
1642 */
1643 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1644 cpi->cpi_mwait.support |= MWAIT_SUPPORT;
1645 add_x86_feature(featureset, X86FSET_MWAIT);
1646 } else {
1647 extern int idle_cpu_assert_cflush_monitor;
1648
1649 /*
1650 * All processors we are aware of which have
1651 * MONITOR/MWAIT also have CLFLUSH.
1652 */
1653 if (idle_cpu_assert_cflush_monitor) {
1654 ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
1655 (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
1656 }
1657 }
1658 }
1659 #endif /* __xpv */
1660
1661 if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
1662 add_x86_feature(featureset, X86FSET_VMX);
1663 }
1664
1665 if (cp->cp_ecx & CPUID_INTC_ECX_RDRAND)
1666 add_x86_feature(featureset, X86FSET_RDRAND);
1667
1668 /*
1669 * Only need it first time, rest of the cpus would follow suit.
1670 * we only capture this for the bootcpu.
1671 */
1672 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1673 add_x86_feature(featureset, X86FSET_CLFSH);
1674 x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
1675 }
1676 if (is_x86_feature(featureset, X86FSET_PAE))
1677 cpi->cpi_pabits = 36;
1678
1679 /*
1680 * Hyperthreading configuration is slightly tricky on Intel
1681 * and pure clones, and even trickier on AMD.
1682 *
1683 * (AMD chose to set the HTT bit on their CMP processors,
1684 * even though they're not actually hyperthreaded. Thus it
1685 * takes a bit more work to figure out what's really going
1686 * on ... see the handling of the CMP_LGCY bit below)
1687 */
1688 if (cp->cp_edx & CPUID_INTC_EDX_HTT) {
1689 cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
1690 if (cpi->cpi_ncpu_per_chip > 1)
1691 add_x86_feature(featureset, X86FSET_HTT);
1692 } else {
1693 cpi->cpi_ncpu_per_chip = 1;
1694 }
1695
1696 if (cpi->cpi_maxeax >= 0xD && !xsave_force_disable) {
1697 struct cpuid_regs r, *ecp;
1698
1699 ecp = &r;
1700 ecp->cp_eax = 0xD;
1701 ecp->cp_ecx = 1;
1702 ecp->cp_edx = ecp->cp_ebx = 0;
1703 (void) __cpuid_insn(ecp);
1704
1705 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEOPT)
1706 add_x86_feature(featureset, X86FSET_XSAVEOPT);
1707 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEC)
1708 add_x86_feature(featureset, X86FSET_XSAVEC);
1709 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVES)
1710 add_x86_feature(featureset, X86FSET_XSAVES);
1711 }
1712
1713 /*
1714 * Work on the "extended" feature information, doing
1715 * some basic initialization for cpuid_pass2()
1716 */
1717 xcpuid = 0;
1718 switch (cpi->cpi_vendor) {
1719 case X86_VENDOR_Intel:
1720 /*
1721 * On KVM we know we will have proper support for extended
1722 * cpuid.
1723 */
1724 if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf ||
1725 (get_hwenv() == HW_KVM && cpi->cpi_family == 6 &&
1726 (cpi->cpi_model == 6 || cpi->cpi_model == 2)))
1727 xcpuid++;
1728 break;
1729 case X86_VENDOR_AMD:
1730 if (cpi->cpi_family > 5 ||
1731 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
1732 xcpuid++;
1733 break;
1734 case X86_VENDOR_Cyrix:
1735 /*
1736 * Only these Cyrix CPUs are -known- to support
1737 * extended cpuid operations.
1738 */
1739 if (x86_type == X86_TYPE_VIA_CYRIX_III ||
1740 x86_type == X86_TYPE_CYRIX_GXm)
1741 xcpuid++;
1742 break;
1743 case X86_VENDOR_Centaur:
1744 case X86_VENDOR_TM:
1745 default:
1746 xcpuid++;
1747 break;
1748 }
1749
1750 if (xcpuid) {
1751 cp = &cpi->cpi_extd[0];
1752 cp->cp_eax = 0x80000000;
1753 cpi->cpi_xmaxeax = __cpuid_insn(cp);
1754 }
1755
1756 if (cpi->cpi_xmaxeax & 0x80000000) {
1757
1758 if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
1759 cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;
1760
1761 switch (cpi->cpi_vendor) {
1762 case X86_VENDOR_Intel:
1763 case X86_VENDOR_AMD:
1764 if (cpi->cpi_xmaxeax < 0x80000001)
1765 break;
1766 cp = &cpi->cpi_extd[1];
1767 cp->cp_eax = 0x80000001;
1768 (void) __cpuid_insn(cp);
1769
1770 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1771 cpi->cpi_family == 5 &&
1772 cpi->cpi_model == 6 &&
1773 cpi->cpi_step == 6) {
1774 /*
1775 * K6 model 6 uses bit 10 to indicate SYSC
1776 * Later models use bit 11. Fix it here.
1777 */
1778 if (cp->cp_edx & 0x400) {
1779 cp->cp_edx &= ~0x400;
1780 cp->cp_edx |= CPUID_AMD_EDX_SYSC;
1781 }
1782 }
1783
1784 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);
1785
1786 /*
1787 * Compute the additions to the kernel's feature word.
1788 */
1789 if (cp->cp_edx & CPUID_AMD_EDX_NX) {
1790 add_x86_feature(featureset, X86FSET_NX);
1791 }
1792
1793 /*
1794 * Regardless whether or not we boot 64-bit,
1795 * we should have a way to identify whether
1796 * the CPU is capable of running 64-bit.
1797 */
1798 if (cp->cp_edx & CPUID_AMD_EDX_LM) {
1799 add_x86_feature(featureset, X86FSET_64);
1800 }
1801
1802 #if defined(__amd64)
1803 /* 1 GB large page - enable only for 64 bit kernel */
1804 if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
1805 add_x86_feature(featureset, X86FSET_1GPG);
1806 }
1807 #endif
1808
1809 if ((cpi->cpi_vendor == X86_VENDOR_AMD) &&
1810 (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
1811 (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
1812 add_x86_feature(featureset, X86FSET_SSE4A);
1813 }
1814
1815 /*
1816 * If both the HTT and CMP_LGCY bits are set,
1817 * then we're not actually HyperThreaded. Read
1818 * "AMD CPUID Specification" for more details.
1819 */
1820 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1821 is_x86_feature(featureset, X86FSET_HTT) &&
1822 (cp->cp_ecx & CPUID_AMD_ECX_CMP_LGCY)) {
1823 remove_x86_feature(featureset, X86FSET_HTT);
1824 add_x86_feature(featureset, X86FSET_CMP);
1825 }
1826 #if defined(__amd64)
1827 /*
1828 * It's really tricky to support syscall/sysret in
1829 * the i386 kernel; we rely on sysenter/sysexit
1830 * instead. In the amd64 kernel, things are -way-
1831 * better.
1832 */
1833 if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
1834 add_x86_feature(featureset, X86FSET_ASYSC);
1835 }
1836
1837 /*
1838 * While we're thinking about system calls, note
1839 * that AMD processors don't support sysenter
1840 * in long mode at all, so don't try to program them.
1841 */
1842 if (x86_vendor == X86_VENDOR_AMD) {
1843 remove_x86_feature(featureset, X86FSET_SEP);
1844 }
1845 #endif
1846 if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
1847 add_x86_feature(featureset, X86FSET_TSCP);
1848 }
1849
1850 if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
1851 add_x86_feature(featureset, X86FSET_SVM);
1852 }
1853
1854 if (cp->cp_ecx & CPUID_AMD_ECX_TOPOEXT) {
1855 add_x86_feature(featureset, X86FSET_TOPOEXT);
1856 }
1857 break;
1858 default:
1859 break;
1860 }
1861
1862 /*
1863 * Get CPUID data about processor cores and hyperthreads.
1864 */
1865 switch (cpi->cpi_vendor) {
1866 case X86_VENDOR_Intel:
1867 if (cpi->cpi_maxeax >= 4) {
1868 cp = &cpi->cpi_std[4];
1869 cp->cp_eax = 4;
1870 cp->cp_ecx = 0;
1871 (void) __cpuid_insn(cp);
1872 platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
1873 }
1874 /*FALLTHROUGH*/
1875 case X86_VENDOR_AMD:
1876 if (cpi->cpi_xmaxeax < 0x80000008)
1877 break;
1878 cp = &cpi->cpi_extd[8];
1879 cp->cp_eax = 0x80000008;
1880 (void) __cpuid_insn(cp);
1881 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, cp);
1882
1883 /*
1884 * Virtual and physical address limits from
1885 * cpuid override previously guessed values.
1886 */
1887 cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
1888 cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
1889 break;
1890 default:
1891 break;
1892 }
1893
1894 /*
1895 * Derive the number of cores per chip
1896 */
1897 switch (cpi->cpi_vendor) {
1898 case X86_VENDOR_Intel:
1899 if (cpi->cpi_maxeax < 4) {
1900 cpi->cpi_ncore_per_chip = 1;
1901 break;
1902 } else {
1903 cpi->cpi_ncore_per_chip =
1904 BITX((cpi)->cpi_std[4].cp_eax, 31, 26) + 1;
1905 }
1906 break;
1907 case X86_VENDOR_AMD:
1908 if (cpi->cpi_xmaxeax < 0x80000008) {
1909 cpi->cpi_ncore_per_chip = 1;
1910 break;
1911 } else {
1912 /*
1913 * On family 0xf cpuid fn 2 ECX[7:0] "NC" is
1914 * 1 less than the number of physical cores on
1915 * the chip. In family 0x10 this value can
1916 * be affected by "downcoring" - it reflects
1917 * 1 less than the number of cores actually
1918 * enabled on this node.
1919 */
1920 cpi->cpi_ncore_per_chip =
1921 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
1922 }
1923 break;
1924 default:
1925 cpi->cpi_ncore_per_chip = 1;
1926 break;
1927 }
1928
1929 /*
1930 * Get CPUID data about TSC Invariance in Deep C-State.
1931 */
1932 switch (cpi->cpi_vendor) {
1933 case X86_VENDOR_Intel:
1934 if (cpi->cpi_maxeax >= 7) {
1935 cp = &cpi->cpi_extd[7];
1936 cp->cp_eax = 0x80000007;
1937 cp->cp_ecx = 0;
1938 (void) __cpuid_insn(cp);
1939 }
1940 break;
1941 default:
1942 break;
1943 }
1944 } else {
1945 cpi->cpi_ncore_per_chip = 1;
1946 }
1947
1948 /*
1949 * If more than one core, then this processor is CMP.
1950 */
1951 if (cpi->cpi_ncore_per_chip > 1) {
1952 add_x86_feature(featureset, X86FSET_CMP);
1953 }
1954
1955 /*
1956 * If the number of cores is the same as the number
1957 * of CPUs, then we cannot have HyperThreading.
1958 */
1959 if (cpi->cpi_ncpu_per_chip == cpi->cpi_ncore_per_chip) {
1960 remove_x86_feature(featureset, X86FSET_HTT);
1961 }
1962
1963 cpi->cpi_apicid = CPI_APIC_ID(cpi);
1964 cpi->cpi_procnodes_per_pkg = 1;
1965 cpi->cpi_cores_per_compunit = 1;
1966 if (is_x86_feature(featureset, X86FSET_HTT) == B_FALSE &&
1967 is_x86_feature(featureset, X86FSET_CMP) == B_FALSE) {
1968 /*
1969 * Single-core single-threaded processors.
1970 */
1971 cpi->cpi_chipid = -1;
1972 cpi->cpi_clogid = 0;
1973 cpi->cpi_coreid = cpu->cpu_id;
1974 cpi->cpi_pkgcoreid = 0;
1975 if (cpi->cpi_vendor == X86_VENDOR_AMD)
1976 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
1977 else
1978 cpi->cpi_procnodeid = cpi->cpi_chipid;
1979 } else if (cpi->cpi_ncpu_per_chip > 1) {
1980 if (cpi->cpi_vendor == X86_VENDOR_Intel)
1981 cpuid_intel_getids(cpu, featureset);
1982 else if (cpi->cpi_vendor == X86_VENDOR_AMD)
1983 cpuid_amd_getids(cpu);
1984 else {
1985 /*
1986 * All other processors are currently
1987 * assumed to have single cores.
1988 */
1989 cpi->cpi_coreid = cpi->cpi_chipid;
1990 cpi->cpi_pkgcoreid = 0;
1991 cpi->cpi_procnodeid = cpi->cpi_chipid;
1992 cpi->cpi_compunitid = cpi->cpi_chipid;
1993 }
1994 }
1995
1996 /*
1997 * Synthesize chip "revision" and socket type
1998 */
1999 cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
2000 cpi->cpi_model, cpi->cpi_step);
2001 cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
2002 cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
2003 cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
2004 cpi->cpi_model, cpi->cpi_step);
2005
2006 /*
2007 * While we're here, check for the AMD "Error Pointer Zero/Restore"
2008 * feature. This can be used to setup the FP save handlers
2009 * appropriately.
2010 */
2011 if (cpi->cpi_vendor == X86_VENDOR_AMD) {
2012 if (cpi->cpi_xmaxeax >= 0x80000008 &&
2013 cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
2014 /* Special handling for AMD FP not necessary. */
2015 cpi->cpi_fp_amd_save = 0;
2016 } else {
2017 cpi->cpi_fp_amd_save = 1;
2018 }
2019 }
2020
2021 /*
2022 * Check the processor leaves that are used for security features.
2023 */
2024 cpuid_scan_security(cpu, featureset);
2025
2026 pass1_done:
2027 cpi->cpi_pass = 1;
2028 }
2029
2030 /*
2031 * Make copies of the cpuid table entries we depend on, in
2032 * part for ease of parsing now, in part so that we have only
2033 * one place to correct any of it, in part for ease of
2034 * later export to userland, and in part so we can look at
2035 * this stuff in a crash dump.
2036 */
2037
2038 /*ARGSUSED*/
2039 void
2040 cpuid_pass2(cpu_t *cpu)
2041 {
2042 uint_t n, nmax;
2043 int i;
2044 struct cpuid_regs *cp;
2045 uint8_t *dp;
2046 uint32_t *iptr;
2047 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2048
2049 ASSERT(cpi->cpi_pass == 1);
2050
2051 if (cpi->cpi_maxeax < 1)
2052 goto pass2_done;
2053
2054 if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
2055 nmax = NMAX_CPI_STD;
2056 /*
2057 * (We already handled n == 0 and n == 1 in pass 1)
2058 */
2059 for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
2060 cp->cp_eax = n;
2061
2062 /*
2063 * n == 7 was handled in pass 1
2064 */
2065 if (n == 7)
2066 continue;
2067
2068 /*
2069 * CPUID function 4 expects %ecx to be initialized
2070 * with an index which indicates which cache to return
2071 * information about. The OS is expected to call function 4
2072 * with %ecx set to 0, 1, 2, ... until it returns with
2073 * EAX[4:0] set to 0, which indicates there are no more
2074 * caches.
2075 *
2076 * Here, populate cpi_std[4] with the information returned by
2077 * function 4 when %ecx == 0, and do the rest in cpuid_pass3()
2078 * when dynamic memory allocation becomes available.
2079 *
2080 * Note: we need to explicitly initialize %ecx here, since
2081 * function 4 may have been previously invoked.
2082 */
2083 if (n == 4)
2084 cp->cp_ecx = 0;
2085
2086 (void) __cpuid_insn(cp);
2087 platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
2088 switch (n) {
2089 case 2:
2090 /*
2091 * "the lower 8 bits of the %eax register
2092 * contain a value that identifies the number
2093 * of times the cpuid [instruction] has to be
2094 * executed to obtain a complete image of the
2095 * processor's caching systems."
2096 *
2097 * How *do* they make this stuff up?
2098 */
2099 cpi->cpi_ncache = sizeof (*cp) *
2100 BITX(cp->cp_eax, 7, 0);
2101 if (cpi->cpi_ncache == 0)
2102 break;
2103 cpi->cpi_ncache--; /* skip count byte */
2104
2105 /*
2106 * Well, for now, rather than attempt to implement
2107 * this slightly dubious algorithm, we just look
2108 * at the first 15 ..
2109 */
2110 if (cpi->cpi_ncache > (sizeof (*cp) - 1))
2111 cpi->cpi_ncache = sizeof (*cp) - 1;
2112
2113 dp = cpi->cpi_cacheinfo;
2114 if (BITX(cp->cp_eax, 31, 31) == 0) {
2115 uint8_t *p = (void *)&cp->cp_eax;
2116 for (i = 1; i < 4; i++)
2117 if (p[i] != 0)
2118 *dp++ = p[i];
2119 }
2120 if (BITX(cp->cp_ebx, 31, 31) == 0) {
2121 uint8_t *p = (void *)&cp->cp_ebx;
2122 for (i = 0; i < 4; i++)
2123 if (p[i] != 0)
2124 *dp++ = p[i];
2125 }
2126 if (BITX(cp->cp_ecx, 31, 31) == 0) {
2127 uint8_t *p = (void *)&cp->cp_ecx;
2128 for (i = 0; i < 4; i++)
2129 if (p[i] != 0)
2130 *dp++ = p[i];
2131 }
2132 if (BITX(cp->cp_edx, 31, 31) == 0) {
2133 uint8_t *p = (void *)&cp->cp_edx;
2134 for (i = 0; i < 4; i++)
2135 if (p[i] != 0)
2136 *dp++ = p[i];
2137 }
2138 break;
2139
2140 case 3: /* Processor serial number, if PSN supported */
2141 break;
2142
2143 case 4: /* Deterministic cache parameters */
2144 break;
2145
2146 case 5: /* Monitor/Mwait parameters */
2147 {
2148 size_t mwait_size;
2149
2150 /*
2151 * check cpi_mwait.support which was set in cpuid_pass1
2152 */
2153 if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
2154 break;
2155
2156 /*
2157 * Protect ourself from insane mwait line size.
2158 * Workaround for incomplete hardware emulator(s).
2159 */
2160 mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
2161 if (mwait_size < sizeof (uint32_t) ||
2162 !ISP2(mwait_size)) {
2163 #if DEBUG
2164 cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
2165 "size %ld", cpu->cpu_id, (long)mwait_size);
2166 #endif
2167 break;
2168 }
2169
2170 cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
2171 cpi->cpi_mwait.mon_max = mwait_size;
2172 if (MWAIT_EXTENSION(cpi)) {
2173 cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
2174 if (MWAIT_INT_ENABLE(cpi))
2175 cpi->cpi_mwait.support |=
2176 MWAIT_ECX_INT_ENABLE;
2177 }
2178 break;
2179 }
2180 default:
2181 break;
2182 }
2183 }
2184
2185 if (cpi->cpi_maxeax >= 0xB && cpi->cpi_vendor == X86_VENDOR_Intel) {
2186 struct cpuid_regs regs;
2187
2188 cp = ®s;
2189 cp->cp_eax = 0xB;
2190 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2191
2192 (void) __cpuid_insn(cp);
2193
2194 /*
2195 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
2196 * indicates that the extended topology enumeration leaf is
2197 * available.
2198 */
2199 if (cp->cp_ebx) {
2200 uint32_t x2apic_id;
2201 uint_t coreid_shift = 0;
2202 uint_t ncpu_per_core = 1;
2203 uint_t chipid_shift = 0;
2204 uint_t ncpu_per_chip = 1;
2205 uint_t i;
2206 uint_t level;
2207
2208 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
2209 cp->cp_eax = 0xB;
2210 cp->cp_ecx = i;
2211
2212 (void) __cpuid_insn(cp);
2213 level = CPI_CPU_LEVEL_TYPE(cp);
2214
2215 if (level == 1) {
2216 x2apic_id = cp->cp_edx;
2217 coreid_shift = BITX(cp->cp_eax, 4, 0);
2218 ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
2219 } else if (level == 2) {
2220 x2apic_id = cp->cp_edx;
2221 chipid_shift = BITX(cp->cp_eax, 4, 0);
2222 ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
2223 }
2224 }
2225
2226 cpi->cpi_apicid = x2apic_id;
2227 cpi->cpi_ncpu_per_chip = ncpu_per_chip;
2228 cpi->cpi_ncore_per_chip = ncpu_per_chip /
2229 ncpu_per_core;
2230 cpi->cpi_chipid = x2apic_id >> chipid_shift;
2231 cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
2232 cpi->cpi_coreid = x2apic_id >> coreid_shift;
2233 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
2234 }
2235
2236 /* Make cp NULL so that we don't stumble on others */
2237 cp = NULL;
2238 }
2239
2240 /*
2241 * XSAVE enumeration
2242 */
2243 if (cpi->cpi_maxeax >= 0xD) {
2244 struct cpuid_regs regs;
2245 boolean_t cpuid_d_valid = B_TRUE;
2246
2247 cp = ®s;
2248 cp->cp_eax = 0xD;
2249 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2250
2251 (void) __cpuid_insn(cp);
2252
2253 /*
2254 * Sanity checks for debug
2255 */
2256 if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
2257 (cp->cp_eax & XFEATURE_SSE) == 0) {
2258 cpuid_d_valid = B_FALSE;
2259 }
2260
2261 cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
2262 cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
2263 cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;
2264
2265 /*
2266 * If the hw supports AVX, get the size and offset in the save
2267 * area for the ymm state.
2268 */
2269 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
2270 cp->cp_eax = 0xD;
2271 cp->cp_ecx = 2;
2272 cp->cp_edx = cp->cp_ebx = 0;
2273
2274 (void) __cpuid_insn(cp);
2275
2276 if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
2277 cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
2278 cpuid_d_valid = B_FALSE;
2279 }
2280
2281 cpi->cpi_xsave.ymm_size = cp->cp_eax;
2282 cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
2283 }
2284
2285 /*
2286 * If the hw supports MPX, get the size and offset in the
2287 * save area for BNDREGS and BNDCSR.
2288 */
2289 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_MPX) {
2290 cp->cp_eax = 0xD;
2291 cp->cp_ecx = 3;
2292 cp->cp_edx = cp->cp_ebx = 0;
2293
2294 (void) __cpuid_insn(cp);
2295
2296 cpi->cpi_xsave.bndregs_size = cp->cp_eax;
2297 cpi->cpi_xsave.bndregs_offset = cp->cp_ebx;
2298
2299 cp->cp_eax = 0xD;
2300 cp->cp_ecx = 4;
2301 cp->cp_edx = cp->cp_ebx = 0;
2302
2303 (void) __cpuid_insn(cp);
2304
2305 cpi->cpi_xsave.bndcsr_size = cp->cp_eax;
2306 cpi->cpi_xsave.bndcsr_offset = cp->cp_ebx;
2307 }
2308
2309 /*
2310 * If the hw supports AVX512, get the size and offset in the
2311 * save area for the opmask registers and zmm state.
2312 */
2313 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX512) {
2314 cp->cp_eax = 0xD;
2315 cp->cp_ecx = 5;
2316 cp->cp_edx = cp->cp_ebx = 0;
2317
2318 (void) __cpuid_insn(cp);
2319
2320 cpi->cpi_xsave.opmask_size = cp->cp_eax;
2321 cpi->cpi_xsave.opmask_offset = cp->cp_ebx;
2322
2323 cp->cp_eax = 0xD;
2324 cp->cp_ecx = 6;
2325 cp->cp_edx = cp->cp_ebx = 0;
2326
2327 (void) __cpuid_insn(cp);
2328
2329 cpi->cpi_xsave.zmmlo_size = cp->cp_eax;
2330 cpi->cpi_xsave.zmmlo_offset = cp->cp_ebx;
2331
2332 cp->cp_eax = 0xD;
2333 cp->cp_ecx = 7;
2334 cp->cp_edx = cp->cp_ebx = 0;
2335
2336 (void) __cpuid_insn(cp);
2337
2338 cpi->cpi_xsave.zmmhi_size = cp->cp_eax;
2339 cpi->cpi_xsave.zmmhi_offset = cp->cp_ebx;
2340 }
2341
2342 if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
2343 xsave_state_size = 0;
2344 } else if (cpuid_d_valid) {
2345 xsave_state_size = cpi->cpi_xsave.xsav_max_size;
2346 } else {
2347 /* Broken CPUID 0xD, probably in HVM */
2348 cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
2349 "value: hw_low = %d, hw_high = %d, xsave_size = %d"
2350 ", ymm_size = %d, ymm_offset = %d\n",
2351 cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
2352 cpi->cpi_xsave.xsav_hw_features_high,
2353 (int)cpi->cpi_xsave.xsav_max_size,
2354 (int)cpi->cpi_xsave.ymm_size,
2355 (int)cpi->cpi_xsave.ymm_offset);
2356
2357 if (xsave_state_size != 0) {
2358 /*
2359 * This must be a non-boot CPU. We cannot
2360 * continue, because boot cpu has already
2361 * enabled XSAVE.
2362 */
2363 ASSERT(cpu->cpu_id != 0);
2364 cmn_err(CE_PANIC, "cpu%d: we have already "
2365 "enabled XSAVE on boot cpu, cannot "
2366 "continue.", cpu->cpu_id);
2367 } else {
2368 /*
2369 * If we reached here on the boot CPU, it's also
2370 * almost certain that we'll reach here on the
2371 * non-boot CPUs. When we're here on a boot CPU
2372 * we should disable the feature, on a non-boot
2373 * CPU we need to confirm that we have.
2374 */
2375 if (cpu->cpu_id == 0) {
2376 remove_x86_feature(x86_featureset,
2377 X86FSET_XSAVE);
2378 remove_x86_feature(x86_featureset,
2379 X86FSET_AVX);
2380 remove_x86_feature(x86_featureset,
2381 X86FSET_F16C);
2382 remove_x86_feature(x86_featureset,
2383 X86FSET_BMI1);
2384 remove_x86_feature(x86_featureset,
2385 X86FSET_BMI2);
2386 remove_x86_feature(x86_featureset,
2387 X86FSET_FMA);
2388 remove_x86_feature(x86_featureset,
2389 X86FSET_AVX2);
2390 remove_x86_feature(x86_featureset,
2391 X86FSET_MPX);
2392 remove_x86_feature(x86_featureset,
2393 X86FSET_AVX512F);
2394 remove_x86_feature(x86_featureset,
2395 X86FSET_AVX512DQ);
2396 remove_x86_feature(x86_featureset,
2397 X86FSET_AVX512PF);
2398 remove_x86_feature(x86_featureset,
2399 X86FSET_AVX512ER);
2400 remove_x86_feature(x86_featureset,
2401 X86FSET_AVX512CD);
2402 remove_x86_feature(x86_featureset,
2403 X86FSET_AVX512BW);
2404 remove_x86_feature(x86_featureset,
2405 X86FSET_AVX512VL);
2406 remove_x86_feature(x86_featureset,
2407 X86FSET_AVX512FMA);
2408 remove_x86_feature(x86_featureset,
2409 X86FSET_AVX512VBMI);
2410 remove_x86_feature(x86_featureset,
2411 X86FSET_AVX512VPOPCDQ);
2412 remove_x86_feature(x86_featureset,
2413 X86FSET_AVX512NNIW);
2414 remove_x86_feature(x86_featureset,
2415 X86FSET_AVX512FMAPS);
2416
2417 CPI_FEATURES_ECX(cpi) &=
2418 ~CPUID_INTC_ECX_XSAVE;
2419 CPI_FEATURES_ECX(cpi) &=
2420 ~CPUID_INTC_ECX_AVX;
2421 CPI_FEATURES_ECX(cpi) &=
2422 ~CPUID_INTC_ECX_F16C;
2423 CPI_FEATURES_ECX(cpi) &=
2424 ~CPUID_INTC_ECX_FMA;
2425 CPI_FEATURES_7_0_EBX(cpi) &=
2426 ~CPUID_INTC_EBX_7_0_BMI1;
2427 CPI_FEATURES_7_0_EBX(cpi) &=
2428 ~CPUID_INTC_EBX_7_0_BMI2;
2429 CPI_FEATURES_7_0_EBX(cpi) &=
2430 ~CPUID_INTC_EBX_7_0_AVX2;
2431 CPI_FEATURES_7_0_EBX(cpi) &=
2432 ~CPUID_INTC_EBX_7_0_MPX;
2433 CPI_FEATURES_7_0_EBX(cpi) &=
2434 ~CPUID_INTC_EBX_7_0_ALL_AVX512;
2435
2436 CPI_FEATURES_7_0_ECX(cpi) &=
2437 ~CPUID_INTC_ECX_7_0_ALL_AVX512;
2438
2439 CPI_FEATURES_7_0_EDX(cpi) &=
2440 ~CPUID_INTC_EDX_7_0_ALL_AVX512;
2441
2442 xsave_force_disable = B_TRUE;
2443 } else {
2444 VERIFY(is_x86_feature(x86_featureset,
2445 X86FSET_XSAVE) == B_FALSE);
2446 }
2447 }
2448 }
2449 }
2450
2451
2452 if ((cpi->cpi_xmaxeax & 0x80000000) == 0)
2453 goto pass2_done;
2454
2455 if ((nmax = cpi->cpi_xmaxeax - 0x80000000 + 1) > NMAX_CPI_EXTD)
2456 nmax = NMAX_CPI_EXTD;
2457 /*
2458 * Copy the extended properties, fixing them as we go.
2459 * (We already handled n == 0 and n == 1 in pass 1)
2460 */
2461 iptr = (void *)cpi->cpi_brandstr;
2462 for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
2463 cp->cp_eax = 0x80000000 + n;
2464 (void) __cpuid_insn(cp);
2465 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000000 + n, cp);
2466 switch (n) {
2467 case 2:
2468 case 3:
2469 case 4:
2470 /*
2471 * Extract the brand string
2472 */
2473 *iptr++ = cp->cp_eax;
2474 *iptr++ = cp->cp_ebx;
2475 *iptr++ = cp->cp_ecx;
2476 *iptr++ = cp->cp_edx;
2477 break;
2478 case 5:
2479 switch (cpi->cpi_vendor) {
2480 case X86_VENDOR_AMD:
2481 /*
2482 * The Athlon and Duron were the first
2483 * parts to report the sizes of the
2484 * TLB for large pages. Before then,
2485 * we don't trust the data.
2486 */
2487 if (cpi->cpi_family < 6 ||
2488 (cpi->cpi_family == 6 &&
2489 cpi->cpi_model < 1))
2490 cp->cp_eax = 0;
2491 break;
2492 default:
2493 break;
2494 }
2495 break;
2496 case 6:
2497 switch (cpi->cpi_vendor) {
2498 case X86_VENDOR_AMD:
2499 /*
2500 * The Athlon and Duron were the first
2501 * AMD parts with L2 TLB's.
2502 * Before then, don't trust the data.
2503 */
2504 if (cpi->cpi_family < 6 ||
2505 cpi->cpi_family == 6 &&
2506 cpi->cpi_model < 1)
2507 cp->cp_eax = cp->cp_ebx = 0;
2508 /*
2509 * AMD Duron rev A0 reports L2
2510 * cache size incorrectly as 1K
2511 * when it is really 64K
2512 */
2513 if (cpi->cpi_family == 6 &&
2514 cpi->cpi_model == 3 &&
2515 cpi->cpi_step == 0) {
2516 cp->cp_ecx &= 0xffff;
2517 cp->cp_ecx |= 0x400000;
2518 }
2519 break;
2520 case X86_VENDOR_Cyrix: /* VIA C3 */
2521 /*
2522 * VIA C3 processors are a bit messed
2523 * up w.r.t. encoding cache sizes in %ecx
2524 */
2525 if (cpi->cpi_family != 6)
2526 break;
2527 /*
2528 * model 7 and 8 were incorrectly encoded
2529 *
2530 * xxx is model 8 really broken?
2531 */
2532 if (cpi->cpi_model == 7 ||
2533 cpi->cpi_model == 8)
2534 cp->cp_ecx =
2535 BITX(cp->cp_ecx, 31, 24) << 16 |
2536 BITX(cp->cp_ecx, 23, 16) << 12 |
2537 BITX(cp->cp_ecx, 15, 8) << 8 |
2538 BITX(cp->cp_ecx, 7, 0);
2539 /*
2540 * model 9 stepping 1 has wrong associativity
2541 */
2542 if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
2543 cp->cp_ecx |= 8 << 12;
2544 break;
2545 case X86_VENDOR_Intel:
2546 /*
2547 * Extended L2 Cache features function.
2548 * First appeared on Prescott.
2549 */
2550 default:
2551 break;
2552 }
2553 break;
2554 default:
2555 break;
2556 }
2557 }
2558
2559 pass2_done:
2560 cpi->cpi_pass = 2;
2561 }
2562
2563 static const char *
2564 intel_cpubrand(const struct cpuid_info *cpi)
2565 {
2566 int i;
2567
2568 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2569 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2570 return ("i486");
2571
2572 switch (cpi->cpi_family) {
2573 case 5:
2574 return ("Intel Pentium(r)");
2575 case 6:
2576 switch (cpi->cpi_model) {
2577 uint_t celeron, xeon;
2578 const struct cpuid_regs *cp;
2579 case 0:
2580 case 1:
2581 case 2:
2582 return ("Intel Pentium(r) Pro");
2583 case 3:
2584 case 4:
2585 return ("Intel Pentium(r) II");
2586 case 6:
2587 return ("Intel Celeron(r)");
2588 case 5:
2589 case 7:
2590 celeron = xeon = 0;
2591 cp = &cpi->cpi_std[2]; /* cache info */
2592
2593 for (i = 1; i < 4; i++) {
2594 uint_t tmp;
2595
2596 tmp = (cp->cp_eax >> (8 * i)) & 0xff;
2597 if (tmp == 0x40)
2598 celeron++;
2599 if (tmp >= 0x44 && tmp <= 0x45)
2600 xeon++;
2601 }
2602
2603 for (i = 0; i < 2; i++) {
2604 uint_t tmp;
2605
2606 tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
2607 if (tmp == 0x40)
2608 celeron++;
2609 else if (tmp >= 0x44 && tmp <= 0x45)
2610 xeon++;
2611 }
2612
2613 for (i = 0; i < 4; i++) {
2614 uint_t tmp;
2615
2616 tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
2617 if (tmp == 0x40)
2618 celeron++;
2619 else if (tmp >= 0x44 && tmp <= 0x45)
2620 xeon++;
2621 }
2622
2623 for (i = 0; i < 4; i++) {
2624 uint_t tmp;
2625
2626 tmp = (cp->cp_edx >> (8 * i)) & 0xff;
2627 if (tmp == 0x40)
2628 celeron++;
2629 else if (tmp >= 0x44 && tmp <= 0x45)
2630 xeon++;
2631 }
2632
2633 if (celeron)
2634 return ("Intel Celeron(r)");
2635 if (xeon)
2636 return (cpi->cpi_model == 5 ?
2637 "Intel Pentium(r) II Xeon(tm)" :
2638 "Intel Pentium(r) III Xeon(tm)");
2639 return (cpi->cpi_model == 5 ?
2640 "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
2641 "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
2642 default:
2643 break;
2644 }
2645 default:
2646 break;
2647 }
2648
2649 /* BrandID is present if the field is nonzero */
2650 if (cpi->cpi_brandid != 0) {
2651 static const struct {
2652 uint_t bt_bid;
2653 const char *bt_str;
2654 } brand_tbl[] = {
2655 { 0x1, "Intel(r) Celeron(r)" },
2656 { 0x2, "Intel(r) Pentium(r) III" },
2657 { 0x3, "Intel(r) Pentium(r) III Xeon(tm)" },
2658 { 0x4, "Intel(r) Pentium(r) III" },
2659 { 0x6, "Mobile Intel(r) Pentium(r) III" },
2660 { 0x7, "Mobile Intel(r) Celeron(r)" },
2661 { 0x8, "Intel(r) Pentium(r) 4" },
2662 { 0x9, "Intel(r) Pentium(r) 4" },
2663 { 0xa, "Intel(r) Celeron(r)" },
2664 { 0xb, "Intel(r) Xeon(tm)" },
2665 { 0xc, "Intel(r) Xeon(tm) MP" },
2666 { 0xe, "Mobile Intel(r) Pentium(r) 4" },
2667 { 0xf, "Mobile Intel(r) Celeron(r)" },
2668 { 0x11, "Mobile Genuine Intel(r)" },
2669 { 0x12, "Intel(r) Celeron(r) M" },
2670 { 0x13, "Mobile Intel(r) Celeron(r)" },
2671 { 0x14, "Intel(r) Celeron(r)" },
2672 { 0x15, "Mobile Genuine Intel(r)" },
2673 { 0x16, "Intel(r) Pentium(r) M" },
2674 { 0x17, "Mobile Intel(r) Celeron(r)" }
2675 };
2676 uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
2677 uint_t sgn;
2678
2679 sgn = (cpi->cpi_family << 8) |
2680 (cpi->cpi_model << 4) | cpi->cpi_step;
2681
2682 for (i = 0; i < btblmax; i++)
2683 if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
2684 break;
2685 if (i < btblmax) {
2686 if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
2687 return ("Intel(r) Celeron(r)");
2688 if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
2689 return ("Intel(r) Xeon(tm) MP");
2690 if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
2691 return ("Intel(r) Xeon(tm)");
2692 return (brand_tbl[i].bt_str);
2693 }
2694 }
2695
2696 return (NULL);
2697 }
2698
2699 static const char *
2700 amd_cpubrand(const struct cpuid_info *cpi)
2701 {
2702 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2703 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2704 return ("i486 compatible");
2705
2706 switch (cpi->cpi_family) {
2707 case 5:
2708 switch (cpi->cpi_model) {
2709 case 0:
2710 case 1:
2711 case 2:
2712 case 3:
2713 case 4:
2714 case 5:
2715 return ("AMD-K5(r)");
2716 case 6:
2717 case 7:
2718 return ("AMD-K6(r)");
2719 case 8:
2720 return ("AMD-K6(r)-2");
2721 case 9:
2722 return ("AMD-K6(r)-III");
2723 default:
2724 return ("AMD (family 5)");
2725 }
2726 case 6:
2727 switch (cpi->cpi_model) {
2728 case 1:
2729 return ("AMD-K7(tm)");
2730 case 0:
2731 case 2:
2732 case 4:
2733 return ("AMD Athlon(tm)");
2734 case 3:
2735 case 7:
2736 return ("AMD Duron(tm)");
2737 case 6:
2738 case 8:
2739 case 10:
2740 /*
2741 * Use the L2 cache size to distinguish
2742 */
2743 return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
2744 "AMD Athlon(tm)" : "AMD Duron(tm)");
2745 default:
2746 return ("AMD (family 6)");
2747 }
2748 default:
2749 break;
2750 }
2751
2752 if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
2753 cpi->cpi_brandid != 0) {
2754 switch (BITX(cpi->cpi_brandid, 7, 5)) {
2755 case 3:
2756 return ("AMD Opteron(tm) UP 1xx");
2757 case 4:
2758 return ("AMD Opteron(tm) DP 2xx");
2759 case 5:
2760 return ("AMD Opteron(tm) MP 8xx");
2761 default:
2762 return ("AMD Opteron(tm)");
2763 }
2764 }
2765
2766 return (NULL);
2767 }
2768
2769 static const char *
2770 cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
2771 {
2772 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2773 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5 ||
2774 type == X86_TYPE_CYRIX_486)
2775 return ("i486 compatible");
2776
2777 switch (type) {
2778 case X86_TYPE_CYRIX_6x86:
2779 return ("Cyrix 6x86");
2780 case X86_TYPE_CYRIX_6x86L:
2781 return ("Cyrix 6x86L");
2782 case X86_TYPE_CYRIX_6x86MX:
2783 return ("Cyrix 6x86MX");
2784 case X86_TYPE_CYRIX_GXm:
2785 return ("Cyrix GXm");
2786 case X86_TYPE_CYRIX_MediaGX:
2787 return ("Cyrix MediaGX");
2788 case X86_TYPE_CYRIX_MII:
2789 return ("Cyrix M2");
2790 case X86_TYPE_VIA_CYRIX_III:
2791 return ("VIA Cyrix M3");
2792 default:
2793 /*
2794 * Have another wild guess ..
2795 */
2796 if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
2797 return ("Cyrix 5x86");
2798 else if (cpi->cpi_family == 5) {
2799 switch (cpi->cpi_model) {
2800 case 2:
2801 return ("Cyrix 6x86"); /* Cyrix M1 */
2802 case 4:
2803 return ("Cyrix MediaGX");
2804 default:
2805 break;
2806 }
2807 } else if (cpi->cpi_family == 6) {
2808 switch (cpi->cpi_model) {
2809 case 0:
2810 return ("Cyrix 6x86MX"); /* Cyrix M2? */
2811 case 5:
2812 case 6:
2813 case 7:
2814 case 8:
2815 case 9:
2816 return ("VIA C3");
2817 default:
2818 break;
2819 }
2820 }
2821 break;
2822 }
2823 return (NULL);
2824 }
2825
2826 /*
2827 * This only gets called in the case that the CPU extended
2828 * feature brand string (0x80000002, 0x80000003, 0x80000004)
2829 * aren't available, or contain null bytes for some reason.
2830 */
2831 static void
2832 fabricate_brandstr(struct cpuid_info *cpi)
2833 {
2834 const char *brand = NULL;
2835
2836 switch (cpi->cpi_vendor) {
2837 case X86_VENDOR_Intel:
2838 brand = intel_cpubrand(cpi);
2839 break;
2840 case X86_VENDOR_AMD:
2841 brand = amd_cpubrand(cpi);
2842 break;
2843 case X86_VENDOR_Cyrix:
2844 brand = cyrix_cpubrand(cpi, x86_type);
2845 break;
2846 case X86_VENDOR_NexGen:
2847 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2848 brand = "NexGen Nx586";
2849 break;
2850 case X86_VENDOR_Centaur:
2851 if (cpi->cpi_family == 5)
2852 switch (cpi->cpi_model) {
2853 case 4:
2854 brand = "Centaur C6";
2855 break;
2856 case 8:
2857 brand = "Centaur C2";
2858 break;
2859 case 9:
2860 brand = "Centaur C3";
2861 break;
2862 default:
2863 break;
2864 }
2865 break;
2866 case X86_VENDOR_Rise:
2867 if (cpi->cpi_family == 5 &&
2868 (cpi->cpi_model == 0 || cpi->cpi_model == 2))
2869 brand = "Rise mP6";
2870 break;
2871 case X86_VENDOR_SiS:
2872 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2873 brand = "SiS 55x";
2874 break;
2875 case X86_VENDOR_TM:
2876 if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
2877 brand = "Transmeta Crusoe TM3x00 or TM5x00";
2878 break;
2879 case X86_VENDOR_NSC:
2880 case X86_VENDOR_UMC:
2881 default:
2882 break;
2883 }
2884 if (brand) {
2885 (void) strcpy((char *)cpi->cpi_brandstr, brand);
2886 return;
2887 }
2888
2889 /*
2890 * If all else fails ...
2891 */
2892 (void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
2893 "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
2894 cpi->cpi_model, cpi->cpi_step);
2895 }
2896
2897 /*
2898 * This routine is called just after kernel memory allocation
2899 * becomes available on cpu0, and as part of mp_startup() on
2900 * the other cpus.
2901 *
2902 * Fixup the brand string, and collect any information from cpuid
2903 * that requires dynamically allocated storage to represent.
2904 */
2905 /*ARGSUSED*/
2906 void
2907 cpuid_pass3(cpu_t *cpu)
2908 {
2909 int i, max, shft, level, size;
2910 struct cpuid_regs regs;
2911 struct cpuid_regs *cp;
2912 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2913
2914 ASSERT(cpi->cpi_pass == 2);
2915
2916 /*
2917 * Function 4: Deterministic cache parameters
2918 *
2919 * Take this opportunity to detect the number of threads
2920 * sharing the last level cache, and construct a corresponding
2921 * cache id. The respective cpuid_info members are initialized
2922 * to the default case of "no last level cache sharing".
2923 */
2924 cpi->cpi_ncpu_shr_last_cache = 1;
2925 cpi->cpi_last_lvl_cacheid = cpu->cpu_id;
2926
2927 if (cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) {
2928
2929 /*
2930 * Find the # of elements (size) returned by fn 4, and along
2931 * the way detect last level cache sharing details.
2932 */
2933 bzero(®s, sizeof (regs));
2934 cp = ®s;
2935 for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
2936 cp->cp_eax = 4;
2937 cp->cp_ecx = i;
2938
2939 (void) __cpuid_insn(cp);
2940
2941 if (CPI_CACHE_TYPE(cp) == 0)
2942 break;
2943 level = CPI_CACHE_LVL(cp);
2944 if (level > max) {
2945 max = level;
2946 cpi->cpi_ncpu_shr_last_cache =
2947 CPI_NTHR_SHR_CACHE(cp) + 1;
2948 }
2949 }
2950 cpi->cpi_std_4_size = size = i;
2951
2952 /*
2953 * Allocate the cpi_std_4 array. The first element
2954 * references the regs for fn 4, %ecx == 0, which
2955 * cpuid_pass2() stashed in cpi->cpi_std[4].
2956 */
2957 if (size > 0) {
2958 cpi->cpi_std_4 =
2959 kmem_alloc(size * sizeof (cp), KM_SLEEP);
2960 cpi->cpi_std_4[0] = &cpi->cpi_std[4];
2961
2962 /*
2963 * Allocate storage to hold the additional regs
2964 * for function 4, %ecx == 1 .. cpi_std_4_size.
2965 *
2966 * The regs for fn 4, %ecx == 0 has already
2967 * been allocated as indicated above.
2968 */
2969 for (i = 1; i < size; i++) {
2970 cp = cpi->cpi_std_4[i] =
2971 kmem_zalloc(sizeof (regs), KM_SLEEP);
2972 cp->cp_eax = 4;
2973 cp->cp_ecx = i;
2974
2975 (void) __cpuid_insn(cp);
2976 }
2977 }
2978 /*
2979 * Determine the number of bits needed to represent
2980 * the number of CPUs sharing the last level cache.
2981 *
2982 * Shift off that number of bits from the APIC id to
2983 * derive the cache id.
2984 */
2985 shft = 0;
2986 for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
2987 shft++;
2988 cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
2989 }
2990
2991 /*
2992 * Now fixup the brand string
2993 */
2994 if ((cpi->cpi_xmaxeax & 0x80000000) == 0) {
2995 fabricate_brandstr(cpi);
2996 } else {
2997
2998 /*
2999 * If we successfully extracted a brand string from the cpuid
3000 * instruction, clean it up by removing leading spaces and
3001 * similar junk.
3002 */
3003 if (cpi->cpi_brandstr[0]) {
3004 size_t maxlen = sizeof (cpi->cpi_brandstr);
3005 char *src, *dst;
3006
3007 dst = src = (char *)cpi->cpi_brandstr;
3008 src[maxlen - 1] = '\0';
3009 /*
3010 * strip leading spaces
3011 */
3012 while (*src == ' ')
3013 src++;
3014 /*
3015 * Remove any 'Genuine' or "Authentic" prefixes
3016 */
3017 if (strncmp(src, "Genuine ", 8) == 0)
3018 src += 8;
3019 if (strncmp(src, "Authentic ", 10) == 0)
3020 src += 10;
3021
3022 /*
3023 * Now do an in-place copy.
3024 * Map (R) to (r) and (TM) to (tm).
3025 * The era of teletypes is long gone, and there's
3026 * -really- no need to shout.
3027 */
3028 while (*src != '\0') {
3029 if (src[0] == '(') {
3030 if (strncmp(src + 1, "R)", 2) == 0) {
3031 (void) strncpy(dst, "(r)", 3);
3032 src += 3;
3033 dst += 3;
3034 continue;
3035 }
3036 if (strncmp(src + 1, "TM)", 3) == 0) {
3037 (void) strncpy(dst, "(tm)", 4);
3038 src += 4;
3039 dst += 4;
3040 continue;
3041 }
3042 }
3043 *dst++ = *src++;
3044 }
3045 *dst = '\0';
3046
3047 /*
3048 * Finally, remove any trailing spaces
3049 */
3050 while (--dst > cpi->cpi_brandstr)
3051 if (*dst == ' ')
3052 *dst = '\0';
3053 else
3054 break;
3055 } else
3056 fabricate_brandstr(cpi);
3057 }
3058 cpi->cpi_pass = 3;
3059 }
3060
3061 /*
3062 * This routine is called out of bind_hwcap() much later in the life
3063 * of the kernel (post_startup()). The job of this routine is to resolve
3064 * the hardware feature support and kernel support for those features into
3065 * what we're actually going to tell applications via the aux vector.
3066 */
3067 void
3068 cpuid_pass4(cpu_t *cpu, uint_t *hwcap_out)
3069 {
3070 struct cpuid_info *cpi;
3071 uint_t hwcap_flags = 0, hwcap_flags_2 = 0;
3072
3073 if (cpu == NULL)
3074 cpu = CPU;
3075 cpi = cpu->cpu_m.mcpu_cpi;
3076
3077 ASSERT(cpi->cpi_pass == 3);
3078
3079 if (cpi->cpi_maxeax >= 1) {
3080 uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
3081 uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];
3082 uint32_t *ebx = &cpi->cpi_support[STD_EBX_FEATURES];
3083
3084 *edx = CPI_FEATURES_EDX(cpi);
3085 *ecx = CPI_FEATURES_ECX(cpi);
3086 *ebx = CPI_FEATURES_7_0_EBX(cpi);
3087
3088 /*
3089 * [these require explicit kernel support]
3090 */
3091 if (!is_x86_feature(x86_featureset, X86FSET_SEP))
3092 *edx &= ~CPUID_INTC_EDX_SEP;
3093
3094 if (!is_x86_feature(x86_featureset, X86FSET_SSE))
3095 *edx &= ~(CPUID_INTC_EDX_FXSR|CPUID_INTC_EDX_SSE);
3096 if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
3097 *edx &= ~CPUID_INTC_EDX_SSE2;
3098
3099 if (!is_x86_feature(x86_featureset, X86FSET_HTT))
3100 *edx &= ~CPUID_INTC_EDX_HTT;
3101
3102 if (!is_x86_feature(x86_featureset, X86FSET_SSE3))
3103 *ecx &= ~CPUID_INTC_ECX_SSE3;
3104
3105 if (!is_x86_feature(x86_featureset, X86FSET_SSSE3))
3106 *ecx &= ~CPUID_INTC_ECX_SSSE3;
3107 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_1))
3108 *ecx &= ~CPUID_INTC_ECX_SSE4_1;
3109 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2))
3110 *ecx &= ~CPUID_INTC_ECX_SSE4_2;
3111 if (!is_x86_feature(x86_featureset, X86FSET_AES))
3112 *ecx &= ~CPUID_INTC_ECX_AES;
3113 if (!is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ))
3114 *ecx &= ~CPUID_INTC_ECX_PCLMULQDQ;
3115 if (!is_x86_feature(x86_featureset, X86FSET_XSAVE))
3116 *ecx &= ~(CPUID_INTC_ECX_XSAVE |
3117 CPUID_INTC_ECX_OSXSAVE);
3118 if (!is_x86_feature(x86_featureset, X86FSET_AVX))
3119 *ecx &= ~CPUID_INTC_ECX_AVX;
3120 if (!is_x86_feature(x86_featureset, X86FSET_F16C))
3121 *ecx &= ~CPUID_INTC_ECX_F16C;
3122 if (!is_x86_feature(x86_featureset, X86FSET_FMA))
3123 *ecx &= ~CPUID_INTC_ECX_FMA;
3124 if (!is_x86_feature(x86_featureset, X86FSET_BMI1))
3125 *ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
3126 if (!is_x86_feature(x86_featureset, X86FSET_BMI2))
3127 *ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
3128 if (!is_x86_feature(x86_featureset, X86FSET_AVX2))
3129 *ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
3130 if (!is_x86_feature(x86_featureset, X86FSET_RDSEED))
3131 *ebx &= ~CPUID_INTC_EBX_7_0_RDSEED;
3132 if (!is_x86_feature(x86_featureset, X86FSET_ADX))
3133 *ebx &= ~CPUID_INTC_EBX_7_0_ADX;
3134
3135 /*
3136 * [no explicit support required beyond x87 fp context]
3137 */
3138 if (!fpu_exists)
3139 *edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);
3140
3141 /*
3142 * Now map the supported feature vector to things that we
3143 * think userland will care about.
3144 */
3145 if (*edx & CPUID_INTC_EDX_SEP)
3146 hwcap_flags |= AV_386_SEP;
3147 if (*edx & CPUID_INTC_EDX_SSE)
3148 hwcap_flags |= AV_386_FXSR | AV_386_SSE;
3149 if (*edx & CPUID_INTC_EDX_SSE2)
3150 hwcap_flags |= AV_386_SSE2;
3151 if (*ecx & CPUID_INTC_ECX_SSE3)
3152 hwcap_flags |= AV_386_SSE3;
3153 if (*ecx & CPUID_INTC_ECX_SSSE3)
3154 hwcap_flags |= AV_386_SSSE3;
3155 if (*ecx & CPUID_INTC_ECX_SSE4_1)
3156 hwcap_flags |= AV_386_SSE4_1;
3157 if (*ecx & CPUID_INTC_ECX_SSE4_2)
3158 hwcap_flags |= AV_386_SSE4_2;
3159 if (*ecx & CPUID_INTC_ECX_MOVBE)
3160 hwcap_flags |= AV_386_MOVBE;
3161 if (*ecx & CPUID_INTC_ECX_AES)
3162 hwcap_flags |= AV_386_AES;
3163 if (*ecx & CPUID_INTC_ECX_PCLMULQDQ)
3164 hwcap_flags |= AV_386_PCLMULQDQ;
3165 if ((*ecx & CPUID_INTC_ECX_XSAVE) &&
3166 (*ecx & CPUID_INTC_ECX_OSXSAVE)) {
3167 hwcap_flags |= AV_386_XSAVE;
3168
3169 if (*ecx & CPUID_INTC_ECX_AVX) {
3170 uint32_t *ecx_7 = &CPI_FEATURES_7_0_ECX(cpi);
3171 uint32_t *edx_7 = &CPI_FEATURES_7_0_EDX(cpi);
3172
3173 hwcap_flags |= AV_386_AVX;
3174 if (*ecx & CPUID_INTC_ECX_F16C)
3175 hwcap_flags_2 |= AV_386_2_F16C;
3176 if (*ecx & CPUID_INTC_ECX_FMA)
3177 hwcap_flags_2 |= AV_386_2_FMA;
3178
3179 if (*ebx & CPUID_INTC_EBX_7_0_BMI1)
3180 hwcap_flags_2 |= AV_386_2_BMI1;
3181 if (*ebx & CPUID_INTC_EBX_7_0_BMI2)
3182 hwcap_flags_2 |= AV_386_2_BMI2;
3183 if (*ebx & CPUID_INTC_EBX_7_0_AVX2)
3184 hwcap_flags_2 |= AV_386_2_AVX2;
3185 if (*ebx & CPUID_INTC_EBX_7_0_AVX512F)
3186 hwcap_flags_2 |= AV_386_2_AVX512F;
3187 if (*ebx & CPUID_INTC_EBX_7_0_AVX512DQ)
3188 hwcap_flags_2 |= AV_386_2_AVX512DQ;
3189 if (*ebx & CPUID_INTC_EBX_7_0_AVX512IFMA)
3190 hwcap_flags_2 |= AV_386_2_AVX512IFMA;
3191 if (*ebx & CPUID_INTC_EBX_7_0_AVX512PF)
3192 hwcap_flags_2 |= AV_386_2_AVX512PF;
3193 if (*ebx & CPUID_INTC_EBX_7_0_AVX512ER)
3194 hwcap_flags_2 |= AV_386_2_AVX512ER;
3195 if (*ebx & CPUID_INTC_EBX_7_0_AVX512CD)
3196 hwcap_flags_2 |= AV_386_2_AVX512CD;
3197 if (*ebx & CPUID_INTC_EBX_7_0_AVX512BW)
3198 hwcap_flags_2 |= AV_386_2_AVX512BW;
3199 if (*ebx & CPUID_INTC_EBX_7_0_AVX512VL)
3200 hwcap_flags_2 |= AV_386_2_AVX512VL;
3201
3202 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VBMI)
3203 hwcap_flags_2 |= AV_386_2_AVX512VBMI;
3204 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
3205 hwcap_flags_2 |= AV_386_2_AVX512VPOPCDQ;
3206
3207 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124NNIW)
3208 hwcap_flags_2 |= AV_386_2_AVX512_4NNIW;
3209 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124FMAPS)
3210 hwcap_flags_2 |= AV_386_2_AVX512_4FMAPS;
3211 }
3212 }
3213 if (*ecx & CPUID_INTC_ECX_VMX)
3214 hwcap_flags |= AV_386_VMX;
3215 if (*ecx & CPUID_INTC_ECX_POPCNT)
3216 hwcap_flags |= AV_386_POPCNT;
3217 if (*edx & CPUID_INTC_EDX_FPU)
3218 hwcap_flags |= AV_386_FPU;
3219 if (*edx & CPUID_INTC_EDX_MMX)
3220 hwcap_flags |= AV_386_MMX;
3221
3222 if (*edx & CPUID_INTC_EDX_TSC)
3223 hwcap_flags |= AV_386_TSC;
3224 if (*edx & CPUID_INTC_EDX_CX8)
3225 hwcap_flags |= AV_386_CX8;
3226 if (*edx & CPUID_INTC_EDX_CMOV)
3227 hwcap_flags |= AV_386_CMOV;
3228 if (*ecx & CPUID_INTC_ECX_CX16)
3229 hwcap_flags |= AV_386_CX16;
3230
3231 if (*ecx & CPUID_INTC_ECX_RDRAND)
3232 hwcap_flags_2 |= AV_386_2_RDRAND;
3233 if (*ebx & CPUID_INTC_EBX_7_0_ADX)
3234 hwcap_flags_2 |= AV_386_2_ADX;
3235 if (*ebx & CPUID_INTC_EBX_7_0_RDSEED)
3236 hwcap_flags_2 |= AV_386_2_RDSEED;
3237 if (*ebx & CPUID_INTC_EBX_7_0_SHA)
3238 hwcap_flags_2 |= AV_386_2_SHA;
3239
3240 }
3241
3242 if (cpi->cpi_xmaxeax < 0x80000001)
3243 goto pass4_done;
3244
3245 switch (cpi->cpi_vendor) {
3246 struct cpuid_regs cp;
3247 uint32_t *edx, *ecx;
3248
3249 case X86_VENDOR_Intel:
3250 /*
3251 * Seems like Intel duplicated what we necessary
3252 * here to make the initial crop of 64-bit OS's work.
3253 * Hopefully, those are the only "extended" bits
3254 * they'll add.
3255 */
3256 /*FALLTHROUGH*/
3257
3258 case X86_VENDOR_AMD:
3259 edx = &cpi->cpi_support[AMD_EDX_FEATURES];
3260 ecx = &cpi->cpi_support[AMD_ECX_FEATURES];
3261
3262 *edx = CPI_FEATURES_XTD_EDX(cpi);
3263 *ecx = CPI_FEATURES_XTD_ECX(cpi);
3264
3265 /*
3266 * [these features require explicit kernel support]
3267 */
3268 switch (cpi->cpi_vendor) {
3269 case X86_VENDOR_Intel:
3270 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3271 *edx &= ~CPUID_AMD_EDX_TSCP;
3272 break;
3273
3274 case X86_VENDOR_AMD:
3275 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3276 *edx &= ~CPUID_AMD_EDX_TSCP;
3277 if (!is_x86_feature(x86_featureset, X86FSET_SSE4A))
3278 *ecx &= ~CPUID_AMD_ECX_SSE4A;
3279 break;
3280
3281 default:
3282 break;
3283 }
3284
3285 /*
3286 * [no explicit support required beyond
3287 * x87 fp context and exception handlers]
3288 */
3289 if (!fpu_exists)
3290 *edx &= ~(CPUID_AMD_EDX_MMXamd |
3291 CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);
3292
3293 if (!is_x86_feature(x86_featureset, X86FSET_NX))
3294 *edx &= ~CPUID_AMD_EDX_NX;
3295 #if !defined(__amd64)
3296 *edx &= ~CPUID_AMD_EDX_LM;
3297 #endif
3298 /*
3299 * Now map the supported feature vector to
3300 * things that we think userland will care about.
3301 */
3302 #if defined(__amd64)
3303 if (*edx & CPUID_AMD_EDX_SYSC)
3304 hwcap_flags |= AV_386_AMD_SYSC;
3305 #endif
3306 if (*edx & CPUID_AMD_EDX_MMXamd)
3307 hwcap_flags |= AV_386_AMD_MMX;
3308 if (*edx & CPUID_AMD_EDX_3DNow)
3309 hwcap_flags |= AV_386_AMD_3DNow;
3310 if (*edx & CPUID_AMD_EDX_3DNowx)
3311 hwcap_flags |= AV_386_AMD_3DNowx;
3312 if (*ecx & CPUID_AMD_ECX_SVM)
3313 hwcap_flags |= AV_386_AMD_SVM;
3314
3315 switch (cpi->cpi_vendor) {
3316 case X86_VENDOR_AMD:
3317 if (*edx & CPUID_AMD_EDX_TSCP)
3318 hwcap_flags |= AV_386_TSCP;
3319 if (*ecx & CPUID_AMD_ECX_AHF64)
3320 hwcap_flags |= AV_386_AHF;
3321 if (*ecx & CPUID_AMD_ECX_SSE4A)
3322 hwcap_flags |= AV_386_AMD_SSE4A;
3323 if (*ecx & CPUID_AMD_ECX_LZCNT)
3324 hwcap_flags |= AV_386_AMD_LZCNT;
3325 break;
3326
3327 case X86_VENDOR_Intel:
3328 if (*edx & CPUID_AMD_EDX_TSCP)
3329 hwcap_flags |= AV_386_TSCP;
3330 /*
3331 * Aarrgh.
3332 * Intel uses a different bit in the same word.
3333 */
3334 if (*ecx & CPUID_INTC_ECX_AHF64)
3335 hwcap_flags |= AV_386_AHF;
3336 break;
3337
3338 default:
3339 break;
3340 }
3341 break;
3342
3343 case X86_VENDOR_TM:
3344 cp.cp_eax = 0x80860001;
3345 (void) __cpuid_insn(&cp);
3346 cpi->cpi_support[TM_EDX_FEATURES] = cp.cp_edx;
3347 break;
3348
3349 default:
3350 break;
3351 }
3352
3353 pass4_done:
3354 cpi->cpi_pass = 4;
3355 if (hwcap_out != NULL) {
3356 hwcap_out[0] = hwcap_flags;
3357 hwcap_out[1] = hwcap_flags_2;
3358 }
3359 }
3360
3361
3362 /*
3363 * Simulate the cpuid instruction using the data we previously
3364 * captured about this CPU. We try our best to return the truth
3365 * about the hardware, independently of kernel support.
3366 */
3367 uint32_t
3368 cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
3369 {
3370 struct cpuid_info *cpi;
3371 struct cpuid_regs *xcp;
3372
3373 if (cpu == NULL)
3374 cpu = CPU;
3375 cpi = cpu->cpu_m.mcpu_cpi;
3376
3377 ASSERT(cpuid_checkpass(cpu, 3));
3378
3379 /*
3380 * CPUID data is cached in two separate places: cpi_std for standard
3381 * CPUID functions, and cpi_extd for extended CPUID functions.
3382 */
3383 if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD)
3384 xcp = &cpi->cpi_std[cp->cp_eax];
3385 else if (cp->cp_eax >= 0x80000000 && cp->cp_eax <= cpi->cpi_xmaxeax &&
3386 cp->cp_eax < 0x80000000 + NMAX_CPI_EXTD)
3387 xcp = &cpi->cpi_extd[cp->cp_eax - 0x80000000];
3388 else
3389 /*
3390 * The caller is asking for data from an input parameter which
3391 * the kernel has not cached. In this case we go fetch from
3392 * the hardware and return the data directly to the user.
3393 */
3394 return (__cpuid_insn(cp));
3395
3396 cp->cp_eax = xcp->cp_eax;
3397 cp->cp_ebx = xcp->cp_ebx;
3398 cp->cp_ecx = xcp->cp_ecx;
3399 cp->cp_edx = xcp->cp_edx;
3400 return (cp->cp_eax);
3401 }
3402
3403 int
3404 cpuid_checkpass(cpu_t *cpu, int pass)
3405 {
3406 return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
3407 cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
3408 }
3409
3410 int
3411 cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
3412 {
3413 ASSERT(cpuid_checkpass(cpu, 3));
3414
3415 return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
3416 }
3417
3418 int
3419 cpuid_is_cmt(cpu_t *cpu)
3420 {
3421 if (cpu == NULL)
3422 cpu = CPU;
3423
3424 ASSERT(cpuid_checkpass(cpu, 1));
3425
3426 return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
3427 }
3428
3429 /*
3430 * AMD and Intel both implement the 64-bit variant of the syscall
3431 * instruction (syscallq), so if there's -any- support for syscall,
3432 * cpuid currently says "yes, we support this".
3433 *
3434 * However, Intel decided to -not- implement the 32-bit variant of the
3435 * syscall instruction, so we provide a predicate to allow our caller
3436 * to test that subtlety here.
3437 *
3438 * XXPV Currently, 32-bit syscall instructions don't work via the hypervisor,
3439 * even in the case where the hardware would in fact support it.
3440 */
3441 /*ARGSUSED*/
3442 int
3443 cpuid_syscall32_insn(cpu_t *cpu)
3444 {
3445 ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), 1));
3446
3447 #if !defined(__xpv)
3448 if (cpu == NULL)
3449 cpu = CPU;
3450
3451 /*CSTYLED*/
3452 {
3453 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3454
3455 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
3456 cpi->cpi_xmaxeax >= 0x80000001 &&
3457 (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
3458 return (1);
3459 }
3460 #endif
3461 return (0);
3462 }
3463
3464 int
3465 cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
3466 {
3467 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3468
3469 static const char fmt[] =
3470 "x86 (%s %X family %d model %d step %d clock %d MHz)";
3471 static const char fmt_ht[] =
3472 "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";
3473
3474 ASSERT(cpuid_checkpass(cpu, 1));
3475
3476 if (cpuid_is_cmt(cpu))
3477 return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
3478 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3479 cpi->cpi_family, cpi->cpi_model,
3480 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3481 return (snprintf(s, n, fmt,
3482 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3483 cpi->cpi_family, cpi->cpi_model,
3484 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3485 }
3486
3487 const char *
3488 cpuid_getvendorstr(cpu_t *cpu)
3489 {
3490 ASSERT(cpuid_checkpass(cpu, 1));
3491 return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
3492 }
3493
3494 uint_t
3495 cpuid_getvendor(cpu_t *cpu)
3496 {
3497 ASSERT(cpuid_checkpass(cpu, 1));
3498 return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
3499 }
3500
3501 uint_t
3502 cpuid_getfamily(cpu_t *cpu)
3503 {
3504 ASSERT(cpuid_checkpass(cpu, 1));
3505 return (cpu->cpu_m.mcpu_cpi->cpi_family);
3506 }
3507
3508 uint_t
3509 cpuid_getmodel(cpu_t *cpu)
3510 {
3511 ASSERT(cpuid_checkpass(cpu, 1));
3512 return (cpu->cpu_m.mcpu_cpi->cpi_model);
3513 }
3514
3515 uint_t
3516 cpuid_get_ncpu_per_chip(cpu_t *cpu)
3517 {
3518 ASSERT(cpuid_checkpass(cpu, 1));
3519 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
3520 }
3521
3522 uint_t
3523 cpuid_get_ncore_per_chip(cpu_t *cpu)
3524 {
3525 ASSERT(cpuid_checkpass(cpu, 1));
3526 return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
3527 }
3528
3529 uint_t
3530 cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
3531 {
3532 ASSERT(cpuid_checkpass(cpu, 2));
3533 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
3534 }
3535
3536 id_t
3537 cpuid_get_last_lvl_cacheid(cpu_t *cpu)
3538 {
3539 ASSERT(cpuid_checkpass(cpu, 2));
3540 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3541 }
3542
3543 uint_t
3544 cpuid_getstep(cpu_t *cpu)
3545 {
3546 ASSERT(cpuid_checkpass(cpu, 1));
3547 return (cpu->cpu_m.mcpu_cpi->cpi_step);
3548 }
3549
3550 uint_t
3551 cpuid_getsig(struct cpu *cpu)
3552 {
3553 ASSERT(cpuid_checkpass(cpu, 1));
3554 return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
3555 }
3556
3557 uint32_t
3558 cpuid_getchiprev(struct cpu *cpu)
3559 {
3560 ASSERT(cpuid_checkpass(cpu, 1));
3561 return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
3562 }
3563
3564 const char *
3565 cpuid_getchiprevstr(struct cpu *cpu)
3566 {
3567 ASSERT(cpuid_checkpass(cpu, 1));
3568 return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
3569 }
3570
3571 uint32_t
3572 cpuid_getsockettype(struct cpu *cpu)
3573 {
3574 ASSERT(cpuid_checkpass(cpu, 1));
3575 return (cpu->cpu_m.mcpu_cpi->cpi_socket);
3576 }
3577
3578 const char *
3579 cpuid_getsocketstr(cpu_t *cpu)
3580 {
3581 static const char *socketstr = NULL;
3582 struct cpuid_info *cpi;
3583
3584 ASSERT(cpuid_checkpass(cpu, 1));
3585 cpi = cpu->cpu_m.mcpu_cpi;
3586
3587 /* Assume that socket types are the same across the system */
3588 if (socketstr == NULL)
3589 socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
3590 cpi->cpi_model, cpi->cpi_step);
3591
3592
3593 return (socketstr);
3594 }
3595
3596 int
3597 cpuid_get_chipid(cpu_t *cpu)
3598 {
3599 ASSERT(cpuid_checkpass(cpu, 1));
3600
3601 if (cpuid_is_cmt(cpu))
3602 return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
3603 return (cpu->cpu_id);
3604 }
3605
3606 id_t
3607 cpuid_get_coreid(cpu_t *cpu)
3608 {
3609 ASSERT(cpuid_checkpass(cpu, 1));
3610 return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
3611 }
3612
3613 int
3614 cpuid_get_pkgcoreid(cpu_t *cpu)
3615 {
3616 ASSERT(cpuid_checkpass(cpu, 1));
3617 return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
3618 }
3619
3620 int
3621 cpuid_get_clogid(cpu_t *cpu)
3622 {
3623 ASSERT(cpuid_checkpass(cpu, 1));
3624 return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
3625 }
3626
3627 int
3628 cpuid_get_cacheid(cpu_t *cpu)
3629 {
3630 ASSERT(cpuid_checkpass(cpu, 1));
3631 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3632 }
3633
3634 uint_t
3635 cpuid_get_procnodeid(cpu_t *cpu)
3636 {
3637 ASSERT(cpuid_checkpass(cpu, 1));
3638 return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
3639 }
3640
3641 uint_t
3642 cpuid_get_procnodes_per_pkg(cpu_t *cpu)
3643 {
3644 ASSERT(cpuid_checkpass(cpu, 1));
3645 return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
3646 }
3647
3648 uint_t
3649 cpuid_get_compunitid(cpu_t *cpu)
3650 {
3651 ASSERT(cpuid_checkpass(cpu, 1));
3652 return (cpu->cpu_m.mcpu_cpi->cpi_compunitid);
3653 }
3654
3655 uint_t
3656 cpuid_get_cores_per_compunit(cpu_t *cpu)
3657 {
3658 ASSERT(cpuid_checkpass(cpu, 1));
3659 return (cpu->cpu_m.mcpu_cpi->cpi_cores_per_compunit);
3660 }
3661
3662 /*ARGSUSED*/
3663 int
3664 cpuid_have_cr8access(cpu_t *cpu)
3665 {
3666 #if defined(__amd64)
3667 return (1);
3668 #else
3669 struct cpuid_info *cpi;
3670
3671 ASSERT(cpu != NULL);
3672 cpi = cpu->cpu_m.mcpu_cpi;
3673 if (cpi->cpi_vendor == X86_VENDOR_AMD && cpi->cpi_maxeax >= 1 &&
3674 (CPI_FEATURES_XTD_ECX(cpi) & CPUID_AMD_ECX_CR8D) != 0)
3675 return (1);
3676 return (0);
3677 #endif
3678 }
3679
3680 uint32_t
3681 cpuid_get_apicid(cpu_t *cpu)
3682 {
3683 ASSERT(cpuid_checkpass(cpu, 1));
3684 if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
3685 return (UINT32_MAX);
3686 } else {
3687 return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
3688 }
3689 }
3690
3691 void
3692 cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
3693 {
3694 struct cpuid_info *cpi;
3695
3696 if (cpu == NULL)
3697 cpu = CPU;
3698 cpi = cpu->cpu_m.mcpu_cpi;
3699
3700 ASSERT(cpuid_checkpass(cpu, 1));
3701
3702 if (pabits)
3703 *pabits = cpi->cpi_pabits;
3704 if (vabits)
3705 *vabits = cpi->cpi_vabits;
3706 }
3707
3708 size_t
3709 cpuid_get_xsave_size()
3710 {
3711 return (MAX(cpuid_info0.cpi_xsave.xsav_max_size,
3712 sizeof (struct xsave_state)));
3713 }
3714
3715 /*
3716 * Return true if the CPUs on this system require 'pointer clearing' for the
3717 * floating point error pointer exception handling. In the past, this has been
3718 * true for all AMD K7 & K8 CPUs, although newer AMD CPUs have been changed to
3719 * behave the same as Intel. This is checked via the CPUID_AMD_EBX_ERR_PTR_ZERO
3720 * feature bit and is reflected in the cpi_fp_amd_save member.
3721 */
3722 boolean_t
3723 cpuid_need_fp_excp_handling()
3724 {
3725 return (cpuid_info0.cpi_vendor == X86_VENDOR_AMD &&
3726 cpuid_info0.cpi_fp_amd_save != 0);
3727 }
3728
3729 /*
3730 * Returns the number of data TLB entries for a corresponding
3731 * pagesize. If it can't be computed, or isn't known, the
3732 * routine returns zero. If you ask about an architecturally
3733 * impossible pagesize, the routine will panic (so that the
3734 * hat implementor knows that things are inconsistent.)
3735 */
3736 uint_t
3737 cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
3738 {
3739 struct cpuid_info *cpi;
3740 uint_t dtlb_nent = 0;
3741
3742 if (cpu == NULL)
3743 cpu = CPU;
3744 cpi = cpu->cpu_m.mcpu_cpi;
3745
3746 ASSERT(cpuid_checkpass(cpu, 1));
3747
3748 /*
3749 * Check the L2 TLB info
3750 */
3751 if (cpi->cpi_xmaxeax >= 0x80000006) {
3752 struct cpuid_regs *cp = &cpi->cpi_extd[6];
3753
3754 switch (pagesize) {
3755
3756 case 4 * 1024:
3757 /*
3758 * All zero in the top 16 bits of the register
3759 * indicates a unified TLB. Size is in low 16 bits.
3760 */
3761 if ((cp->cp_ebx & 0xffff0000) == 0)
3762 dtlb_nent = cp->cp_ebx & 0x0000ffff;
3763 else
3764 dtlb_nent = BITX(cp->cp_ebx, 27, 16);
3765 break;
3766
3767 case 2 * 1024 * 1024:
3768 if ((cp->cp_eax & 0xffff0000) == 0)
3769 dtlb_nent = cp->cp_eax & 0x0000ffff;
3770 else
3771 dtlb_nent = BITX(cp->cp_eax, 27, 16);
3772 break;
3773
3774 default:
3775 panic("unknown L2 pagesize");
3776 /*NOTREACHED*/
3777 }
3778 }
3779
3780 if (dtlb_nent != 0)
3781 return (dtlb_nent);
3782
3783 /*
3784 * No L2 TLB support for this size, try L1.
3785 */
3786 if (cpi->cpi_xmaxeax >= 0x80000005) {
3787 struct cpuid_regs *cp = &cpi->cpi_extd[5];
3788
3789 switch (pagesize) {
3790 case 4 * 1024:
3791 dtlb_nent = BITX(cp->cp_ebx, 23, 16);
3792 break;
3793 case 2 * 1024 * 1024:
3794 dtlb_nent = BITX(cp->cp_eax, 23, 16);
3795 break;
3796 default:
3797 panic("unknown L1 d-TLB pagesize");
3798 /*NOTREACHED*/
3799 }
3800 }
3801
3802 return (dtlb_nent);
3803 }
3804
3805 /*
3806 * Return 0 if the erratum is not present or not applicable, positive
3807 * if it is, and negative if the status of the erratum is unknown.
3808 *
3809 * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
3810 * Processors" #25759, Rev 3.57, August 2005
3811 */
3812 int
3813 cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
3814 {
3815 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3816 uint_t eax;
3817
3818 /*
3819 * Bail out if this CPU isn't an AMD CPU, or if it's
3820 * a legacy (32-bit) AMD CPU.
3821 */
3822 if (cpi->cpi_vendor != X86_VENDOR_AMD ||
3823 cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
3824 cpi->cpi_family == 6)
3825
3826 return (0);
3827
3828 eax = cpi->cpi_std[1].cp_eax;
3829
3830 #define SH_B0(eax) (eax == 0xf40 || eax == 0xf50)
3831 #define SH_B3(eax) (eax == 0xf51)
3832 #define B(eax) (SH_B0(eax) || SH_B3(eax))
3833
3834 #define SH_C0(eax) (eax == 0xf48 || eax == 0xf58)
3835
3836 #define SH_CG(eax) (eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
3837 #define DH_CG(eax) (eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
3838 #define CH_CG(eax) (eax == 0xf82 || eax == 0xfb2)
3839 #define CG(eax) (SH_CG(eax) || DH_CG(eax) || CH_CG(eax))
3840
3841 #define SH_D0(eax) (eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
3842 #define DH_D0(eax) (eax == 0x10fc0 || eax == 0x10ff0)
3843 #define CH_D0(eax) (eax == 0x10f80 || eax == 0x10fb0)
3844 #define D0(eax) (SH_D0(eax) || DH_D0(eax) || CH_D0(eax))
3845
3846 #define SH_E0(eax) (eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
3847 #define JH_E1(eax) (eax == 0x20f10) /* JH8_E0 had 0x20f30 */
3848 #define DH_E3(eax) (eax == 0x20fc0 || eax == 0x20ff0)
3849 #define SH_E4(eax) (eax == 0x20f51 || eax == 0x20f71)
3850 #define BH_E4(eax) (eax == 0x20fb1)
3851 #define SH_E5(eax) (eax == 0x20f42)
3852 #define DH_E6(eax) (eax == 0x20ff2 || eax == 0x20fc2)
3853 #define JH_E6(eax) (eax == 0x20f12 || eax == 0x20f32)
3854 #define EX(eax) (SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
3855 SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
3856 DH_E6(eax) || JH_E6(eax))
3857
3858 #define DR_AX(eax) (eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
3859 #define DR_B0(eax) (eax == 0x100f20)
3860 #define DR_B1(eax) (eax == 0x100f21)
3861 #define DR_BA(eax) (eax == 0x100f2a)
3862 #define DR_B2(eax) (eax == 0x100f22)
3863 #define DR_B3(eax) (eax == 0x100f23)
3864 #define RB_C0(eax) (eax == 0x100f40)
3865
3866 switch (erratum) {
3867 case 1:
3868 return (cpi->cpi_family < 0x10);
3869 case 51: /* what does the asterisk mean? */
3870 return (B(eax) || SH_C0(eax) || CG(eax));
3871 case 52:
3872 return (B(eax));
3873 case 57:
3874 return (cpi->cpi_family <= 0x11);
3875 case 58:
3876 return (B(eax));
3877 case 60:
3878 return (cpi->cpi_family <= 0x11);
3879 case 61:
3880 case 62:
3881 case 63:
3882 case 64:
3883 case 65:
3884 case 66:
3885 case 68:
3886 case 69:
3887 case 70:
3888 case 71:
3889 return (B(eax));
3890 case 72:
3891 return (SH_B0(eax));
3892 case 74:
3893 return (B(eax));
3894 case 75:
3895 return (cpi->cpi_family < 0x10);
3896 case 76:
3897 return (B(eax));
3898 case 77:
3899 return (cpi->cpi_family <= 0x11);
3900 case 78:
3901 return (B(eax) || SH_C0(eax));
3902 case 79:
3903 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3904 case 80:
3905 case 81:
3906 case 82:
3907 return (B(eax));
3908 case 83:
3909 return (B(eax) || SH_C0(eax) || CG(eax));
3910 case 85:
3911 return (cpi->cpi_family < 0x10);
3912 case 86:
3913 return (SH_C0(eax) || CG(eax));
3914 case 88:
3915 #if !defined(__amd64)
3916 return (0);
3917 #else
3918 return (B(eax) || SH_C0(eax));
3919 #endif
3920 case 89:
3921 return (cpi->cpi_family < 0x10);
3922 case 90:
3923 return (B(eax) || SH_C0(eax) || CG(eax));
3924 case 91:
3925 case 92:
3926 return (B(eax) || SH_C0(eax));
3927 case 93:
3928 return (SH_C0(eax));
3929 case 94:
3930 return (B(eax) || SH_C0(eax) || CG(eax));
3931 case 95:
3932 #if !defined(__amd64)
3933 return (0);
3934 #else
3935 return (B(eax) || SH_C0(eax));
3936 #endif
3937 case 96:
3938 return (B(eax) || SH_C0(eax) || CG(eax));
3939 case 97:
3940 case 98:
3941 return (SH_C0(eax) || CG(eax));
3942 case 99:
3943 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3944 case 100:
3945 return (B(eax) || SH_C0(eax));
3946 case 101:
3947 case 103:
3948 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3949 case 104:
3950 return (SH_C0(eax) || CG(eax) || D0(eax));
3951 case 105:
3952 case 106:
3953 case 107:
3954 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3955 case 108:
3956 return (DH_CG(eax));
3957 case 109:
3958 return (SH_C0(eax) || CG(eax) || D0(eax));
3959 case 110:
3960 return (D0(eax) || EX(eax));
3961 case 111:
3962 return (CG(eax));
3963 case 112:
3964 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3965 case 113:
3966 return (eax == 0x20fc0);
3967 case 114:
3968 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3969 case 115:
3970 return (SH_E0(eax) || JH_E1(eax));
3971 case 116:
3972 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3973 case 117:
3974 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3975 case 118:
3976 return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
3977 JH_E6(eax));
3978 case 121:
3979 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3980 case 122:
3981 return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
3982 case 123:
3983 return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
3984 case 131:
3985 return (cpi->cpi_family < 0x10);
3986 case 6336786:
3987 /*
3988 * Test for AdvPowerMgmtInfo.TscPStateInvariant
3989 * if this is a K8 family or newer processor
3990 */
3991 if (CPI_FAMILY(cpi) == 0xf) {
3992 struct cpuid_regs regs;
3993 regs.cp_eax = 0x80000007;
3994 (void) __cpuid_insn(®s);
3995 return (!(regs.cp_edx & 0x100));
3996 }
3997 return (0);
3998 case 6323525:
3999 return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
4000 (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);
4001
4002 case 6671130:
4003 /*
4004 * check for processors (pre-Shanghai) that do not provide
4005 * optimal management of 1gb ptes in its tlb.
4006 */
4007 return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);
4008
4009 case 298:
4010 return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
4011 DR_B2(eax) || RB_C0(eax));
4012
4013 case 721:
4014 #if defined(__amd64)
4015 return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);
4016 #else
4017 return (0);
4018 #endif
4019
4020 default:
4021 return (-1);
4022
4023 }
4024 }
4025
4026 /*
4027 * Determine if specified erratum is present via OSVW (OS Visible Workaround).
4028 * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
4029 */
4030 int
4031 osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
4032 {
4033 struct cpuid_info *cpi;
4034 uint_t osvwid;
4035 static int osvwfeature = -1;
4036 uint64_t osvwlength;
4037
4038
4039 cpi = cpu->cpu_m.mcpu_cpi;
4040
4041 /* confirm OSVW supported */
4042 if (osvwfeature == -1) {
4043 osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
4044 } else {
4045 /* assert that osvw feature setting is consistent on all cpus */
4046 ASSERT(osvwfeature ==
4047 (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
4048 }
4049 if (!osvwfeature)
4050 return (-1);
4051
4052 osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;
4053
4054 switch (erratum) {
4055 case 298: /* osvwid is 0 */
4056 osvwid = 0;
4057 if (osvwlength <= (uint64_t)osvwid) {
4058 /* osvwid 0 is unknown */
4059 return (-1);
4060 }
4061
4062 /*
4063 * Check the OSVW STATUS MSR to determine the state
4064 * of the erratum where:
4065 * 0 - fixed by HW
4066 * 1 - BIOS has applied the workaround when BIOS
4067 * workaround is available. (Or for other errata,
4068 * OS workaround is required.)
4069 * For a value of 1, caller will confirm that the
4070 * erratum 298 workaround has indeed been applied by BIOS.
4071 *
4072 * A 1 may be set in cpus that have a HW fix
4073 * in a mixed cpu system. Regarding erratum 298:
4074 * In a multiprocessor platform, the workaround above
4075 * should be applied to all processors regardless of
4076 * silicon revision when an affected processor is
4077 * present.
4078 */
4079
4080 return (rdmsr(MSR_AMD_OSVW_STATUS +
4081 (osvwid / OSVW_ID_CNT_PER_MSR)) &
4082 (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));
4083
4084 default:
4085 return (-1);
4086 }
4087 }
4088
4089 static const char assoc_str[] = "associativity";
4090 static const char line_str[] = "line-size";
4091 static const char size_str[] = "size";
4092
4093 static void
4094 add_cache_prop(dev_info_t *devi, const char *label, const char *type,
4095 uint32_t val)
4096 {
4097 char buf[128];
4098
4099 /*
4100 * ndi_prop_update_int() is used because it is desirable for
4101 * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
4102 */
4103 if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
4104 (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
4105 }
4106
4107 /*
4108 * Intel-style cache/tlb description
4109 *
4110 * Standard cpuid level 2 gives a randomly ordered
4111 * selection of tags that index into a table that describes
4112 * cache and tlb properties.
4113 */
4114
4115 static const char l1_icache_str[] = "l1-icache";
4116 static const char l1_dcache_str[] = "l1-dcache";
4117 static const char l2_cache_str[] = "l2-cache";
4118 static const char l3_cache_str[] = "l3-cache";
4119 static const char itlb4k_str[] = "itlb-4K";
4120 static const char dtlb4k_str[] = "dtlb-4K";
4121 static const char itlb2M_str[] = "itlb-2M";
4122 static const char itlb4M_str[] = "itlb-4M";
4123 static const char dtlb4M_str[] = "dtlb-4M";
4124 static const char dtlb24_str[] = "dtlb0-2M-4M";
4125 static const char itlb424_str[] = "itlb-4K-2M-4M";
4126 static const char itlb24_str[] = "itlb-2M-4M";
4127 static const char dtlb44_str[] = "dtlb-4K-4M";
4128 static const char sl1_dcache_str[] = "sectored-l1-dcache";
4129 static const char sl2_cache_str[] = "sectored-l2-cache";
4130 static const char itrace_str[] = "itrace-cache";
4131 static const char sl3_cache_str[] = "sectored-l3-cache";
4132 static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";
4133
4134 static const struct cachetab {
4135 uint8_t ct_code;
4136 uint8_t ct_assoc;
4137 uint16_t ct_line_size;
4138 size_t ct_size;
4139 const char *ct_label;
4140 } intel_ctab[] = {
4141 /*
4142 * maintain descending order!
4143 *
4144 * Codes ignored - Reason
4145 * ----------------------
4146 * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
4147 * f0H/f1H - Currently we do not interpret prefetch size by design
4148 */
4149 { 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
4150 { 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
4151 { 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
4152 { 0xde, 12, 64, 6*1024*1024, l3_cache_str},
4153 { 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
4154 { 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
4155 { 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
4156 { 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
4157 { 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
4158 { 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
4159 { 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
4160 { 0xd0, 4, 64, 512*1024, l3_cache_str},
4161 { 0xca, 4, 0, 512, sh_l2_tlb4k_str},
4162 { 0xc0, 4, 0, 8, dtlb44_str },
4163 { 0xba, 4, 0, 64, dtlb4k_str },
4164 { 0xb4, 4, 0, 256, dtlb4k_str },
4165 { 0xb3, 4, 0, 128, dtlb4k_str },
4166 { 0xb2, 4, 0, 64, itlb4k_str },
4167 { 0xb0, 4, 0, 128, itlb4k_str },
4168 { 0x87, 8, 64, 1024*1024, l2_cache_str},
4169 { 0x86, 4, 64, 512*1024, l2_cache_str},
4170 { 0x85, 8, 32, 2*1024*1024, l2_cache_str},
4171 { 0x84, 8, 32, 1024*1024, l2_cache_str},
4172 { 0x83, 8, 32, 512*1024, l2_cache_str},
4173 { 0x82, 8, 32, 256*1024, l2_cache_str},
4174 { 0x80, 8, 64, 512*1024, l2_cache_str},
4175 { 0x7f, 2, 64, 512*1024, l2_cache_str},
4176 { 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
4177 { 0x7c, 8, 64, 1024*1024, sl2_cache_str},
4178 { 0x7b, 8, 64, 512*1024, sl2_cache_str},
4179 { 0x7a, 8, 64, 256*1024, sl2_cache_str},
4180 { 0x79, 8, 64, 128*1024, sl2_cache_str},
4181 { 0x78, 8, 64, 1024*1024, l2_cache_str},
4182 { 0x73, 8, 0, 64*1024, itrace_str},
4183 { 0x72, 8, 0, 32*1024, itrace_str},
4184 { 0x71, 8, 0, 16*1024, itrace_str},
4185 { 0x70, 8, 0, 12*1024, itrace_str},
4186 { 0x68, 4, 64, 32*1024, sl1_dcache_str},
4187 { 0x67, 4, 64, 16*1024, sl1_dcache_str},
4188 { 0x66, 4, 64, 8*1024, sl1_dcache_str},
4189 { 0x60, 8, 64, 16*1024, sl1_dcache_str},
4190 { 0x5d, 0, 0, 256, dtlb44_str},
4191 { 0x5c, 0, 0, 128, dtlb44_str},
4192 { 0x5b, 0, 0, 64, dtlb44_str},
4193 { 0x5a, 4, 0, 32, dtlb24_str},
4194 { 0x59, 0, 0, 16, dtlb4k_str},
4195 { 0x57, 4, 0, 16, dtlb4k_str},
4196 { 0x56, 4, 0, 16, dtlb4M_str},
4197 { 0x55, 0, 0, 7, itlb24_str},
4198 { 0x52, 0, 0, 256, itlb424_str},
4199 { 0x51, 0, 0, 128, itlb424_str},
4200 { 0x50, 0, 0, 64, itlb424_str},
4201 { 0x4f, 0, 0, 32, itlb4k_str},
4202 { 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
4203 { 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
4204 { 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
4205 { 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
4206 { 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
4207 { 0x49, 16, 64, 4*1024*1024, l3_cache_str},
4208 { 0x48, 12, 64, 3*1024*1024, l2_cache_str},
4209 { 0x47, 8, 64, 8*1024*1024, l3_cache_str},
4210 { 0x46, 4, 64, 4*1024*1024, l3_cache_str},
4211 { 0x45, 4, 32, 2*1024*1024, l2_cache_str},
4212 { 0x44, 4, 32, 1024*1024, l2_cache_str},
4213 { 0x43, 4, 32, 512*1024, l2_cache_str},
4214 { 0x42, 4, 32, 256*1024, l2_cache_str},
4215 { 0x41, 4, 32, 128*1024, l2_cache_str},
4216 { 0x3e, 4, 64, 512*1024, sl2_cache_str},
4217 { 0x3d, 6, 64, 384*1024, sl2_cache_str},
4218 { 0x3c, 4, 64, 256*1024, sl2_cache_str},
4219 { 0x3b, 2, 64, 128*1024, sl2_cache_str},
4220 { 0x3a, 6, 64, 192*1024, sl2_cache_str},
4221 { 0x39, 4, 64, 128*1024, sl2_cache_str},
4222 { 0x30, 8, 64, 32*1024, l1_icache_str},
4223 { 0x2c, 8, 64, 32*1024, l1_dcache_str},
4224 { 0x29, 8, 64, 4096*1024, sl3_cache_str},
4225 { 0x25, 8, 64, 2048*1024, sl3_cache_str},
4226 { 0x23, 8, 64, 1024*1024, sl3_cache_str},
4227 { 0x22, 4, 64, 512*1024, sl3_cache_str},
4228 { 0x0e, 6, 64, 24*1024, l1_dcache_str},
4229 { 0x0d, 4, 32, 16*1024, l1_dcache_str},
4230 { 0x0c, 4, 32, 16*1024, l1_dcache_str},
4231 { 0x0b, 4, 0, 4, itlb4M_str},
4232 { 0x0a, 2, 32, 8*1024, l1_dcache_str},
4233 { 0x08, 4, 32, 16*1024, l1_icache_str},
4234 { 0x06, 4, 32, 8*1024, l1_icache_str},
4235 { 0x05, 4, 0, 32, dtlb4M_str},
4236 { 0x04, 4, 0, 8, dtlb4M_str},
4237 { 0x03, 4, 0, 64, dtlb4k_str},
4238 { 0x02, 4, 0, 2, itlb4M_str},
4239 { 0x01, 4, 0, 32, itlb4k_str},
4240 { 0 }
4241 };
4242
4243 static const struct cachetab cyrix_ctab[] = {
4244 { 0x70, 4, 0, 32, "tlb-4K" },
4245 { 0x80, 4, 16, 16*1024, "l1-cache" },
4246 { 0 }
4247 };
4248
4249 /*
4250 * Search a cache table for a matching entry
4251 */
4252 static const struct cachetab *
4253 find_cacheent(const struct cachetab *ct, uint_t code)
4254 {
4255 if (code != 0) {
4256 for (; ct->ct_code != 0; ct++)
4257 if (ct->ct_code <= code)
4258 break;
4259 if (ct->ct_code == code)
4260 return (ct);
4261 }
4262 return (NULL);
4263 }
4264
4265 /*
4266 * Populate cachetab entry with L2 or L3 cache-information using
4267 * cpuid function 4. This function is called from intel_walk_cacheinfo()
4268 * when descriptor 0x49 is encountered. It returns 0 if no such cache
4269 * information is found.
4270 */
4271 static int
4272 intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
4273 {
4274 uint32_t level, i;
4275 int ret = 0;
4276
4277 for (i = 0; i < cpi->cpi_std_4_size; i++) {
4278 level = CPI_CACHE_LVL(cpi->cpi_std_4[i]);
4279
4280 if (level == 2 || level == 3) {
4281 ct->ct_assoc = CPI_CACHE_WAYS(cpi->cpi_std_4[i]) + 1;
4282 ct->ct_line_size =
4283 CPI_CACHE_COH_LN_SZ(cpi->cpi_std_4[i]) + 1;
4284 ct->ct_size = ct->ct_assoc *
4285 (CPI_CACHE_PARTS(cpi->cpi_std_4[i]) + 1) *
4286 ct->ct_line_size *
4287 (cpi->cpi_std_4[i]->cp_ecx + 1);
4288
4289 if (level == 2) {
4290 ct->ct_label = l2_cache_str;
4291 } else if (level == 3) {
4292 ct->ct_label = l3_cache_str;
4293 }
4294 ret = 1;
4295 }
4296 }
4297
4298 return (ret);
4299 }
4300
4301 /*
4302 * Walk the cacheinfo descriptor, applying 'func' to every valid element
4303 * The walk is terminated if the walker returns non-zero.
4304 */
4305 static void
4306 intel_walk_cacheinfo(struct cpuid_info *cpi,
4307 void *arg, int (*func)(void *, const struct cachetab *))
4308 {
4309 const struct cachetab *ct;
4310 struct cachetab des_49_ct, des_b1_ct;
4311 uint8_t *dp;
4312 int i;
4313
4314 if ((dp = cpi->cpi_cacheinfo) == NULL)
4315 return;
4316 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4317 /*
4318 * For overloaded descriptor 0x49 we use cpuid function 4
4319 * if supported by the current processor, to create
4320 * cache information.
4321 * For overloaded descriptor 0xb1 we use X86_PAE flag
4322 * to disambiguate the cache information.
4323 */
4324 if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
4325 intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
4326 ct = &des_49_ct;
4327 } else if (*dp == 0xb1) {
4328 des_b1_ct.ct_code = 0xb1;
4329 des_b1_ct.ct_assoc = 4;
4330 des_b1_ct.ct_line_size = 0;
4331 if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
4332 des_b1_ct.ct_size = 8;
4333 des_b1_ct.ct_label = itlb2M_str;
4334 } else {
4335 des_b1_ct.ct_size = 4;
4336 des_b1_ct.ct_label = itlb4M_str;
4337 }
4338 ct = &des_b1_ct;
4339 } else {
4340 if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
4341 continue;
4342 }
4343 }
4344
4345 if (func(arg, ct) != 0) {
4346 break;
4347 }
4348 }
4349 }
4350
4351 /*
4352 * (Like the Intel one, except for Cyrix CPUs)
4353 */
4354 static void
4355 cyrix_walk_cacheinfo(struct cpuid_info *cpi,
4356 void *arg, int (*func)(void *, const struct cachetab *))
4357 {
4358 const struct cachetab *ct;
4359 uint8_t *dp;
4360 int i;
4361
4362 if ((dp = cpi->cpi_cacheinfo) == NULL)
4363 return;
4364 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4365 /*
4366 * Search Cyrix-specific descriptor table first ..
4367 */
4368 if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
4369 if (func(arg, ct) != 0)
4370 break;
4371 continue;
4372 }
4373 /*
4374 * .. else fall back to the Intel one
4375 */
4376 if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
4377 if (func(arg, ct) != 0)
4378 break;
4379 continue;
4380 }
4381 }
4382 }
4383
4384 /*
4385 * A cacheinfo walker that adds associativity, line-size, and size properties
4386 * to the devinfo node it is passed as an argument.
4387 */
4388 static int
4389 add_cacheent_props(void *arg, const struct cachetab *ct)
4390 {
4391 dev_info_t *devi = arg;
4392
4393 add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
4394 if (ct->ct_line_size != 0)
4395 add_cache_prop(devi, ct->ct_label, line_str,
4396 ct->ct_line_size);
4397 add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
4398 return (0);
4399 }
4400
4401
4402 static const char fully_assoc[] = "fully-associative?";
4403
4404 /*
4405 * AMD style cache/tlb description
4406 *
4407 * Extended functions 5 and 6 directly describe properties of
4408 * tlbs and various cache levels.
4409 */
4410 static void
4411 add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4412 {
4413 switch (assoc) {
4414 case 0: /* reserved; ignore */
4415 break;
4416 default:
4417 add_cache_prop(devi, label, assoc_str, assoc);
4418 break;
4419 case 0xff:
4420 add_cache_prop(devi, label, fully_assoc, 1);
4421 break;
4422 }
4423 }
4424
4425 static void
4426 add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4427 {
4428 if (size == 0)
4429 return;
4430 add_cache_prop(devi, label, size_str, size);
4431 add_amd_assoc(devi, label, assoc);
4432 }
4433
4434 static void
4435 add_amd_cache(dev_info_t *devi, const char *label,
4436 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4437 {
4438 if (size == 0 || line_size == 0)
4439 return;
4440 add_amd_assoc(devi, label, assoc);
4441 /*
4442 * Most AMD parts have a sectored cache. Multiple cache lines are
4443 * associated with each tag. A sector consists of all cache lines
4444 * associated with a tag. For example, the AMD K6-III has a sector
4445 * size of 2 cache lines per tag.
4446 */
4447 if (lines_per_tag != 0)
4448 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4449 add_cache_prop(devi, label, line_str, line_size);
4450 add_cache_prop(devi, label, size_str, size * 1024);
4451 }
4452
4453 static void
4454 add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4455 {
4456 switch (assoc) {
4457 case 0: /* off */
4458 break;
4459 case 1:
4460 case 2:
4461 case 4:
4462 add_cache_prop(devi, label, assoc_str, assoc);
4463 break;
4464 case 6:
4465 add_cache_prop(devi, label, assoc_str, 8);
4466 break;
4467 case 8:
4468 add_cache_prop(devi, label, assoc_str, 16);
4469 break;
4470 case 0xf:
4471 add_cache_prop(devi, label, fully_assoc, 1);
4472 break;
4473 default: /* reserved; ignore */
4474 break;
4475 }
4476 }
4477
4478 static void
4479 add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4480 {
4481 if (size == 0 || assoc == 0)
4482 return;
4483 add_amd_l2_assoc(devi, label, assoc);
4484 add_cache_prop(devi, label, size_str, size);
4485 }
4486
4487 static void
4488 add_amd_l2_cache(dev_info_t *devi, const char *label,
4489 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4490 {
4491 if (size == 0 || assoc == 0 || line_size == 0)
4492 return;
4493 add_amd_l2_assoc(devi, label, assoc);
4494 if (lines_per_tag != 0)
4495 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4496 add_cache_prop(devi, label, line_str, line_size);
4497 add_cache_prop(devi, label, size_str, size * 1024);
4498 }
4499
4500 static void
4501 amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
4502 {
4503 struct cpuid_regs *cp;
4504
4505 if (cpi->cpi_xmaxeax < 0x80000005)
4506 return;
4507 cp = &cpi->cpi_extd[5];
4508
4509 /*
4510 * 4M/2M L1 TLB configuration
4511 *
4512 * We report the size for 2M pages because AMD uses two
4513 * TLB entries for one 4M page.
4514 */
4515 add_amd_tlb(devi, "dtlb-2M",
4516 BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
4517 add_amd_tlb(devi, "itlb-2M",
4518 BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));
4519
4520 /*
4521 * 4K L1 TLB configuration
4522 */
4523
4524 switch (cpi->cpi_vendor) {
4525 uint_t nentries;
4526 case X86_VENDOR_TM:
4527 if (cpi->cpi_family >= 5) {
4528 /*
4529 * Crusoe processors have 256 TLB entries, but
4530 * cpuid data format constrains them to only
4531 * reporting 255 of them.
4532 */
4533 if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
4534 nentries = 256;
4535 /*
4536 * Crusoe processors also have a unified TLB
4537 */
4538 add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
4539 nentries);
4540 break;
4541 }
4542 /*FALLTHROUGH*/
4543 default:
4544 add_amd_tlb(devi, itlb4k_str,
4545 BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
4546 add_amd_tlb(devi, dtlb4k_str,
4547 BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
4548 break;
4549 }
4550
4551 /*
4552 * data L1 cache configuration
4553 */
4554
4555 add_amd_cache(devi, l1_dcache_str,
4556 BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
4557 BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));
4558
4559 /*
4560 * code L1 cache configuration
4561 */
4562
4563 add_amd_cache(devi, l1_icache_str,
4564 BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
4565 BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));
4566
4567 if (cpi->cpi_xmaxeax < 0x80000006)
4568 return;
4569 cp = &cpi->cpi_extd[6];
4570
4571 /* Check for a unified L2 TLB for large pages */
4572
4573 if (BITX(cp->cp_eax, 31, 16) == 0)
4574 add_amd_l2_tlb(devi, "l2-tlb-2M",
4575 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4576 else {
4577 add_amd_l2_tlb(devi, "l2-dtlb-2M",
4578 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4579 add_amd_l2_tlb(devi, "l2-itlb-2M",
4580 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4581 }
4582
4583 /* Check for a unified L2 TLB for 4K pages */
4584
4585 if (BITX(cp->cp_ebx, 31, 16) == 0) {
4586 add_amd_l2_tlb(devi, "l2-tlb-4K",
4587 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4588 } else {
4589 add_amd_l2_tlb(devi, "l2-dtlb-4K",
4590 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4591 add_amd_l2_tlb(devi, "l2-itlb-4K",
4592 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4593 }
4594
4595 add_amd_l2_cache(devi, l2_cache_str,
4596 BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
4597 BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
4598 }
4599
4600 /*
4601 * There are two basic ways that the x86 world describes it cache
4602 * and tlb architecture - Intel's way and AMD's way.
4603 *
4604 * Return which flavor of cache architecture we should use
4605 */
4606 static int
4607 x86_which_cacheinfo(struct cpuid_info *cpi)
4608 {
4609 switch (cpi->cpi_vendor) {
4610 case X86_VENDOR_Intel:
4611 if (cpi->cpi_maxeax >= 2)
4612 return (X86_VENDOR_Intel);
4613 break;
4614 case X86_VENDOR_AMD:
4615 /*
4616 * The K5 model 1 was the first part from AMD that reported
4617 * cache sizes via extended cpuid functions.
4618 */
4619 if (cpi->cpi_family > 5 ||
4620 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
4621 return (X86_VENDOR_AMD);
4622 break;
4623 case X86_VENDOR_TM:
4624 if (cpi->cpi_family >= 5)
4625 return (X86_VENDOR_AMD);
4626 /*FALLTHROUGH*/
4627 default:
4628 /*
4629 * If they have extended CPU data for 0x80000005
4630 * then we assume they have AMD-format cache
4631 * information.
4632 *
4633 * If not, and the vendor happens to be Cyrix,
4634 * then try our-Cyrix specific handler.
4635 *
4636 * If we're not Cyrix, then assume we're using Intel's
4637 * table-driven format instead.
4638 */
4639 if (cpi->cpi_xmaxeax >= 0x80000005)
4640 return (X86_VENDOR_AMD);
4641 else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
4642 return (X86_VENDOR_Cyrix);
4643 else if (cpi->cpi_maxeax >= 2)
4644 return (X86_VENDOR_Intel);
4645 break;
4646 }
4647 return (-1);
4648 }
4649
4650 void
4651 cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
4652 struct cpuid_info *cpi)
4653 {
4654 dev_info_t *cpu_devi;
4655 int create;
4656
4657 cpu_devi = (dev_info_t *)dip;
4658
4659 /* device_type */
4660 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4661 "device_type", "cpu");
4662
4663 /* reg */
4664 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4665 "reg", cpu_id);
4666
4667 /* cpu-mhz, and clock-frequency */
4668 if (cpu_freq > 0) {
4669 long long mul;
4670
4671 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4672 "cpu-mhz", cpu_freq);
4673 if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
4674 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4675 "clock-frequency", (int)mul);
4676 }
4677
4678 if (!is_x86_feature(x86_featureset, X86FSET_CPUID)) {
4679 return;
4680 }
4681
4682 /* vendor-id */
4683 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4684 "vendor-id", cpi->cpi_vendorstr);
4685
4686 if (cpi->cpi_maxeax == 0) {
4687 return;
4688 }
4689
4690 /*
4691 * family, model, and step
4692 */
4693 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4694 "family", CPI_FAMILY(cpi));
4695 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4696 "cpu-model", CPI_MODEL(cpi));
4697 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4698 "stepping-id", CPI_STEP(cpi));
4699
4700 /* type */
4701 switch (cpi->cpi_vendor) {
4702 case X86_VENDOR_Intel:
4703 create = 1;
4704 break;
4705 default:
4706 create = 0;
4707 break;
4708 }
4709 if (create)
4710 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4711 "type", CPI_TYPE(cpi));
4712
4713 /* ext-family */
4714 switch (cpi->cpi_vendor) {
4715 case X86_VENDOR_Intel:
4716 case X86_VENDOR_AMD:
4717 create = cpi->cpi_family >= 0xf;
4718 break;
4719 default:
4720 create = 0;
4721 break;
4722 }
4723 if (create)
4724 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4725 "ext-family", CPI_FAMILY_XTD(cpi));
4726
4727 /* ext-model */
4728 switch (cpi->cpi_vendor) {
4729 case X86_VENDOR_Intel:
4730 create = IS_EXTENDED_MODEL_INTEL(cpi);
4731 break;
4732 case X86_VENDOR_AMD:
4733 create = CPI_FAMILY(cpi) == 0xf;
4734 break;
4735 default:
4736 create = 0;
4737 break;
4738 }
4739 if (create)
4740 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4741 "ext-model", CPI_MODEL_XTD(cpi));
4742
4743 /* generation */
4744 switch (cpi->cpi_vendor) {
4745 case X86_VENDOR_AMD:
4746 /*
4747 * AMD K5 model 1 was the first part to support this
4748 */
4749 create = cpi->cpi_xmaxeax >= 0x80000001;
4750 break;
4751 default:
4752 create = 0;
4753 break;
4754 }
4755 if (create)
4756 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4757 "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));
4758
4759 /* brand-id */
4760 switch (cpi->cpi_vendor) {
4761 case X86_VENDOR_Intel:
4762 /*
4763 * brand id first appeared on Pentium III Xeon model 8,
4764 * and Celeron model 8 processors and Opteron
4765 */
4766 create = cpi->cpi_family > 6 ||
4767 (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
4768 break;
4769 case X86_VENDOR_AMD:
4770 create = cpi->cpi_family >= 0xf;
4771 break;
4772 default:
4773 create = 0;
4774 break;
4775 }
4776 if (create && cpi->cpi_brandid != 0) {
4777 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4778 "brand-id", cpi->cpi_brandid);
4779 }
4780
4781 /* chunks, and apic-id */
4782 switch (cpi->cpi_vendor) {
4783 /*
4784 * first available on Pentium IV and Opteron (K8)
4785 */
4786 case X86_VENDOR_Intel:
4787 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4788 break;
4789 case X86_VENDOR_AMD:
4790 create = cpi->cpi_family >= 0xf;
4791 break;
4792 default:
4793 create = 0;
4794 break;
4795 }
4796 if (create) {
4797 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4798 "chunks", CPI_CHUNKS(cpi));
4799 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4800 "apic-id", cpi->cpi_apicid);
4801 if (cpi->cpi_chipid >= 0) {
4802 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4803 "chip#", cpi->cpi_chipid);
4804 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4805 "clog#", cpi->cpi_clogid);
4806 }
4807 }
4808
4809 /* cpuid-features */
4810 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4811 "cpuid-features", CPI_FEATURES_EDX(cpi));
4812
4813
4814 /* cpuid-features-ecx */
4815 switch (cpi->cpi_vendor) {
4816 case X86_VENDOR_Intel:
4817 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4818 break;
4819 case X86_VENDOR_AMD:
4820 create = cpi->cpi_family >= 0xf;
4821 break;
4822 default:
4823 create = 0;
4824 break;
4825 }
4826 if (create)
4827 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4828 "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));
4829
4830 /* ext-cpuid-features */
4831 switch (cpi->cpi_vendor) {
4832 case X86_VENDOR_Intel:
4833 case X86_VENDOR_AMD:
4834 case X86_VENDOR_Cyrix:
4835 case X86_VENDOR_TM:
4836 case X86_VENDOR_Centaur:
4837 create = cpi->cpi_xmaxeax >= 0x80000001;
4838 break;
4839 default:
4840 create = 0;
4841 break;
4842 }
4843 if (create) {
4844 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4845 "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
4846 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4847 "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
4848 }
4849
4850 /*
4851 * Brand String first appeared in Intel Pentium IV, AMD K5
4852 * model 1, and Cyrix GXm. On earlier models we try and
4853 * simulate something similar .. so this string should always
4854 * same -something- about the processor, however lame.
4855 */
4856 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4857 "brand-string", cpi->cpi_brandstr);
4858
4859 /*
4860 * Finally, cache and tlb information
4861 */
4862 switch (x86_which_cacheinfo(cpi)) {
4863 case X86_VENDOR_Intel:
4864 intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4865 break;
4866 case X86_VENDOR_Cyrix:
4867 cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4868 break;
4869 case X86_VENDOR_AMD:
4870 amd_cache_info(cpi, cpu_devi);
4871 break;
4872 default:
4873 break;
4874 }
4875 }
4876
4877 struct l2info {
4878 int *l2i_csz;
4879 int *l2i_lsz;
4880 int *l2i_assoc;
4881 int l2i_ret;
4882 };
4883
4884 /*
4885 * A cacheinfo walker that fetches the size, line-size and associativity
4886 * of the L2 cache
4887 */
4888 static int
4889 intel_l2cinfo(void *arg, const struct cachetab *ct)
4890 {
4891 struct l2info *l2i = arg;
4892 int *ip;
4893
4894 if (ct->ct_label != l2_cache_str &&
4895 ct->ct_label != sl2_cache_str)
4896 return (0); /* not an L2 -- keep walking */
4897
4898 if ((ip = l2i->l2i_csz) != NULL)
4899 *ip = ct->ct_size;
4900 if ((ip = l2i->l2i_lsz) != NULL)
4901 *ip = ct->ct_line_size;
4902 if ((ip = l2i->l2i_assoc) != NULL)
4903 *ip = ct->ct_assoc;
4904 l2i->l2i_ret = ct->ct_size;
4905 return (1); /* was an L2 -- terminate walk */
4906 }
4907
4908 /*
4909 * AMD L2/L3 Cache and TLB Associativity Field Definition:
4910 *
4911 * Unlike the associativity for the L1 cache and tlb where the 8 bit
4912 * value is the associativity, the associativity for the L2 cache and
4913 * tlb is encoded in the following table. The 4 bit L2 value serves as
4914 * an index into the amd_afd[] array to determine the associativity.
4915 * -1 is undefined. 0 is fully associative.
4916 */
4917
4918 static int amd_afd[] =
4919 {-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};
4920
4921 static void
4922 amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
4923 {
4924 struct cpuid_regs *cp;
4925 uint_t size, assoc;
4926 int i;
4927 int *ip;
4928
4929 if (cpi->cpi_xmaxeax < 0x80000006)
4930 return;
4931 cp = &cpi->cpi_extd[6];
4932
4933 if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
4934 (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
4935 uint_t cachesz = size * 1024;
4936 assoc = amd_afd[i];
4937
4938 ASSERT(assoc != -1);
4939
4940 if ((ip = l2i->l2i_csz) != NULL)
4941 *ip = cachesz;
4942 if ((ip = l2i->l2i_lsz) != NULL)
4943 *ip = BITX(cp->cp_ecx, 7, 0);
4944 if ((ip = l2i->l2i_assoc) != NULL)
4945 *ip = assoc;
4946 l2i->l2i_ret = cachesz;
4947 }
4948 }
4949
4950 int
4951 getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
4952 {
4953 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
4954 struct l2info __l2info, *l2i = &__l2info;
4955
4956 l2i->l2i_csz = csz;
4957 l2i->l2i_lsz = lsz;
4958 l2i->l2i_assoc = assoc;
4959 l2i->l2i_ret = -1;
4960
4961 switch (x86_which_cacheinfo(cpi)) {
4962 case X86_VENDOR_Intel:
4963 intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4964 break;
4965 case X86_VENDOR_Cyrix:
4966 cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4967 break;
4968 case X86_VENDOR_AMD:
4969 amd_l2cacheinfo(cpi, l2i);
4970 break;
4971 default:
4972 break;
4973 }
4974 return (l2i->l2i_ret);
4975 }
4976
4977 #if !defined(__xpv)
4978
4979 uint32_t *
4980 cpuid_mwait_alloc(cpu_t *cpu)
4981 {
4982 uint32_t *ret;
4983 size_t mwait_size;
4984
4985 ASSERT(cpuid_checkpass(CPU, 2));
4986
4987 mwait_size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
4988 if (mwait_size == 0)
4989 return (NULL);
4990
4991 /*
4992 * kmem_alloc() returns cache line size aligned data for mwait_size
4993 * allocations. mwait_size is currently cache line sized. Neither
4994 * of these implementation details are guarantied to be true in the
4995 * future.
4996 *
4997 * First try allocating mwait_size as kmem_alloc() currently returns
4998 * correctly aligned memory. If kmem_alloc() does not return
4999 * mwait_size aligned memory, then use mwait_size ROUNDUP.
5000 *
5001 * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
5002 * decide to free this memory.
5003 */
5004 ret = kmem_zalloc(mwait_size, KM_SLEEP);
5005 if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) {
5006 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
5007 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size;
5008 *ret = MWAIT_RUNNING;
5009 return (ret);
5010 } else {
5011 kmem_free(ret, mwait_size);
5012 ret = kmem_zalloc(mwait_size * 2, KM_SLEEP);
5013 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
5014 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size * 2;
5015 ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size);
5016 *ret = MWAIT_RUNNING;
5017 return (ret);
5018 }
5019 }
5020
5021 void
5022 cpuid_mwait_free(cpu_t *cpu)
5023 {
5024 if (cpu->cpu_m.mcpu_cpi == NULL) {
5025 return;
5026 }
5027
5028 if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
5029 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
5030 kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
5031 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
5032 }
5033
5034 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
5035 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
5036 }
5037
5038 void
5039 patch_tsc_read(int flag)
5040 {
5041 size_t cnt;
5042
5043 switch (flag) {
5044 case TSC_NONE:
5045 cnt = &_no_rdtsc_end - &_no_rdtsc_start;
5046 (void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
5047 break;
5048 case TSC_RDTSC_MFENCE:
5049 cnt = &_tsc_mfence_end - &_tsc_mfence_start;
5050 (void) memcpy((void *)tsc_read,
5051 (void *)&_tsc_mfence_start, cnt);
5052 break;
5053 case TSC_RDTSC_LFENCE:
5054 cnt = &_tsc_lfence_end - &_tsc_lfence_start;
5055 (void) memcpy((void *)tsc_read,
5056 (void *)&_tsc_lfence_start, cnt);
5057 break;
5058 case TSC_TSCP:
5059 cnt = &_tscp_end - &_tscp_start;
5060 (void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
5061 break;
5062 default:
5063 /* Bail for unexpected TSC types. (TSC_NONE covers 0) */
5064 cmn_err(CE_PANIC, "Unrecogized TSC type: %d", flag);
5065 break;
5066 }
5067 tsc_type = flag;
5068 }
5069
5070 int
5071 cpuid_deep_cstates_supported(void)
5072 {
5073 struct cpuid_info *cpi;
5074 struct cpuid_regs regs;
5075
5076 ASSERT(cpuid_checkpass(CPU, 1));
5077
5078 cpi = CPU->cpu_m.mcpu_cpi;
5079
5080 if (!is_x86_feature(x86_featureset, X86FSET_CPUID))
5081 return (0);
5082
5083 switch (cpi->cpi_vendor) {
5084 case X86_VENDOR_Intel:
5085 if (cpi->cpi_xmaxeax < 0x80000007)
5086 return (0);
5087
5088 /*
5089 * TSC run at a constant rate in all ACPI C-states?
5090 */
5091 regs.cp_eax = 0x80000007;
5092 (void) __cpuid_insn(®s);
5093 return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);
5094
5095 default:
5096 return (0);
5097 }
5098 }
5099
5100 #endif /* !__xpv */
5101
5102 void
5103 post_startup_cpu_fixups(void)
5104 {
5105 #ifndef __xpv
5106 /*
5107 * Some AMD processors support C1E state. Entering this state will
5108 * cause the local APIC timer to stop, which we can't deal with at
5109 * this time.
5110 */
5111 if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
5112 on_trap_data_t otd;
5113 uint64_t reg;
5114
5115 if (!on_trap(&otd, OT_DATA_ACCESS)) {
5116 reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
5117 /* Disable C1E state if it is enabled by BIOS */
5118 if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
5119 AMD_ACTONCMPHALT_MASK) {
5120 reg &= ~(AMD_ACTONCMPHALT_MASK <<
5121 AMD_ACTONCMPHALT_SHIFT);
5122 wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
5123 }
5124 }
5125 no_trap();
5126 }
5127 #endif /* !__xpv */
5128 }
5129
5130 void
5131 enable_pcid(void)
5132 {
5133 if (x86_use_pcid == -1)
5134 x86_use_pcid = is_x86_feature(x86_featureset, X86FSET_PCID);
5135
5136 if (x86_use_invpcid == -1) {
5137 x86_use_invpcid = is_x86_feature(x86_featureset,
5138 X86FSET_INVPCID);
5139 }
5140
5141 if (!x86_use_pcid)
5142 return;
5143
5144 /*
5145 * Intel say that on setting PCIDE, it immediately starts using the PCID
5146 * bits; better make sure there's nothing there.
5147 */
5148 ASSERT((getcr3() & MMU_PAGEOFFSET) == PCID_NONE);
5149
5150 setcr4(getcr4() | CR4_PCIDE);
5151 }
5152
5153 /*
5154 * Setup necessary registers to enable XSAVE feature on this processor.
5155 * This function needs to be called early enough, so that no xsave/xrstor
5156 * ops will execute on the processor before the MSRs are properly set up.
5157 *
5158 * Current implementation has the following assumption:
5159 * - cpuid_pass1() is done, so that X86 features are known.
5160 * - fpu_probe() is done, so that fp_save_mech is chosen.
5161 */
5162 void
5163 xsave_setup_msr(cpu_t *cpu)
5164 {
5165 ASSERT(fp_save_mech == FP_XSAVE);
5166 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
5167
5168 /* Enable OSXSAVE in CR4. */
5169 setcr4(getcr4() | CR4_OSXSAVE);
5170 /*
5171 * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
5172 * correct value.
5173 */
5174 cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
5175 setup_xfem();
5176 }
5177
5178 /*
5179 * Starting with the Westmere processor the local
5180 * APIC timer will continue running in all C-states,
5181 * including the deepest C-states.
5182 */
5183 int
5184 cpuid_arat_supported(void)
5185 {
5186 struct cpuid_info *cpi;
5187 struct cpuid_regs regs;
5188
5189 ASSERT(cpuid_checkpass(CPU, 1));
5190 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5191
5192 cpi = CPU->cpu_m.mcpu_cpi;
5193
5194 switch (cpi->cpi_vendor) {
5195 case X86_VENDOR_Intel:
5196 /*
5197 * Always-running Local APIC Timer is
5198 * indicated by CPUID.6.EAX[2].
5199 */
5200 if (cpi->cpi_maxeax >= 6) {
5201 regs.cp_eax = 6;
5202 (void) cpuid_insn(NULL, ®s);
5203 return (regs.cp_eax & CPUID_CSTATE_ARAT);
5204 } else {
5205 return (0);
5206 }
5207 default:
5208 return (0);
5209 }
5210 }
5211
5212 /*
5213 * Check support for Intel ENERGY_PERF_BIAS feature
5214 */
5215 int
5216 cpuid_iepb_supported(struct cpu *cp)
5217 {
5218 struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
5219 struct cpuid_regs regs;
5220
5221 ASSERT(cpuid_checkpass(cp, 1));
5222
5223 if (!(is_x86_feature(x86_featureset, X86FSET_CPUID)) ||
5224 !(is_x86_feature(x86_featureset, X86FSET_MSR))) {
5225 return (0);
5226 }
5227
5228 /*
5229 * Intel ENERGY_PERF_BIAS MSR is indicated by
5230 * capability bit CPUID.6.ECX.3
5231 */
5232 if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
5233 return (0);
5234
5235 regs.cp_eax = 0x6;
5236 (void) cpuid_insn(NULL, ®s);
5237 return (regs.cp_ecx & CPUID_EPB_SUPPORT);
5238 }
5239
5240 /*
5241 * Check support for TSC deadline timer
5242 *
5243 * TSC deadline timer provides a superior software programming
5244 * model over local APIC timer that eliminates "time drifts".
5245 * Instead of specifying a relative time, software specifies an
5246 * absolute time as the target at which the processor should
5247 * generate a timer event.
5248 */
5249 int
5250 cpuid_deadline_tsc_supported(void)
5251 {
5252 struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
5253 struct cpuid_regs regs;
5254
5255 ASSERT(cpuid_checkpass(CPU, 1));
5256 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5257
5258 switch (cpi->cpi_vendor) {
5259 case X86_VENDOR_Intel:
5260 if (cpi->cpi_maxeax >= 1) {
5261 regs.cp_eax = 1;
5262 (void) cpuid_insn(NULL, ®s);
5263 return (regs.cp_ecx & CPUID_DEADLINE_TSC);
5264 } else {
5265 return (0);
5266 }
5267 default:
5268 return (0);
5269 }
5270 }
5271
5272 #if defined(__amd64) && !defined(__xpv)
5273 /*
5274 * Patch in versions of bcopy for high performance Intel Nhm processors
5275 * and later...
5276 */
5277 void
5278 patch_memops(uint_t vendor)
5279 {
5280 size_t cnt, i;
5281 caddr_t to, from;
5282
5283 if ((vendor == X86_VENDOR_Intel) &&
5284 is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
5285 cnt = &bcopy_patch_end - &bcopy_patch_start;
5286 to = &bcopy_ck_size;
5287 from = &bcopy_patch_start;
5288 for (i = 0; i < cnt; i++) {
5289 *to++ = *from++;
5290 }
5291 }
5292 }
5293 #endif /* __amd64 && !__xpv */
5294
5295 /*
5296 * This function finds the number of bits to represent the number of cores per
5297 * chip and the number of strands per core for the Intel platforms.
5298 * It re-uses the x2APIC cpuid code of the cpuid_pass2().
5299 */
5300 void
5301 cpuid_get_ext_topo(uint_t vendor, uint_t *core_nbits, uint_t *strand_nbits)
5302 {
5303 struct cpuid_regs regs;
5304 struct cpuid_regs *cp = ®s;
5305
5306 if (vendor != X86_VENDOR_Intel) {
5307 return;
5308 }
5309
5310 /* if the cpuid level is 0xB, extended topo is available. */
5311 cp->cp_eax = 0;
5312 if (__cpuid_insn(cp) >= 0xB) {
5313
5314 cp->cp_eax = 0xB;
5315 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
5316 (void) __cpuid_insn(cp);
5317
5318 /*
5319 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
5320 * indicates that the extended topology enumeration leaf is
5321 * available.
5322 */
5323 if (cp->cp_ebx) {
5324 uint_t coreid_shift = 0;
5325 uint_t chipid_shift = 0;
5326 uint_t i;
5327 uint_t level;
5328
5329 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
5330 cp->cp_eax = 0xB;
5331 cp->cp_ecx = i;
5332
5333 (void) __cpuid_insn(cp);
5334 level = CPI_CPU_LEVEL_TYPE(cp);
5335
5336 if (level == 1) {
5337 /*
5338 * Thread level processor topology
5339 * Number of bits shift right APIC ID
5340 * to get the coreid.
5341 */
5342 coreid_shift = BITX(cp->cp_eax, 4, 0);
5343 } else if (level == 2) {
5344 /*
5345 * Core level processor topology
5346 * Number of bits shift right APIC ID
5347 * to get the chipid.
5348 */
5349 chipid_shift = BITX(cp->cp_eax, 4, 0);
5350 }
5351 }
5352
5353 if (coreid_shift > 0 && chipid_shift > coreid_shift) {
5354 *strand_nbits = coreid_shift;
5355 *core_nbits = chipid_shift - coreid_shift;
5356 }
5357 }
5358 }
5359 }
5360
5361 void
5362 cpuid_pass_ucode(cpu_t *cpu, uchar_t *fset)
5363 {
5364 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
5365 struct cpuid_regs cp;
5366
5367 /*
5368 * Reread the CPUID portions that we need for various security
5369 * information.
5370 */
5371 if (cpi->cpi_vendor == X86_VENDOR_Intel) {
5372 /*
5373 * Check if we now have leaf 7 available to us.
5374 */
5375 if (cpi->cpi_maxeax < 7) {
5376 bzero(&cp, sizeof (cp));
5377 cp.cp_eax = 0;
5378 cpi->cpi_maxeax = __cpuid_insn(&cp);
5379 if (cpi->cpi_maxeax < 7)
5380 return;
5381 }
5382
5383 bzero(&cp, sizeof (cp));
5384 cp.cp_eax = 7;
5385 cp.cp_ecx = 0;
5386 (void) __cpuid_insn(&cp);
5387 cpi->cpi_std[7] = cp;
5388 } else if (cpi->cpi_vendor == X86_VENDOR_AMD) {
5389 /* No xcpuid support */
5390 if (cpi->cpi_family < 5 ||
5391 (cpi->cpi_family == 5 && cpi->cpi_model < 1))
5392 return;
5393
5394 if (cpi->cpi_xmaxeax < 0x80000008) {
5395 bzero(&cp, sizeof (cp));
5396 cp.cp_eax = 0x80000000;
5397 cpi->cpi_xmaxeax = __cpuid_insn(&cp);
5398 if (cpi->cpi_xmaxeax < 0x80000008) {
5399 return;
5400 }
5401 }
5402
5403 bzero(&cp, sizeof (cp));
5404 cp.cp_eax = 0x80000008;
5405 (void) __cpuid_insn(&cp);
5406 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, &cp);
5407 cpi->cpi_extd[8] = cp;
5408 } else {
5409 /*
5410 * Nothing to do here. Return an empty set which has already
5411 * been zeroed for us.
5412 */
5413 return;
5414 }
5415 cpuid_scan_security(cpu, fset);
5416 }
5417
5418 /* ARGSUSED */
5419 static int
5420 cpuid_post_ucodeadm_xc(xc_arg_t arg0, xc_arg_t arg1, xc_arg_t arg2)
5421 {
5422 uchar_t *fset;
5423
5424 fset = (uchar_t *)(arg0 + sizeof (x86_featureset) * CPU->cpu_id);
5425 cpuid_pass_ucode(CPU, fset);
5426
5427 return (0);
5428 }
5429
5430 /*
5431 * After a microcode update where the version has changed, then we need to
5432 * rescan CPUID. To do this we check every CPU to make sure that they have the
5433 * same microcode. Then we perform a cross call to all such CPUs. It's the
5434 * caller's job to make sure that no one else can end up doing an update while
5435 * this is going on.
5436 *
5437 * We assume that the system is microcode capable if we're called.
5438 */
5439 void
5440 cpuid_post_ucodeadm(void)
5441 {
5442 uint32_t rev;
5443 int i;
5444 struct cpu *cpu;
5445 cpuset_t cpuset;
5446 void *argdata;
5447 uchar_t *f0;
5448
5449 argdata = kmem_zalloc(sizeof (x86_featureset) * NCPU, KM_SLEEP);
5450
5451 mutex_enter(&cpu_lock);
5452 cpu = cpu_get(0);
5453 rev = cpu->cpu_m.mcpu_ucode_info->cui_rev;
5454 CPUSET_ONLY(cpuset, 0);
5455 for (i = 1; i < max_ncpus; i++) {
5456 if ((cpu = cpu_get(i)) == NULL)
5457 continue;
5458
5459 if (cpu->cpu_m.mcpu_ucode_info->cui_rev != rev) {
5460 panic("post microcode update CPU %d has differing "
5461 "microcode revision (%u) from CPU 0 (%u)",
5462 i, cpu->cpu_m.mcpu_ucode_info->cui_rev, rev);
5463 }
5464 CPUSET_ADD(cpuset, i);
5465 }
5466
5467 kpreempt_disable();
5468 xc_sync((xc_arg_t)argdata, 0, 0, CPUSET2BV(cpuset),
5469 cpuid_post_ucodeadm_xc);
5470 kpreempt_enable();
5471
5472 /*
5473 * OK, now look at each CPU and see if their feature sets are equal.
5474 */
5475 f0 = argdata;
5476 for (i = 1; i < max_ncpus; i++) {
5477 uchar_t *fset;
5478 if (!CPU_IN_SET(cpuset, i))
5479 continue;
5480
5481 fset = (uchar_t *)((uintptr_t)argdata +
5482 sizeof (x86_featureset) * i);
5483
5484 if (!compare_x86_featureset(f0, fset)) {
5485 panic("Post microcode update CPU %d has "
5486 "differing security feature (%p) set from CPU 0 "
5487 "(%p), not appending to feature set", i,
5488 (void *)fset, (void *)f0);
5489 }
5490 }
5491
5492 mutex_exit(&cpu_lock);
5493
5494 for (i = 0; i < NUM_X86_FEATURES; i++) {
5495 cmn_err(CE_CONT, "?post-ucode x86_feature: %s\n",
5496 x86_feature_names[i]);
5497 if (is_x86_feature(f0, i)) {
5498 add_x86_feature(x86_featureset, i);
5499 }
5500 }
5501 kmem_free(argdata, sizeof (x86_featureset) * NCPU);
5502 }