1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
26 */
27 /*
28 * Copyright (c) 2010, Intel Corporation.
29 * All rights reserved.
30 */
31 /*
32 * Portions Copyright 2009 Advanced Micro Devices, Inc.
33 */
34 /*
35 * Copyright 2017 Joyent, Inc.
36 */
37 /*
38 * Various routines to handle identification
39 * and classification of x86 processors.
40 */
41
42 #include <sys/types.h>
43 #include <sys/archsystm.h>
44 #include <sys/x86_archext.h>
45 #include <sys/kmem.h>
46 #include <sys/systm.h>
47 #include <sys/cmn_err.h>
48 #include <sys/sunddi.h>
49 #include <sys/sunndi.h>
50 #include <sys/cpuvar.h>
51 #include <sys/processor.h>
52 #include <sys/sysmacros.h>
53 #include <sys/pg.h>
54 #include <sys/fp.h>
55 #include <sys/controlregs.h>
56 #include <sys/bitmap.h>
57 #include <sys/auxv_386.h>
58 #include <sys/memnode.h>
59 #include <sys/pci_cfgspace.h>
60 #include <sys/comm_page.h>
61 #include <sys/tsc.h>
62
63 #ifdef __xpv
64 #include <sys/hypervisor.h>
65 #else
66 #include <sys/ontrap.h>
67 #endif
68
69 /*
70 * Pass 0 of cpuid feature analysis happens in locore. It contains special code
71 * to recognize Cyrix processors that are not cpuid-compliant, and to deal with
72 * them accordingly. For most modern processors, feature detection occurs here
73 * in pass 1.
74 *
75 * Pass 1 of cpuid feature analysis happens just at the beginning of mlsetup()
76 * for the boot CPU and does the basic analysis that the early kernel needs.
77 * x86_featureset is set based on the return value of cpuid_pass1() of the boot
78 * CPU.
79 *
80 * Pass 1 includes:
81 *
82 * o Determining vendor/model/family/stepping and setting x86_type and
83 * x86_vendor accordingly.
84 * o Processing the feature flags returned by the cpuid instruction while
85 * applying any workarounds or tricks for the specific processor.
86 * o Mapping the feature flags into Solaris feature bits (X86_*).
87 * o Processing extended feature flags if supported by the processor,
88 * again while applying specific processor knowledge.
89 * o Determining the CMT characteristics of the system.
90 *
91 * Pass 1 is done on non-boot CPUs during their initialization and the results
92 * are used only as a meager attempt at ensuring that all processors within the
93 * system support the same features.
94 *
95 * Pass 2 of cpuid feature analysis happens just at the beginning
96 * of startup(). It just copies in and corrects the remainder
97 * of the cpuid data we depend on: standard cpuid functions that we didn't
98 * need for pass1 feature analysis, and extended cpuid functions beyond the
99 * simple feature processing done in pass1.
100 *
101 * Pass 3 of cpuid analysis is invoked after basic kernel services; in
102 * particular kernel memory allocation has been made available. It creates a
103 * readable brand string based on the data collected in the first two passes.
104 *
105 * Pass 4 of cpuid analysis is invoked after post_startup() when all
106 * the support infrastructure for various hardware features has been
107 * initialized. It determines which processor features will be reported
108 * to userland via the aux vector.
109 *
110 * All passes are executed on all CPUs, but only the boot CPU determines what
111 * features the kernel will use.
112 *
113 * Much of the worst junk in this file is for the support of processors
114 * that didn't really implement the cpuid instruction properly.
115 *
116 * NOTE: The accessor functions (cpuid_get*) are aware of, and ASSERT upon,
117 * the pass numbers. Accordingly, changes to the pass code may require changes
118 * to the accessor code.
119 */
120
121 uint_t x86_vendor = X86_VENDOR_IntelClone;
122 uint_t x86_type = X86_TYPE_OTHER;
123 uint_t x86_clflush_size = 0;
124
125 uint_t pentiumpro_bug4046376;
126
127 uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
128
129 static char *x86_feature_names[NUM_X86_FEATURES] = {
130 "lgpg",
131 "tsc",
132 "msr",
133 "mtrr",
134 "pge",
135 "de",
136 "cmov",
137 "mmx",
138 "mca",
139 "pae",
140 "cv8",
141 "pat",
142 "sep",
143 "sse",
144 "sse2",
145 "htt",
146 "asysc",
147 "nx",
148 "sse3",
149 "cx16",
150 "cmp",
151 "tscp",
152 "mwait",
153 "sse4a",
154 "cpuid",
155 "ssse3",
156 "sse4_1",
157 "sse4_2",
158 "1gpg",
159 "clfsh",
160 "64",
161 "aes",
162 "pclmulqdq",
163 "xsave",
164 "avx",
165 "vmx",
166 "svm",
167 "topoext",
168 "f16c",
169 "rdrand",
170 "x2apic",
171 "avx2",
172 "bmi1",
173 "bmi2",
174 "fma",
175 "smep",
176 "smap",
177 "adx",
178 "rdseed",
179 "mpx",
180 "avx512f",
181 "avx512dq",
182 "avx512pf",
183 "avx512er",
184 "avx512cd",
185 "avx512bw",
186 "avx512vl",
187 "avx512fma",
188 "avx512vbmi",
189 "avx512_vpopcntdq",
190 "avx512_4vnniw",
191 "avx512_4fmaps",
192 "xsaveopt",
193 "xsavec",
194 "xsaves",
195 "sha",
196 "umip",
197 "pku",
198 "ospke",
199 };
200
201 boolean_t
202 is_x86_feature(void *featureset, uint_t feature)
203 {
204 ASSERT(feature < NUM_X86_FEATURES);
205 return (BT_TEST((ulong_t *)featureset, feature));
206 }
207
208 void
209 add_x86_feature(void *featureset, uint_t feature)
210 {
211 ASSERT(feature < NUM_X86_FEATURES);
212 BT_SET((ulong_t *)featureset, feature);
213 }
214
215 void
216 remove_x86_feature(void *featureset, uint_t feature)
217 {
218 ASSERT(feature < NUM_X86_FEATURES);
219 BT_CLEAR((ulong_t *)featureset, feature);
220 }
221
222 boolean_t
223 compare_x86_featureset(void *setA, void *setB)
224 {
225 /*
226 * We assume that the unused bits of the bitmap are always zero.
227 */
228 if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
229 return (B_TRUE);
230 } else {
231 return (B_FALSE);
232 }
233 }
234
235 void
236 print_x86_featureset(void *featureset)
237 {
238 uint_t i;
239
240 for (i = 0; i < NUM_X86_FEATURES; i++) {
241 if (is_x86_feature(featureset, i)) {
242 cmn_err(CE_CONT, "?x86_feature: %s\n",
243 x86_feature_names[i]);
244 }
245 }
246 }
247
248 /* Note: This is the maximum size for the CPU, not the size of the structure. */
249 static size_t xsave_state_size = 0;
250 uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
251 boolean_t xsave_force_disable = B_FALSE;
252 extern int disable_smap;
253
254 /*
255 * This is set to platform type we are running on.
256 */
257 static int platform_type = -1;
258
259 #if !defined(__xpv)
260 /*
261 * Variable to patch if hypervisor platform detection needs to be
262 * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
263 */
264 int enable_platform_detection = 1;
265 #endif
266
267 /*
268 * monitor/mwait info.
269 *
270 * size_actual and buf_actual are the real address and size allocated to get
271 * proper mwait_buf alignement. buf_actual and size_actual should be passed
272 * to kmem_free(). Currently kmem_alloc() and mwait happen to both use
273 * processor cache-line alignment, but this is not guarantied in the furture.
274 */
275 struct mwait_info {
276 size_t mon_min; /* min size to avoid missed wakeups */
277 size_t mon_max; /* size to avoid false wakeups */
278 size_t size_actual; /* size actually allocated */
279 void *buf_actual; /* memory actually allocated */
280 uint32_t support; /* processor support of monitor/mwait */
281 };
282
283 /*
284 * xsave/xrestor info.
285 *
286 * This structure contains HW feature bits and the size of the xsave save area.
287 * Note: the kernel declares a fixed size (AVX_XSAVE_SIZE) structure
288 * (xsave_state) to describe the xsave layout. However, at runtime the
289 * per-lwp xsave area is dynamically allocated based on xsav_max_size. The
290 * xsave_state structure simply represents the legacy layout of the beginning
291 * of the xsave area.
292 */
293 struct xsave_info {
294 uint32_t xsav_hw_features_low; /* Supported HW features */
295 uint32_t xsav_hw_features_high; /* Supported HW features */
296 size_t xsav_max_size; /* max size save area for HW features */
297 size_t ymm_size; /* AVX: size of ymm save area */
298 size_t ymm_offset; /* AVX: offset for ymm save area */
299 size_t bndregs_size; /* MPX: size of bndregs save area */
300 size_t bndregs_offset; /* MPX: offset for bndregs save area */
301 size_t bndcsr_size; /* MPX: size of bndcsr save area */
302 size_t bndcsr_offset; /* MPX: offset for bndcsr save area */
303 size_t opmask_size; /* AVX512: size of opmask save */
304 size_t opmask_offset; /* AVX512: offset for opmask save */
305 size_t zmmlo_size; /* AVX512: size of zmm 256 save */
306 size_t zmmlo_offset; /* AVX512: offset for zmm 256 save */
307 size_t zmmhi_size; /* AVX512: size of zmm hi reg save */
308 size_t zmmhi_offset; /* AVX512: offset for zmm hi reg save */
309 };
310
311
312 /*
313 * These constants determine how many of the elements of the
314 * cpuid we cache in the cpuid_info data structure; the
315 * remaining elements are accessible via the cpuid instruction.
316 */
317
318 #define NMAX_CPI_STD 8 /* eax = 0 .. 7 */
319 #define NMAX_CPI_EXTD 0x1f /* eax = 0x80000000 .. 0x8000001e */
320
321 /*
322 * Some terminology needs to be explained:
323 * - Socket: Something that can be plugged into a motherboard.
324 * - Package: Same as socket
325 * - Chip: Same as socket. Note that AMD's documentation uses term "chip"
326 * differently: there, chip is the same as processor node (below)
327 * - Processor node: Some AMD processors have more than one
328 * "subprocessor" embedded in a package. These subprocessors (nodes)
329 * are fully-functional processors themselves with cores, caches,
330 * memory controllers, PCI configuration spaces. They are connected
331 * inside the package with Hypertransport links. On single-node
332 * processors, processor node is equivalent to chip/socket/package.
333 * - Compute Unit: Some AMD processors pair cores in "compute units" that
334 * share the FPU and the I$ and L2 caches.
335 */
336
337 struct cpuid_info {
338 uint_t cpi_pass; /* last pass completed */
339 /*
340 * standard function information
341 */
342 uint_t cpi_maxeax; /* fn 0: %eax */
343 char cpi_vendorstr[13]; /* fn 0: %ebx:%ecx:%edx */
344 uint_t cpi_vendor; /* enum of cpi_vendorstr */
345
346 uint_t cpi_family; /* fn 1: extended family */
347 uint_t cpi_model; /* fn 1: extended model */
348 uint_t cpi_step; /* fn 1: stepping */
349 chipid_t cpi_chipid; /* fn 1: %ebx: Intel: chip # */
350 /* AMD: package/socket # */
351 uint_t cpi_brandid; /* fn 1: %ebx: brand ID */
352 int cpi_clogid; /* fn 1: %ebx: thread # */
353 uint_t cpi_ncpu_per_chip; /* fn 1: %ebx: logical cpu count */
354 uint8_t cpi_cacheinfo[16]; /* fn 2: intel-style cache desc */
355 uint_t cpi_ncache; /* fn 2: number of elements */
356 uint_t cpi_ncpu_shr_last_cache; /* fn 4: %eax: ncpus sharing cache */
357 id_t cpi_last_lvl_cacheid; /* fn 4: %eax: derived cache id */
358 uint_t cpi_std_4_size; /* fn 4: number of fn 4 elements */
359 struct cpuid_regs **cpi_std_4; /* fn 4: %ecx == 0 .. fn4_size */
360 struct cpuid_regs cpi_std[NMAX_CPI_STD]; /* 0 .. 7 */
361 /*
362 * extended function information
363 */
364 uint_t cpi_xmaxeax; /* fn 0x80000000: %eax */
365 char cpi_brandstr[49]; /* fn 0x8000000[234] */
366 uint8_t cpi_pabits; /* fn 0x80000006: %eax */
367 uint8_t cpi_vabits; /* fn 0x80000006: %eax */
368 uint8_t cpi_fp_amd_save; /* AMD: FP error pointer save rqd. */
369 struct cpuid_regs cpi_extd[NMAX_CPI_EXTD]; /* 0x800000XX */
370
371 id_t cpi_coreid; /* same coreid => strands share core */
372 int cpi_pkgcoreid; /* core number within single package */
373 uint_t cpi_ncore_per_chip; /* AMD: fn 0x80000008: %ecx[7-0] */
374 /* Intel: fn 4: %eax[31-26] */
375 /*
376 * supported feature information
377 */
378 uint32_t cpi_support[6];
379 #define STD_EDX_FEATURES 0
380 #define AMD_EDX_FEATURES 1
381 #define TM_EDX_FEATURES 2
382 #define STD_ECX_FEATURES 3
383 #define AMD_ECX_FEATURES 4
384 #define STD_EBX_FEATURES 5
385 /*
386 * Synthesized information, where known.
387 */
388 uint32_t cpi_chiprev; /* See X86_CHIPREV_* in x86_archext.h */
389 const char *cpi_chiprevstr; /* May be NULL if chiprev unknown */
390 uint32_t cpi_socket; /* Chip package/socket type */
391
392 struct mwait_info cpi_mwait; /* fn 5: monitor/mwait info */
393 uint32_t cpi_apicid;
394 uint_t cpi_procnodeid; /* AMD: nodeID on HT, Intel: chipid */
395 uint_t cpi_procnodes_per_pkg; /* AMD: # of nodes in the package */
396 /* Intel: 1 */
397 uint_t cpi_compunitid; /* AMD: ComputeUnit ID, Intel: coreid */
398 uint_t cpi_cores_per_compunit; /* AMD: # of cores in the ComputeUnit */
399
400 struct xsave_info cpi_xsave; /* fn D: xsave/xrestor info */
401 };
402
403
404 static struct cpuid_info cpuid_info0;
405
406 /*
407 * These bit fields are defined by the Intel Application Note AP-485
408 * "Intel Processor Identification and the CPUID Instruction"
409 */
410 #define CPI_FAMILY_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
411 #define CPI_MODEL_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
412 #define CPI_TYPE(cpi) BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
413 #define CPI_FAMILY(cpi) BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
414 #define CPI_STEP(cpi) BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
415 #define CPI_MODEL(cpi) BITX((cpi)->cpi_std[1].cp_eax, 7, 4)
416
417 #define CPI_FEATURES_EDX(cpi) ((cpi)->cpi_std[1].cp_edx)
418 #define CPI_FEATURES_ECX(cpi) ((cpi)->cpi_std[1].cp_ecx)
419 #define CPI_FEATURES_XTD_EDX(cpi) ((cpi)->cpi_extd[1].cp_edx)
420 #define CPI_FEATURES_XTD_ECX(cpi) ((cpi)->cpi_extd[1].cp_ecx)
421 #define CPI_FEATURES_7_0_EBX(cpi) ((cpi)->cpi_std[7].cp_ebx)
422 #define CPI_FEATURES_7_0_ECX(cpi) ((cpi)->cpi_std[7].cp_ecx)
423 #define CPI_FEATURES_7_0_EDX(cpi) ((cpi)->cpi_std[7].cp_edx)
424
425 #define CPI_BRANDID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
426 #define CPI_CHUNKS(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
427 #define CPI_CPU_COUNT(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
428 #define CPI_APIC_ID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)
429
430 #define CPI_MAXEAX_MAX 0x100 /* sanity control */
431 #define CPI_XMAXEAX_MAX 0x80000100
432 #define CPI_FN4_ECX_MAX 0x20 /* sanity: max fn 4 levels */
433 #define CPI_FNB_ECX_MAX 0x20 /* sanity: max fn B levels */
434
435 /*
436 * Function 4 (Deterministic Cache Parameters) macros
437 * Defined by Intel Application Note AP-485
438 */
439 #define CPI_NUM_CORES(regs) BITX((regs)->cp_eax, 31, 26)
440 #define CPI_NTHR_SHR_CACHE(regs) BITX((regs)->cp_eax, 25, 14)
441 #define CPI_FULL_ASSOC_CACHE(regs) BITX((regs)->cp_eax, 9, 9)
442 #define CPI_SELF_INIT_CACHE(regs) BITX((regs)->cp_eax, 8, 8)
443 #define CPI_CACHE_LVL(regs) BITX((regs)->cp_eax, 7, 5)
444 #define CPI_CACHE_TYPE(regs) BITX((regs)->cp_eax, 4, 0)
445 #define CPI_CPU_LEVEL_TYPE(regs) BITX((regs)->cp_ecx, 15, 8)
446
447 #define CPI_CACHE_WAYS(regs) BITX((regs)->cp_ebx, 31, 22)
448 #define CPI_CACHE_PARTS(regs) BITX((regs)->cp_ebx, 21, 12)
449 #define CPI_CACHE_COH_LN_SZ(regs) BITX((regs)->cp_ebx, 11, 0)
450
451 #define CPI_CACHE_SETS(regs) BITX((regs)->cp_ecx, 31, 0)
452
453 #define CPI_PREFCH_STRIDE(regs) BITX((regs)->cp_edx, 9, 0)
454
455
456 /*
457 * A couple of shorthand macros to identify "later" P6-family chips
458 * like the Pentium M and Core. First, the "older" P6-based stuff
459 * (loosely defined as "pre-Pentium-4"):
460 * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
461 */
462 #define IS_LEGACY_P6(cpi) ( \
463 cpi->cpi_family == 6 && \
464 (cpi->cpi_model == 1 || \
465 cpi->cpi_model == 3 || \
466 cpi->cpi_model == 5 || \
467 cpi->cpi_model == 6 || \
468 cpi->cpi_model == 7 || \
469 cpi->cpi_model == 8 || \
470 cpi->cpi_model == 0xA || \
471 cpi->cpi_model == 0xB) \
472 )
473
474 /* A "new F6" is everything with family 6 that's not the above */
475 #define IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))
476
477 /* Extended family/model support */
478 #define IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
479 cpi->cpi_family >= 0xf)
480
481 /*
482 * Info for monitor/mwait idle loop.
483 *
484 * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
485 * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
486 * 2006.
487 * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
488 * Documentation Updates" #33633, Rev 2.05, December 2006.
489 */
490 #define MWAIT_SUPPORT (0x00000001) /* mwait supported */
491 #define MWAIT_EXTENSIONS (0x00000002) /* extenstion supported */
492 #define MWAIT_ECX_INT_ENABLE (0x00000004) /* ecx 1 extension supported */
493 #define MWAIT_SUPPORTED(cpi) ((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
494 #define MWAIT_INT_ENABLE(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x2)
495 #define MWAIT_EXTENSION(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x1)
496 #define MWAIT_SIZE_MIN(cpi) BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
497 #define MWAIT_SIZE_MAX(cpi) BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
498 /*
499 * Number of sub-cstates for a given c-state.
500 */
501 #define MWAIT_NUM_SUBC_STATES(cpi, c_state) \
502 BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)
503
504 /*
505 * XSAVE leaf 0xD enumeration
506 */
507 #define CPUID_LEAFD_2_YMM_OFFSET 576
508 #define CPUID_LEAFD_2_YMM_SIZE 256
509
510 /*
511 * Functions we consune from cpuid_subr.c; don't publish these in a header
512 * file to try and keep people using the expected cpuid_* interfaces.
513 */
514 extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
515 extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
516 extern uint32_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
517 extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
518 extern uint_t _cpuid_vendorstr_to_vendorcode(char *);
519
520 /*
521 * Apply up various platform-dependent restrictions where the
522 * underlying platform restrictions mean the CPU can be marked
523 * as less capable than its cpuid instruction would imply.
524 */
525 #if defined(__xpv)
526 static void
527 platform_cpuid_mangle(uint_t vendor, uint32_t eax, struct cpuid_regs *cp)
528 {
529 switch (eax) {
530 case 1: {
531 uint32_t mcamask = DOMAIN_IS_INITDOMAIN(xen_info) ?
532 0 : CPUID_INTC_EDX_MCA;
533 cp->cp_edx &=
534 ~(mcamask |
535 CPUID_INTC_EDX_PSE |
536 CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
537 CPUID_INTC_EDX_SEP | CPUID_INTC_EDX_MTRR |
538 CPUID_INTC_EDX_PGE | CPUID_INTC_EDX_PAT |
539 CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
540 CPUID_INTC_EDX_PSE36 | CPUID_INTC_EDX_HTT);
541 break;
542 }
543
544 case 0x80000001:
545 cp->cp_edx &=
546 ~(CPUID_AMD_EDX_PSE |
547 CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
548 CPUID_AMD_EDX_MTRR | CPUID_AMD_EDX_PGE |
549 CPUID_AMD_EDX_PAT | CPUID_AMD_EDX_PSE36 |
550 CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
551 CPUID_AMD_EDX_TSCP);
552 cp->cp_ecx &= ~CPUID_AMD_ECX_CMP_LGCY;
553 break;
554 default:
555 break;
556 }
557
558 switch (vendor) {
559 case X86_VENDOR_Intel:
560 switch (eax) {
561 case 4:
562 /*
563 * Zero out the (ncores-per-chip - 1) field
564 */
565 cp->cp_eax &= 0x03fffffff;
566 break;
567 default:
568 break;
569 }
570 break;
571 case X86_VENDOR_AMD:
572 switch (eax) {
573
574 case 0x80000001:
575 cp->cp_ecx &= ~CPUID_AMD_ECX_CR8D;
576 break;
577
578 case 0x80000008:
579 /*
580 * Zero out the (ncores-per-chip - 1) field
581 */
582 cp->cp_ecx &= 0xffffff00;
583 break;
584 default:
585 break;
586 }
587 break;
588 default:
589 break;
590 }
591 }
592 #else
593 #define platform_cpuid_mangle(vendor, eax, cp) /* nothing */
594 #endif
595
596 /*
597 * Some undocumented ways of patching the results of the cpuid
598 * instruction to permit running Solaris 10 on future cpus that
599 * we don't currently support. Could be set to non-zero values
600 * via settings in eeprom.
601 */
602
603 uint32_t cpuid_feature_ecx_include;
604 uint32_t cpuid_feature_ecx_exclude;
605 uint32_t cpuid_feature_edx_include;
606 uint32_t cpuid_feature_edx_exclude;
607
608 /*
609 * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
610 */
611 void
612 cpuid_alloc_space(cpu_t *cpu)
613 {
614 /*
615 * By convention, cpu0 is the boot cpu, which is set up
616 * before memory allocation is available. All other cpus get
617 * their cpuid_info struct allocated here.
618 */
619 ASSERT(cpu->cpu_id != 0);
620 ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
621 cpu->cpu_m.mcpu_cpi =
622 kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
623 }
624
625 void
626 cpuid_free_space(cpu_t *cpu)
627 {
628 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
629 int i;
630
631 ASSERT(cpi != NULL);
632 ASSERT(cpi != &cpuid_info0);
633
634 /*
635 * Free up any function 4 related dynamic storage
636 */
637 for (i = 1; i < cpi->cpi_std_4_size; i++)
638 kmem_free(cpi->cpi_std_4[i], sizeof (struct cpuid_regs));
639 if (cpi->cpi_std_4_size > 0)
640 kmem_free(cpi->cpi_std_4,
641 cpi->cpi_std_4_size * sizeof (struct cpuid_regs *));
642
643 kmem_free(cpi, sizeof (*cpi));
644 cpu->cpu_m.mcpu_cpi = NULL;
645 }
646
647 #if !defined(__xpv)
648 /*
649 * Determine the type of the underlying platform. This is used to customize
650 * initialization of various subsystems (e.g. TSC). determine_platform() must
651 * only ever be called once to prevent two processors from seeing different
652 * values of platform_type. Must be called before cpuid_pass1(), the earliest
653 * consumer to execute (uses _cpuid_chiprev --> synth_amd_info --> get_hwenv).
654 */
655 void
656 determine_platform(void)
657 {
658 struct cpuid_regs cp;
659 uint32_t base;
660 uint32_t regs[4];
661 char *hvstr = (char *)regs;
662
663 ASSERT(platform_type == -1);
664
665 platform_type = HW_NATIVE;
666
667 if (!enable_platform_detection)
668 return;
669
670 /*
671 * If Hypervisor CPUID bit is set, try to determine hypervisor
672 * vendor signature, and set platform type accordingly.
673 *
674 * References:
675 * http://lkml.org/lkml/2008/10/1/246
676 * http://kb.vmware.com/kb/1009458
677 */
678 cp.cp_eax = 0x1;
679 (void) __cpuid_insn(&cp);
680 if ((cp.cp_ecx & CPUID_INTC_ECX_HV) != 0) {
681 cp.cp_eax = 0x40000000;
682 (void) __cpuid_insn(&cp);
683 regs[0] = cp.cp_ebx;
684 regs[1] = cp.cp_ecx;
685 regs[2] = cp.cp_edx;
686 regs[3] = 0;
687 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0) {
688 platform_type = HW_XEN_HVM;
689 return;
690 }
691 if (strcmp(hvstr, HVSIG_VMWARE) == 0) {
692 platform_type = HW_VMWARE;
693 return;
694 }
695 if (strcmp(hvstr, HVSIG_KVM) == 0) {
696 platform_type = HW_KVM;
697 return;
698 }
699 if (strcmp(hvstr, HVSIG_MICROSOFT) == 0)
700 platform_type = HW_MICROSOFT;
701 } else {
702 /*
703 * Check older VMware hardware versions. VMware hypervisor is
704 * detected by performing an IN operation to VMware hypervisor
705 * port and checking that value returned in %ebx is VMware
706 * hypervisor magic value.
707 *
708 * References: http://kb.vmware.com/kb/1009458
709 */
710 vmware_port(VMWARE_HVCMD_GETVERSION, regs);
711 if (regs[1] == VMWARE_HVMAGIC) {
712 platform_type = HW_VMWARE;
713 return;
714 }
715 }
716
717 /*
718 * Check Xen hypervisor. In a fully virtualized domain,
719 * Xen's pseudo-cpuid function returns a string representing the
720 * Xen signature in %ebx, %ecx, and %edx. %eax contains the maximum
721 * supported cpuid function. We need at least a (base + 2) leaf value
722 * to do what we want to do. Try different base values, since the
723 * hypervisor might use a different one depending on whether Hyper-V
724 * emulation is switched on by default or not.
725 */
726 for (base = 0x40000000; base < 0x40010000; base += 0x100) {
727 cp.cp_eax = base;
728 (void) __cpuid_insn(&cp);
729 regs[0] = cp.cp_ebx;
730 regs[1] = cp.cp_ecx;
731 regs[2] = cp.cp_edx;
732 regs[3] = 0;
733 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0 &&
734 cp.cp_eax >= (base + 2)) {
735 platform_type &= ~HW_NATIVE;
736 platform_type |= HW_XEN_HVM;
737 return;
738 }
739 }
740 }
741
742 int
743 get_hwenv(void)
744 {
745 ASSERT(platform_type != -1);
746 return (platform_type);
747 }
748
749 int
750 is_controldom(void)
751 {
752 return (0);
753 }
754
755 #else
756
757 int
758 get_hwenv(void)
759 {
760 return (HW_XEN_PV);
761 }
762
763 int
764 is_controldom(void)
765 {
766 return (DOMAIN_IS_INITDOMAIN(xen_info));
767 }
768
769 #endif /* __xpv */
770
771 static void
772 cpuid_intel_getids(cpu_t *cpu, void *feature)
773 {
774 uint_t i;
775 uint_t chipid_shift = 0;
776 uint_t coreid_shift = 0;
777 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
778
779 for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
780 chipid_shift++;
781
782 cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
783 cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);
784
785 if (is_x86_feature(feature, X86FSET_CMP)) {
786 /*
787 * Multi-core (and possibly multi-threaded)
788 * processors.
789 */
790 uint_t ncpu_per_core;
791 if (cpi->cpi_ncore_per_chip == 1)
792 ncpu_per_core = cpi->cpi_ncpu_per_chip;
793 else if (cpi->cpi_ncore_per_chip > 1)
794 ncpu_per_core = cpi->cpi_ncpu_per_chip /
795 cpi->cpi_ncore_per_chip;
796 /*
797 * 8bit APIC IDs on dual core Pentiums
798 * look like this:
799 *
800 * +-----------------------+------+------+
801 * | Physical Package ID | MC | HT |
802 * +-----------------------+------+------+
803 * <------- chipid -------->
804 * <------- coreid --------------->
805 * <--- clogid -->
806 * <------>
807 * pkgcoreid
808 *
809 * Where the number of bits necessary to
810 * represent MC and HT fields together equals
811 * to the minimum number of bits necessary to
812 * store the value of cpi->cpi_ncpu_per_chip.
813 * Of those bits, the MC part uses the number
814 * of bits necessary to store the value of
815 * cpi->cpi_ncore_per_chip.
816 */
817 for (i = 1; i < ncpu_per_core; i <<= 1)
818 coreid_shift++;
819 cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
820 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
821 } else if (is_x86_feature(feature, X86FSET_HTT)) {
822 /*
823 * Single-core multi-threaded processors.
824 */
825 cpi->cpi_coreid = cpi->cpi_chipid;
826 cpi->cpi_pkgcoreid = 0;
827 }
828 cpi->cpi_procnodeid = cpi->cpi_chipid;
829 cpi->cpi_compunitid = cpi->cpi_coreid;
830 }
831
832 static void
833 cpuid_amd_getids(cpu_t *cpu)
834 {
835 int i, first_half, coreidsz;
836 uint32_t nb_caps_reg;
837 uint_t node2_1;
838 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
839 struct cpuid_regs *cp;
840
841 /*
842 * AMD CMP chips currently have a single thread per core.
843 *
844 * Since no two cpus share a core we must assign a distinct coreid
845 * per cpu, and we do this by using the cpu_id. This scheme does not,
846 * however, guarantee that sibling cores of a chip will have sequential
847 * coreids starting at a multiple of the number of cores per chip -
848 * that is usually the case, but if the ACPI MADT table is presented
849 * in a different order then we need to perform a few more gymnastics
850 * for the pkgcoreid.
851 *
852 * All processors in the system have the same number of enabled
853 * cores. Cores within a processor are always numbered sequentially
854 * from 0 regardless of how many or which are disabled, and there
855 * is no way for operating system to discover the real core id when some
856 * are disabled.
857 *
858 * In family 0x15, the cores come in pairs called compute units. They
859 * share I$ and L2 caches and the FPU. Enumeration of this feature is
860 * simplified by the new topology extensions CPUID leaf, indicated by
861 * the X86 feature X86FSET_TOPOEXT.
862 */
863
864 cpi->cpi_coreid = cpu->cpu_id;
865 cpi->cpi_compunitid = cpu->cpu_id;
866
867 if (cpi->cpi_xmaxeax >= 0x80000008) {
868
869 coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);
870
871 /*
872 * In AMD parlance chip is really a node while Solaris
873 * sees chip as equivalent to socket/package.
874 */
875 cpi->cpi_ncore_per_chip =
876 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
877 if (coreidsz == 0) {
878 /* Use legacy method */
879 for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
880 coreidsz++;
881 if (coreidsz == 0)
882 coreidsz = 1;
883 }
884 } else {
885 /* Assume single-core part */
886 cpi->cpi_ncore_per_chip = 1;
887 coreidsz = 1;
888 }
889
890 cpi->cpi_clogid = cpi->cpi_pkgcoreid =
891 cpi->cpi_apicid & ((1<<coreidsz) - 1);
892 cpi->cpi_ncpu_per_chip = cpi->cpi_ncore_per_chip;
893
894 /* Get node ID, compute unit ID */
895 if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
896 cpi->cpi_xmaxeax >= 0x8000001e) {
897 cp = &cpi->cpi_extd[0x1e];
898 cp->cp_eax = 0x8000001e;
899 (void) __cpuid_insn(cp);
900
901 cpi->cpi_procnodes_per_pkg = BITX(cp->cp_ecx, 10, 8) + 1;
902 cpi->cpi_procnodeid = BITX(cp->cp_ecx, 7, 0);
903 cpi->cpi_cores_per_compunit = BITX(cp->cp_ebx, 15, 8) + 1;
904 cpi->cpi_compunitid = BITX(cp->cp_ebx, 7, 0)
905 + (cpi->cpi_ncore_per_chip / cpi->cpi_cores_per_compunit)
906 * (cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg);
907 } else if (cpi->cpi_family == 0xf || cpi->cpi_family >= 0x11) {
908 cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
909 } else if (cpi->cpi_family == 0x10) {
910 /*
911 * See if we are a multi-node processor.
912 * All processors in the system have the same number of nodes
913 */
914 nb_caps_reg = pci_getl_func(0, 24, 3, 0xe8);
915 if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
916 /* Single-node */
917 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
918 coreidsz);
919 } else {
920
921 /*
922 * Multi-node revision D (2 nodes per package
923 * are supported)
924 */
925 cpi->cpi_procnodes_per_pkg = 2;
926
927 first_half = (cpi->cpi_pkgcoreid <=
928 (cpi->cpi_ncore_per_chip/2 - 1));
929
930 if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
931 /* We are BSP */
932 cpi->cpi_procnodeid = (first_half ? 0 : 1);
933 } else {
934
935 /* We are AP */
936 /* NodeId[2:1] bits to use for reading F3xe8 */
937 node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;
938
939 nb_caps_reg =
940 pci_getl_func(0, 24 + node2_1, 3, 0xe8);
941
942 /*
943 * Check IntNodeNum bit (31:30, but bit 31 is
944 * always 0 on dual-node processors)
945 */
946 if (BITX(nb_caps_reg, 30, 30) == 0)
947 cpi->cpi_procnodeid = node2_1 +
948 !first_half;
949 else
950 cpi->cpi_procnodeid = node2_1 +
951 first_half;
952 }
953 }
954 } else {
955 cpi->cpi_procnodeid = 0;
956 }
957
958 cpi->cpi_chipid =
959 cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg;
960 }
961
962 /*
963 * Setup XFeature_Enabled_Mask register. Required by xsave feature.
964 */
965 void
966 setup_xfem(void)
967 {
968 uint64_t flags = XFEATURE_LEGACY_FP;
969
970 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
971
972 if (is_x86_feature(x86_featureset, X86FSET_SSE))
973 flags |= XFEATURE_SSE;
974
975 if (is_x86_feature(x86_featureset, X86FSET_AVX))
976 flags |= XFEATURE_AVX;
977
978 if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
979 flags |= XFEATURE_AVX512;
980
981 set_xcr(XFEATURE_ENABLED_MASK, flags);
982
983 xsave_bv_all = flags;
984 }
985
986 void
987 cpuid_pass1(cpu_t *cpu, uchar_t *featureset)
988 {
989 uint32_t mask_ecx, mask_edx;
990 struct cpuid_info *cpi;
991 struct cpuid_regs *cp;
992 int xcpuid;
993 #if !defined(__xpv)
994 extern int idle_cpu_prefer_mwait;
995 #endif
996
997 /*
998 * Space statically allocated for BSP, ensure pointer is set
999 */
1000 if (cpu->cpu_id == 0) {
1001 if (cpu->cpu_m.mcpu_cpi == NULL)
1002 cpu->cpu_m.mcpu_cpi = &cpuid_info0;
1003 }
1004
1005 add_x86_feature(featureset, X86FSET_CPUID);
1006
1007 cpi = cpu->cpu_m.mcpu_cpi;
1008 ASSERT(cpi != NULL);
1009 cp = &cpi->cpi_std[0];
1010 cp->cp_eax = 0;
1011 cpi->cpi_maxeax = __cpuid_insn(cp);
1012 {
1013 uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
1014 *iptr++ = cp->cp_ebx;
1015 *iptr++ = cp->cp_edx;
1016 *iptr++ = cp->cp_ecx;
1017 *(char *)&cpi->cpi_vendorstr[12] = '\0';
1018 }
1019
1020 cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
1021 x86_vendor = cpi->cpi_vendor; /* for compatibility */
1022
1023 /*
1024 * Limit the range in case of weird hardware
1025 */
1026 if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
1027 cpi->cpi_maxeax = CPI_MAXEAX_MAX;
1028 if (cpi->cpi_maxeax < 1)
1029 goto pass1_done;
1030
1031 cp = &cpi->cpi_std[1];
1032 cp->cp_eax = 1;
1033 (void) __cpuid_insn(cp);
1034
1035 /*
1036 * Extract identifying constants for easy access.
1037 */
1038 cpi->cpi_model = CPI_MODEL(cpi);
1039 cpi->cpi_family = CPI_FAMILY(cpi);
1040
1041 if (cpi->cpi_family == 0xf)
1042 cpi->cpi_family += CPI_FAMILY_XTD(cpi);
1043
1044 /*
1045 * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
1046 * Intel, and presumably everyone else, uses model == 0xf, as
1047 * one would expect (max value means possible overflow). Sigh.
1048 */
1049
1050 switch (cpi->cpi_vendor) {
1051 case X86_VENDOR_Intel:
1052 if (IS_EXTENDED_MODEL_INTEL(cpi))
1053 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1054 break;
1055 case X86_VENDOR_AMD:
1056 if (CPI_FAMILY(cpi) == 0xf)
1057 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1058 break;
1059 default:
1060 if (cpi->cpi_model == 0xf)
1061 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1062 break;
1063 }
1064
1065 cpi->cpi_step = CPI_STEP(cpi);
1066 cpi->cpi_brandid = CPI_BRANDID(cpi);
1067
1068 /*
1069 * *default* assumptions:
1070 * - believe %edx feature word
1071 * - ignore %ecx feature word
1072 * - 32-bit virtual and physical addressing
1073 */
1074 mask_edx = 0xffffffff;
1075 mask_ecx = 0;
1076
1077 cpi->cpi_pabits = cpi->cpi_vabits = 32;
1078
1079 switch (cpi->cpi_vendor) {
1080 case X86_VENDOR_Intel:
1081 if (cpi->cpi_family == 5)
1082 x86_type = X86_TYPE_P5;
1083 else if (IS_LEGACY_P6(cpi)) {
1084 x86_type = X86_TYPE_P6;
1085 pentiumpro_bug4046376 = 1;
1086 /*
1087 * Clear the SEP bit when it was set erroneously
1088 */
1089 if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
1090 cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
1091 } else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
1092 x86_type = X86_TYPE_P4;
1093 /*
1094 * We don't currently depend on any of the %ecx
1095 * features until Prescott, so we'll only check
1096 * this from P4 onwards. We might want to revisit
1097 * that idea later.
1098 */
1099 mask_ecx = 0xffffffff;
1100 } else if (cpi->cpi_family > 0xf)
1101 mask_ecx = 0xffffffff;
1102 /*
1103 * We don't support MONITOR/MWAIT if leaf 5 is not available
1104 * to obtain the monitor linesize.
1105 */
1106 if (cpi->cpi_maxeax < 5)
1107 mask_ecx &= ~CPUID_INTC_ECX_MON;
1108 break;
1109 case X86_VENDOR_IntelClone:
1110 default:
1111 break;
1112 case X86_VENDOR_AMD:
1113 #if defined(OPTERON_ERRATUM_108)
1114 if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
1115 cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
1116 cpi->cpi_model = 0xc;
1117 } else
1118 #endif
1119 if (cpi->cpi_family == 5) {
1120 /*
1121 * AMD K5 and K6
1122 *
1123 * These CPUs have an incomplete implementation
1124 * of MCA/MCE which we mask away.
1125 */
1126 mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);
1127
1128 /*
1129 * Model 0 uses the wrong (APIC) bit
1130 * to indicate PGE. Fix it here.
1131 */
1132 if (cpi->cpi_model == 0) {
1133 if (cp->cp_edx & 0x200) {
1134 cp->cp_edx &= ~0x200;
1135 cp->cp_edx |= CPUID_INTC_EDX_PGE;
1136 }
1137 }
1138
1139 /*
1140 * Early models had problems w/ MMX; disable.
1141 */
1142 if (cpi->cpi_model < 6)
1143 mask_edx &= ~CPUID_INTC_EDX_MMX;
1144 }
1145
1146 /*
1147 * For newer families, SSE3 and CX16, at least, are valid;
1148 * enable all
1149 */
1150 if (cpi->cpi_family >= 0xf)
1151 mask_ecx = 0xffffffff;
1152 /*
1153 * We don't support MONITOR/MWAIT if leaf 5 is not available
1154 * to obtain the monitor linesize.
1155 */
1156 if (cpi->cpi_maxeax < 5)
1157 mask_ecx &= ~CPUID_INTC_ECX_MON;
1158
1159 #if !defined(__xpv)
1160 /*
1161 * Do not use MONITOR/MWAIT to halt in the idle loop on any AMD
1162 * processors. AMD does not intend MWAIT to be used in the cpu
1163 * idle loop on current and future processors. 10h and future
1164 * AMD processors use more power in MWAIT than HLT.
1165 * Pre-family-10h Opterons do not have the MWAIT instruction.
1166 */
1167 idle_cpu_prefer_mwait = 0;
1168 #endif
1169
1170 break;
1171 case X86_VENDOR_TM:
1172 /*
1173 * workaround the NT workaround in CMS 4.1
1174 */
1175 if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
1176 (cpi->cpi_step == 2 || cpi->cpi_step == 3))
1177 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1178 break;
1179 case X86_VENDOR_Centaur:
1180 /*
1181 * workaround the NT workarounds again
1182 */
1183 if (cpi->cpi_family == 6)
1184 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1185 break;
1186 case X86_VENDOR_Cyrix:
1187 /*
1188 * We rely heavily on the probing in locore
1189 * to actually figure out what parts, if any,
1190 * of the Cyrix cpuid instruction to believe.
1191 */
1192 switch (x86_type) {
1193 case X86_TYPE_CYRIX_486:
1194 mask_edx = 0;
1195 break;
1196 case X86_TYPE_CYRIX_6x86:
1197 mask_edx = 0;
1198 break;
1199 case X86_TYPE_CYRIX_6x86L:
1200 mask_edx =
1201 CPUID_INTC_EDX_DE |
1202 CPUID_INTC_EDX_CX8;
1203 break;
1204 case X86_TYPE_CYRIX_6x86MX:
1205 mask_edx =
1206 CPUID_INTC_EDX_DE |
1207 CPUID_INTC_EDX_MSR |
1208 CPUID_INTC_EDX_CX8 |
1209 CPUID_INTC_EDX_PGE |
1210 CPUID_INTC_EDX_CMOV |
1211 CPUID_INTC_EDX_MMX;
1212 break;
1213 case X86_TYPE_CYRIX_GXm:
1214 mask_edx =
1215 CPUID_INTC_EDX_MSR |
1216 CPUID_INTC_EDX_CX8 |
1217 CPUID_INTC_EDX_CMOV |
1218 CPUID_INTC_EDX_MMX;
1219 break;
1220 case X86_TYPE_CYRIX_MediaGX:
1221 break;
1222 case X86_TYPE_CYRIX_MII:
1223 case X86_TYPE_VIA_CYRIX_III:
1224 mask_edx =
1225 CPUID_INTC_EDX_DE |
1226 CPUID_INTC_EDX_TSC |
1227 CPUID_INTC_EDX_MSR |
1228 CPUID_INTC_EDX_CX8 |
1229 CPUID_INTC_EDX_PGE |
1230 CPUID_INTC_EDX_CMOV |
1231 CPUID_INTC_EDX_MMX;
1232 break;
1233 default:
1234 break;
1235 }
1236 break;
1237 }
1238
1239 #if defined(__xpv)
1240 /*
1241 * Do not support MONITOR/MWAIT under a hypervisor
1242 */
1243 mask_ecx &= ~CPUID_INTC_ECX_MON;
1244 /*
1245 * Do not support XSAVE under a hypervisor for now
1246 */
1247 xsave_force_disable = B_TRUE;
1248
1249 #endif /* __xpv */
1250
1251 if (xsave_force_disable) {
1252 mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
1253 mask_ecx &= ~CPUID_INTC_ECX_AVX;
1254 mask_ecx &= ~CPUID_INTC_ECX_F16C;
1255 mask_ecx &= ~CPUID_INTC_ECX_FMA;
1256 }
1257
1258 /*
1259 * Now we've figured out the masks that determine
1260 * which bits we choose to believe, apply the masks
1261 * to the feature words, then map the kernel's view
1262 * of these feature words into its feature word.
1263 */
1264 cp->cp_edx &= mask_edx;
1265 cp->cp_ecx &= mask_ecx;
1266
1267 /*
1268 * apply any platform restrictions (we don't call this
1269 * immediately after __cpuid_insn here, because we need the
1270 * workarounds applied above first)
1271 */
1272 platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);
1273
1274 /*
1275 * In addition to ecx and edx, Intel is storing a bunch of instruction
1276 * set extensions in leaf 7's ebx, ecx, and edx.
1277 */
1278 if (cpi->cpi_vendor == X86_VENDOR_Intel && cpi->cpi_maxeax >= 7) {
1279 struct cpuid_regs *ecp;
1280 ecp = &cpi->cpi_std[7];
1281 ecp->cp_eax = 7;
1282 ecp->cp_ecx = 0;
1283 (void) __cpuid_insn(ecp);
1284 /*
1285 * If XSAVE has been disabled, just ignore all of the
1286 * extended-save-area dependent flags here.
1287 */
1288 if (xsave_force_disable) {
1289 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
1290 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
1291 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
1292 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_MPX;
1293 ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_ALL_AVX512;
1294 ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_ALL_AVX512;
1295 ecp->cp_edx &= ~CPUID_INTC_EDX_7_0_ALL_AVX512;
1296 }
1297
1298 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMEP)
1299 add_x86_feature(featureset, X86FSET_SMEP);
1300
1301 /*
1302 * We check disable_smap here in addition to in startup_smap()
1303 * to ensure CPUs that aren't the boot CPU don't accidentally
1304 * include it in the feature set and thus generate a mismatched
1305 * x86 feature set across CPUs. Note that at this time we only
1306 * enable SMAP for the 64-bit kernel.
1307 */
1308 #if defined(__amd64)
1309 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMAP &&
1310 disable_smap == 0)
1311 add_x86_feature(featureset, X86FSET_SMAP);
1312 #endif
1313 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_MPX)
1314 add_x86_feature(featureset, X86FSET_MPX);
1315
1316 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_RDSEED)
1317 add_x86_feature(featureset, X86FSET_RDSEED);
1318
1319 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_ADX)
1320 add_x86_feature(featureset, X86FSET_ADX);
1321 }
1322
1323 /*
1324 * fold in overrides from the "eeprom" mechanism
1325 */
1326 cp->cp_edx |= cpuid_feature_edx_include;
1327 cp->cp_edx &= ~cpuid_feature_edx_exclude;
1328
1329 cp->cp_ecx |= cpuid_feature_ecx_include;
1330 cp->cp_ecx &= ~cpuid_feature_ecx_exclude;
1331
1332 if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
1333 add_x86_feature(featureset, X86FSET_LARGEPAGE);
1334 }
1335 if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
1336 add_x86_feature(featureset, X86FSET_TSC);
1337 }
1338 if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
1339 add_x86_feature(featureset, X86FSET_MSR);
1340 }
1341 if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
1342 add_x86_feature(featureset, X86FSET_MTRR);
1343 }
1344 if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
1345 add_x86_feature(featureset, X86FSET_PGE);
1346 }
1347 if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
1348 add_x86_feature(featureset, X86FSET_CMOV);
1349 }
1350 if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
1351 add_x86_feature(featureset, X86FSET_MMX);
1352 }
1353 if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
1354 (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
1355 add_x86_feature(featureset, X86FSET_MCA);
1356 }
1357 if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
1358 add_x86_feature(featureset, X86FSET_PAE);
1359 }
1360 if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
1361 add_x86_feature(featureset, X86FSET_CX8);
1362 }
1363 if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
1364 add_x86_feature(featureset, X86FSET_CX16);
1365 }
1366 if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
1367 add_x86_feature(featureset, X86FSET_PAT);
1368 }
1369 if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
1370 add_x86_feature(featureset, X86FSET_SEP);
1371 }
1372 if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
1373 /*
1374 * In our implementation, fxsave/fxrstor
1375 * are prerequisites before we'll even
1376 * try and do SSE things.
1377 */
1378 if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
1379 add_x86_feature(featureset, X86FSET_SSE);
1380 }
1381 if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
1382 add_x86_feature(featureset, X86FSET_SSE2);
1383 }
1384 if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
1385 add_x86_feature(featureset, X86FSET_SSE3);
1386 }
1387 if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
1388 add_x86_feature(featureset, X86FSET_SSSE3);
1389 }
1390 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
1391 add_x86_feature(featureset, X86FSET_SSE4_1);
1392 }
1393 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
1394 add_x86_feature(featureset, X86FSET_SSE4_2);
1395 }
1396 if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
1397 add_x86_feature(featureset, X86FSET_AES);
1398 }
1399 if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
1400 add_x86_feature(featureset, X86FSET_PCLMULQDQ);
1401 }
1402
1403 if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_SHA)
1404 add_x86_feature(featureset, X86FSET_SHA);
1405
1406 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_UMIP)
1407 add_x86_feature(featureset, X86FSET_UMIP);
1408 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_PKU)
1409 add_x86_feature(featureset, X86FSET_PKU);
1410 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_OSPKE)
1411 add_x86_feature(featureset, X86FSET_OSPKE);
1412
1413 if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
1414 add_x86_feature(featureset, X86FSET_XSAVE);
1415
1416 /* We only test AVX & AVX512 when there is XSAVE */
1417
1418 if (cp->cp_ecx & CPUID_INTC_ECX_AVX) {
1419 add_x86_feature(featureset,
1420 X86FSET_AVX);
1421
1422 /*
1423 * Intel says we can't check these without also
1424 * checking AVX.
1425 */
1426 if (cp->cp_ecx & CPUID_INTC_ECX_F16C)
1427 add_x86_feature(featureset,
1428 X86FSET_F16C);
1429
1430 if (cp->cp_ecx & CPUID_INTC_ECX_FMA)
1431 add_x86_feature(featureset,
1432 X86FSET_FMA);
1433
1434 if (cpi->cpi_std[7].cp_ebx &
1435 CPUID_INTC_EBX_7_0_BMI1)
1436 add_x86_feature(featureset,
1437 X86FSET_BMI1);
1438
1439 if (cpi->cpi_std[7].cp_ebx &
1440 CPUID_INTC_EBX_7_0_BMI2)
1441 add_x86_feature(featureset,
1442 X86FSET_BMI2);
1443
1444 if (cpi->cpi_std[7].cp_ebx &
1445 CPUID_INTC_EBX_7_0_AVX2)
1446 add_x86_feature(featureset,
1447 X86FSET_AVX2);
1448 }
1449
1450 if (cpi->cpi_std[7].cp_ebx &
1451 CPUID_INTC_EBX_7_0_AVX512F) {
1452 add_x86_feature(featureset, X86FSET_AVX512F);
1453
1454 if (cpi->cpi_std[7].cp_ebx &
1455 CPUID_INTC_EBX_7_0_AVX512DQ)
1456 add_x86_feature(featureset,
1457 X86FSET_AVX512DQ);
1458 if (cpi->cpi_std[7].cp_ebx &
1459 CPUID_INTC_EBX_7_0_AVX512IFMA)
1460 add_x86_feature(featureset,
1461 X86FSET_AVX512FMA);
1462 if (cpi->cpi_std[7].cp_ebx &
1463 CPUID_INTC_EBX_7_0_AVX512PF)
1464 add_x86_feature(featureset,
1465 X86FSET_AVX512PF);
1466 if (cpi->cpi_std[7].cp_ebx &
1467 CPUID_INTC_EBX_7_0_AVX512ER)
1468 add_x86_feature(featureset,
1469 X86FSET_AVX512ER);
1470 if (cpi->cpi_std[7].cp_ebx &
1471 CPUID_INTC_EBX_7_0_AVX512CD)
1472 add_x86_feature(featureset,
1473 X86FSET_AVX512CD);
1474 if (cpi->cpi_std[7].cp_ebx &
1475 CPUID_INTC_EBX_7_0_AVX512BW)
1476 add_x86_feature(featureset,
1477 X86FSET_AVX512BW);
1478 if (cpi->cpi_std[7].cp_ebx &
1479 CPUID_INTC_EBX_7_0_AVX512VL)
1480 add_x86_feature(featureset,
1481 X86FSET_AVX512VL);
1482
1483 if (cpi->cpi_std[7].cp_ecx &
1484 CPUID_INTC_ECX_7_0_AVX512VBMI)
1485 add_x86_feature(featureset,
1486 X86FSET_AVX512VBMI);
1487 if (cpi->cpi_std[7].cp_ecx &
1488 CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
1489 add_x86_feature(featureset,
1490 X86FSET_AVX512VPOPCDQ);
1491
1492 if (cpi->cpi_std[7].cp_edx &
1493 CPUID_INTC_EDX_7_0_AVX5124NNIW)
1494 add_x86_feature(featureset,
1495 X86FSET_AVX512NNIW);
1496 if (cpi->cpi_std[7].cp_edx &
1497 CPUID_INTC_EDX_7_0_AVX5124FMAPS)
1498 add_x86_feature(featureset,
1499 X86FSET_AVX512FMAPS);
1500 }
1501 }
1502 }
1503 if (cp->cp_ecx & CPUID_INTC_ECX_X2APIC) {
1504 add_x86_feature(featureset, X86FSET_X2APIC);
1505 }
1506 if (cp->cp_edx & CPUID_INTC_EDX_DE) {
1507 add_x86_feature(featureset, X86FSET_DE);
1508 }
1509 #if !defined(__xpv)
1510 if (cp->cp_ecx & CPUID_INTC_ECX_MON) {
1511
1512 /*
1513 * We require the CLFLUSH instruction for erratum workaround
1514 * to use MONITOR/MWAIT.
1515 */
1516 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1517 cpi->cpi_mwait.support |= MWAIT_SUPPORT;
1518 add_x86_feature(featureset, X86FSET_MWAIT);
1519 } else {
1520 extern int idle_cpu_assert_cflush_monitor;
1521
1522 /*
1523 * All processors we are aware of which have
1524 * MONITOR/MWAIT also have CLFLUSH.
1525 */
1526 if (idle_cpu_assert_cflush_monitor) {
1527 ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
1528 (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
1529 }
1530 }
1531 }
1532 #endif /* __xpv */
1533
1534 if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
1535 add_x86_feature(featureset, X86FSET_VMX);
1536 }
1537
1538 if (cp->cp_ecx & CPUID_INTC_ECX_RDRAND)
1539 add_x86_feature(featureset, X86FSET_RDRAND);
1540
1541 /*
1542 * Only need it first time, rest of the cpus would follow suit.
1543 * we only capture this for the bootcpu.
1544 */
1545 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1546 add_x86_feature(featureset, X86FSET_CLFSH);
1547 x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
1548 }
1549 if (is_x86_feature(featureset, X86FSET_PAE))
1550 cpi->cpi_pabits = 36;
1551
1552 /*
1553 * Hyperthreading configuration is slightly tricky on Intel
1554 * and pure clones, and even trickier on AMD.
1555 *
1556 * (AMD chose to set the HTT bit on their CMP processors,
1557 * even though they're not actually hyperthreaded. Thus it
1558 * takes a bit more work to figure out what's really going
1559 * on ... see the handling of the CMP_LGCY bit below)
1560 */
1561 if (cp->cp_edx & CPUID_INTC_EDX_HTT) {
1562 cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
1563 if (cpi->cpi_ncpu_per_chip > 1)
1564 add_x86_feature(featureset, X86FSET_HTT);
1565 } else {
1566 cpi->cpi_ncpu_per_chip = 1;
1567 }
1568
1569 if (cpi->cpi_vendor == X86_VENDOR_Intel && cpi->cpi_maxeax >= 0xD &&
1570 !xsave_force_disable) {
1571 struct cpuid_regs r, *ecp;
1572
1573 ecp = &r;
1574 ecp->cp_eax = 0xD;
1575 ecp->cp_ecx = 1;
1576 ecp->cp_edx = ecp->cp_ebx = 0;
1577 (void) __cpuid_insn(ecp);
1578
1579 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEOPT)
1580 add_x86_feature(featureset, X86FSET_XSAVEOPT);
1581 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEC)
1582 add_x86_feature(featureset, X86FSET_XSAVEC);
1583 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVES)
1584 add_x86_feature(featureset, X86FSET_XSAVES);
1585 }
1586
1587 /*
1588 * Work on the "extended" feature information, doing
1589 * some basic initialization for cpuid_pass2()
1590 */
1591 xcpuid = 0;
1592 switch (cpi->cpi_vendor) {
1593 case X86_VENDOR_Intel:
1594 /*
1595 * On KVM we know we will have proper support for extended
1596 * cpuid.
1597 */
1598 if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf ||
1599 (get_hwenv() == HW_KVM && cpi->cpi_family == 6 &&
1600 (cpi->cpi_model == 6 || cpi->cpi_model == 2)))
1601 xcpuid++;
1602 break;
1603 case X86_VENDOR_AMD:
1604 if (cpi->cpi_family > 5 ||
1605 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
1606 xcpuid++;
1607 break;
1608 case X86_VENDOR_Cyrix:
1609 /*
1610 * Only these Cyrix CPUs are -known- to support
1611 * extended cpuid operations.
1612 */
1613 if (x86_type == X86_TYPE_VIA_CYRIX_III ||
1614 x86_type == X86_TYPE_CYRIX_GXm)
1615 xcpuid++;
1616 break;
1617 case X86_VENDOR_Centaur:
1618 case X86_VENDOR_TM:
1619 default:
1620 xcpuid++;
1621 break;
1622 }
1623
1624 if (xcpuid) {
1625 cp = &cpi->cpi_extd[0];
1626 cp->cp_eax = 0x80000000;
1627 cpi->cpi_xmaxeax = __cpuid_insn(cp);
1628 }
1629
1630 if (cpi->cpi_xmaxeax & 0x80000000) {
1631
1632 if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
1633 cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;
1634
1635 switch (cpi->cpi_vendor) {
1636 case X86_VENDOR_Intel:
1637 case X86_VENDOR_AMD:
1638 if (cpi->cpi_xmaxeax < 0x80000001)
1639 break;
1640 cp = &cpi->cpi_extd[1];
1641 cp->cp_eax = 0x80000001;
1642 (void) __cpuid_insn(cp);
1643
1644 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1645 cpi->cpi_family == 5 &&
1646 cpi->cpi_model == 6 &&
1647 cpi->cpi_step == 6) {
1648 /*
1649 * K6 model 6 uses bit 10 to indicate SYSC
1650 * Later models use bit 11. Fix it here.
1651 */
1652 if (cp->cp_edx & 0x400) {
1653 cp->cp_edx &= ~0x400;
1654 cp->cp_edx |= CPUID_AMD_EDX_SYSC;
1655 }
1656 }
1657
1658 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);
1659
1660 /*
1661 * Compute the additions to the kernel's feature word.
1662 */
1663 if (cp->cp_edx & CPUID_AMD_EDX_NX) {
1664 add_x86_feature(featureset, X86FSET_NX);
1665 }
1666
1667 /*
1668 * Regardless whether or not we boot 64-bit,
1669 * we should have a way to identify whether
1670 * the CPU is capable of running 64-bit.
1671 */
1672 if (cp->cp_edx & CPUID_AMD_EDX_LM) {
1673 add_x86_feature(featureset, X86FSET_64);
1674 }
1675
1676 #if defined(__amd64)
1677 /* 1 GB large page - enable only for 64 bit kernel */
1678 if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
1679 add_x86_feature(featureset, X86FSET_1GPG);
1680 }
1681 #endif
1682
1683 if ((cpi->cpi_vendor == X86_VENDOR_AMD) &&
1684 (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
1685 (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
1686 add_x86_feature(featureset, X86FSET_SSE4A);
1687 }
1688
1689 /*
1690 * If both the HTT and CMP_LGCY bits are set,
1691 * then we're not actually HyperThreaded. Read
1692 * "AMD CPUID Specification" for more details.
1693 */
1694 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1695 is_x86_feature(featureset, X86FSET_HTT) &&
1696 (cp->cp_ecx & CPUID_AMD_ECX_CMP_LGCY)) {
1697 remove_x86_feature(featureset, X86FSET_HTT);
1698 add_x86_feature(featureset, X86FSET_CMP);
1699 }
1700 #if defined(__amd64)
1701 /*
1702 * It's really tricky to support syscall/sysret in
1703 * the i386 kernel; we rely on sysenter/sysexit
1704 * instead. In the amd64 kernel, things are -way-
1705 * better.
1706 */
1707 if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
1708 add_x86_feature(featureset, X86FSET_ASYSC);
1709 }
1710
1711 /*
1712 * While we're thinking about system calls, note
1713 * that AMD processors don't support sysenter
1714 * in long mode at all, so don't try to program them.
1715 */
1716 if (x86_vendor == X86_VENDOR_AMD) {
1717 remove_x86_feature(featureset, X86FSET_SEP);
1718 }
1719 #endif
1720 if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
1721 add_x86_feature(featureset, X86FSET_TSCP);
1722 }
1723
1724 if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
1725 add_x86_feature(featureset, X86FSET_SVM);
1726 }
1727
1728 if (cp->cp_ecx & CPUID_AMD_ECX_TOPOEXT) {
1729 add_x86_feature(featureset, X86FSET_TOPOEXT);
1730 }
1731 break;
1732 default:
1733 break;
1734 }
1735
1736 /*
1737 * Get CPUID data about processor cores and hyperthreads.
1738 */
1739 switch (cpi->cpi_vendor) {
1740 case X86_VENDOR_Intel:
1741 if (cpi->cpi_maxeax >= 4) {
1742 cp = &cpi->cpi_std[4];
1743 cp->cp_eax = 4;
1744 cp->cp_ecx = 0;
1745 (void) __cpuid_insn(cp);
1746 platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
1747 }
1748 /*FALLTHROUGH*/
1749 case X86_VENDOR_AMD:
1750 if (cpi->cpi_xmaxeax < 0x80000008)
1751 break;
1752 cp = &cpi->cpi_extd[8];
1753 cp->cp_eax = 0x80000008;
1754 (void) __cpuid_insn(cp);
1755 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, cp);
1756
1757 /*
1758 * Virtual and physical address limits from
1759 * cpuid override previously guessed values.
1760 */
1761 cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
1762 cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
1763 break;
1764 default:
1765 break;
1766 }
1767
1768 /*
1769 * Derive the number of cores per chip
1770 */
1771 switch (cpi->cpi_vendor) {
1772 case X86_VENDOR_Intel:
1773 if (cpi->cpi_maxeax < 4) {
1774 cpi->cpi_ncore_per_chip = 1;
1775 break;
1776 } else {
1777 cpi->cpi_ncore_per_chip =
1778 BITX((cpi)->cpi_std[4].cp_eax, 31, 26) + 1;
1779 }
1780 break;
1781 case X86_VENDOR_AMD:
1782 if (cpi->cpi_xmaxeax < 0x80000008) {
1783 cpi->cpi_ncore_per_chip = 1;
1784 break;
1785 } else {
1786 /*
1787 * On family 0xf cpuid fn 2 ECX[7:0] "NC" is
1788 * 1 less than the number of physical cores on
1789 * the chip. In family 0x10 this value can
1790 * be affected by "downcoring" - it reflects
1791 * 1 less than the number of cores actually
1792 * enabled on this node.
1793 */
1794 cpi->cpi_ncore_per_chip =
1795 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
1796 }
1797 break;
1798 default:
1799 cpi->cpi_ncore_per_chip = 1;
1800 break;
1801 }
1802
1803 /*
1804 * Get CPUID data about TSC Invariance in Deep C-State.
1805 */
1806 switch (cpi->cpi_vendor) {
1807 case X86_VENDOR_Intel:
1808 if (cpi->cpi_maxeax >= 7) {
1809 cp = &cpi->cpi_extd[7];
1810 cp->cp_eax = 0x80000007;
1811 cp->cp_ecx = 0;
1812 (void) __cpuid_insn(cp);
1813 }
1814 break;
1815 default:
1816 break;
1817 }
1818 } else {
1819 cpi->cpi_ncore_per_chip = 1;
1820 }
1821
1822 /*
1823 * If more than one core, then this processor is CMP.
1824 */
1825 if (cpi->cpi_ncore_per_chip > 1) {
1826 add_x86_feature(featureset, X86FSET_CMP);
1827 }
1828
1829 /*
1830 * If the number of cores is the same as the number
1831 * of CPUs, then we cannot have HyperThreading.
1832 */
1833 if (cpi->cpi_ncpu_per_chip == cpi->cpi_ncore_per_chip) {
1834 remove_x86_feature(featureset, X86FSET_HTT);
1835 }
1836
1837 cpi->cpi_apicid = CPI_APIC_ID(cpi);
1838 cpi->cpi_procnodes_per_pkg = 1;
1839 cpi->cpi_cores_per_compunit = 1;
1840 if (is_x86_feature(featureset, X86FSET_HTT) == B_FALSE &&
1841 is_x86_feature(featureset, X86FSET_CMP) == B_FALSE) {
1842 /*
1843 * Single-core single-threaded processors.
1844 */
1845 cpi->cpi_chipid = -1;
1846 cpi->cpi_clogid = 0;
1847 cpi->cpi_coreid = cpu->cpu_id;
1848 cpi->cpi_pkgcoreid = 0;
1849 if (cpi->cpi_vendor == X86_VENDOR_AMD)
1850 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
1851 else
1852 cpi->cpi_procnodeid = cpi->cpi_chipid;
1853 } else if (cpi->cpi_ncpu_per_chip > 1) {
1854 if (cpi->cpi_vendor == X86_VENDOR_Intel)
1855 cpuid_intel_getids(cpu, featureset);
1856 else if (cpi->cpi_vendor == X86_VENDOR_AMD)
1857 cpuid_amd_getids(cpu);
1858 else {
1859 /*
1860 * All other processors are currently
1861 * assumed to have single cores.
1862 */
1863 cpi->cpi_coreid = cpi->cpi_chipid;
1864 cpi->cpi_pkgcoreid = 0;
1865 cpi->cpi_procnodeid = cpi->cpi_chipid;
1866 cpi->cpi_compunitid = cpi->cpi_chipid;
1867 }
1868 }
1869
1870 /*
1871 * Synthesize chip "revision" and socket type
1872 */
1873 cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
1874 cpi->cpi_model, cpi->cpi_step);
1875 cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
1876 cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
1877 cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
1878 cpi->cpi_model, cpi->cpi_step);
1879
1880 /*
1881 * While we're here, check for the AMD "Error Pointer Zero/Restore"
1882 * feature. This can be used to setup the FP save handlers
1883 * appropriately.
1884 */
1885 if (cpi->cpi_vendor == X86_VENDOR_AMD) {
1886 if (cpi->cpi_xmaxeax >= 0x80000008 &&
1887 cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
1888 /* Special handling for AMD FP not necessary. */
1889 cpi->cpi_fp_amd_save = 0;
1890 } else {
1891 cpi->cpi_fp_amd_save = 1;
1892 }
1893 }
1894
1895 pass1_done:
1896 cpi->cpi_pass = 1;
1897 }
1898
1899 /*
1900 * Make copies of the cpuid table entries we depend on, in
1901 * part for ease of parsing now, in part so that we have only
1902 * one place to correct any of it, in part for ease of
1903 * later export to userland, and in part so we can look at
1904 * this stuff in a crash dump.
1905 */
1906
1907 /*ARGSUSED*/
1908 void
1909 cpuid_pass2(cpu_t *cpu)
1910 {
1911 uint_t n, nmax;
1912 int i;
1913 struct cpuid_regs *cp;
1914 uint8_t *dp;
1915 uint32_t *iptr;
1916 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
1917
1918 ASSERT(cpi->cpi_pass == 1);
1919
1920 if (cpi->cpi_maxeax < 1)
1921 goto pass2_done;
1922
1923 if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
1924 nmax = NMAX_CPI_STD;
1925 /*
1926 * (We already handled n == 0 and n == 1 in pass 1)
1927 */
1928 for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
1929 cp->cp_eax = n;
1930
1931 /*
1932 * CPUID function 4 expects %ecx to be initialized
1933 * with an index which indicates which cache to return
1934 * information about. The OS is expected to call function 4
1935 * with %ecx set to 0, 1, 2, ... until it returns with
1936 * EAX[4:0] set to 0, which indicates there are no more
1937 * caches.
1938 *
1939 * Here, populate cpi_std[4] with the information returned by
1940 * function 4 when %ecx == 0, and do the rest in cpuid_pass3()
1941 * when dynamic memory allocation becomes available.
1942 *
1943 * Note: we need to explicitly initialize %ecx here, since
1944 * function 4 may have been previously invoked.
1945 *
1946 * The same is all true for CPUID function 7.
1947 */
1948 if (n == 4 || n == 7)
1949 cp->cp_ecx = 0;
1950
1951 (void) __cpuid_insn(cp);
1952 platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
1953 switch (n) {
1954 case 2:
1955 /*
1956 * "the lower 8 bits of the %eax register
1957 * contain a value that identifies the number
1958 * of times the cpuid [instruction] has to be
1959 * executed to obtain a complete image of the
1960 * processor's caching systems."
1961 *
1962 * How *do* they make this stuff up?
1963 */
1964 cpi->cpi_ncache = sizeof (*cp) *
1965 BITX(cp->cp_eax, 7, 0);
1966 if (cpi->cpi_ncache == 0)
1967 break;
1968 cpi->cpi_ncache--; /* skip count byte */
1969
1970 /*
1971 * Well, for now, rather than attempt to implement
1972 * this slightly dubious algorithm, we just look
1973 * at the first 15 ..
1974 */
1975 if (cpi->cpi_ncache > (sizeof (*cp) - 1))
1976 cpi->cpi_ncache = sizeof (*cp) - 1;
1977
1978 dp = cpi->cpi_cacheinfo;
1979 if (BITX(cp->cp_eax, 31, 31) == 0) {
1980 uint8_t *p = (void *)&cp->cp_eax;
1981 for (i = 1; i < 4; i++)
1982 if (p[i] != 0)
1983 *dp++ = p[i];
1984 }
1985 if (BITX(cp->cp_ebx, 31, 31) == 0) {
1986 uint8_t *p = (void *)&cp->cp_ebx;
1987 for (i = 0; i < 4; i++)
1988 if (p[i] != 0)
1989 *dp++ = p[i];
1990 }
1991 if (BITX(cp->cp_ecx, 31, 31) == 0) {
1992 uint8_t *p = (void *)&cp->cp_ecx;
1993 for (i = 0; i < 4; i++)
1994 if (p[i] != 0)
1995 *dp++ = p[i];
1996 }
1997 if (BITX(cp->cp_edx, 31, 31) == 0) {
1998 uint8_t *p = (void *)&cp->cp_edx;
1999 for (i = 0; i < 4; i++)
2000 if (p[i] != 0)
2001 *dp++ = p[i];
2002 }
2003 break;
2004
2005 case 3: /* Processor serial number, if PSN supported */
2006 break;
2007
2008 case 4: /* Deterministic cache parameters */
2009 break;
2010
2011 case 5: /* Monitor/Mwait parameters */
2012 {
2013 size_t mwait_size;
2014
2015 /*
2016 * check cpi_mwait.support which was set in cpuid_pass1
2017 */
2018 if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
2019 break;
2020
2021 /*
2022 * Protect ourself from insane mwait line size.
2023 * Workaround for incomplete hardware emulator(s).
2024 */
2025 mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
2026 if (mwait_size < sizeof (uint32_t) ||
2027 !ISP2(mwait_size)) {
2028 #if DEBUG
2029 cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
2030 "size %ld", cpu->cpu_id, (long)mwait_size);
2031 #endif
2032 break;
2033 }
2034
2035 cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
2036 cpi->cpi_mwait.mon_max = mwait_size;
2037 if (MWAIT_EXTENSION(cpi)) {
2038 cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
2039 if (MWAIT_INT_ENABLE(cpi))
2040 cpi->cpi_mwait.support |=
2041 MWAIT_ECX_INT_ENABLE;
2042 }
2043 break;
2044 }
2045 default:
2046 break;
2047 }
2048 }
2049
2050 if (cpi->cpi_maxeax >= 0xB && cpi->cpi_vendor == X86_VENDOR_Intel) {
2051 struct cpuid_regs regs;
2052
2053 cp = ®s;
2054 cp->cp_eax = 0xB;
2055 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2056
2057 (void) __cpuid_insn(cp);
2058
2059 /*
2060 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
2061 * indicates that the extended topology enumeration leaf is
2062 * available.
2063 */
2064 if (cp->cp_ebx) {
2065 uint32_t x2apic_id;
2066 uint_t coreid_shift = 0;
2067 uint_t ncpu_per_core = 1;
2068 uint_t chipid_shift = 0;
2069 uint_t ncpu_per_chip = 1;
2070 uint_t i;
2071 uint_t level;
2072
2073 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
2074 cp->cp_eax = 0xB;
2075 cp->cp_ecx = i;
2076
2077 (void) __cpuid_insn(cp);
2078 level = CPI_CPU_LEVEL_TYPE(cp);
2079
2080 if (level == 1) {
2081 x2apic_id = cp->cp_edx;
2082 coreid_shift = BITX(cp->cp_eax, 4, 0);
2083 ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
2084 } else if (level == 2) {
2085 x2apic_id = cp->cp_edx;
2086 chipid_shift = BITX(cp->cp_eax, 4, 0);
2087 ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
2088 }
2089 }
2090
2091 cpi->cpi_apicid = x2apic_id;
2092 cpi->cpi_ncpu_per_chip = ncpu_per_chip;
2093 cpi->cpi_ncore_per_chip = ncpu_per_chip /
2094 ncpu_per_core;
2095 cpi->cpi_chipid = x2apic_id >> chipid_shift;
2096 cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
2097 cpi->cpi_coreid = x2apic_id >> coreid_shift;
2098 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
2099 }
2100
2101 /* Make cp NULL so that we don't stumble on others */
2102 cp = NULL;
2103 }
2104
2105 /*
2106 * XSAVE enumeration
2107 */
2108 if (cpi->cpi_maxeax >= 0xD) {
2109 struct cpuid_regs regs;
2110 boolean_t cpuid_d_valid = B_TRUE;
2111
2112 cp = ®s;
2113 cp->cp_eax = 0xD;
2114 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2115
2116 (void) __cpuid_insn(cp);
2117
2118 /*
2119 * Sanity checks for debug
2120 */
2121 if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
2122 (cp->cp_eax & XFEATURE_SSE) == 0) {
2123 cpuid_d_valid = B_FALSE;
2124 }
2125
2126 cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
2127 cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
2128 cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;
2129
2130 /*
2131 * If the hw supports AVX, get the size and offset in the save
2132 * area for the ymm state.
2133 */
2134 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
2135 cp->cp_eax = 0xD;
2136 cp->cp_ecx = 2;
2137 cp->cp_edx = cp->cp_ebx = 0;
2138
2139 (void) __cpuid_insn(cp);
2140
2141 if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
2142 cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
2143 cpuid_d_valid = B_FALSE;
2144 }
2145
2146 cpi->cpi_xsave.ymm_size = cp->cp_eax;
2147 cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
2148 }
2149
2150 /*
2151 * If the hw supports MPX, get the size and offset in the
2152 * save area for BNDREGS and BNDCSR.
2153 */
2154 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_MPX) {
2155 cp->cp_eax = 0xD;
2156 cp->cp_ecx = 3;
2157 cp->cp_edx = cp->cp_ebx = 0;
2158
2159 (void) __cpuid_insn(cp);
2160
2161 cpi->cpi_xsave.bndregs_size = cp->cp_eax;
2162 cpi->cpi_xsave.bndregs_offset = cp->cp_ebx;
2163
2164 cp->cp_eax = 0xD;
2165 cp->cp_ecx = 4;
2166 cp->cp_edx = cp->cp_ebx = 0;
2167
2168 (void) __cpuid_insn(cp);
2169
2170 cpi->cpi_xsave.bndcsr_size = cp->cp_eax;
2171 cpi->cpi_xsave.bndcsr_offset = cp->cp_ebx;
2172 }
2173
2174 /*
2175 * If the hw supports AVX512, get the size and offset in the
2176 * save area for the opmask registers and zmm state.
2177 */
2178 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX512) {
2179 cp->cp_eax = 0xD;
2180 cp->cp_ecx = 5;
2181 cp->cp_edx = cp->cp_ebx = 0;
2182
2183 (void) __cpuid_insn(cp);
2184
2185 cpi->cpi_xsave.opmask_size = cp->cp_eax;
2186 cpi->cpi_xsave.opmask_offset = cp->cp_ebx;
2187
2188 cp->cp_eax = 0xD;
2189 cp->cp_ecx = 6;
2190 cp->cp_edx = cp->cp_ebx = 0;
2191
2192 (void) __cpuid_insn(cp);
2193
2194 cpi->cpi_xsave.zmmlo_size = cp->cp_eax;
2195 cpi->cpi_xsave.zmmlo_offset = cp->cp_ebx;
2196
2197 cp->cp_eax = 0xD;
2198 cp->cp_ecx = 7;
2199 cp->cp_edx = cp->cp_ebx = 0;
2200
2201 (void) __cpuid_insn(cp);
2202
2203 cpi->cpi_xsave.zmmhi_size = cp->cp_eax;
2204 cpi->cpi_xsave.zmmhi_offset = cp->cp_ebx;
2205 }
2206
2207 if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
2208 xsave_state_size = 0;
2209 } else if (cpuid_d_valid) {
2210 xsave_state_size = cpi->cpi_xsave.xsav_max_size;
2211 } else {
2212 /* Broken CPUID 0xD, probably in HVM */
2213 cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
2214 "value: hw_low = %d, hw_high = %d, xsave_size = %d"
2215 ", ymm_size = %d, ymm_offset = %d\n",
2216 cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
2217 cpi->cpi_xsave.xsav_hw_features_high,
2218 (int)cpi->cpi_xsave.xsav_max_size,
2219 (int)cpi->cpi_xsave.ymm_size,
2220 (int)cpi->cpi_xsave.ymm_offset);
2221
2222 if (xsave_state_size != 0) {
2223 /*
2224 * This must be a non-boot CPU. We cannot
2225 * continue, because boot cpu has already
2226 * enabled XSAVE.
2227 */
2228 ASSERT(cpu->cpu_id != 0);
2229 cmn_err(CE_PANIC, "cpu%d: we have already "
2230 "enabled XSAVE on boot cpu, cannot "
2231 "continue.", cpu->cpu_id);
2232 } else {
2233 /*
2234 * If we reached here on the boot CPU, it's also
2235 * almost certain that we'll reach here on the
2236 * non-boot CPUs. When we're here on a boot CPU
2237 * we should disable the feature, on a non-boot
2238 * CPU we need to confirm that we have.
2239 */
2240 if (cpu->cpu_id == 0) {
2241 remove_x86_feature(x86_featureset,
2242 X86FSET_XSAVE);
2243 remove_x86_feature(x86_featureset,
2244 X86FSET_AVX);
2245 remove_x86_feature(x86_featureset,
2246 X86FSET_F16C);
2247 remove_x86_feature(x86_featureset,
2248 X86FSET_BMI1);
2249 remove_x86_feature(x86_featureset,
2250 X86FSET_BMI2);
2251 remove_x86_feature(x86_featureset,
2252 X86FSET_FMA);
2253 remove_x86_feature(x86_featureset,
2254 X86FSET_AVX2);
2255 remove_x86_feature(x86_featureset,
2256 X86FSET_MPX);
2257 remove_x86_feature(x86_featureset,
2258 X86FSET_AVX512F);
2259 remove_x86_feature(x86_featureset,
2260 X86FSET_AVX512DQ);
2261 remove_x86_feature(x86_featureset,
2262 X86FSET_AVX512PF);
2263 remove_x86_feature(x86_featureset,
2264 X86FSET_AVX512ER);
2265 remove_x86_feature(x86_featureset,
2266 X86FSET_AVX512CD);
2267 remove_x86_feature(x86_featureset,
2268 X86FSET_AVX512BW);
2269 remove_x86_feature(x86_featureset,
2270 X86FSET_AVX512VL);
2271 remove_x86_feature(x86_featureset,
2272 X86FSET_AVX512FMA);
2273 remove_x86_feature(x86_featureset,
2274 X86FSET_AVX512VBMI);
2275 remove_x86_feature(x86_featureset,
2276 X86FSET_AVX512VPOPCDQ);
2277 remove_x86_feature(x86_featureset,
2278 X86FSET_AVX512NNIW);
2279 remove_x86_feature(x86_featureset,
2280 X86FSET_AVX512FMAPS);
2281
2282 CPI_FEATURES_ECX(cpi) &=
2283 ~CPUID_INTC_ECX_XSAVE;
2284 CPI_FEATURES_ECX(cpi) &=
2285 ~CPUID_INTC_ECX_AVX;
2286 CPI_FEATURES_ECX(cpi) &=
2287 ~CPUID_INTC_ECX_F16C;
2288 CPI_FEATURES_ECX(cpi) &=
2289 ~CPUID_INTC_ECX_FMA;
2290 CPI_FEATURES_7_0_EBX(cpi) &=
2291 ~CPUID_INTC_EBX_7_0_BMI1;
2292 CPI_FEATURES_7_0_EBX(cpi) &=
2293 ~CPUID_INTC_EBX_7_0_BMI2;
2294 CPI_FEATURES_7_0_EBX(cpi) &=
2295 ~CPUID_INTC_EBX_7_0_AVX2;
2296 CPI_FEATURES_7_0_EBX(cpi) &=
2297 ~CPUID_INTC_EBX_7_0_MPX;
2298 CPI_FEATURES_7_0_EBX(cpi) &=
2299 ~CPUID_INTC_EBX_7_0_ALL_AVX512;
2300
2301 CPI_FEATURES_7_0_ECX(cpi) &=
2302 ~CPUID_INTC_ECX_7_0_ALL_AVX512;
2303
2304 CPI_FEATURES_7_0_EDX(cpi) &=
2305 ~CPUID_INTC_EDX_7_0_ALL_AVX512;
2306
2307 xsave_force_disable = B_TRUE;
2308 } else {
2309 VERIFY(is_x86_feature(x86_featureset,
2310 X86FSET_XSAVE) == B_FALSE);
2311 }
2312 }
2313 }
2314 }
2315
2316
2317 if ((cpi->cpi_xmaxeax & 0x80000000) == 0)
2318 goto pass2_done;
2319
2320 if ((nmax = cpi->cpi_xmaxeax - 0x80000000 + 1) > NMAX_CPI_EXTD)
2321 nmax = NMAX_CPI_EXTD;
2322 /*
2323 * Copy the extended properties, fixing them as we go.
2324 * (We already handled n == 0 and n == 1 in pass 1)
2325 */
2326 iptr = (void *)cpi->cpi_brandstr;
2327 for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
2328 cp->cp_eax = 0x80000000 + n;
2329 (void) __cpuid_insn(cp);
2330 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000000 + n, cp);
2331 switch (n) {
2332 case 2:
2333 case 3:
2334 case 4:
2335 /*
2336 * Extract the brand string
2337 */
2338 *iptr++ = cp->cp_eax;
2339 *iptr++ = cp->cp_ebx;
2340 *iptr++ = cp->cp_ecx;
2341 *iptr++ = cp->cp_edx;
2342 break;
2343 case 5:
2344 switch (cpi->cpi_vendor) {
2345 case X86_VENDOR_AMD:
2346 /*
2347 * The Athlon and Duron were the first
2348 * parts to report the sizes of the
2349 * TLB for large pages. Before then,
2350 * we don't trust the data.
2351 */
2352 if (cpi->cpi_family < 6 ||
2353 (cpi->cpi_family == 6 &&
2354 cpi->cpi_model < 1))
2355 cp->cp_eax = 0;
2356 break;
2357 default:
2358 break;
2359 }
2360 break;
2361 case 6:
2362 switch (cpi->cpi_vendor) {
2363 case X86_VENDOR_AMD:
2364 /*
2365 * The Athlon and Duron were the first
2366 * AMD parts with L2 TLB's.
2367 * Before then, don't trust the data.
2368 */
2369 if (cpi->cpi_family < 6 ||
2370 cpi->cpi_family == 6 &&
2371 cpi->cpi_model < 1)
2372 cp->cp_eax = cp->cp_ebx = 0;
2373 /*
2374 * AMD Duron rev A0 reports L2
2375 * cache size incorrectly as 1K
2376 * when it is really 64K
2377 */
2378 if (cpi->cpi_family == 6 &&
2379 cpi->cpi_model == 3 &&
2380 cpi->cpi_step == 0) {
2381 cp->cp_ecx &= 0xffff;
2382 cp->cp_ecx |= 0x400000;
2383 }
2384 break;
2385 case X86_VENDOR_Cyrix: /* VIA C3 */
2386 /*
2387 * VIA C3 processors are a bit messed
2388 * up w.r.t. encoding cache sizes in %ecx
2389 */
2390 if (cpi->cpi_family != 6)
2391 break;
2392 /*
2393 * model 7 and 8 were incorrectly encoded
2394 *
2395 * xxx is model 8 really broken?
2396 */
2397 if (cpi->cpi_model == 7 ||
2398 cpi->cpi_model == 8)
2399 cp->cp_ecx =
2400 BITX(cp->cp_ecx, 31, 24) << 16 |
2401 BITX(cp->cp_ecx, 23, 16) << 12 |
2402 BITX(cp->cp_ecx, 15, 8) << 8 |
2403 BITX(cp->cp_ecx, 7, 0);
2404 /*
2405 * model 9 stepping 1 has wrong associativity
2406 */
2407 if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
2408 cp->cp_ecx |= 8 << 12;
2409 break;
2410 case X86_VENDOR_Intel:
2411 /*
2412 * Extended L2 Cache features function.
2413 * First appeared on Prescott.
2414 */
2415 default:
2416 break;
2417 }
2418 break;
2419 default:
2420 break;
2421 }
2422 }
2423
2424 pass2_done:
2425 cpi->cpi_pass = 2;
2426 }
2427
2428 static const char *
2429 intel_cpubrand(const struct cpuid_info *cpi)
2430 {
2431 int i;
2432
2433 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2434 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2435 return ("i486");
2436
2437 switch (cpi->cpi_family) {
2438 case 5:
2439 return ("Intel Pentium(r)");
2440 case 6:
2441 switch (cpi->cpi_model) {
2442 uint_t celeron, xeon;
2443 const struct cpuid_regs *cp;
2444 case 0:
2445 case 1:
2446 case 2:
2447 return ("Intel Pentium(r) Pro");
2448 case 3:
2449 case 4:
2450 return ("Intel Pentium(r) II");
2451 case 6:
2452 return ("Intel Celeron(r)");
2453 case 5:
2454 case 7:
2455 celeron = xeon = 0;
2456 cp = &cpi->cpi_std[2]; /* cache info */
2457
2458 for (i = 1; i < 4; i++) {
2459 uint_t tmp;
2460
2461 tmp = (cp->cp_eax >> (8 * i)) & 0xff;
2462 if (tmp == 0x40)
2463 celeron++;
2464 if (tmp >= 0x44 && tmp <= 0x45)
2465 xeon++;
2466 }
2467
2468 for (i = 0; i < 2; i++) {
2469 uint_t tmp;
2470
2471 tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
2472 if (tmp == 0x40)
2473 celeron++;
2474 else if (tmp >= 0x44 && tmp <= 0x45)
2475 xeon++;
2476 }
2477
2478 for (i = 0; i < 4; i++) {
2479 uint_t tmp;
2480
2481 tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
2482 if (tmp == 0x40)
2483 celeron++;
2484 else if (tmp >= 0x44 && tmp <= 0x45)
2485 xeon++;
2486 }
2487
2488 for (i = 0; i < 4; i++) {
2489 uint_t tmp;
2490
2491 tmp = (cp->cp_edx >> (8 * i)) & 0xff;
2492 if (tmp == 0x40)
2493 celeron++;
2494 else if (tmp >= 0x44 && tmp <= 0x45)
2495 xeon++;
2496 }
2497
2498 if (celeron)
2499 return ("Intel Celeron(r)");
2500 if (xeon)
2501 return (cpi->cpi_model == 5 ?
2502 "Intel Pentium(r) II Xeon(tm)" :
2503 "Intel Pentium(r) III Xeon(tm)");
2504 return (cpi->cpi_model == 5 ?
2505 "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
2506 "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
2507 default:
2508 break;
2509 }
2510 default:
2511 break;
2512 }
2513
2514 /* BrandID is present if the field is nonzero */
2515 if (cpi->cpi_brandid != 0) {
2516 static const struct {
2517 uint_t bt_bid;
2518 const char *bt_str;
2519 } brand_tbl[] = {
2520 { 0x1, "Intel(r) Celeron(r)" },
2521 { 0x2, "Intel(r) Pentium(r) III" },
2522 { 0x3, "Intel(r) Pentium(r) III Xeon(tm)" },
2523 { 0x4, "Intel(r) Pentium(r) III" },
2524 { 0x6, "Mobile Intel(r) Pentium(r) III" },
2525 { 0x7, "Mobile Intel(r) Celeron(r)" },
2526 { 0x8, "Intel(r) Pentium(r) 4" },
2527 { 0x9, "Intel(r) Pentium(r) 4" },
2528 { 0xa, "Intel(r) Celeron(r)" },
2529 { 0xb, "Intel(r) Xeon(tm)" },
2530 { 0xc, "Intel(r) Xeon(tm) MP" },
2531 { 0xe, "Mobile Intel(r) Pentium(r) 4" },
2532 { 0xf, "Mobile Intel(r) Celeron(r)" },
2533 { 0x11, "Mobile Genuine Intel(r)" },
2534 { 0x12, "Intel(r) Celeron(r) M" },
2535 { 0x13, "Mobile Intel(r) Celeron(r)" },
2536 { 0x14, "Intel(r) Celeron(r)" },
2537 { 0x15, "Mobile Genuine Intel(r)" },
2538 { 0x16, "Intel(r) Pentium(r) M" },
2539 { 0x17, "Mobile Intel(r) Celeron(r)" }
2540 };
2541 uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
2542 uint_t sgn;
2543
2544 sgn = (cpi->cpi_family << 8) |
2545 (cpi->cpi_model << 4) | cpi->cpi_step;
2546
2547 for (i = 0; i < btblmax; i++)
2548 if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
2549 break;
2550 if (i < btblmax) {
2551 if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
2552 return ("Intel(r) Celeron(r)");
2553 if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
2554 return ("Intel(r) Xeon(tm) MP");
2555 if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
2556 return ("Intel(r) Xeon(tm)");
2557 return (brand_tbl[i].bt_str);
2558 }
2559 }
2560
2561 return (NULL);
2562 }
2563
2564 static const char *
2565 amd_cpubrand(const struct cpuid_info *cpi)
2566 {
2567 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2568 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2569 return ("i486 compatible");
2570
2571 switch (cpi->cpi_family) {
2572 case 5:
2573 switch (cpi->cpi_model) {
2574 case 0:
2575 case 1:
2576 case 2:
2577 case 3:
2578 case 4:
2579 case 5:
2580 return ("AMD-K5(r)");
2581 case 6:
2582 case 7:
2583 return ("AMD-K6(r)");
2584 case 8:
2585 return ("AMD-K6(r)-2");
2586 case 9:
2587 return ("AMD-K6(r)-III");
2588 default:
2589 return ("AMD (family 5)");
2590 }
2591 case 6:
2592 switch (cpi->cpi_model) {
2593 case 1:
2594 return ("AMD-K7(tm)");
2595 case 0:
2596 case 2:
2597 case 4:
2598 return ("AMD Athlon(tm)");
2599 case 3:
2600 case 7:
2601 return ("AMD Duron(tm)");
2602 case 6:
2603 case 8:
2604 case 10:
2605 /*
2606 * Use the L2 cache size to distinguish
2607 */
2608 return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
2609 "AMD Athlon(tm)" : "AMD Duron(tm)");
2610 default:
2611 return ("AMD (family 6)");
2612 }
2613 default:
2614 break;
2615 }
2616
2617 if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
2618 cpi->cpi_brandid != 0) {
2619 switch (BITX(cpi->cpi_brandid, 7, 5)) {
2620 case 3:
2621 return ("AMD Opteron(tm) UP 1xx");
2622 case 4:
2623 return ("AMD Opteron(tm) DP 2xx");
2624 case 5:
2625 return ("AMD Opteron(tm) MP 8xx");
2626 default:
2627 return ("AMD Opteron(tm)");
2628 }
2629 }
2630
2631 return (NULL);
2632 }
2633
2634 static const char *
2635 cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
2636 {
2637 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2638 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5 ||
2639 type == X86_TYPE_CYRIX_486)
2640 return ("i486 compatible");
2641
2642 switch (type) {
2643 case X86_TYPE_CYRIX_6x86:
2644 return ("Cyrix 6x86");
2645 case X86_TYPE_CYRIX_6x86L:
2646 return ("Cyrix 6x86L");
2647 case X86_TYPE_CYRIX_6x86MX:
2648 return ("Cyrix 6x86MX");
2649 case X86_TYPE_CYRIX_GXm:
2650 return ("Cyrix GXm");
2651 case X86_TYPE_CYRIX_MediaGX:
2652 return ("Cyrix MediaGX");
2653 case X86_TYPE_CYRIX_MII:
2654 return ("Cyrix M2");
2655 case X86_TYPE_VIA_CYRIX_III:
2656 return ("VIA Cyrix M3");
2657 default:
2658 /*
2659 * Have another wild guess ..
2660 */
2661 if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
2662 return ("Cyrix 5x86");
2663 else if (cpi->cpi_family == 5) {
2664 switch (cpi->cpi_model) {
2665 case 2:
2666 return ("Cyrix 6x86"); /* Cyrix M1 */
2667 case 4:
2668 return ("Cyrix MediaGX");
2669 default:
2670 break;
2671 }
2672 } else if (cpi->cpi_family == 6) {
2673 switch (cpi->cpi_model) {
2674 case 0:
2675 return ("Cyrix 6x86MX"); /* Cyrix M2? */
2676 case 5:
2677 case 6:
2678 case 7:
2679 case 8:
2680 case 9:
2681 return ("VIA C3");
2682 default:
2683 break;
2684 }
2685 }
2686 break;
2687 }
2688 return (NULL);
2689 }
2690
2691 /*
2692 * This only gets called in the case that the CPU extended
2693 * feature brand string (0x80000002, 0x80000003, 0x80000004)
2694 * aren't available, or contain null bytes for some reason.
2695 */
2696 static void
2697 fabricate_brandstr(struct cpuid_info *cpi)
2698 {
2699 const char *brand = NULL;
2700
2701 switch (cpi->cpi_vendor) {
2702 case X86_VENDOR_Intel:
2703 brand = intel_cpubrand(cpi);
2704 break;
2705 case X86_VENDOR_AMD:
2706 brand = amd_cpubrand(cpi);
2707 break;
2708 case X86_VENDOR_Cyrix:
2709 brand = cyrix_cpubrand(cpi, x86_type);
2710 break;
2711 case X86_VENDOR_NexGen:
2712 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2713 brand = "NexGen Nx586";
2714 break;
2715 case X86_VENDOR_Centaur:
2716 if (cpi->cpi_family == 5)
2717 switch (cpi->cpi_model) {
2718 case 4:
2719 brand = "Centaur C6";
2720 break;
2721 case 8:
2722 brand = "Centaur C2";
2723 break;
2724 case 9:
2725 brand = "Centaur C3";
2726 break;
2727 default:
2728 break;
2729 }
2730 break;
2731 case X86_VENDOR_Rise:
2732 if (cpi->cpi_family == 5 &&
2733 (cpi->cpi_model == 0 || cpi->cpi_model == 2))
2734 brand = "Rise mP6";
2735 break;
2736 case X86_VENDOR_SiS:
2737 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2738 brand = "SiS 55x";
2739 break;
2740 case X86_VENDOR_TM:
2741 if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
2742 brand = "Transmeta Crusoe TM3x00 or TM5x00";
2743 break;
2744 case X86_VENDOR_NSC:
2745 case X86_VENDOR_UMC:
2746 default:
2747 break;
2748 }
2749 if (brand) {
2750 (void) strcpy((char *)cpi->cpi_brandstr, brand);
2751 return;
2752 }
2753
2754 /*
2755 * If all else fails ...
2756 */
2757 (void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
2758 "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
2759 cpi->cpi_model, cpi->cpi_step);
2760 }
2761
2762 /*
2763 * This routine is called just after kernel memory allocation
2764 * becomes available on cpu0, and as part of mp_startup() on
2765 * the other cpus.
2766 *
2767 * Fixup the brand string, and collect any information from cpuid
2768 * that requires dynamically allocated storage to represent.
2769 */
2770 /*ARGSUSED*/
2771 void
2772 cpuid_pass3(cpu_t *cpu)
2773 {
2774 int i, max, shft, level, size;
2775 struct cpuid_regs regs;
2776 struct cpuid_regs *cp;
2777 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2778
2779 ASSERT(cpi->cpi_pass == 2);
2780
2781 /*
2782 * Function 4: Deterministic cache parameters
2783 *
2784 * Take this opportunity to detect the number of threads
2785 * sharing the last level cache, and construct a corresponding
2786 * cache id. The respective cpuid_info members are initialized
2787 * to the default case of "no last level cache sharing".
2788 */
2789 cpi->cpi_ncpu_shr_last_cache = 1;
2790 cpi->cpi_last_lvl_cacheid = cpu->cpu_id;
2791
2792 if (cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) {
2793
2794 /*
2795 * Find the # of elements (size) returned by fn 4, and along
2796 * the way detect last level cache sharing details.
2797 */
2798 bzero(®s, sizeof (regs));
2799 cp = ®s;
2800 for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
2801 cp->cp_eax = 4;
2802 cp->cp_ecx = i;
2803
2804 (void) __cpuid_insn(cp);
2805
2806 if (CPI_CACHE_TYPE(cp) == 0)
2807 break;
2808 level = CPI_CACHE_LVL(cp);
2809 if (level > max) {
2810 max = level;
2811 cpi->cpi_ncpu_shr_last_cache =
2812 CPI_NTHR_SHR_CACHE(cp) + 1;
2813 }
2814 }
2815 cpi->cpi_std_4_size = size = i;
2816
2817 /*
2818 * Allocate the cpi_std_4 array. The first element
2819 * references the regs for fn 4, %ecx == 0, which
2820 * cpuid_pass2() stashed in cpi->cpi_std[4].
2821 */
2822 if (size > 0) {
2823 cpi->cpi_std_4 =
2824 kmem_alloc(size * sizeof (cp), KM_SLEEP);
2825 cpi->cpi_std_4[0] = &cpi->cpi_std[4];
2826
2827 /*
2828 * Allocate storage to hold the additional regs
2829 * for function 4, %ecx == 1 .. cpi_std_4_size.
2830 *
2831 * The regs for fn 4, %ecx == 0 has already
2832 * been allocated as indicated above.
2833 */
2834 for (i = 1; i < size; i++) {
2835 cp = cpi->cpi_std_4[i] =
2836 kmem_zalloc(sizeof (regs), KM_SLEEP);
2837 cp->cp_eax = 4;
2838 cp->cp_ecx = i;
2839
2840 (void) __cpuid_insn(cp);
2841 }
2842 }
2843 /*
2844 * Determine the number of bits needed to represent
2845 * the number of CPUs sharing the last level cache.
2846 *
2847 * Shift off that number of bits from the APIC id to
2848 * derive the cache id.
2849 */
2850 shft = 0;
2851 for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
2852 shft++;
2853 cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
2854 }
2855
2856 /*
2857 * Now fixup the brand string
2858 */
2859 if ((cpi->cpi_xmaxeax & 0x80000000) == 0) {
2860 fabricate_brandstr(cpi);
2861 } else {
2862
2863 /*
2864 * If we successfully extracted a brand string from the cpuid
2865 * instruction, clean it up by removing leading spaces and
2866 * similar junk.
2867 */
2868 if (cpi->cpi_brandstr[0]) {
2869 size_t maxlen = sizeof (cpi->cpi_brandstr);
2870 char *src, *dst;
2871
2872 dst = src = (char *)cpi->cpi_brandstr;
2873 src[maxlen - 1] = '\0';
2874 /*
2875 * strip leading spaces
2876 */
2877 while (*src == ' ')
2878 src++;
2879 /*
2880 * Remove any 'Genuine' or "Authentic" prefixes
2881 */
2882 if (strncmp(src, "Genuine ", 8) == 0)
2883 src += 8;
2884 if (strncmp(src, "Authentic ", 10) == 0)
2885 src += 10;
2886
2887 /*
2888 * Now do an in-place copy.
2889 * Map (R) to (r) and (TM) to (tm).
2890 * The era of teletypes is long gone, and there's
2891 * -really- no need to shout.
2892 */
2893 while (*src != '\0') {
2894 if (src[0] == '(') {
2895 if (strncmp(src + 1, "R)", 2) == 0) {
2896 (void) strncpy(dst, "(r)", 3);
2897 src += 3;
2898 dst += 3;
2899 continue;
2900 }
2901 if (strncmp(src + 1, "TM)", 3) == 0) {
2902 (void) strncpy(dst, "(tm)", 4);
2903 src += 4;
2904 dst += 4;
2905 continue;
2906 }
2907 }
2908 *dst++ = *src++;
2909 }
2910 *dst = '\0';
2911
2912 /*
2913 * Finally, remove any trailing spaces
2914 */
2915 while (--dst > cpi->cpi_brandstr)
2916 if (*dst == ' ')
2917 *dst = '\0';
2918 else
2919 break;
2920 } else
2921 fabricate_brandstr(cpi);
2922 }
2923 cpi->cpi_pass = 3;
2924 }
2925
2926 /*
2927 * This routine is called out of bind_hwcap() much later in the life
2928 * of the kernel (post_startup()). The job of this routine is to resolve
2929 * the hardware feature support and kernel support for those features into
2930 * what we're actually going to tell applications via the aux vector.
2931 */
2932 void
2933 cpuid_pass4(cpu_t *cpu, uint_t *hwcap_out)
2934 {
2935 struct cpuid_info *cpi;
2936 uint_t hwcap_flags = 0, hwcap_flags_2 = 0;
2937
2938 if (cpu == NULL)
2939 cpu = CPU;
2940 cpi = cpu->cpu_m.mcpu_cpi;
2941
2942 ASSERT(cpi->cpi_pass == 3);
2943
2944 if (cpi->cpi_maxeax >= 1) {
2945 uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
2946 uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];
2947 uint32_t *ebx = &cpi->cpi_support[STD_EBX_FEATURES];
2948
2949 *edx = CPI_FEATURES_EDX(cpi);
2950 *ecx = CPI_FEATURES_ECX(cpi);
2951 *ebx = CPI_FEATURES_7_0_EBX(cpi);
2952
2953 /*
2954 * [these require explicit kernel support]
2955 */
2956 if (!is_x86_feature(x86_featureset, X86FSET_SEP))
2957 *edx &= ~CPUID_INTC_EDX_SEP;
2958
2959 if (!is_x86_feature(x86_featureset, X86FSET_SSE))
2960 *edx &= ~(CPUID_INTC_EDX_FXSR|CPUID_INTC_EDX_SSE);
2961 if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
2962 *edx &= ~CPUID_INTC_EDX_SSE2;
2963
2964 if (!is_x86_feature(x86_featureset, X86FSET_HTT))
2965 *edx &= ~CPUID_INTC_EDX_HTT;
2966
2967 if (!is_x86_feature(x86_featureset, X86FSET_SSE3))
2968 *ecx &= ~CPUID_INTC_ECX_SSE3;
2969
2970 if (!is_x86_feature(x86_featureset, X86FSET_SSSE3))
2971 *ecx &= ~CPUID_INTC_ECX_SSSE3;
2972 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_1))
2973 *ecx &= ~CPUID_INTC_ECX_SSE4_1;
2974 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2))
2975 *ecx &= ~CPUID_INTC_ECX_SSE4_2;
2976 if (!is_x86_feature(x86_featureset, X86FSET_AES))
2977 *ecx &= ~CPUID_INTC_ECX_AES;
2978 if (!is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ))
2979 *ecx &= ~CPUID_INTC_ECX_PCLMULQDQ;
2980 if (!is_x86_feature(x86_featureset, X86FSET_XSAVE))
2981 *ecx &= ~(CPUID_INTC_ECX_XSAVE |
2982 CPUID_INTC_ECX_OSXSAVE);
2983 if (!is_x86_feature(x86_featureset, X86FSET_AVX))
2984 *ecx &= ~CPUID_INTC_ECX_AVX;
2985 if (!is_x86_feature(x86_featureset, X86FSET_F16C))
2986 *ecx &= ~CPUID_INTC_ECX_F16C;
2987 if (!is_x86_feature(x86_featureset, X86FSET_FMA))
2988 *ecx &= ~CPUID_INTC_ECX_FMA;
2989 if (!is_x86_feature(x86_featureset, X86FSET_BMI1))
2990 *ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
2991 if (!is_x86_feature(x86_featureset, X86FSET_BMI2))
2992 *ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
2993 if (!is_x86_feature(x86_featureset, X86FSET_AVX2))
2994 *ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
2995 if (!is_x86_feature(x86_featureset, X86FSET_RDSEED))
2996 *ebx &= ~CPUID_INTC_EBX_7_0_RDSEED;
2997 if (!is_x86_feature(x86_featureset, X86FSET_ADX))
2998 *ebx &= ~CPUID_INTC_EBX_7_0_ADX;
2999
3000 /*
3001 * [no explicit support required beyond x87 fp context]
3002 */
3003 if (!fpu_exists)
3004 *edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);
3005
3006 /*
3007 * Now map the supported feature vector to things that we
3008 * think userland will care about.
3009 */
3010 if (*edx & CPUID_INTC_EDX_SEP)
3011 hwcap_flags |= AV_386_SEP;
3012 if (*edx & CPUID_INTC_EDX_SSE)
3013 hwcap_flags |= AV_386_FXSR | AV_386_SSE;
3014 if (*edx & CPUID_INTC_EDX_SSE2)
3015 hwcap_flags |= AV_386_SSE2;
3016 if (*ecx & CPUID_INTC_ECX_SSE3)
3017 hwcap_flags |= AV_386_SSE3;
3018 if (*ecx & CPUID_INTC_ECX_SSSE3)
3019 hwcap_flags |= AV_386_SSSE3;
3020 if (*ecx & CPUID_INTC_ECX_SSE4_1)
3021 hwcap_flags |= AV_386_SSE4_1;
3022 if (*ecx & CPUID_INTC_ECX_SSE4_2)
3023 hwcap_flags |= AV_386_SSE4_2;
3024 if (*ecx & CPUID_INTC_ECX_MOVBE)
3025 hwcap_flags |= AV_386_MOVBE;
3026 if (*ecx & CPUID_INTC_ECX_AES)
3027 hwcap_flags |= AV_386_AES;
3028 if (*ecx & CPUID_INTC_ECX_PCLMULQDQ)
3029 hwcap_flags |= AV_386_PCLMULQDQ;
3030 if ((*ecx & CPUID_INTC_ECX_XSAVE) &&
3031 (*ecx & CPUID_INTC_ECX_OSXSAVE)) {
3032 hwcap_flags |= AV_386_XSAVE;
3033
3034 if (*ecx & CPUID_INTC_ECX_AVX) {
3035 uint32_t *ecx_7 = &CPI_FEATURES_7_0_ECX(cpi);
3036 uint32_t *edx_7 = &CPI_FEATURES_7_0_EDX(cpi);
3037
3038 hwcap_flags |= AV_386_AVX;
3039 if (*ecx & CPUID_INTC_ECX_F16C)
3040 hwcap_flags_2 |= AV_386_2_F16C;
3041 if (*ecx & CPUID_INTC_ECX_FMA)
3042 hwcap_flags_2 |= AV_386_2_FMA;
3043
3044 if (*ebx & CPUID_INTC_EBX_7_0_BMI1)
3045 hwcap_flags_2 |= AV_386_2_BMI1;
3046 if (*ebx & CPUID_INTC_EBX_7_0_BMI2)
3047 hwcap_flags_2 |= AV_386_2_BMI2;
3048 if (*ebx & CPUID_INTC_EBX_7_0_AVX2)
3049 hwcap_flags_2 |= AV_386_2_AVX2;
3050 if (*ebx & CPUID_INTC_EBX_7_0_AVX512F)
3051 hwcap_flags_2 |= AV_386_2_AVX512F;
3052 if (*ebx & CPUID_INTC_EBX_7_0_AVX512DQ)
3053 hwcap_flags_2 |= AV_386_2_AVX512DQ;
3054 if (*ebx & CPUID_INTC_EBX_7_0_AVX512IFMA)
3055 hwcap_flags_2 |= AV_386_2_AVX512IFMA;
3056 if (*ebx & CPUID_INTC_EBX_7_0_AVX512PF)
3057 hwcap_flags_2 |= AV_386_2_AVX512PF;
3058 if (*ebx & CPUID_INTC_EBX_7_0_AVX512ER)
3059 hwcap_flags_2 |= AV_386_2_AVX512ER;
3060 if (*ebx & CPUID_INTC_EBX_7_0_AVX512CD)
3061 hwcap_flags_2 |= AV_386_2_AVX512CD;
3062 if (*ebx & CPUID_INTC_EBX_7_0_AVX512BW)
3063 hwcap_flags_2 |= AV_386_2_AVX512BW;
3064 if (*ebx & CPUID_INTC_EBX_7_0_AVX512VL)
3065 hwcap_flags_2 |= AV_386_2_AVX512VL;
3066
3067 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VBMI)
3068 hwcap_flags_2 |= AV_386_2_AVX512VBMI;
3069 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
3070 hwcap_flags_2 |= AV_386_2_AVX512VPOPCDQ;
3071
3072 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124NNIW)
3073 hwcap_flags_2 |= AV_386_2_AVX512_4NNIW;
3074 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124FMAPS)
3075 hwcap_flags_2 |= AV_386_2_AVX512_4FMAPS;
3076 }
3077 }
3078 if (*ecx & CPUID_INTC_ECX_VMX)
3079 hwcap_flags |= AV_386_VMX;
3080 if (*ecx & CPUID_INTC_ECX_POPCNT)
3081 hwcap_flags |= AV_386_POPCNT;
3082 if (*edx & CPUID_INTC_EDX_FPU)
3083 hwcap_flags |= AV_386_FPU;
3084 if (*edx & CPUID_INTC_EDX_MMX)
3085 hwcap_flags |= AV_386_MMX;
3086
3087 if (*edx & CPUID_INTC_EDX_TSC)
3088 hwcap_flags |= AV_386_TSC;
3089 if (*edx & CPUID_INTC_EDX_CX8)
3090 hwcap_flags |= AV_386_CX8;
3091 if (*edx & CPUID_INTC_EDX_CMOV)
3092 hwcap_flags |= AV_386_CMOV;
3093 if (*ecx & CPUID_INTC_ECX_CX16)
3094 hwcap_flags |= AV_386_CX16;
3095
3096 if (*ecx & CPUID_INTC_ECX_RDRAND)
3097 hwcap_flags_2 |= AV_386_2_RDRAND;
3098 if (*ebx & CPUID_INTC_EBX_7_0_ADX)
3099 hwcap_flags_2 |= AV_386_2_ADX;
3100 if (*ebx & CPUID_INTC_EBX_7_0_RDSEED)
3101 hwcap_flags_2 |= AV_386_2_RDSEED;
3102
3103 }
3104
3105 if (cpi->cpi_xmaxeax < 0x80000001)
3106 goto pass4_done;
3107
3108 switch (cpi->cpi_vendor) {
3109 struct cpuid_regs cp;
3110 uint32_t *edx, *ecx;
3111
3112 case X86_VENDOR_Intel:
3113 /*
3114 * Seems like Intel duplicated what we necessary
3115 * here to make the initial crop of 64-bit OS's work.
3116 * Hopefully, those are the only "extended" bits
3117 * they'll add.
3118 */
3119 /*FALLTHROUGH*/
3120
3121 case X86_VENDOR_AMD:
3122 edx = &cpi->cpi_support[AMD_EDX_FEATURES];
3123 ecx = &cpi->cpi_support[AMD_ECX_FEATURES];
3124
3125 *edx = CPI_FEATURES_XTD_EDX(cpi);
3126 *ecx = CPI_FEATURES_XTD_ECX(cpi);
3127
3128 /*
3129 * [these features require explicit kernel support]
3130 */
3131 switch (cpi->cpi_vendor) {
3132 case X86_VENDOR_Intel:
3133 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3134 *edx &= ~CPUID_AMD_EDX_TSCP;
3135 break;
3136
3137 case X86_VENDOR_AMD:
3138 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3139 *edx &= ~CPUID_AMD_EDX_TSCP;
3140 if (!is_x86_feature(x86_featureset, X86FSET_SSE4A))
3141 *ecx &= ~CPUID_AMD_ECX_SSE4A;
3142 break;
3143
3144 default:
3145 break;
3146 }
3147
3148 /*
3149 * [no explicit support required beyond
3150 * x87 fp context and exception handlers]
3151 */
3152 if (!fpu_exists)
3153 *edx &= ~(CPUID_AMD_EDX_MMXamd |
3154 CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);
3155
3156 if (!is_x86_feature(x86_featureset, X86FSET_NX))
3157 *edx &= ~CPUID_AMD_EDX_NX;
3158 #if !defined(__amd64)
3159 *edx &= ~CPUID_AMD_EDX_LM;
3160 #endif
3161 /*
3162 * Now map the supported feature vector to
3163 * things that we think userland will care about.
3164 */
3165 #if defined(__amd64)
3166 if (*edx & CPUID_AMD_EDX_SYSC)
3167 hwcap_flags |= AV_386_AMD_SYSC;
3168 #endif
3169 if (*edx & CPUID_AMD_EDX_MMXamd)
3170 hwcap_flags |= AV_386_AMD_MMX;
3171 if (*edx & CPUID_AMD_EDX_3DNow)
3172 hwcap_flags |= AV_386_AMD_3DNow;
3173 if (*edx & CPUID_AMD_EDX_3DNowx)
3174 hwcap_flags |= AV_386_AMD_3DNowx;
3175 if (*ecx & CPUID_AMD_ECX_SVM)
3176 hwcap_flags |= AV_386_AMD_SVM;
3177
3178 switch (cpi->cpi_vendor) {
3179 case X86_VENDOR_AMD:
3180 if (*edx & CPUID_AMD_EDX_TSCP)
3181 hwcap_flags |= AV_386_TSCP;
3182 if (*ecx & CPUID_AMD_ECX_AHF64)
3183 hwcap_flags |= AV_386_AHF;
3184 if (*ecx & CPUID_AMD_ECX_SSE4A)
3185 hwcap_flags |= AV_386_AMD_SSE4A;
3186 if (*ecx & CPUID_AMD_ECX_LZCNT)
3187 hwcap_flags |= AV_386_AMD_LZCNT;
3188 break;
3189
3190 case X86_VENDOR_Intel:
3191 if (*edx & CPUID_AMD_EDX_TSCP)
3192 hwcap_flags |= AV_386_TSCP;
3193 /*
3194 * Aarrgh.
3195 * Intel uses a different bit in the same word.
3196 */
3197 if (*ecx & CPUID_INTC_ECX_AHF64)
3198 hwcap_flags |= AV_386_AHF;
3199 break;
3200
3201 default:
3202 break;
3203 }
3204 break;
3205
3206 case X86_VENDOR_TM:
3207 cp.cp_eax = 0x80860001;
3208 (void) __cpuid_insn(&cp);
3209 cpi->cpi_support[TM_EDX_FEATURES] = cp.cp_edx;
3210 break;
3211
3212 default:
3213 break;
3214 }
3215
3216 pass4_done:
3217 cpi->cpi_pass = 4;
3218 if (hwcap_out != NULL) {
3219 hwcap_out[0] = hwcap_flags;
3220 hwcap_out[1] = hwcap_flags_2;
3221 }
3222 }
3223
3224
3225 /*
3226 * Simulate the cpuid instruction using the data we previously
3227 * captured about this CPU. We try our best to return the truth
3228 * about the hardware, independently of kernel support.
3229 */
3230 uint32_t
3231 cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
3232 {
3233 struct cpuid_info *cpi;
3234 struct cpuid_regs *xcp;
3235
3236 if (cpu == NULL)
3237 cpu = CPU;
3238 cpi = cpu->cpu_m.mcpu_cpi;
3239
3240 ASSERT(cpuid_checkpass(cpu, 3));
3241
3242 /*
3243 * CPUID data is cached in two separate places: cpi_std for standard
3244 * CPUID functions, and cpi_extd for extended CPUID functions.
3245 */
3246 if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD)
3247 xcp = &cpi->cpi_std[cp->cp_eax];
3248 else if (cp->cp_eax >= 0x80000000 && cp->cp_eax <= cpi->cpi_xmaxeax &&
3249 cp->cp_eax < 0x80000000 + NMAX_CPI_EXTD)
3250 xcp = &cpi->cpi_extd[cp->cp_eax - 0x80000000];
3251 else
3252 /*
3253 * The caller is asking for data from an input parameter which
3254 * the kernel has not cached. In this case we go fetch from
3255 * the hardware and return the data directly to the user.
3256 */
3257 return (__cpuid_insn(cp));
3258
3259 cp->cp_eax = xcp->cp_eax;
3260 cp->cp_ebx = xcp->cp_ebx;
3261 cp->cp_ecx = xcp->cp_ecx;
3262 cp->cp_edx = xcp->cp_edx;
3263 return (cp->cp_eax);
3264 }
3265
3266 int
3267 cpuid_checkpass(cpu_t *cpu, int pass)
3268 {
3269 return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
3270 cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
3271 }
3272
3273 int
3274 cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
3275 {
3276 ASSERT(cpuid_checkpass(cpu, 3));
3277
3278 return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
3279 }
3280
3281 int
3282 cpuid_is_cmt(cpu_t *cpu)
3283 {
3284 if (cpu == NULL)
3285 cpu = CPU;
3286
3287 ASSERT(cpuid_checkpass(cpu, 1));
3288
3289 return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
3290 }
3291
3292 /*
3293 * AMD and Intel both implement the 64-bit variant of the syscall
3294 * instruction (syscallq), so if there's -any- support for syscall,
3295 * cpuid currently says "yes, we support this".
3296 *
3297 * However, Intel decided to -not- implement the 32-bit variant of the
3298 * syscall instruction, so we provide a predicate to allow our caller
3299 * to test that subtlety here.
3300 *
3301 * XXPV Currently, 32-bit syscall instructions don't work via the hypervisor,
3302 * even in the case where the hardware would in fact support it.
3303 */
3304 /*ARGSUSED*/
3305 int
3306 cpuid_syscall32_insn(cpu_t *cpu)
3307 {
3308 ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), 1));
3309
3310 #if !defined(__xpv)
3311 if (cpu == NULL)
3312 cpu = CPU;
3313
3314 /*CSTYLED*/
3315 {
3316 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3317
3318 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
3319 cpi->cpi_xmaxeax >= 0x80000001 &&
3320 (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
3321 return (1);
3322 }
3323 #endif
3324 return (0);
3325 }
3326
3327 int
3328 cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
3329 {
3330 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3331
3332 static const char fmt[] =
3333 "x86 (%s %X family %d model %d step %d clock %d MHz)";
3334 static const char fmt_ht[] =
3335 "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";
3336
3337 ASSERT(cpuid_checkpass(cpu, 1));
3338
3339 if (cpuid_is_cmt(cpu))
3340 return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
3341 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3342 cpi->cpi_family, cpi->cpi_model,
3343 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3344 return (snprintf(s, n, fmt,
3345 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3346 cpi->cpi_family, cpi->cpi_model,
3347 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3348 }
3349
3350 const char *
3351 cpuid_getvendorstr(cpu_t *cpu)
3352 {
3353 ASSERT(cpuid_checkpass(cpu, 1));
3354 return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
3355 }
3356
3357 uint_t
3358 cpuid_getvendor(cpu_t *cpu)
3359 {
3360 ASSERT(cpuid_checkpass(cpu, 1));
3361 return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
3362 }
3363
3364 uint_t
3365 cpuid_getfamily(cpu_t *cpu)
3366 {
3367 ASSERT(cpuid_checkpass(cpu, 1));
3368 return (cpu->cpu_m.mcpu_cpi->cpi_family);
3369 }
3370
3371 uint_t
3372 cpuid_getmodel(cpu_t *cpu)
3373 {
3374 ASSERT(cpuid_checkpass(cpu, 1));
3375 return (cpu->cpu_m.mcpu_cpi->cpi_model);
3376 }
3377
3378 uint_t
3379 cpuid_get_ncpu_per_chip(cpu_t *cpu)
3380 {
3381 ASSERT(cpuid_checkpass(cpu, 1));
3382 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
3383 }
3384
3385 uint_t
3386 cpuid_get_ncore_per_chip(cpu_t *cpu)
3387 {
3388 ASSERT(cpuid_checkpass(cpu, 1));
3389 return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
3390 }
3391
3392 uint_t
3393 cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
3394 {
3395 ASSERT(cpuid_checkpass(cpu, 2));
3396 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
3397 }
3398
3399 id_t
3400 cpuid_get_last_lvl_cacheid(cpu_t *cpu)
3401 {
3402 ASSERT(cpuid_checkpass(cpu, 2));
3403 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3404 }
3405
3406 uint_t
3407 cpuid_getstep(cpu_t *cpu)
3408 {
3409 ASSERT(cpuid_checkpass(cpu, 1));
3410 return (cpu->cpu_m.mcpu_cpi->cpi_step);
3411 }
3412
3413 uint_t
3414 cpuid_getsig(struct cpu *cpu)
3415 {
3416 ASSERT(cpuid_checkpass(cpu, 1));
3417 return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
3418 }
3419
3420 uint32_t
3421 cpuid_getchiprev(struct cpu *cpu)
3422 {
3423 ASSERT(cpuid_checkpass(cpu, 1));
3424 return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
3425 }
3426
3427 const char *
3428 cpuid_getchiprevstr(struct cpu *cpu)
3429 {
3430 ASSERT(cpuid_checkpass(cpu, 1));
3431 return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
3432 }
3433
3434 uint32_t
3435 cpuid_getsockettype(struct cpu *cpu)
3436 {
3437 ASSERT(cpuid_checkpass(cpu, 1));
3438 return (cpu->cpu_m.mcpu_cpi->cpi_socket);
3439 }
3440
3441 const char *
3442 cpuid_getsocketstr(cpu_t *cpu)
3443 {
3444 static const char *socketstr = NULL;
3445 struct cpuid_info *cpi;
3446
3447 ASSERT(cpuid_checkpass(cpu, 1));
3448 cpi = cpu->cpu_m.mcpu_cpi;
3449
3450 /* Assume that socket types are the same across the system */
3451 if (socketstr == NULL)
3452 socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
3453 cpi->cpi_model, cpi->cpi_step);
3454
3455
3456 return (socketstr);
3457 }
3458
3459 int
3460 cpuid_get_chipid(cpu_t *cpu)
3461 {
3462 ASSERT(cpuid_checkpass(cpu, 1));
3463
3464 if (cpuid_is_cmt(cpu))
3465 return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
3466 return (cpu->cpu_id);
3467 }
3468
3469 id_t
3470 cpuid_get_coreid(cpu_t *cpu)
3471 {
3472 ASSERT(cpuid_checkpass(cpu, 1));
3473 return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
3474 }
3475
3476 int
3477 cpuid_get_pkgcoreid(cpu_t *cpu)
3478 {
3479 ASSERT(cpuid_checkpass(cpu, 1));
3480 return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
3481 }
3482
3483 int
3484 cpuid_get_clogid(cpu_t *cpu)
3485 {
3486 ASSERT(cpuid_checkpass(cpu, 1));
3487 return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
3488 }
3489
3490 int
3491 cpuid_get_cacheid(cpu_t *cpu)
3492 {
3493 ASSERT(cpuid_checkpass(cpu, 1));
3494 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3495 }
3496
3497 uint_t
3498 cpuid_get_procnodeid(cpu_t *cpu)
3499 {
3500 ASSERT(cpuid_checkpass(cpu, 1));
3501 return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
3502 }
3503
3504 uint_t
3505 cpuid_get_procnodes_per_pkg(cpu_t *cpu)
3506 {
3507 ASSERT(cpuid_checkpass(cpu, 1));
3508 return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
3509 }
3510
3511 uint_t
3512 cpuid_get_compunitid(cpu_t *cpu)
3513 {
3514 ASSERT(cpuid_checkpass(cpu, 1));
3515 return (cpu->cpu_m.mcpu_cpi->cpi_compunitid);
3516 }
3517
3518 uint_t
3519 cpuid_get_cores_per_compunit(cpu_t *cpu)
3520 {
3521 ASSERT(cpuid_checkpass(cpu, 1));
3522 return (cpu->cpu_m.mcpu_cpi->cpi_cores_per_compunit);
3523 }
3524
3525 /*ARGSUSED*/
3526 int
3527 cpuid_have_cr8access(cpu_t *cpu)
3528 {
3529 #if defined(__amd64)
3530 return (1);
3531 #else
3532 struct cpuid_info *cpi;
3533
3534 ASSERT(cpu != NULL);
3535 cpi = cpu->cpu_m.mcpu_cpi;
3536 if (cpi->cpi_vendor == X86_VENDOR_AMD && cpi->cpi_maxeax >= 1 &&
3537 (CPI_FEATURES_XTD_ECX(cpi) & CPUID_AMD_ECX_CR8D) != 0)
3538 return (1);
3539 return (0);
3540 #endif
3541 }
3542
3543 uint32_t
3544 cpuid_get_apicid(cpu_t *cpu)
3545 {
3546 ASSERT(cpuid_checkpass(cpu, 1));
3547 if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
3548 return (UINT32_MAX);
3549 } else {
3550 return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
3551 }
3552 }
3553
3554 void
3555 cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
3556 {
3557 struct cpuid_info *cpi;
3558
3559 if (cpu == NULL)
3560 cpu = CPU;
3561 cpi = cpu->cpu_m.mcpu_cpi;
3562
3563 ASSERT(cpuid_checkpass(cpu, 1));
3564
3565 if (pabits)
3566 *pabits = cpi->cpi_pabits;
3567 if (vabits)
3568 *vabits = cpi->cpi_vabits;
3569 }
3570
3571 size_t
3572 cpuid_get_xsave_size()
3573 {
3574 return (MAX(cpuid_info0.cpi_xsave.xsav_max_size,
3575 sizeof (struct xsave_state)));
3576 }
3577
3578 /*
3579 * Return true if the CPUs on this system require 'pointer clearing' for the
3580 * floating point error pointer exception handling. In the past, this has been
3581 * true for all AMD K7 & K8 CPUs, although newer AMD CPUs have been changed to
3582 * behave the same as Intel. This is checked via the CPUID_AMD_EBX_ERR_PTR_ZERO
3583 * feature bit and is reflected in the cpi_fp_amd_save member. Once this has
3584 * been confirmed on hardware which supports that feature, this test should be
3585 * narrowed. In the meantime, we always follow the existing behavior on any AMD
3586 * CPU.
3587 */
3588 boolean_t
3589 cpuid_need_fp_excp_handling()
3590 {
3591 return (cpuid_info0.cpi_vendor == X86_VENDOR_AMD);
3592 }
3593
3594 /*
3595 * Returns the number of data TLB entries for a corresponding
3596 * pagesize. If it can't be computed, or isn't known, the
3597 * routine returns zero. If you ask about an architecturally
3598 * impossible pagesize, the routine will panic (so that the
3599 * hat implementor knows that things are inconsistent.)
3600 */
3601 uint_t
3602 cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
3603 {
3604 struct cpuid_info *cpi;
3605 uint_t dtlb_nent = 0;
3606
3607 if (cpu == NULL)
3608 cpu = CPU;
3609 cpi = cpu->cpu_m.mcpu_cpi;
3610
3611 ASSERT(cpuid_checkpass(cpu, 1));
3612
3613 /*
3614 * Check the L2 TLB info
3615 */
3616 if (cpi->cpi_xmaxeax >= 0x80000006) {
3617 struct cpuid_regs *cp = &cpi->cpi_extd[6];
3618
3619 switch (pagesize) {
3620
3621 case 4 * 1024:
3622 /*
3623 * All zero in the top 16 bits of the register
3624 * indicates a unified TLB. Size is in low 16 bits.
3625 */
3626 if ((cp->cp_ebx & 0xffff0000) == 0)
3627 dtlb_nent = cp->cp_ebx & 0x0000ffff;
3628 else
3629 dtlb_nent = BITX(cp->cp_ebx, 27, 16);
3630 break;
3631
3632 case 2 * 1024 * 1024:
3633 if ((cp->cp_eax & 0xffff0000) == 0)
3634 dtlb_nent = cp->cp_eax & 0x0000ffff;
3635 else
3636 dtlb_nent = BITX(cp->cp_eax, 27, 16);
3637 break;
3638
3639 default:
3640 panic("unknown L2 pagesize");
3641 /*NOTREACHED*/
3642 }
3643 }
3644
3645 if (dtlb_nent != 0)
3646 return (dtlb_nent);
3647
3648 /*
3649 * No L2 TLB support for this size, try L1.
3650 */
3651 if (cpi->cpi_xmaxeax >= 0x80000005) {
3652 struct cpuid_regs *cp = &cpi->cpi_extd[5];
3653
3654 switch (pagesize) {
3655 case 4 * 1024:
3656 dtlb_nent = BITX(cp->cp_ebx, 23, 16);
3657 break;
3658 case 2 * 1024 * 1024:
3659 dtlb_nent = BITX(cp->cp_eax, 23, 16);
3660 break;
3661 default:
3662 panic("unknown L1 d-TLB pagesize");
3663 /*NOTREACHED*/
3664 }
3665 }
3666
3667 return (dtlb_nent);
3668 }
3669
3670 /*
3671 * Return 0 if the erratum is not present or not applicable, positive
3672 * if it is, and negative if the status of the erratum is unknown.
3673 *
3674 * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
3675 * Processors" #25759, Rev 3.57, August 2005
3676 */
3677 int
3678 cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
3679 {
3680 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3681 uint_t eax;
3682
3683 /*
3684 * Bail out if this CPU isn't an AMD CPU, or if it's
3685 * a legacy (32-bit) AMD CPU.
3686 */
3687 if (cpi->cpi_vendor != X86_VENDOR_AMD ||
3688 cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
3689 cpi->cpi_family == 6)
3690
3691 return (0);
3692
3693 eax = cpi->cpi_std[1].cp_eax;
3694
3695 #define SH_B0(eax) (eax == 0xf40 || eax == 0xf50)
3696 #define SH_B3(eax) (eax == 0xf51)
3697 #define B(eax) (SH_B0(eax) || SH_B3(eax))
3698
3699 #define SH_C0(eax) (eax == 0xf48 || eax == 0xf58)
3700
3701 #define SH_CG(eax) (eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
3702 #define DH_CG(eax) (eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
3703 #define CH_CG(eax) (eax == 0xf82 || eax == 0xfb2)
3704 #define CG(eax) (SH_CG(eax) || DH_CG(eax) || CH_CG(eax))
3705
3706 #define SH_D0(eax) (eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
3707 #define DH_D0(eax) (eax == 0x10fc0 || eax == 0x10ff0)
3708 #define CH_D0(eax) (eax == 0x10f80 || eax == 0x10fb0)
3709 #define D0(eax) (SH_D0(eax) || DH_D0(eax) || CH_D0(eax))
3710
3711 #define SH_E0(eax) (eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
3712 #define JH_E1(eax) (eax == 0x20f10) /* JH8_E0 had 0x20f30 */
3713 #define DH_E3(eax) (eax == 0x20fc0 || eax == 0x20ff0)
3714 #define SH_E4(eax) (eax == 0x20f51 || eax == 0x20f71)
3715 #define BH_E4(eax) (eax == 0x20fb1)
3716 #define SH_E5(eax) (eax == 0x20f42)
3717 #define DH_E6(eax) (eax == 0x20ff2 || eax == 0x20fc2)
3718 #define JH_E6(eax) (eax == 0x20f12 || eax == 0x20f32)
3719 #define EX(eax) (SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
3720 SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
3721 DH_E6(eax) || JH_E6(eax))
3722
3723 #define DR_AX(eax) (eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
3724 #define DR_B0(eax) (eax == 0x100f20)
3725 #define DR_B1(eax) (eax == 0x100f21)
3726 #define DR_BA(eax) (eax == 0x100f2a)
3727 #define DR_B2(eax) (eax == 0x100f22)
3728 #define DR_B3(eax) (eax == 0x100f23)
3729 #define RB_C0(eax) (eax == 0x100f40)
3730
3731 switch (erratum) {
3732 case 1:
3733 return (cpi->cpi_family < 0x10);
3734 case 51: /* what does the asterisk mean? */
3735 return (B(eax) || SH_C0(eax) || CG(eax));
3736 case 52:
3737 return (B(eax));
3738 case 57:
3739 return (cpi->cpi_family <= 0x11);
3740 case 58:
3741 return (B(eax));
3742 case 60:
3743 return (cpi->cpi_family <= 0x11);
3744 case 61:
3745 case 62:
3746 case 63:
3747 case 64:
3748 case 65:
3749 case 66:
3750 case 68:
3751 case 69:
3752 case 70:
3753 case 71:
3754 return (B(eax));
3755 case 72:
3756 return (SH_B0(eax));
3757 case 74:
3758 return (B(eax));
3759 case 75:
3760 return (cpi->cpi_family < 0x10);
3761 case 76:
3762 return (B(eax));
3763 case 77:
3764 return (cpi->cpi_family <= 0x11);
3765 case 78:
3766 return (B(eax) || SH_C0(eax));
3767 case 79:
3768 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3769 case 80:
3770 case 81:
3771 case 82:
3772 return (B(eax));
3773 case 83:
3774 return (B(eax) || SH_C0(eax) || CG(eax));
3775 case 85:
3776 return (cpi->cpi_family < 0x10);
3777 case 86:
3778 return (SH_C0(eax) || CG(eax));
3779 case 88:
3780 #if !defined(__amd64)
3781 return (0);
3782 #else
3783 return (B(eax) || SH_C0(eax));
3784 #endif
3785 case 89:
3786 return (cpi->cpi_family < 0x10);
3787 case 90:
3788 return (B(eax) || SH_C0(eax) || CG(eax));
3789 case 91:
3790 case 92:
3791 return (B(eax) || SH_C0(eax));
3792 case 93:
3793 return (SH_C0(eax));
3794 case 94:
3795 return (B(eax) || SH_C0(eax) || CG(eax));
3796 case 95:
3797 #if !defined(__amd64)
3798 return (0);
3799 #else
3800 return (B(eax) || SH_C0(eax));
3801 #endif
3802 case 96:
3803 return (B(eax) || SH_C0(eax) || CG(eax));
3804 case 97:
3805 case 98:
3806 return (SH_C0(eax) || CG(eax));
3807 case 99:
3808 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3809 case 100:
3810 return (B(eax) || SH_C0(eax));
3811 case 101:
3812 case 103:
3813 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3814 case 104:
3815 return (SH_C0(eax) || CG(eax) || D0(eax));
3816 case 105:
3817 case 106:
3818 case 107:
3819 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3820 case 108:
3821 return (DH_CG(eax));
3822 case 109:
3823 return (SH_C0(eax) || CG(eax) || D0(eax));
3824 case 110:
3825 return (D0(eax) || EX(eax));
3826 case 111:
3827 return (CG(eax));
3828 case 112:
3829 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3830 case 113:
3831 return (eax == 0x20fc0);
3832 case 114:
3833 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3834 case 115:
3835 return (SH_E0(eax) || JH_E1(eax));
3836 case 116:
3837 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3838 case 117:
3839 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3840 case 118:
3841 return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
3842 JH_E6(eax));
3843 case 121:
3844 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3845 case 122:
3846 return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
3847 case 123:
3848 return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
3849 case 131:
3850 return (cpi->cpi_family < 0x10);
3851 case 6336786:
3852 /*
3853 * Test for AdvPowerMgmtInfo.TscPStateInvariant
3854 * if this is a K8 family or newer processor
3855 */
3856 if (CPI_FAMILY(cpi) == 0xf) {
3857 struct cpuid_regs regs;
3858 regs.cp_eax = 0x80000007;
3859 (void) __cpuid_insn(®s);
3860 return (!(regs.cp_edx & 0x100));
3861 }
3862 return (0);
3863 case 6323525:
3864 return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
3865 (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);
3866
3867 case 6671130:
3868 /*
3869 * check for processors (pre-Shanghai) that do not provide
3870 * optimal management of 1gb ptes in its tlb.
3871 */
3872 return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);
3873
3874 case 298:
3875 return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
3876 DR_B2(eax) || RB_C0(eax));
3877
3878 case 721:
3879 #if defined(__amd64)
3880 return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);
3881 #else
3882 return (0);
3883 #endif
3884
3885 default:
3886 return (-1);
3887
3888 }
3889 }
3890
3891 /*
3892 * Determine if specified erratum is present via OSVW (OS Visible Workaround).
3893 * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
3894 */
3895 int
3896 osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
3897 {
3898 struct cpuid_info *cpi;
3899 uint_t osvwid;
3900 static int osvwfeature = -1;
3901 uint64_t osvwlength;
3902
3903
3904 cpi = cpu->cpu_m.mcpu_cpi;
3905
3906 /* confirm OSVW supported */
3907 if (osvwfeature == -1) {
3908 osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
3909 } else {
3910 /* assert that osvw feature setting is consistent on all cpus */
3911 ASSERT(osvwfeature ==
3912 (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
3913 }
3914 if (!osvwfeature)
3915 return (-1);
3916
3917 osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;
3918
3919 switch (erratum) {
3920 case 298: /* osvwid is 0 */
3921 osvwid = 0;
3922 if (osvwlength <= (uint64_t)osvwid) {
3923 /* osvwid 0 is unknown */
3924 return (-1);
3925 }
3926
3927 /*
3928 * Check the OSVW STATUS MSR to determine the state
3929 * of the erratum where:
3930 * 0 - fixed by HW
3931 * 1 - BIOS has applied the workaround when BIOS
3932 * workaround is available. (Or for other errata,
3933 * OS workaround is required.)
3934 * For a value of 1, caller will confirm that the
3935 * erratum 298 workaround has indeed been applied by BIOS.
3936 *
3937 * A 1 may be set in cpus that have a HW fix
3938 * in a mixed cpu system. Regarding erratum 298:
3939 * In a multiprocessor platform, the workaround above
3940 * should be applied to all processors regardless of
3941 * silicon revision when an affected processor is
3942 * present.
3943 */
3944
3945 return (rdmsr(MSR_AMD_OSVW_STATUS +
3946 (osvwid / OSVW_ID_CNT_PER_MSR)) &
3947 (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));
3948
3949 default:
3950 return (-1);
3951 }
3952 }
3953
3954 static const char assoc_str[] = "associativity";
3955 static const char line_str[] = "line-size";
3956 static const char size_str[] = "size";
3957
3958 static void
3959 add_cache_prop(dev_info_t *devi, const char *label, const char *type,
3960 uint32_t val)
3961 {
3962 char buf[128];
3963
3964 /*
3965 * ndi_prop_update_int() is used because it is desirable for
3966 * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
3967 */
3968 if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
3969 (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
3970 }
3971
3972 /*
3973 * Intel-style cache/tlb description
3974 *
3975 * Standard cpuid level 2 gives a randomly ordered
3976 * selection of tags that index into a table that describes
3977 * cache and tlb properties.
3978 */
3979
3980 static const char l1_icache_str[] = "l1-icache";
3981 static const char l1_dcache_str[] = "l1-dcache";
3982 static const char l2_cache_str[] = "l2-cache";
3983 static const char l3_cache_str[] = "l3-cache";
3984 static const char itlb4k_str[] = "itlb-4K";
3985 static const char dtlb4k_str[] = "dtlb-4K";
3986 static const char itlb2M_str[] = "itlb-2M";
3987 static const char itlb4M_str[] = "itlb-4M";
3988 static const char dtlb4M_str[] = "dtlb-4M";
3989 static const char dtlb24_str[] = "dtlb0-2M-4M";
3990 static const char itlb424_str[] = "itlb-4K-2M-4M";
3991 static const char itlb24_str[] = "itlb-2M-4M";
3992 static const char dtlb44_str[] = "dtlb-4K-4M";
3993 static const char sl1_dcache_str[] = "sectored-l1-dcache";
3994 static const char sl2_cache_str[] = "sectored-l2-cache";
3995 static const char itrace_str[] = "itrace-cache";
3996 static const char sl3_cache_str[] = "sectored-l3-cache";
3997 static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";
3998
3999 static const struct cachetab {
4000 uint8_t ct_code;
4001 uint8_t ct_assoc;
4002 uint16_t ct_line_size;
4003 size_t ct_size;
4004 const char *ct_label;
4005 } intel_ctab[] = {
4006 /*
4007 * maintain descending order!
4008 *
4009 * Codes ignored - Reason
4010 * ----------------------
4011 * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
4012 * f0H/f1H - Currently we do not interpret prefetch size by design
4013 */
4014 { 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
4015 { 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
4016 { 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
4017 { 0xde, 12, 64, 6*1024*1024, l3_cache_str},
4018 { 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
4019 { 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
4020 { 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
4021 { 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
4022 { 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
4023 { 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
4024 { 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
4025 { 0xd0, 4, 64, 512*1024, l3_cache_str},
4026 { 0xca, 4, 0, 512, sh_l2_tlb4k_str},
4027 { 0xc0, 4, 0, 8, dtlb44_str },
4028 { 0xba, 4, 0, 64, dtlb4k_str },
4029 { 0xb4, 4, 0, 256, dtlb4k_str },
4030 { 0xb3, 4, 0, 128, dtlb4k_str },
4031 { 0xb2, 4, 0, 64, itlb4k_str },
4032 { 0xb0, 4, 0, 128, itlb4k_str },
4033 { 0x87, 8, 64, 1024*1024, l2_cache_str},
4034 { 0x86, 4, 64, 512*1024, l2_cache_str},
4035 { 0x85, 8, 32, 2*1024*1024, l2_cache_str},
4036 { 0x84, 8, 32, 1024*1024, l2_cache_str},
4037 { 0x83, 8, 32, 512*1024, l2_cache_str},
4038 { 0x82, 8, 32, 256*1024, l2_cache_str},
4039 { 0x80, 8, 64, 512*1024, l2_cache_str},
4040 { 0x7f, 2, 64, 512*1024, l2_cache_str},
4041 { 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
4042 { 0x7c, 8, 64, 1024*1024, sl2_cache_str},
4043 { 0x7b, 8, 64, 512*1024, sl2_cache_str},
4044 { 0x7a, 8, 64, 256*1024, sl2_cache_str},
4045 { 0x79, 8, 64, 128*1024, sl2_cache_str},
4046 { 0x78, 8, 64, 1024*1024, l2_cache_str},
4047 { 0x73, 8, 0, 64*1024, itrace_str},
4048 { 0x72, 8, 0, 32*1024, itrace_str},
4049 { 0x71, 8, 0, 16*1024, itrace_str},
4050 { 0x70, 8, 0, 12*1024, itrace_str},
4051 { 0x68, 4, 64, 32*1024, sl1_dcache_str},
4052 { 0x67, 4, 64, 16*1024, sl1_dcache_str},
4053 { 0x66, 4, 64, 8*1024, sl1_dcache_str},
4054 { 0x60, 8, 64, 16*1024, sl1_dcache_str},
4055 { 0x5d, 0, 0, 256, dtlb44_str},
4056 { 0x5c, 0, 0, 128, dtlb44_str},
4057 { 0x5b, 0, 0, 64, dtlb44_str},
4058 { 0x5a, 4, 0, 32, dtlb24_str},
4059 { 0x59, 0, 0, 16, dtlb4k_str},
4060 { 0x57, 4, 0, 16, dtlb4k_str},
4061 { 0x56, 4, 0, 16, dtlb4M_str},
4062 { 0x55, 0, 0, 7, itlb24_str},
4063 { 0x52, 0, 0, 256, itlb424_str},
4064 { 0x51, 0, 0, 128, itlb424_str},
4065 { 0x50, 0, 0, 64, itlb424_str},
4066 { 0x4f, 0, 0, 32, itlb4k_str},
4067 { 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
4068 { 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
4069 { 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
4070 { 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
4071 { 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
4072 { 0x49, 16, 64, 4*1024*1024, l3_cache_str},
4073 { 0x48, 12, 64, 3*1024*1024, l2_cache_str},
4074 { 0x47, 8, 64, 8*1024*1024, l3_cache_str},
4075 { 0x46, 4, 64, 4*1024*1024, l3_cache_str},
4076 { 0x45, 4, 32, 2*1024*1024, l2_cache_str},
4077 { 0x44, 4, 32, 1024*1024, l2_cache_str},
4078 { 0x43, 4, 32, 512*1024, l2_cache_str},
4079 { 0x42, 4, 32, 256*1024, l2_cache_str},
4080 { 0x41, 4, 32, 128*1024, l2_cache_str},
4081 { 0x3e, 4, 64, 512*1024, sl2_cache_str},
4082 { 0x3d, 6, 64, 384*1024, sl2_cache_str},
4083 { 0x3c, 4, 64, 256*1024, sl2_cache_str},
4084 { 0x3b, 2, 64, 128*1024, sl2_cache_str},
4085 { 0x3a, 6, 64, 192*1024, sl2_cache_str},
4086 { 0x39, 4, 64, 128*1024, sl2_cache_str},
4087 { 0x30, 8, 64, 32*1024, l1_icache_str},
4088 { 0x2c, 8, 64, 32*1024, l1_dcache_str},
4089 { 0x29, 8, 64, 4096*1024, sl3_cache_str},
4090 { 0x25, 8, 64, 2048*1024, sl3_cache_str},
4091 { 0x23, 8, 64, 1024*1024, sl3_cache_str},
4092 { 0x22, 4, 64, 512*1024, sl3_cache_str},
4093 { 0x0e, 6, 64, 24*1024, l1_dcache_str},
4094 { 0x0d, 4, 32, 16*1024, l1_dcache_str},
4095 { 0x0c, 4, 32, 16*1024, l1_dcache_str},
4096 { 0x0b, 4, 0, 4, itlb4M_str},
4097 { 0x0a, 2, 32, 8*1024, l1_dcache_str},
4098 { 0x08, 4, 32, 16*1024, l1_icache_str},
4099 { 0x06, 4, 32, 8*1024, l1_icache_str},
4100 { 0x05, 4, 0, 32, dtlb4M_str},
4101 { 0x04, 4, 0, 8, dtlb4M_str},
4102 { 0x03, 4, 0, 64, dtlb4k_str},
4103 { 0x02, 4, 0, 2, itlb4M_str},
4104 { 0x01, 4, 0, 32, itlb4k_str},
4105 { 0 }
4106 };
4107
4108 static const struct cachetab cyrix_ctab[] = {
4109 { 0x70, 4, 0, 32, "tlb-4K" },
4110 { 0x80, 4, 16, 16*1024, "l1-cache" },
4111 { 0 }
4112 };
4113
4114 /*
4115 * Search a cache table for a matching entry
4116 */
4117 static const struct cachetab *
4118 find_cacheent(const struct cachetab *ct, uint_t code)
4119 {
4120 if (code != 0) {
4121 for (; ct->ct_code != 0; ct++)
4122 if (ct->ct_code <= code)
4123 break;
4124 if (ct->ct_code == code)
4125 return (ct);
4126 }
4127 return (NULL);
4128 }
4129
4130 /*
4131 * Populate cachetab entry with L2 or L3 cache-information using
4132 * cpuid function 4. This function is called from intel_walk_cacheinfo()
4133 * when descriptor 0x49 is encountered. It returns 0 if no such cache
4134 * information is found.
4135 */
4136 static int
4137 intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
4138 {
4139 uint32_t level, i;
4140 int ret = 0;
4141
4142 for (i = 0; i < cpi->cpi_std_4_size; i++) {
4143 level = CPI_CACHE_LVL(cpi->cpi_std_4[i]);
4144
4145 if (level == 2 || level == 3) {
4146 ct->ct_assoc = CPI_CACHE_WAYS(cpi->cpi_std_4[i]) + 1;
4147 ct->ct_line_size =
4148 CPI_CACHE_COH_LN_SZ(cpi->cpi_std_4[i]) + 1;
4149 ct->ct_size = ct->ct_assoc *
4150 (CPI_CACHE_PARTS(cpi->cpi_std_4[i]) + 1) *
4151 ct->ct_line_size *
4152 (cpi->cpi_std_4[i]->cp_ecx + 1);
4153
4154 if (level == 2) {
4155 ct->ct_label = l2_cache_str;
4156 } else if (level == 3) {
4157 ct->ct_label = l3_cache_str;
4158 }
4159 ret = 1;
4160 }
4161 }
4162
4163 return (ret);
4164 }
4165
4166 /*
4167 * Walk the cacheinfo descriptor, applying 'func' to every valid element
4168 * The walk is terminated if the walker returns non-zero.
4169 */
4170 static void
4171 intel_walk_cacheinfo(struct cpuid_info *cpi,
4172 void *arg, int (*func)(void *, const struct cachetab *))
4173 {
4174 const struct cachetab *ct;
4175 struct cachetab des_49_ct, des_b1_ct;
4176 uint8_t *dp;
4177 int i;
4178
4179 if ((dp = cpi->cpi_cacheinfo) == NULL)
4180 return;
4181 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4182 /*
4183 * For overloaded descriptor 0x49 we use cpuid function 4
4184 * if supported by the current processor, to create
4185 * cache information.
4186 * For overloaded descriptor 0xb1 we use X86_PAE flag
4187 * to disambiguate the cache information.
4188 */
4189 if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
4190 intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
4191 ct = &des_49_ct;
4192 } else if (*dp == 0xb1) {
4193 des_b1_ct.ct_code = 0xb1;
4194 des_b1_ct.ct_assoc = 4;
4195 des_b1_ct.ct_line_size = 0;
4196 if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
4197 des_b1_ct.ct_size = 8;
4198 des_b1_ct.ct_label = itlb2M_str;
4199 } else {
4200 des_b1_ct.ct_size = 4;
4201 des_b1_ct.ct_label = itlb4M_str;
4202 }
4203 ct = &des_b1_ct;
4204 } else {
4205 if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
4206 continue;
4207 }
4208 }
4209
4210 if (func(arg, ct) != 0) {
4211 break;
4212 }
4213 }
4214 }
4215
4216 /*
4217 * (Like the Intel one, except for Cyrix CPUs)
4218 */
4219 static void
4220 cyrix_walk_cacheinfo(struct cpuid_info *cpi,
4221 void *arg, int (*func)(void *, const struct cachetab *))
4222 {
4223 const struct cachetab *ct;
4224 uint8_t *dp;
4225 int i;
4226
4227 if ((dp = cpi->cpi_cacheinfo) == NULL)
4228 return;
4229 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4230 /*
4231 * Search Cyrix-specific descriptor table first ..
4232 */
4233 if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
4234 if (func(arg, ct) != 0)
4235 break;
4236 continue;
4237 }
4238 /*
4239 * .. else fall back to the Intel one
4240 */
4241 if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
4242 if (func(arg, ct) != 0)
4243 break;
4244 continue;
4245 }
4246 }
4247 }
4248
4249 /*
4250 * A cacheinfo walker that adds associativity, line-size, and size properties
4251 * to the devinfo node it is passed as an argument.
4252 */
4253 static int
4254 add_cacheent_props(void *arg, const struct cachetab *ct)
4255 {
4256 dev_info_t *devi = arg;
4257
4258 add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
4259 if (ct->ct_line_size != 0)
4260 add_cache_prop(devi, ct->ct_label, line_str,
4261 ct->ct_line_size);
4262 add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
4263 return (0);
4264 }
4265
4266
4267 static const char fully_assoc[] = "fully-associative?";
4268
4269 /*
4270 * AMD style cache/tlb description
4271 *
4272 * Extended functions 5 and 6 directly describe properties of
4273 * tlbs and various cache levels.
4274 */
4275 static void
4276 add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4277 {
4278 switch (assoc) {
4279 case 0: /* reserved; ignore */
4280 break;
4281 default:
4282 add_cache_prop(devi, label, assoc_str, assoc);
4283 break;
4284 case 0xff:
4285 add_cache_prop(devi, label, fully_assoc, 1);
4286 break;
4287 }
4288 }
4289
4290 static void
4291 add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4292 {
4293 if (size == 0)
4294 return;
4295 add_cache_prop(devi, label, size_str, size);
4296 add_amd_assoc(devi, label, assoc);
4297 }
4298
4299 static void
4300 add_amd_cache(dev_info_t *devi, const char *label,
4301 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4302 {
4303 if (size == 0 || line_size == 0)
4304 return;
4305 add_amd_assoc(devi, label, assoc);
4306 /*
4307 * Most AMD parts have a sectored cache. Multiple cache lines are
4308 * associated with each tag. A sector consists of all cache lines
4309 * associated with a tag. For example, the AMD K6-III has a sector
4310 * size of 2 cache lines per tag.
4311 */
4312 if (lines_per_tag != 0)
4313 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4314 add_cache_prop(devi, label, line_str, line_size);
4315 add_cache_prop(devi, label, size_str, size * 1024);
4316 }
4317
4318 static void
4319 add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4320 {
4321 switch (assoc) {
4322 case 0: /* off */
4323 break;
4324 case 1:
4325 case 2:
4326 case 4:
4327 add_cache_prop(devi, label, assoc_str, assoc);
4328 break;
4329 case 6:
4330 add_cache_prop(devi, label, assoc_str, 8);
4331 break;
4332 case 8:
4333 add_cache_prop(devi, label, assoc_str, 16);
4334 break;
4335 case 0xf:
4336 add_cache_prop(devi, label, fully_assoc, 1);
4337 break;
4338 default: /* reserved; ignore */
4339 break;
4340 }
4341 }
4342
4343 static void
4344 add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4345 {
4346 if (size == 0 || assoc == 0)
4347 return;
4348 add_amd_l2_assoc(devi, label, assoc);
4349 add_cache_prop(devi, label, size_str, size);
4350 }
4351
4352 static void
4353 add_amd_l2_cache(dev_info_t *devi, const char *label,
4354 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4355 {
4356 if (size == 0 || assoc == 0 || line_size == 0)
4357 return;
4358 add_amd_l2_assoc(devi, label, assoc);
4359 if (lines_per_tag != 0)
4360 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4361 add_cache_prop(devi, label, line_str, line_size);
4362 add_cache_prop(devi, label, size_str, size * 1024);
4363 }
4364
4365 static void
4366 amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
4367 {
4368 struct cpuid_regs *cp;
4369
4370 if (cpi->cpi_xmaxeax < 0x80000005)
4371 return;
4372 cp = &cpi->cpi_extd[5];
4373
4374 /*
4375 * 4M/2M L1 TLB configuration
4376 *
4377 * We report the size for 2M pages because AMD uses two
4378 * TLB entries for one 4M page.
4379 */
4380 add_amd_tlb(devi, "dtlb-2M",
4381 BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
4382 add_amd_tlb(devi, "itlb-2M",
4383 BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));
4384
4385 /*
4386 * 4K L1 TLB configuration
4387 */
4388
4389 switch (cpi->cpi_vendor) {
4390 uint_t nentries;
4391 case X86_VENDOR_TM:
4392 if (cpi->cpi_family >= 5) {
4393 /*
4394 * Crusoe processors have 256 TLB entries, but
4395 * cpuid data format constrains them to only
4396 * reporting 255 of them.
4397 */
4398 if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
4399 nentries = 256;
4400 /*
4401 * Crusoe processors also have a unified TLB
4402 */
4403 add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
4404 nentries);
4405 break;
4406 }
4407 /*FALLTHROUGH*/
4408 default:
4409 add_amd_tlb(devi, itlb4k_str,
4410 BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
4411 add_amd_tlb(devi, dtlb4k_str,
4412 BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
4413 break;
4414 }
4415
4416 /*
4417 * data L1 cache configuration
4418 */
4419
4420 add_amd_cache(devi, l1_dcache_str,
4421 BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
4422 BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));
4423
4424 /*
4425 * code L1 cache configuration
4426 */
4427
4428 add_amd_cache(devi, l1_icache_str,
4429 BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
4430 BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));
4431
4432 if (cpi->cpi_xmaxeax < 0x80000006)
4433 return;
4434 cp = &cpi->cpi_extd[6];
4435
4436 /* Check for a unified L2 TLB for large pages */
4437
4438 if (BITX(cp->cp_eax, 31, 16) == 0)
4439 add_amd_l2_tlb(devi, "l2-tlb-2M",
4440 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4441 else {
4442 add_amd_l2_tlb(devi, "l2-dtlb-2M",
4443 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4444 add_amd_l2_tlb(devi, "l2-itlb-2M",
4445 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4446 }
4447
4448 /* Check for a unified L2 TLB for 4K pages */
4449
4450 if (BITX(cp->cp_ebx, 31, 16) == 0) {
4451 add_amd_l2_tlb(devi, "l2-tlb-4K",
4452 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4453 } else {
4454 add_amd_l2_tlb(devi, "l2-dtlb-4K",
4455 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4456 add_amd_l2_tlb(devi, "l2-itlb-4K",
4457 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4458 }
4459
4460 add_amd_l2_cache(devi, l2_cache_str,
4461 BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
4462 BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
4463 }
4464
4465 /*
4466 * There are two basic ways that the x86 world describes it cache
4467 * and tlb architecture - Intel's way and AMD's way.
4468 *
4469 * Return which flavor of cache architecture we should use
4470 */
4471 static int
4472 x86_which_cacheinfo(struct cpuid_info *cpi)
4473 {
4474 switch (cpi->cpi_vendor) {
4475 case X86_VENDOR_Intel:
4476 if (cpi->cpi_maxeax >= 2)
4477 return (X86_VENDOR_Intel);
4478 break;
4479 case X86_VENDOR_AMD:
4480 /*
4481 * The K5 model 1 was the first part from AMD that reported
4482 * cache sizes via extended cpuid functions.
4483 */
4484 if (cpi->cpi_family > 5 ||
4485 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
4486 return (X86_VENDOR_AMD);
4487 break;
4488 case X86_VENDOR_TM:
4489 if (cpi->cpi_family >= 5)
4490 return (X86_VENDOR_AMD);
4491 /*FALLTHROUGH*/
4492 default:
4493 /*
4494 * If they have extended CPU data for 0x80000005
4495 * then we assume they have AMD-format cache
4496 * information.
4497 *
4498 * If not, and the vendor happens to be Cyrix,
4499 * then try our-Cyrix specific handler.
4500 *
4501 * If we're not Cyrix, then assume we're using Intel's
4502 * table-driven format instead.
4503 */
4504 if (cpi->cpi_xmaxeax >= 0x80000005)
4505 return (X86_VENDOR_AMD);
4506 else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
4507 return (X86_VENDOR_Cyrix);
4508 else if (cpi->cpi_maxeax >= 2)
4509 return (X86_VENDOR_Intel);
4510 break;
4511 }
4512 return (-1);
4513 }
4514
4515 void
4516 cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
4517 struct cpuid_info *cpi)
4518 {
4519 dev_info_t *cpu_devi;
4520 int create;
4521
4522 cpu_devi = (dev_info_t *)dip;
4523
4524 /* device_type */
4525 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4526 "device_type", "cpu");
4527
4528 /* reg */
4529 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4530 "reg", cpu_id);
4531
4532 /* cpu-mhz, and clock-frequency */
4533 if (cpu_freq > 0) {
4534 long long mul;
4535
4536 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4537 "cpu-mhz", cpu_freq);
4538 if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
4539 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4540 "clock-frequency", (int)mul);
4541 }
4542
4543 if (!is_x86_feature(x86_featureset, X86FSET_CPUID)) {
4544 return;
4545 }
4546
4547 /* vendor-id */
4548 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4549 "vendor-id", cpi->cpi_vendorstr);
4550
4551 if (cpi->cpi_maxeax == 0) {
4552 return;
4553 }
4554
4555 /*
4556 * family, model, and step
4557 */
4558 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4559 "family", CPI_FAMILY(cpi));
4560 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4561 "cpu-model", CPI_MODEL(cpi));
4562 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4563 "stepping-id", CPI_STEP(cpi));
4564
4565 /* type */
4566 switch (cpi->cpi_vendor) {
4567 case X86_VENDOR_Intel:
4568 create = 1;
4569 break;
4570 default:
4571 create = 0;
4572 break;
4573 }
4574 if (create)
4575 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4576 "type", CPI_TYPE(cpi));
4577
4578 /* ext-family */
4579 switch (cpi->cpi_vendor) {
4580 case X86_VENDOR_Intel:
4581 case X86_VENDOR_AMD:
4582 create = cpi->cpi_family >= 0xf;
4583 break;
4584 default:
4585 create = 0;
4586 break;
4587 }
4588 if (create)
4589 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4590 "ext-family", CPI_FAMILY_XTD(cpi));
4591
4592 /* ext-model */
4593 switch (cpi->cpi_vendor) {
4594 case X86_VENDOR_Intel:
4595 create = IS_EXTENDED_MODEL_INTEL(cpi);
4596 break;
4597 case X86_VENDOR_AMD:
4598 create = CPI_FAMILY(cpi) == 0xf;
4599 break;
4600 default:
4601 create = 0;
4602 break;
4603 }
4604 if (create)
4605 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4606 "ext-model", CPI_MODEL_XTD(cpi));
4607
4608 /* generation */
4609 switch (cpi->cpi_vendor) {
4610 case X86_VENDOR_AMD:
4611 /*
4612 * AMD K5 model 1 was the first part to support this
4613 */
4614 create = cpi->cpi_xmaxeax >= 0x80000001;
4615 break;
4616 default:
4617 create = 0;
4618 break;
4619 }
4620 if (create)
4621 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4622 "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));
4623
4624 /* brand-id */
4625 switch (cpi->cpi_vendor) {
4626 case X86_VENDOR_Intel:
4627 /*
4628 * brand id first appeared on Pentium III Xeon model 8,
4629 * and Celeron model 8 processors and Opteron
4630 */
4631 create = cpi->cpi_family > 6 ||
4632 (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
4633 break;
4634 case X86_VENDOR_AMD:
4635 create = cpi->cpi_family >= 0xf;
4636 break;
4637 default:
4638 create = 0;
4639 break;
4640 }
4641 if (create && cpi->cpi_brandid != 0) {
4642 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4643 "brand-id", cpi->cpi_brandid);
4644 }
4645
4646 /* chunks, and apic-id */
4647 switch (cpi->cpi_vendor) {
4648 /*
4649 * first available on Pentium IV and Opteron (K8)
4650 */
4651 case X86_VENDOR_Intel:
4652 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4653 break;
4654 case X86_VENDOR_AMD:
4655 create = cpi->cpi_family >= 0xf;
4656 break;
4657 default:
4658 create = 0;
4659 break;
4660 }
4661 if (create) {
4662 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4663 "chunks", CPI_CHUNKS(cpi));
4664 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4665 "apic-id", cpi->cpi_apicid);
4666 if (cpi->cpi_chipid >= 0) {
4667 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4668 "chip#", cpi->cpi_chipid);
4669 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4670 "clog#", cpi->cpi_clogid);
4671 }
4672 }
4673
4674 /* cpuid-features */
4675 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4676 "cpuid-features", CPI_FEATURES_EDX(cpi));
4677
4678
4679 /* cpuid-features-ecx */
4680 switch (cpi->cpi_vendor) {
4681 case X86_VENDOR_Intel:
4682 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4683 break;
4684 case X86_VENDOR_AMD:
4685 create = cpi->cpi_family >= 0xf;
4686 break;
4687 default:
4688 create = 0;
4689 break;
4690 }
4691 if (create)
4692 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4693 "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));
4694
4695 /* ext-cpuid-features */
4696 switch (cpi->cpi_vendor) {
4697 case X86_VENDOR_Intel:
4698 case X86_VENDOR_AMD:
4699 case X86_VENDOR_Cyrix:
4700 case X86_VENDOR_TM:
4701 case X86_VENDOR_Centaur:
4702 create = cpi->cpi_xmaxeax >= 0x80000001;
4703 break;
4704 default:
4705 create = 0;
4706 break;
4707 }
4708 if (create) {
4709 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4710 "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
4711 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4712 "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
4713 }
4714
4715 /*
4716 * Brand String first appeared in Intel Pentium IV, AMD K5
4717 * model 1, and Cyrix GXm. On earlier models we try and
4718 * simulate something similar .. so this string should always
4719 * same -something- about the processor, however lame.
4720 */
4721 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4722 "brand-string", cpi->cpi_brandstr);
4723
4724 /*
4725 * Finally, cache and tlb information
4726 */
4727 switch (x86_which_cacheinfo(cpi)) {
4728 case X86_VENDOR_Intel:
4729 intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4730 break;
4731 case X86_VENDOR_Cyrix:
4732 cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4733 break;
4734 case X86_VENDOR_AMD:
4735 amd_cache_info(cpi, cpu_devi);
4736 break;
4737 default:
4738 break;
4739 }
4740 }
4741
4742 struct l2info {
4743 int *l2i_csz;
4744 int *l2i_lsz;
4745 int *l2i_assoc;
4746 int l2i_ret;
4747 };
4748
4749 /*
4750 * A cacheinfo walker that fetches the size, line-size and associativity
4751 * of the L2 cache
4752 */
4753 static int
4754 intel_l2cinfo(void *arg, const struct cachetab *ct)
4755 {
4756 struct l2info *l2i = arg;
4757 int *ip;
4758
4759 if (ct->ct_label != l2_cache_str &&
4760 ct->ct_label != sl2_cache_str)
4761 return (0); /* not an L2 -- keep walking */
4762
4763 if ((ip = l2i->l2i_csz) != NULL)
4764 *ip = ct->ct_size;
4765 if ((ip = l2i->l2i_lsz) != NULL)
4766 *ip = ct->ct_line_size;
4767 if ((ip = l2i->l2i_assoc) != NULL)
4768 *ip = ct->ct_assoc;
4769 l2i->l2i_ret = ct->ct_size;
4770 return (1); /* was an L2 -- terminate walk */
4771 }
4772
4773 /*
4774 * AMD L2/L3 Cache and TLB Associativity Field Definition:
4775 *
4776 * Unlike the associativity for the L1 cache and tlb where the 8 bit
4777 * value is the associativity, the associativity for the L2 cache and
4778 * tlb is encoded in the following table. The 4 bit L2 value serves as
4779 * an index into the amd_afd[] array to determine the associativity.
4780 * -1 is undefined. 0 is fully associative.
4781 */
4782
4783 static int amd_afd[] =
4784 {-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};
4785
4786 static void
4787 amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
4788 {
4789 struct cpuid_regs *cp;
4790 uint_t size, assoc;
4791 int i;
4792 int *ip;
4793
4794 if (cpi->cpi_xmaxeax < 0x80000006)
4795 return;
4796 cp = &cpi->cpi_extd[6];
4797
4798 if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
4799 (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
4800 uint_t cachesz = size * 1024;
4801 assoc = amd_afd[i];
4802
4803 ASSERT(assoc != -1);
4804
4805 if ((ip = l2i->l2i_csz) != NULL)
4806 *ip = cachesz;
4807 if ((ip = l2i->l2i_lsz) != NULL)
4808 *ip = BITX(cp->cp_ecx, 7, 0);
4809 if ((ip = l2i->l2i_assoc) != NULL)
4810 *ip = assoc;
4811 l2i->l2i_ret = cachesz;
4812 }
4813 }
4814
4815 int
4816 getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
4817 {
4818 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
4819 struct l2info __l2info, *l2i = &__l2info;
4820
4821 l2i->l2i_csz = csz;
4822 l2i->l2i_lsz = lsz;
4823 l2i->l2i_assoc = assoc;
4824 l2i->l2i_ret = -1;
4825
4826 switch (x86_which_cacheinfo(cpi)) {
4827 case X86_VENDOR_Intel:
4828 intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4829 break;
4830 case X86_VENDOR_Cyrix:
4831 cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4832 break;
4833 case X86_VENDOR_AMD:
4834 amd_l2cacheinfo(cpi, l2i);
4835 break;
4836 default:
4837 break;
4838 }
4839 return (l2i->l2i_ret);
4840 }
4841
4842 #if !defined(__xpv)
4843
4844 uint32_t *
4845 cpuid_mwait_alloc(cpu_t *cpu)
4846 {
4847 uint32_t *ret;
4848 size_t mwait_size;
4849
4850 ASSERT(cpuid_checkpass(CPU, 2));
4851
4852 mwait_size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
4853 if (mwait_size == 0)
4854 return (NULL);
4855
4856 /*
4857 * kmem_alloc() returns cache line size aligned data for mwait_size
4858 * allocations. mwait_size is currently cache line sized. Neither
4859 * of these implementation details are guarantied to be true in the
4860 * future.
4861 *
4862 * First try allocating mwait_size as kmem_alloc() currently returns
4863 * correctly aligned memory. If kmem_alloc() does not return
4864 * mwait_size aligned memory, then use mwait_size ROUNDUP.
4865 *
4866 * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
4867 * decide to free this memory.
4868 */
4869 ret = kmem_zalloc(mwait_size, KM_SLEEP);
4870 if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) {
4871 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4872 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size;
4873 *ret = MWAIT_RUNNING;
4874 return (ret);
4875 } else {
4876 kmem_free(ret, mwait_size);
4877 ret = kmem_zalloc(mwait_size * 2, KM_SLEEP);
4878 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4879 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size * 2;
4880 ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size);
4881 *ret = MWAIT_RUNNING;
4882 return (ret);
4883 }
4884 }
4885
4886 void
4887 cpuid_mwait_free(cpu_t *cpu)
4888 {
4889 if (cpu->cpu_m.mcpu_cpi == NULL) {
4890 return;
4891 }
4892
4893 if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
4894 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
4895 kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
4896 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
4897 }
4898
4899 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
4900 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
4901 }
4902
4903 void
4904 patch_tsc_read(int flag)
4905 {
4906 size_t cnt;
4907
4908 switch (flag) {
4909 case TSC_NONE:
4910 cnt = &_no_rdtsc_end - &_no_rdtsc_start;
4911 (void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
4912 break;
4913 case TSC_RDTSC_MFENCE:
4914 cnt = &_tsc_mfence_end - &_tsc_mfence_start;
4915 (void) memcpy((void *)tsc_read,
4916 (void *)&_tsc_mfence_start, cnt);
4917 break;
4918 case TSC_RDTSC_LFENCE:
4919 cnt = &_tsc_lfence_end - &_tsc_lfence_start;
4920 (void) memcpy((void *)tsc_read,
4921 (void *)&_tsc_lfence_start, cnt);
4922 break;
4923 case TSC_TSCP:
4924 cnt = &_tscp_end - &_tscp_start;
4925 (void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
4926 break;
4927 default:
4928 /* Bail for unexpected TSC types. (TSC_NONE covers 0) */
4929 cmn_err(CE_PANIC, "Unrecogized TSC type: %d", flag);
4930 break;
4931 }
4932 tsc_type = flag;
4933 }
4934
4935 int
4936 cpuid_deep_cstates_supported(void)
4937 {
4938 struct cpuid_info *cpi;
4939 struct cpuid_regs regs;
4940
4941 ASSERT(cpuid_checkpass(CPU, 1));
4942
4943 cpi = CPU->cpu_m.mcpu_cpi;
4944
4945 if (!is_x86_feature(x86_featureset, X86FSET_CPUID))
4946 return (0);
4947
4948 switch (cpi->cpi_vendor) {
4949 case X86_VENDOR_Intel:
4950 if (cpi->cpi_xmaxeax < 0x80000007)
4951 return (0);
4952
4953 /*
4954 * TSC run at a constant rate in all ACPI C-states?
4955 */
4956 regs.cp_eax = 0x80000007;
4957 (void) __cpuid_insn(®s);
4958 return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);
4959
4960 default:
4961 return (0);
4962 }
4963 }
4964
4965 #endif /* !__xpv */
4966
4967 void
4968 post_startup_cpu_fixups(void)
4969 {
4970 #ifndef __xpv
4971 /*
4972 * Some AMD processors support C1E state. Entering this state will
4973 * cause the local APIC timer to stop, which we can't deal with at
4974 * this time.
4975 */
4976 if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
4977 on_trap_data_t otd;
4978 uint64_t reg;
4979
4980 if (!on_trap(&otd, OT_DATA_ACCESS)) {
4981 reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
4982 /* Disable C1E state if it is enabled by BIOS */
4983 if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
4984 AMD_ACTONCMPHALT_MASK) {
4985 reg &= ~(AMD_ACTONCMPHALT_MASK <<
4986 AMD_ACTONCMPHALT_SHIFT);
4987 wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
4988 }
4989 }
4990 no_trap();
4991 }
4992 #endif /* !__xpv */
4993 }
4994
4995 /*
4996 * Setup necessary registers to enable XSAVE feature on this processor.
4997 * This function needs to be called early enough, so that no xsave/xrstor
4998 * ops will execute on the processor before the MSRs are properly set up.
4999 *
5000 * Current implementation has the following assumption:
5001 * - cpuid_pass1() is done, so that X86 features are known.
5002 * - fpu_probe() is done, so that fp_save_mech is chosen.
5003 */
5004 void
5005 xsave_setup_msr(cpu_t *cpu)
5006 {
5007 ASSERT(fp_save_mech == FP_XSAVE);
5008 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
5009
5010 /* Enable OSXSAVE in CR4. */
5011 setcr4(getcr4() | CR4_OSXSAVE);
5012 /*
5013 * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
5014 * correct value.
5015 */
5016 cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
5017 setup_xfem();
5018 }
5019
5020 /*
5021 * Starting with the Westmere processor the local
5022 * APIC timer will continue running in all C-states,
5023 * including the deepest C-states.
5024 */
5025 int
5026 cpuid_arat_supported(void)
5027 {
5028 struct cpuid_info *cpi;
5029 struct cpuid_regs regs;
5030
5031 ASSERT(cpuid_checkpass(CPU, 1));
5032 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5033
5034 cpi = CPU->cpu_m.mcpu_cpi;
5035
5036 switch (cpi->cpi_vendor) {
5037 case X86_VENDOR_Intel:
5038 /*
5039 * Always-running Local APIC Timer is
5040 * indicated by CPUID.6.EAX[2].
5041 */
5042 if (cpi->cpi_maxeax >= 6) {
5043 regs.cp_eax = 6;
5044 (void) cpuid_insn(NULL, ®s);
5045 return (regs.cp_eax & CPUID_CSTATE_ARAT);
5046 } else {
5047 return (0);
5048 }
5049 default:
5050 return (0);
5051 }
5052 }
5053
5054 /*
5055 * Check support for Intel ENERGY_PERF_BIAS feature
5056 */
5057 int
5058 cpuid_iepb_supported(struct cpu *cp)
5059 {
5060 struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
5061 struct cpuid_regs regs;
5062
5063 ASSERT(cpuid_checkpass(cp, 1));
5064
5065 if (!(is_x86_feature(x86_featureset, X86FSET_CPUID)) ||
5066 !(is_x86_feature(x86_featureset, X86FSET_MSR))) {
5067 return (0);
5068 }
5069
5070 /*
5071 * Intel ENERGY_PERF_BIAS MSR is indicated by
5072 * capability bit CPUID.6.ECX.3
5073 */
5074 if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
5075 return (0);
5076
5077 regs.cp_eax = 0x6;
5078 (void) cpuid_insn(NULL, ®s);
5079 return (regs.cp_ecx & CPUID_EPB_SUPPORT);
5080 }
5081
5082 /*
5083 * Check support for TSC deadline timer
5084 *
5085 * TSC deadline timer provides a superior software programming
5086 * model over local APIC timer that eliminates "time drifts".
5087 * Instead of specifying a relative time, software specifies an
5088 * absolute time as the target at which the processor should
5089 * generate a timer event.
5090 */
5091 int
5092 cpuid_deadline_tsc_supported(void)
5093 {
5094 struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
5095 struct cpuid_regs regs;
5096
5097 ASSERT(cpuid_checkpass(CPU, 1));
5098 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5099
5100 switch (cpi->cpi_vendor) {
5101 case X86_VENDOR_Intel:
5102 if (cpi->cpi_maxeax >= 1) {
5103 regs.cp_eax = 1;
5104 (void) cpuid_insn(NULL, ®s);
5105 return (regs.cp_ecx & CPUID_DEADLINE_TSC);
5106 } else {
5107 return (0);
5108 }
5109 default:
5110 return (0);
5111 }
5112 }
5113
5114 #if defined(__amd64) && !defined(__xpv)
5115 /*
5116 * Patch in versions of bcopy for high performance Intel Nhm processors
5117 * and later...
5118 */
5119 void
5120 patch_memops(uint_t vendor)
5121 {
5122 size_t cnt, i;
5123 caddr_t to, from;
5124
5125 if ((vendor == X86_VENDOR_Intel) &&
5126 is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
5127 cnt = &bcopy_patch_end - &bcopy_patch_start;
5128 to = &bcopy_ck_size;
5129 from = &bcopy_patch_start;
5130 for (i = 0; i < cnt; i++) {
5131 *to++ = *from++;
5132 }
5133 }
5134 }
5135 #endif /* __amd64 && !__xpv */
5136
5137 /*
5138 * This function finds the number of bits to represent the number of cores per
5139 * chip and the number of strands per core for the Intel platforms.
5140 * It re-uses the x2APIC cpuid code of the cpuid_pass2().
5141 */
5142 void
5143 cpuid_get_ext_topo(uint_t vendor, uint_t *core_nbits, uint_t *strand_nbits)
5144 {
5145 struct cpuid_regs regs;
5146 struct cpuid_regs *cp = ®s;
5147
5148 if (vendor != X86_VENDOR_Intel) {
5149 return;
5150 }
5151
5152 /* if the cpuid level is 0xB, extended topo is available. */
5153 cp->cp_eax = 0;
5154 if (__cpuid_insn(cp) >= 0xB) {
5155
5156 cp->cp_eax = 0xB;
5157 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
5158 (void) __cpuid_insn(cp);
5159
5160 /*
5161 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
5162 * indicates that the extended topology enumeration leaf is
5163 * available.
5164 */
5165 if (cp->cp_ebx) {
5166 uint_t coreid_shift = 0;
5167 uint_t chipid_shift = 0;
5168 uint_t i;
5169 uint_t level;
5170
5171 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
5172 cp->cp_eax = 0xB;
5173 cp->cp_ecx = i;
5174
5175 (void) __cpuid_insn(cp);
5176 level = CPI_CPU_LEVEL_TYPE(cp);
5177
5178 if (level == 1) {
5179 /*
5180 * Thread level processor topology
5181 * Number of bits shift right APIC ID
5182 * to get the coreid.
5183 */
5184 coreid_shift = BITX(cp->cp_eax, 4, 0);
5185 } else if (level == 2) {
5186 /*
5187 * Core level processor topology
5188 * Number of bits shift right APIC ID
5189 * to get the chipid.
5190 */
5191 chipid_shift = BITX(cp->cp_eax, 4, 0);
5192 }
5193 }
5194
5195 if (coreid_shift > 0 && chipid_shift > coreid_shift) {
5196 *strand_nbits = coreid_shift;
5197 *core_nbits = chipid_shift - coreid_shift;
5198 }
5199 }
5200 }
5201 }