1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
  24  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
  26  */
  27 /*
  28  * Copyright (c) 2010, Intel Corporation.
  29  * All rights reserved.
  30  */
  31 /*
  32  * Portions Copyright 2009 Advanced Micro Devices, Inc.
  33  */
  34 /*
  35  * Copyright 2017 Joyent, Inc.
  36  */
  37 /*
  38  * Various routines to handle identification
  39  * and classification of x86 processors.
  40  */
  41 
  42 #include <sys/types.h>
  43 #include <sys/archsystm.h>
  44 #include <sys/x86_archext.h>
  45 #include <sys/kmem.h>
  46 #include <sys/systm.h>
  47 #include <sys/cmn_err.h>
  48 #include <sys/sunddi.h>
  49 #include <sys/sunndi.h>
  50 #include <sys/cpuvar.h>
  51 #include <sys/processor.h>
  52 #include <sys/sysmacros.h>
  53 #include <sys/pg.h>
  54 #include <sys/fp.h>
  55 #include <sys/controlregs.h>
  56 #include <sys/bitmap.h>
  57 #include <sys/auxv_386.h>
  58 #include <sys/memnode.h>
  59 #include <sys/pci_cfgspace.h>
  60 #include <sys/comm_page.h>
  61 #include <sys/tsc.h>
  62 
  63 #ifdef __xpv
  64 #include <sys/hypervisor.h>
  65 #else
  66 #include <sys/ontrap.h>
  67 #endif
  68 
  69 /*
  70  * Pass 0 of cpuid feature analysis happens in locore. It contains special code
  71  * to recognize Cyrix processors that are not cpuid-compliant, and to deal with
  72  * them accordingly. For most modern processors, feature detection occurs here
  73  * in pass 1.
  74  *
  75  * Pass 1 of cpuid feature analysis happens just at the beginning of mlsetup()
  76  * for the boot CPU and does the basic analysis that the early kernel needs.
  77  * x86_featureset is set based on the return value of cpuid_pass1() of the boot
  78  * CPU.
  79  *
  80  * Pass 1 includes:
  81  *
  82  *      o Determining vendor/model/family/stepping and setting x86_type and
  83  *        x86_vendor accordingly.
  84  *      o Processing the feature flags returned by the cpuid instruction while
  85  *        applying any workarounds or tricks for the specific processor.
  86  *      o Mapping the feature flags into Solaris feature bits (X86_*).
  87  *      o Processing extended feature flags if supported by the processor,
  88  *        again while applying specific processor knowledge.
  89  *      o Determining the CMT characteristics of the system.
  90  *
  91  * Pass 1 is done on non-boot CPUs during their initialization and the results
  92  * are used only as a meager attempt at ensuring that all processors within the
  93  * system support the same features.
  94  *
  95  * Pass 2 of cpuid feature analysis happens just at the beginning
  96  * of startup().  It just copies in and corrects the remainder
  97  * of the cpuid data we depend on: standard cpuid functions that we didn't
  98  * need for pass1 feature analysis, and extended cpuid functions beyond the
  99  * simple feature processing done in pass1.
 100  *
 101  * Pass 3 of cpuid analysis is invoked after basic kernel services; in
 102  * particular kernel memory allocation has been made available. It creates a
 103  * readable brand string based on the data collected in the first two passes.
 104  *
 105  * Pass 4 of cpuid analysis is invoked after post_startup() when all
 106  * the support infrastructure for various hardware features has been
 107  * initialized. It determines which processor features will be reported
 108  * to userland via the aux vector.
 109  *
 110  * All passes are executed on all CPUs, but only the boot CPU determines what
 111  * features the kernel will use.
 112  *
 113  * Much of the worst junk in this file is for the support of processors
 114  * that didn't really implement the cpuid instruction properly.
 115  *
 116  * NOTE: The accessor functions (cpuid_get*) are aware of, and ASSERT upon,
 117  * the pass numbers.  Accordingly, changes to the pass code may require changes
 118  * to the accessor code.
 119  */
 120 
 121 uint_t x86_vendor = X86_VENDOR_IntelClone;
 122 uint_t x86_type = X86_TYPE_OTHER;
 123 uint_t x86_clflush_size = 0;
 124 
 125 uint_t pentiumpro_bug4046376;
 126 
 127 uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
 128 
 129 static char *x86_feature_names[NUM_X86_FEATURES] = {
 130         "lgpg",
 131         "tsc",
 132         "msr",
 133         "mtrr",
 134         "pge",
 135         "de",
 136         "cmov",
 137         "mmx",
 138         "mca",
 139         "pae",
 140         "cv8",
 141         "pat",
 142         "sep",
 143         "sse",
 144         "sse2",
 145         "htt",
 146         "asysc",
 147         "nx",
 148         "sse3",
 149         "cx16",
 150         "cmp",
 151         "tscp",
 152         "mwait",
 153         "sse4a",
 154         "cpuid",
 155         "ssse3",
 156         "sse4_1",
 157         "sse4_2",
 158         "1gpg",
 159         "clfsh",
 160         "64",
 161         "aes",
 162         "pclmulqdq",
 163         "xsave",
 164         "avx",
 165         "vmx",
 166         "svm",
 167         "topoext",
 168         "f16c",
 169         "rdrand",
 170         "x2apic",
 171         "avx2",
 172         "bmi1",
 173         "bmi2",
 174         "fma",
 175         "smep",
 176         "smap",
 177         "adx",
 178         "rdseed",
 179         "mpx",
 180         "avx512f",
 181         "avx512dq",
 182         "avx512pf",
 183         "avx512er",
 184         "avx512cd",
 185         "avx512bw",
 186         "avx512vl",
 187         "avx512fma",
 188         "avx512vbmi",
 189         "avx512_vpopcntdq",
 190         "avx512_4vnniw",
 191         "avx512_4fmaps",
 192         "xsaveopt",
 193         "xsavec",
 194         "xsaves",
 195         "sha",
 196         "umip",
 197         "pku",
 198         "ospke",
 199 };
 200 
 201 boolean_t
 202 is_x86_feature(void *featureset, uint_t feature)
 203 {
 204         ASSERT(feature < NUM_X86_FEATURES);
 205         return (BT_TEST((ulong_t *)featureset, feature));
 206 }
 207 
 208 void
 209 add_x86_feature(void *featureset, uint_t feature)
 210 {
 211         ASSERT(feature < NUM_X86_FEATURES);
 212         BT_SET((ulong_t *)featureset, feature);
 213 }
 214 
 215 void
 216 remove_x86_feature(void *featureset, uint_t feature)
 217 {
 218         ASSERT(feature < NUM_X86_FEATURES);
 219         BT_CLEAR((ulong_t *)featureset, feature);
 220 }
 221 
 222 boolean_t
 223 compare_x86_featureset(void *setA, void *setB)
 224 {
 225         /*
 226          * We assume that the unused bits of the bitmap are always zero.
 227          */
 228         if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
 229                 return (B_TRUE);
 230         } else {
 231                 return (B_FALSE);
 232         }
 233 }
 234 
 235 void
 236 print_x86_featureset(void *featureset)
 237 {
 238         uint_t i;
 239 
 240         for (i = 0; i < NUM_X86_FEATURES; i++) {
 241                 if (is_x86_feature(featureset, i)) {
 242                         cmn_err(CE_CONT, "?x86_feature: %s\n",
 243                             x86_feature_names[i]);
 244                 }
 245         }
 246 }
 247 
 248 /* Note: This is the maximum size for the CPU, not the size of the structure. */
 249 static size_t xsave_state_size = 0;
 250 uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
 251 boolean_t xsave_force_disable = B_FALSE;
 252 extern int disable_smap;
 253 
 254 /*
 255  * This is set to platform type we are running on.
 256  */
 257 static int platform_type = -1;
 258 
 259 #if !defined(__xpv)
 260 /*
 261  * Variable to patch if hypervisor platform detection needs to be
 262  * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
 263  */
 264 int enable_platform_detection = 1;
 265 #endif
 266 
 267 /*
 268  * monitor/mwait info.
 269  *
 270  * size_actual and buf_actual are the real address and size allocated to get
 271  * proper mwait_buf alignement.  buf_actual and size_actual should be passed
 272  * to kmem_free().  Currently kmem_alloc() and mwait happen to both use
 273  * processor cache-line alignment, but this is not guarantied in the furture.
 274  */
 275 struct mwait_info {
 276         size_t          mon_min;        /* min size to avoid missed wakeups */
 277         size_t          mon_max;        /* size to avoid false wakeups */
 278         size_t          size_actual;    /* size actually allocated */
 279         void            *buf_actual;    /* memory actually allocated */
 280         uint32_t        support;        /* processor support of monitor/mwait */
 281 };
 282 
 283 /*
 284  * xsave/xrestor info.
 285  *
 286  * This structure contains HW feature bits and the size of the xsave save area.
 287  * Note: the kernel declares a fixed size (AVX_XSAVE_SIZE) structure
 288  * (xsave_state) to describe the xsave layout. However, at runtime the
 289  * per-lwp xsave area is dynamically allocated based on xsav_max_size. The
 290  * xsave_state structure simply represents the legacy layout of the beginning
 291  * of the xsave area.
 292  */
 293 struct xsave_info {
 294         uint32_t        xsav_hw_features_low;   /* Supported HW features */
 295         uint32_t        xsav_hw_features_high;  /* Supported HW features */
 296         size_t          xsav_max_size;  /* max size save area for HW features */
 297         size_t          ymm_size;       /* AVX: size of ymm save area */
 298         size_t          ymm_offset;     /* AVX: offset for ymm save area */
 299         size_t          bndregs_size;   /* MPX: size of bndregs save area */
 300         size_t          bndregs_offset; /* MPX: offset for bndregs save area */
 301         size_t          bndcsr_size;    /* MPX: size of bndcsr save area */
 302         size_t          bndcsr_offset;  /* MPX: offset for bndcsr save area */
 303         size_t          opmask_size;    /* AVX512: size of opmask save */
 304         size_t          opmask_offset;  /* AVX512: offset for opmask save */
 305         size_t          zmmlo_size;     /* AVX512: size of zmm 256 save */
 306         size_t          zmmlo_offset;   /* AVX512: offset for zmm 256 save */
 307         size_t          zmmhi_size;     /* AVX512: size of zmm hi reg save */
 308         size_t          zmmhi_offset;   /* AVX512: offset for zmm hi reg save */
 309 };
 310 
 311 
 312 /*
 313  * These constants determine how many of the elements of the
 314  * cpuid we cache in the cpuid_info data structure; the
 315  * remaining elements are accessible via the cpuid instruction.
 316  */
 317 
 318 #define NMAX_CPI_STD    8               /* eax = 0 .. 7 */
 319 #define NMAX_CPI_EXTD   0x1f            /* eax = 0x80000000 .. 0x8000001e */
 320 
 321 /*
 322  * Some terminology needs to be explained:
 323  *  - Socket: Something that can be plugged into a motherboard.
 324  *  - Package: Same as socket
 325  *  - Chip: Same as socket. Note that AMD's documentation uses term "chip"
 326  *    differently: there, chip is the same as processor node (below)
 327  *  - Processor node: Some AMD processors have more than one
 328  *    "subprocessor" embedded in a package. These subprocessors (nodes)
 329  *    are fully-functional processors themselves with cores, caches,
 330  *    memory controllers, PCI configuration spaces. They are connected
 331  *    inside the package with Hypertransport links. On single-node
 332  *    processors, processor node is equivalent to chip/socket/package.
 333  *  - Compute Unit: Some AMD processors pair cores in "compute units" that
 334  *    share the FPU and the I$ and L2 caches.
 335  */
 336 
 337 struct cpuid_info {
 338         uint_t cpi_pass;                /* last pass completed */
 339         /*
 340          * standard function information
 341          */
 342         uint_t cpi_maxeax;              /* fn 0: %eax */
 343         char cpi_vendorstr[13];         /* fn 0: %ebx:%ecx:%edx */
 344         uint_t cpi_vendor;              /* enum of cpi_vendorstr */
 345 
 346         uint_t cpi_family;              /* fn 1: extended family */
 347         uint_t cpi_model;               /* fn 1: extended model */
 348         uint_t cpi_step;                /* fn 1: stepping */
 349         chipid_t cpi_chipid;            /* fn 1: %ebx:  Intel: chip # */
 350                                         /*              AMD: package/socket # */
 351         uint_t cpi_brandid;             /* fn 1: %ebx: brand ID */
 352         int cpi_clogid;                 /* fn 1: %ebx: thread # */
 353         uint_t cpi_ncpu_per_chip;       /* fn 1: %ebx: logical cpu count */
 354         uint8_t cpi_cacheinfo[16];      /* fn 2: intel-style cache desc */
 355         uint_t cpi_ncache;              /* fn 2: number of elements */
 356         uint_t cpi_ncpu_shr_last_cache; /* fn 4: %eax: ncpus sharing cache */
 357         id_t cpi_last_lvl_cacheid;      /* fn 4: %eax: derived cache id */
 358         uint_t cpi_std_4_size;          /* fn 4: number of fn 4 elements */
 359         struct cpuid_regs **cpi_std_4;  /* fn 4: %ecx == 0 .. fn4_size */
 360         struct cpuid_regs cpi_std[NMAX_CPI_STD];        /* 0 .. 7 */
 361         /*
 362          * extended function information
 363          */
 364         uint_t cpi_xmaxeax;             /* fn 0x80000000: %eax */
 365         char cpi_brandstr[49];          /* fn 0x8000000[234] */
 366         uint8_t cpi_pabits;             /* fn 0x80000006: %eax */
 367         uint8_t cpi_vabits;             /* fn 0x80000006: %eax */
 368         uint8_t cpi_fp_amd_save;        /* AMD: FP error pointer save rqd. */
 369         struct  cpuid_regs cpi_extd[NMAX_CPI_EXTD];     /* 0x800000XX */
 370 
 371         id_t cpi_coreid;                /* same coreid => strands share core */
 372         int cpi_pkgcoreid;              /* core number within single package */
 373         uint_t cpi_ncore_per_chip;      /* AMD: fn 0x80000008: %ecx[7-0] */
 374                                         /* Intel: fn 4: %eax[31-26] */
 375         /*
 376          * supported feature information
 377          */
 378         uint32_t cpi_support[6];
 379 #define STD_EDX_FEATURES        0
 380 #define AMD_EDX_FEATURES        1
 381 #define TM_EDX_FEATURES         2
 382 #define STD_ECX_FEATURES        3
 383 #define AMD_ECX_FEATURES        4
 384 #define STD_EBX_FEATURES        5
 385         /*
 386          * Synthesized information, where known.
 387          */
 388         uint32_t cpi_chiprev;           /* See X86_CHIPREV_* in x86_archext.h */
 389         const char *cpi_chiprevstr;     /* May be NULL if chiprev unknown */
 390         uint32_t cpi_socket;            /* Chip package/socket type */
 391 
 392         struct mwait_info cpi_mwait;    /* fn 5: monitor/mwait info */
 393         uint32_t cpi_apicid;
 394         uint_t cpi_procnodeid;          /* AMD: nodeID on HT, Intel: chipid */
 395         uint_t cpi_procnodes_per_pkg;   /* AMD: # of nodes in the package */
 396                                         /* Intel: 1 */
 397         uint_t cpi_compunitid;          /* AMD: ComputeUnit ID, Intel: coreid */
 398         uint_t cpi_cores_per_compunit;  /* AMD: # of cores in the ComputeUnit */
 399 
 400         struct xsave_info cpi_xsave;    /* fn D: xsave/xrestor info */
 401 };
 402 
 403 
 404 static struct cpuid_info cpuid_info0;
 405 
 406 /*
 407  * These bit fields are defined by the Intel Application Note AP-485
 408  * "Intel Processor Identification and the CPUID Instruction"
 409  */
 410 #define CPI_FAMILY_XTD(cpi)     BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
 411 #define CPI_MODEL_XTD(cpi)      BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
 412 #define CPI_TYPE(cpi)           BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
 413 #define CPI_FAMILY(cpi)         BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
 414 #define CPI_STEP(cpi)           BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
 415 #define CPI_MODEL(cpi)          BITX((cpi)->cpi_std[1].cp_eax, 7, 4)
 416 
 417 #define CPI_FEATURES_EDX(cpi)           ((cpi)->cpi_std[1].cp_edx)
 418 #define CPI_FEATURES_ECX(cpi)           ((cpi)->cpi_std[1].cp_ecx)
 419 #define CPI_FEATURES_XTD_EDX(cpi)       ((cpi)->cpi_extd[1].cp_edx)
 420 #define CPI_FEATURES_XTD_ECX(cpi)       ((cpi)->cpi_extd[1].cp_ecx)
 421 #define CPI_FEATURES_7_0_EBX(cpi)       ((cpi)->cpi_std[7].cp_ebx)
 422 #define CPI_FEATURES_7_0_ECX(cpi)       ((cpi)->cpi_std[7].cp_ecx)
 423 #define CPI_FEATURES_7_0_EDX(cpi)       ((cpi)->cpi_std[7].cp_edx)
 424 
 425 #define CPI_BRANDID(cpi)        BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
 426 #define CPI_CHUNKS(cpi)         BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
 427 #define CPI_CPU_COUNT(cpi)      BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
 428 #define CPI_APIC_ID(cpi)        BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)
 429 
 430 #define CPI_MAXEAX_MAX          0x100           /* sanity control */
 431 #define CPI_XMAXEAX_MAX         0x80000100
 432 #define CPI_FN4_ECX_MAX         0x20            /* sanity: max fn 4 levels */
 433 #define CPI_FNB_ECX_MAX         0x20            /* sanity: max fn B levels */
 434 
 435 /*
 436  * Function 4 (Deterministic Cache Parameters) macros
 437  * Defined by Intel Application Note AP-485
 438  */
 439 #define CPI_NUM_CORES(regs)             BITX((regs)->cp_eax, 31, 26)
 440 #define CPI_NTHR_SHR_CACHE(regs)        BITX((regs)->cp_eax, 25, 14)
 441 #define CPI_FULL_ASSOC_CACHE(regs)      BITX((regs)->cp_eax, 9, 9)
 442 #define CPI_SELF_INIT_CACHE(regs)       BITX((regs)->cp_eax, 8, 8)
 443 #define CPI_CACHE_LVL(regs)             BITX((regs)->cp_eax, 7, 5)
 444 #define CPI_CACHE_TYPE(regs)            BITX((regs)->cp_eax, 4, 0)
 445 #define CPI_CPU_LEVEL_TYPE(regs)        BITX((regs)->cp_ecx, 15, 8)
 446 
 447 #define CPI_CACHE_WAYS(regs)            BITX((regs)->cp_ebx, 31, 22)
 448 #define CPI_CACHE_PARTS(regs)           BITX((regs)->cp_ebx, 21, 12)
 449 #define CPI_CACHE_COH_LN_SZ(regs)       BITX((regs)->cp_ebx, 11, 0)
 450 
 451 #define CPI_CACHE_SETS(regs)            BITX((regs)->cp_ecx, 31, 0)
 452 
 453 #define CPI_PREFCH_STRIDE(regs)         BITX((regs)->cp_edx, 9, 0)
 454 
 455 
 456 /*
 457  * A couple of shorthand macros to identify "later" P6-family chips
 458  * like the Pentium M and Core.  First, the "older" P6-based stuff
 459  * (loosely defined as "pre-Pentium-4"):
 460  * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
 461  */
 462 #define IS_LEGACY_P6(cpi) (                     \
 463         cpi->cpi_family == 6 &&                      \
 464                 (cpi->cpi_model == 1 ||              \
 465                 cpi->cpi_model == 3 ||               \
 466                 cpi->cpi_model == 5 ||               \
 467                 cpi->cpi_model == 6 ||               \
 468                 cpi->cpi_model == 7 ||               \
 469                 cpi->cpi_model == 8 ||               \
 470                 cpi->cpi_model == 0xA ||     \
 471                 cpi->cpi_model == 0xB)               \
 472 )
 473 
 474 /* A "new F6" is everything with family 6 that's not the above */
 475 #define IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))
 476 
 477 /* Extended family/model support */
 478 #define IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
 479         cpi->cpi_family >= 0xf)
 480 
 481 /*
 482  * Info for monitor/mwait idle loop.
 483  *
 484  * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
 485  * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
 486  * 2006.
 487  * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
 488  * Documentation Updates" #33633, Rev 2.05, December 2006.
 489  */
 490 #define MWAIT_SUPPORT           (0x00000001)    /* mwait supported */
 491 #define MWAIT_EXTENSIONS        (0x00000002)    /* extenstion supported */
 492 #define MWAIT_ECX_INT_ENABLE    (0x00000004)    /* ecx 1 extension supported */
 493 #define MWAIT_SUPPORTED(cpi)    ((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
 494 #define MWAIT_INT_ENABLE(cpi)   ((cpi)->cpi_std[5].cp_ecx & 0x2)
 495 #define MWAIT_EXTENSION(cpi)    ((cpi)->cpi_std[5].cp_ecx & 0x1)
 496 #define MWAIT_SIZE_MIN(cpi)     BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
 497 #define MWAIT_SIZE_MAX(cpi)     BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
 498 /*
 499  * Number of sub-cstates for a given c-state.
 500  */
 501 #define MWAIT_NUM_SUBC_STATES(cpi, c_state)                     \
 502         BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)
 503 
 504 /*
 505  * XSAVE leaf 0xD enumeration
 506  */
 507 #define CPUID_LEAFD_2_YMM_OFFSET        576
 508 #define CPUID_LEAFD_2_YMM_SIZE          256
 509 
 510 /*
 511  * Functions we consune from cpuid_subr.c;  don't publish these in a header
 512  * file to try and keep people using the expected cpuid_* interfaces.
 513  */
 514 extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
 515 extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
 516 extern uint32_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
 517 extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
 518 extern uint_t _cpuid_vendorstr_to_vendorcode(char *);
 519 
 520 /*
 521  * Apply up various platform-dependent restrictions where the
 522  * underlying platform restrictions mean the CPU can be marked
 523  * as less capable than its cpuid instruction would imply.
 524  */
 525 #if defined(__xpv)
 526 static void
 527 platform_cpuid_mangle(uint_t vendor, uint32_t eax, struct cpuid_regs *cp)
 528 {
 529         switch (eax) {
 530         case 1: {
 531                 uint32_t mcamask = DOMAIN_IS_INITDOMAIN(xen_info) ?
 532                     0 : CPUID_INTC_EDX_MCA;
 533                 cp->cp_edx &=
 534                     ~(mcamask |
 535                     CPUID_INTC_EDX_PSE |
 536                     CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
 537                     CPUID_INTC_EDX_SEP | CPUID_INTC_EDX_MTRR |
 538                     CPUID_INTC_EDX_PGE | CPUID_INTC_EDX_PAT |
 539                     CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
 540                     CPUID_INTC_EDX_PSE36 | CPUID_INTC_EDX_HTT);
 541                 break;
 542         }
 543 
 544         case 0x80000001:
 545                 cp->cp_edx &=
 546                     ~(CPUID_AMD_EDX_PSE |
 547                     CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
 548                     CPUID_AMD_EDX_MTRR | CPUID_AMD_EDX_PGE |
 549                     CPUID_AMD_EDX_PAT | CPUID_AMD_EDX_PSE36 |
 550                     CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
 551                     CPUID_AMD_EDX_TSCP);
 552                 cp->cp_ecx &= ~CPUID_AMD_ECX_CMP_LGCY;
 553                 break;
 554         default:
 555                 break;
 556         }
 557 
 558         switch (vendor) {
 559         case X86_VENDOR_Intel:
 560                 switch (eax) {
 561                 case 4:
 562                         /*
 563                          * Zero out the (ncores-per-chip - 1) field
 564                          */
 565                         cp->cp_eax &= 0x03fffffff;
 566                         break;
 567                 default:
 568                         break;
 569                 }
 570                 break;
 571         case X86_VENDOR_AMD:
 572                 switch (eax) {
 573 
 574                 case 0x80000001:
 575                         cp->cp_ecx &= ~CPUID_AMD_ECX_CR8D;
 576                         break;
 577 
 578                 case 0x80000008:
 579                         /*
 580                          * Zero out the (ncores-per-chip - 1) field
 581                          */
 582                         cp->cp_ecx &= 0xffffff00;
 583                         break;
 584                 default:
 585                         break;
 586                 }
 587                 break;
 588         default:
 589                 break;
 590         }
 591 }
 592 #else
 593 #define platform_cpuid_mangle(vendor, eax, cp)  /* nothing */
 594 #endif
 595 
 596 /*
 597  *  Some undocumented ways of patching the results of the cpuid
 598  *  instruction to permit running Solaris 10 on future cpus that
 599  *  we don't currently support.  Could be set to non-zero values
 600  *  via settings in eeprom.
 601  */
 602 
 603 uint32_t cpuid_feature_ecx_include;
 604 uint32_t cpuid_feature_ecx_exclude;
 605 uint32_t cpuid_feature_edx_include;
 606 uint32_t cpuid_feature_edx_exclude;
 607 
 608 /*
 609  * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
 610  */
 611 void
 612 cpuid_alloc_space(cpu_t *cpu)
 613 {
 614         /*
 615          * By convention, cpu0 is the boot cpu, which is set up
 616          * before memory allocation is available.  All other cpus get
 617          * their cpuid_info struct allocated here.
 618          */
 619         ASSERT(cpu->cpu_id != 0);
 620         ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
 621         cpu->cpu_m.mcpu_cpi =
 622             kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
 623 }
 624 
 625 void
 626 cpuid_free_space(cpu_t *cpu)
 627 {
 628         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
 629         int i;
 630 
 631         ASSERT(cpi != NULL);
 632         ASSERT(cpi != &cpuid_info0);
 633 
 634         /*
 635          * Free up any function 4 related dynamic storage
 636          */
 637         for (i = 1; i < cpi->cpi_std_4_size; i++)
 638                 kmem_free(cpi->cpi_std_4[i], sizeof (struct cpuid_regs));
 639         if (cpi->cpi_std_4_size > 0)
 640                 kmem_free(cpi->cpi_std_4,
 641                     cpi->cpi_std_4_size * sizeof (struct cpuid_regs *));
 642 
 643         kmem_free(cpi, sizeof (*cpi));
 644         cpu->cpu_m.mcpu_cpi = NULL;
 645 }
 646 
 647 #if !defined(__xpv)
 648 /*
 649  * Determine the type of the underlying platform. This is used to customize
 650  * initialization of various subsystems (e.g. TSC). determine_platform() must
 651  * only ever be called once to prevent two processors from seeing different
 652  * values of platform_type. Must be called before cpuid_pass1(), the earliest
 653  * consumer to execute (uses _cpuid_chiprev --> synth_amd_info --> get_hwenv).
 654  */
 655 void
 656 determine_platform(void)
 657 {
 658         struct cpuid_regs cp;
 659         uint32_t base;
 660         uint32_t regs[4];
 661         char *hvstr = (char *)regs;
 662 
 663         ASSERT(platform_type == -1);
 664 
 665         platform_type = HW_NATIVE;
 666 
 667         if (!enable_platform_detection)
 668                 return;
 669 
 670         /*
 671          * If Hypervisor CPUID bit is set, try to determine hypervisor
 672          * vendor signature, and set platform type accordingly.
 673          *
 674          * References:
 675          * http://lkml.org/lkml/2008/10/1/246
 676          * http://kb.vmware.com/kb/1009458
 677          */
 678         cp.cp_eax = 0x1;
 679         (void) __cpuid_insn(&cp);
 680         if ((cp.cp_ecx & CPUID_INTC_ECX_HV) != 0) {
 681                 cp.cp_eax = 0x40000000;
 682                 (void) __cpuid_insn(&cp);
 683                 regs[0] = cp.cp_ebx;
 684                 regs[1] = cp.cp_ecx;
 685                 regs[2] = cp.cp_edx;
 686                 regs[3] = 0;
 687                 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0) {
 688                         platform_type = HW_XEN_HVM;
 689                         return;
 690                 }
 691                 if (strcmp(hvstr, HVSIG_VMWARE) == 0) {
 692                         platform_type = HW_VMWARE;
 693                         return;
 694                 }
 695                 if (strcmp(hvstr, HVSIG_KVM) == 0) {
 696                         platform_type = HW_KVM;
 697                         return;
 698                 }
 699                 if (strcmp(hvstr, HVSIG_MICROSOFT) == 0)
 700                         platform_type = HW_MICROSOFT;
 701         } else {
 702                 /*
 703                  * Check older VMware hardware versions. VMware hypervisor is
 704                  * detected by performing an IN operation to VMware hypervisor
 705                  * port and checking that value returned in %ebx is VMware
 706                  * hypervisor magic value.
 707                  *
 708                  * References: http://kb.vmware.com/kb/1009458
 709                  */
 710                 vmware_port(VMWARE_HVCMD_GETVERSION, regs);
 711                 if (regs[1] == VMWARE_HVMAGIC) {
 712                         platform_type = HW_VMWARE;
 713                         return;
 714                 }
 715         }
 716 
 717         /*
 718          * Check Xen hypervisor. In a fully virtualized domain,
 719          * Xen's pseudo-cpuid function returns a string representing the
 720          * Xen signature in %ebx, %ecx, and %edx. %eax contains the maximum
 721          * supported cpuid function. We need at least a (base + 2) leaf value
 722          * to do what we want to do. Try different base values, since the
 723          * hypervisor might use a different one depending on whether Hyper-V
 724          * emulation is switched on by default or not.
 725          */
 726         for (base = 0x40000000; base < 0x40010000; base += 0x100) {
 727                 cp.cp_eax = base;
 728                 (void) __cpuid_insn(&cp);
 729                 regs[0] = cp.cp_ebx;
 730                 regs[1] = cp.cp_ecx;
 731                 regs[2] = cp.cp_edx;
 732                 regs[3] = 0;
 733                 if (strcmp(hvstr, HVSIG_XEN_HVM) == 0 &&
 734                     cp.cp_eax >= (base + 2)) {
 735                         platform_type &= ~HW_NATIVE;
 736                         platform_type |= HW_XEN_HVM;
 737                         return;
 738                 }
 739         }
 740 }
 741 
 742 int
 743 get_hwenv(void)
 744 {
 745         ASSERT(platform_type != -1);
 746         return (platform_type);
 747 }
 748 
 749 int
 750 is_controldom(void)
 751 {
 752         return (0);
 753 }
 754 
 755 #else
 756 
 757 int
 758 get_hwenv(void)
 759 {
 760         return (HW_XEN_PV);
 761 }
 762 
 763 int
 764 is_controldom(void)
 765 {
 766         return (DOMAIN_IS_INITDOMAIN(xen_info));
 767 }
 768 
 769 #endif  /* __xpv */
 770 
 771 static void
 772 cpuid_intel_getids(cpu_t *cpu, void *feature)
 773 {
 774         uint_t i;
 775         uint_t chipid_shift = 0;
 776         uint_t coreid_shift = 0;
 777         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
 778 
 779         for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
 780                 chipid_shift++;
 781 
 782         cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
 783         cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);
 784 
 785         if (is_x86_feature(feature, X86FSET_CMP)) {
 786                 /*
 787                  * Multi-core (and possibly multi-threaded)
 788                  * processors.
 789                  */
 790                 uint_t ncpu_per_core;
 791                 if (cpi->cpi_ncore_per_chip == 1)
 792                         ncpu_per_core = cpi->cpi_ncpu_per_chip;
 793                 else if (cpi->cpi_ncore_per_chip > 1)
 794                         ncpu_per_core = cpi->cpi_ncpu_per_chip /
 795                             cpi->cpi_ncore_per_chip;
 796                 /*
 797                  * 8bit APIC IDs on dual core Pentiums
 798                  * look like this:
 799                  *
 800                  * +-----------------------+------+------+
 801                  * | Physical Package ID   |  MC  |  HT  |
 802                  * +-----------------------+------+------+
 803                  * <------- chipid -------->
 804                  * <------- coreid --------------->
 805                  *                         <--- clogid -->
 806                  *                         <------>
 807                  *                         pkgcoreid
 808                  *
 809                  * Where the number of bits necessary to
 810                  * represent MC and HT fields together equals
 811                  * to the minimum number of bits necessary to
 812                  * store the value of cpi->cpi_ncpu_per_chip.
 813                  * Of those bits, the MC part uses the number
 814                  * of bits necessary to store the value of
 815                  * cpi->cpi_ncore_per_chip.
 816                  */
 817                 for (i = 1; i < ncpu_per_core; i <<= 1)
 818                         coreid_shift++;
 819                 cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
 820                 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
 821         } else if (is_x86_feature(feature, X86FSET_HTT)) {
 822                 /*
 823                  * Single-core multi-threaded processors.
 824                  */
 825                 cpi->cpi_coreid = cpi->cpi_chipid;
 826                 cpi->cpi_pkgcoreid = 0;
 827         }
 828         cpi->cpi_procnodeid = cpi->cpi_chipid;
 829         cpi->cpi_compunitid = cpi->cpi_coreid;
 830 }
 831 
 832 static void
 833 cpuid_amd_getids(cpu_t *cpu)
 834 {
 835         int i, first_half, coreidsz;
 836         uint32_t nb_caps_reg;
 837         uint_t node2_1;
 838         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
 839         struct cpuid_regs *cp;
 840 
 841         /*
 842          * AMD CMP chips currently have a single thread per core.
 843          *
 844          * Since no two cpus share a core we must assign a distinct coreid
 845          * per cpu, and we do this by using the cpu_id.  This scheme does not,
 846          * however, guarantee that sibling cores of a chip will have sequential
 847          * coreids starting at a multiple of the number of cores per chip -
 848          * that is usually the case, but if the ACPI MADT table is presented
 849          * in a different order then we need to perform a few more gymnastics
 850          * for the pkgcoreid.
 851          *
 852          * All processors in the system have the same number of enabled
 853          * cores. Cores within a processor are always numbered sequentially
 854          * from 0 regardless of how many or which are disabled, and there
 855          * is no way for operating system to discover the real core id when some
 856          * are disabled.
 857          *
 858          * In family 0x15, the cores come in pairs called compute units. They
 859          * share I$ and L2 caches and the FPU. Enumeration of this feature is
 860          * simplified by the new topology extensions CPUID leaf, indicated by
 861          * the X86 feature X86FSET_TOPOEXT.
 862          */
 863 
 864         cpi->cpi_coreid = cpu->cpu_id;
 865         cpi->cpi_compunitid = cpu->cpu_id;
 866 
 867         if (cpi->cpi_xmaxeax >= 0x80000008) {
 868 
 869                 coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);
 870 
 871                 /*
 872                  * In AMD parlance chip is really a node while Solaris
 873                  * sees chip as equivalent to socket/package.
 874                  */
 875                 cpi->cpi_ncore_per_chip =
 876                     BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
 877                 if (coreidsz == 0) {
 878                         /* Use legacy method */
 879                         for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
 880                                 coreidsz++;
 881                         if (coreidsz == 0)
 882                                 coreidsz = 1;
 883                 }
 884         } else {
 885                 /* Assume single-core part */
 886                 cpi->cpi_ncore_per_chip = 1;
 887                 coreidsz = 1;
 888         }
 889 
 890         cpi->cpi_clogid = cpi->cpi_pkgcoreid =
 891             cpi->cpi_apicid & ((1<<coreidsz) - 1);
 892         cpi->cpi_ncpu_per_chip = cpi->cpi_ncore_per_chip;
 893 
 894         /* Get node ID, compute unit ID */
 895         if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
 896             cpi->cpi_xmaxeax >= 0x8000001e) {
 897                 cp = &cpi->cpi_extd[0x1e];
 898                 cp->cp_eax = 0x8000001e;
 899                 (void) __cpuid_insn(cp);
 900 
 901                 cpi->cpi_procnodes_per_pkg = BITX(cp->cp_ecx, 10, 8) + 1;
 902                 cpi->cpi_procnodeid = BITX(cp->cp_ecx, 7, 0);
 903                 cpi->cpi_cores_per_compunit = BITX(cp->cp_ebx, 15, 8) + 1;
 904                 cpi->cpi_compunitid = BITX(cp->cp_ebx, 7, 0)
 905                     + (cpi->cpi_ncore_per_chip / cpi->cpi_cores_per_compunit)
 906                     * (cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg);
 907         } else if (cpi->cpi_family == 0xf || cpi->cpi_family >= 0x11) {
 908                 cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
 909         } else if (cpi->cpi_family == 0x10) {
 910                 /*
 911                  * See if we are a multi-node processor.
 912                  * All processors in the system have the same number of nodes
 913                  */
 914                 nb_caps_reg =  pci_getl_func(0, 24, 3, 0xe8);
 915                 if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
 916                         /* Single-node */
 917                         cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
 918                             coreidsz);
 919                 } else {
 920 
 921                         /*
 922                          * Multi-node revision D (2 nodes per package
 923                          * are supported)
 924                          */
 925                         cpi->cpi_procnodes_per_pkg = 2;
 926 
 927                         first_half = (cpi->cpi_pkgcoreid <=
 928                             (cpi->cpi_ncore_per_chip/2 - 1));
 929 
 930                         if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
 931                                 /* We are BSP */
 932                                 cpi->cpi_procnodeid = (first_half ? 0 : 1);
 933                         } else {
 934 
 935                                 /* We are AP */
 936                                 /* NodeId[2:1] bits to use for reading F3xe8 */
 937                                 node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;
 938 
 939                                 nb_caps_reg =
 940                                     pci_getl_func(0, 24 + node2_1, 3, 0xe8);
 941 
 942                                 /*
 943                                  * Check IntNodeNum bit (31:30, but bit 31 is
 944                                  * always 0 on dual-node processors)
 945                                  */
 946                                 if (BITX(nb_caps_reg, 30, 30) == 0)
 947                                         cpi->cpi_procnodeid = node2_1 +
 948                                             !first_half;
 949                                 else
 950                                         cpi->cpi_procnodeid = node2_1 +
 951                                             first_half;
 952                         }
 953                 }
 954         } else {
 955                 cpi->cpi_procnodeid = 0;
 956         }
 957 
 958         cpi->cpi_chipid =
 959             cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg;
 960 }
 961 
 962 /*
 963  * Setup XFeature_Enabled_Mask register. Required by xsave feature.
 964  */
 965 void
 966 setup_xfem(void)
 967 {
 968         uint64_t flags = XFEATURE_LEGACY_FP;
 969 
 970         ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
 971 
 972         if (is_x86_feature(x86_featureset, X86FSET_SSE))
 973                 flags |= XFEATURE_SSE;
 974 
 975         if (is_x86_feature(x86_featureset, X86FSET_AVX))
 976                 flags |= XFEATURE_AVX;
 977 
 978         if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
 979                 flags |= XFEATURE_AVX512;
 980 
 981         set_xcr(XFEATURE_ENABLED_MASK, flags);
 982 
 983         xsave_bv_all = flags;
 984 }
 985 
 986 void
 987 cpuid_pass1(cpu_t *cpu, uchar_t *featureset)
 988 {
 989         uint32_t mask_ecx, mask_edx;
 990         struct cpuid_info *cpi;
 991         struct cpuid_regs *cp;
 992         int xcpuid;
 993 #if !defined(__xpv)
 994         extern int idle_cpu_prefer_mwait;
 995 #endif
 996 
 997         /*
 998          * Space statically allocated for BSP, ensure pointer is set
 999          */
1000         if (cpu->cpu_id == 0) {
1001                 if (cpu->cpu_m.mcpu_cpi == NULL)
1002                         cpu->cpu_m.mcpu_cpi = &cpuid_info0;
1003         }
1004 
1005         add_x86_feature(featureset, X86FSET_CPUID);
1006 
1007         cpi = cpu->cpu_m.mcpu_cpi;
1008         ASSERT(cpi != NULL);
1009         cp = &cpi->cpi_std[0];
1010         cp->cp_eax = 0;
1011         cpi->cpi_maxeax = __cpuid_insn(cp);
1012         {
1013                 uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
1014                 *iptr++ = cp->cp_ebx;
1015                 *iptr++ = cp->cp_edx;
1016                 *iptr++ = cp->cp_ecx;
1017                 *(char *)&cpi->cpi_vendorstr[12] = '\0';
1018         }
1019 
1020         cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
1021         x86_vendor = cpi->cpi_vendor; /* for compatibility */
1022 
1023         /*
1024          * Limit the range in case of weird hardware
1025          */
1026         if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
1027                 cpi->cpi_maxeax = CPI_MAXEAX_MAX;
1028         if (cpi->cpi_maxeax < 1)
1029                 goto pass1_done;
1030 
1031         cp = &cpi->cpi_std[1];
1032         cp->cp_eax = 1;
1033         (void) __cpuid_insn(cp);
1034 
1035         /*
1036          * Extract identifying constants for easy access.
1037          */
1038         cpi->cpi_model = CPI_MODEL(cpi);
1039         cpi->cpi_family = CPI_FAMILY(cpi);
1040 
1041         if (cpi->cpi_family == 0xf)
1042                 cpi->cpi_family += CPI_FAMILY_XTD(cpi);
1043 
1044         /*
1045          * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
1046          * Intel, and presumably everyone else, uses model == 0xf, as
1047          * one would expect (max value means possible overflow).  Sigh.
1048          */
1049 
1050         switch (cpi->cpi_vendor) {
1051         case X86_VENDOR_Intel:
1052                 if (IS_EXTENDED_MODEL_INTEL(cpi))
1053                         cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1054                 break;
1055         case X86_VENDOR_AMD:
1056                 if (CPI_FAMILY(cpi) == 0xf)
1057                         cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1058                 break;
1059         default:
1060                 if (cpi->cpi_model == 0xf)
1061                         cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
1062                 break;
1063         }
1064 
1065         cpi->cpi_step = CPI_STEP(cpi);
1066         cpi->cpi_brandid = CPI_BRANDID(cpi);
1067 
1068         /*
1069          * *default* assumptions:
1070          * - believe %edx feature word
1071          * - ignore %ecx feature word
1072          * - 32-bit virtual and physical addressing
1073          */
1074         mask_edx = 0xffffffff;
1075         mask_ecx = 0;
1076 
1077         cpi->cpi_pabits = cpi->cpi_vabits = 32;
1078 
1079         switch (cpi->cpi_vendor) {
1080         case X86_VENDOR_Intel:
1081                 if (cpi->cpi_family == 5)
1082                         x86_type = X86_TYPE_P5;
1083                 else if (IS_LEGACY_P6(cpi)) {
1084                         x86_type = X86_TYPE_P6;
1085                         pentiumpro_bug4046376 = 1;
1086                         /*
1087                          * Clear the SEP bit when it was set erroneously
1088                          */
1089                         if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
1090                                 cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
1091                 } else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
1092                         x86_type = X86_TYPE_P4;
1093                         /*
1094                          * We don't currently depend on any of the %ecx
1095                          * features until Prescott, so we'll only check
1096                          * this from P4 onwards.  We might want to revisit
1097                          * that idea later.
1098                          */
1099                         mask_ecx = 0xffffffff;
1100                 } else if (cpi->cpi_family > 0xf)
1101                         mask_ecx = 0xffffffff;
1102                 /*
1103                  * We don't support MONITOR/MWAIT if leaf 5 is not available
1104                  * to obtain the monitor linesize.
1105                  */
1106                 if (cpi->cpi_maxeax < 5)
1107                         mask_ecx &= ~CPUID_INTC_ECX_MON;
1108                 break;
1109         case X86_VENDOR_IntelClone:
1110         default:
1111                 break;
1112         case X86_VENDOR_AMD:
1113 #if defined(OPTERON_ERRATUM_108)
1114                 if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
1115                         cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
1116                         cpi->cpi_model = 0xc;
1117                 } else
1118 #endif
1119                 if (cpi->cpi_family == 5) {
1120                         /*
1121                          * AMD K5 and K6
1122                          *
1123                          * These CPUs have an incomplete implementation
1124                          * of MCA/MCE which we mask away.
1125                          */
1126                         mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);
1127 
1128                         /*
1129                          * Model 0 uses the wrong (APIC) bit
1130                          * to indicate PGE.  Fix it here.
1131                          */
1132                         if (cpi->cpi_model == 0) {
1133                                 if (cp->cp_edx & 0x200) {
1134                                         cp->cp_edx &= ~0x200;
1135                                         cp->cp_edx |= CPUID_INTC_EDX_PGE;
1136                                 }
1137                         }
1138 
1139                         /*
1140                          * Early models had problems w/ MMX; disable.
1141                          */
1142                         if (cpi->cpi_model < 6)
1143                                 mask_edx &= ~CPUID_INTC_EDX_MMX;
1144                 }
1145 
1146                 /*
1147                  * For newer families, SSE3 and CX16, at least, are valid;
1148                  * enable all
1149                  */
1150                 if (cpi->cpi_family >= 0xf)
1151                         mask_ecx = 0xffffffff;
1152                 /*
1153                  * We don't support MONITOR/MWAIT if leaf 5 is not available
1154                  * to obtain the monitor linesize.
1155                  */
1156                 if (cpi->cpi_maxeax < 5)
1157                         mask_ecx &= ~CPUID_INTC_ECX_MON;
1158 
1159 #if !defined(__xpv)
1160                 /*
1161                  * Do not use MONITOR/MWAIT to halt in the idle loop on any AMD
1162                  * processors.  AMD does not intend MWAIT to be used in the cpu
1163                  * idle loop on current and future processors.  10h and future
1164                  * AMD processors use more power in MWAIT than HLT.
1165                  * Pre-family-10h Opterons do not have the MWAIT instruction.
1166                  */
1167                 idle_cpu_prefer_mwait = 0;
1168 #endif
1169 
1170                 break;
1171         case X86_VENDOR_TM:
1172                 /*
1173                  * workaround the NT workaround in CMS 4.1
1174                  */
1175                 if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
1176                     (cpi->cpi_step == 2 || cpi->cpi_step == 3))
1177                         cp->cp_edx |= CPUID_INTC_EDX_CX8;
1178                 break;
1179         case X86_VENDOR_Centaur:
1180                 /*
1181                  * workaround the NT workarounds again
1182                  */
1183                 if (cpi->cpi_family == 6)
1184                         cp->cp_edx |= CPUID_INTC_EDX_CX8;
1185                 break;
1186         case X86_VENDOR_Cyrix:
1187                 /*
1188                  * We rely heavily on the probing in locore
1189                  * to actually figure out what parts, if any,
1190                  * of the Cyrix cpuid instruction to believe.
1191                  */
1192                 switch (x86_type) {
1193                 case X86_TYPE_CYRIX_486:
1194                         mask_edx = 0;
1195                         break;
1196                 case X86_TYPE_CYRIX_6x86:
1197                         mask_edx = 0;
1198                         break;
1199                 case X86_TYPE_CYRIX_6x86L:
1200                         mask_edx =
1201                             CPUID_INTC_EDX_DE |
1202                             CPUID_INTC_EDX_CX8;
1203                         break;
1204                 case X86_TYPE_CYRIX_6x86MX:
1205                         mask_edx =
1206                             CPUID_INTC_EDX_DE |
1207                             CPUID_INTC_EDX_MSR |
1208                             CPUID_INTC_EDX_CX8 |
1209                             CPUID_INTC_EDX_PGE |
1210                             CPUID_INTC_EDX_CMOV |
1211                             CPUID_INTC_EDX_MMX;
1212                         break;
1213                 case X86_TYPE_CYRIX_GXm:
1214                         mask_edx =
1215                             CPUID_INTC_EDX_MSR |
1216                             CPUID_INTC_EDX_CX8 |
1217                             CPUID_INTC_EDX_CMOV |
1218                             CPUID_INTC_EDX_MMX;
1219                         break;
1220                 case X86_TYPE_CYRIX_MediaGX:
1221                         break;
1222                 case X86_TYPE_CYRIX_MII:
1223                 case X86_TYPE_VIA_CYRIX_III:
1224                         mask_edx =
1225                             CPUID_INTC_EDX_DE |
1226                             CPUID_INTC_EDX_TSC |
1227                             CPUID_INTC_EDX_MSR |
1228                             CPUID_INTC_EDX_CX8 |
1229                             CPUID_INTC_EDX_PGE |
1230                             CPUID_INTC_EDX_CMOV |
1231                             CPUID_INTC_EDX_MMX;
1232                         break;
1233                 default:
1234                         break;
1235                 }
1236                 break;
1237         }
1238 
1239 #if defined(__xpv)
1240         /*
1241          * Do not support MONITOR/MWAIT under a hypervisor
1242          */
1243         mask_ecx &= ~CPUID_INTC_ECX_MON;
1244         /*
1245          * Do not support XSAVE under a hypervisor for now
1246          */
1247         xsave_force_disable = B_TRUE;
1248 
1249 #endif  /* __xpv */
1250 
1251         if (xsave_force_disable) {
1252                 mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
1253                 mask_ecx &= ~CPUID_INTC_ECX_AVX;
1254                 mask_ecx &= ~CPUID_INTC_ECX_F16C;
1255                 mask_ecx &= ~CPUID_INTC_ECX_FMA;
1256         }
1257 
1258         /*
1259          * Now we've figured out the masks that determine
1260          * which bits we choose to believe, apply the masks
1261          * to the feature words, then map the kernel's view
1262          * of these feature words into its feature word.
1263          */
1264         cp->cp_edx &= mask_edx;
1265         cp->cp_ecx &= mask_ecx;
1266 
1267         /*
1268          * apply any platform restrictions (we don't call this
1269          * immediately after __cpuid_insn here, because we need the
1270          * workarounds applied above first)
1271          */
1272         platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);
1273 
1274         /*
1275          * In addition to ecx and edx, Intel is storing a bunch of instruction
1276          * set extensions in leaf 7's ebx, ecx, and edx.
1277          */
1278         if (cpi->cpi_vendor == X86_VENDOR_Intel && cpi->cpi_maxeax >= 7) {
1279                 struct cpuid_regs *ecp;
1280                 ecp = &cpi->cpi_std[7];
1281                 ecp->cp_eax = 7;
1282                 ecp->cp_ecx = 0;
1283                 (void) __cpuid_insn(ecp);
1284                 /*
1285                  * If XSAVE has been disabled, just ignore all of the
1286                  * extended-save-area dependent flags here.
1287                  */
1288                 if (xsave_force_disable) {
1289                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
1290                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
1291                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
1292                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_MPX;
1293                         ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_ALL_AVX512;
1294                         ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_ALL_AVX512;
1295                         ecp->cp_edx &= ~CPUID_INTC_EDX_7_0_ALL_AVX512;
1296                 }
1297 
1298                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMEP)
1299                         add_x86_feature(featureset, X86FSET_SMEP);
1300 
1301                 /*
1302                  * We check disable_smap here in addition to in startup_smap()
1303                  * to ensure CPUs that aren't the boot CPU don't accidentally
1304                  * include it in the feature set and thus generate a mismatched
1305                  * x86 feature set across CPUs. Note that at this time we only
1306                  * enable SMAP for the 64-bit kernel.
1307                  */
1308 #if defined(__amd64)
1309                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMAP &&
1310                     disable_smap == 0)
1311                         add_x86_feature(featureset, X86FSET_SMAP);
1312 #endif
1313                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_MPX)
1314                         add_x86_feature(featureset, X86FSET_MPX);
1315 
1316                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_RDSEED)
1317                         add_x86_feature(featureset, X86FSET_RDSEED);
1318 
1319                 if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_ADX)
1320                         add_x86_feature(featureset, X86FSET_ADX);
1321         }
1322 
1323         /*
1324          * fold in overrides from the "eeprom" mechanism
1325          */
1326         cp->cp_edx |= cpuid_feature_edx_include;
1327         cp->cp_edx &= ~cpuid_feature_edx_exclude;
1328 
1329         cp->cp_ecx |= cpuid_feature_ecx_include;
1330         cp->cp_ecx &= ~cpuid_feature_ecx_exclude;
1331 
1332         if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
1333                 add_x86_feature(featureset, X86FSET_LARGEPAGE);
1334         }
1335         if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
1336                 add_x86_feature(featureset, X86FSET_TSC);
1337         }
1338         if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
1339                 add_x86_feature(featureset, X86FSET_MSR);
1340         }
1341         if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
1342                 add_x86_feature(featureset, X86FSET_MTRR);
1343         }
1344         if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
1345                 add_x86_feature(featureset, X86FSET_PGE);
1346         }
1347         if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
1348                 add_x86_feature(featureset, X86FSET_CMOV);
1349         }
1350         if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
1351                 add_x86_feature(featureset, X86FSET_MMX);
1352         }
1353         if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
1354             (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
1355                 add_x86_feature(featureset, X86FSET_MCA);
1356         }
1357         if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
1358                 add_x86_feature(featureset, X86FSET_PAE);
1359         }
1360         if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
1361                 add_x86_feature(featureset, X86FSET_CX8);
1362         }
1363         if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
1364                 add_x86_feature(featureset, X86FSET_CX16);
1365         }
1366         if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
1367                 add_x86_feature(featureset, X86FSET_PAT);
1368         }
1369         if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
1370                 add_x86_feature(featureset, X86FSET_SEP);
1371         }
1372         if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
1373                 /*
1374                  * In our implementation, fxsave/fxrstor
1375                  * are prerequisites before we'll even
1376                  * try and do SSE things.
1377                  */
1378                 if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
1379                         add_x86_feature(featureset, X86FSET_SSE);
1380                 }
1381                 if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
1382                         add_x86_feature(featureset, X86FSET_SSE2);
1383                 }
1384                 if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
1385                         add_x86_feature(featureset, X86FSET_SSE3);
1386                 }
1387                 if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
1388                         add_x86_feature(featureset, X86FSET_SSSE3);
1389                 }
1390                 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
1391                         add_x86_feature(featureset, X86FSET_SSE4_1);
1392                 }
1393                 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
1394                         add_x86_feature(featureset, X86FSET_SSE4_2);
1395                 }
1396                 if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
1397                         add_x86_feature(featureset, X86FSET_AES);
1398                 }
1399                 if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
1400                         add_x86_feature(featureset, X86FSET_PCLMULQDQ);
1401                 }
1402 
1403                 if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_SHA)
1404                         add_x86_feature(featureset, X86FSET_SHA);
1405 
1406                 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_UMIP)
1407                         add_x86_feature(featureset, X86FSET_UMIP);
1408                 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_PKU)
1409                         add_x86_feature(featureset, X86FSET_PKU);
1410                 if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_OSPKE)
1411                         add_x86_feature(featureset, X86FSET_OSPKE);
1412 
1413                 if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
1414                         add_x86_feature(featureset, X86FSET_XSAVE);
1415 
1416                         /* We only test AVX & AVX512 when there is XSAVE */
1417 
1418                         if (cp->cp_ecx & CPUID_INTC_ECX_AVX) {
1419                                 add_x86_feature(featureset,
1420                                     X86FSET_AVX);
1421 
1422                                 /*
1423                                  * Intel says we can't check these without also
1424                                  * checking AVX.
1425                                  */
1426                                 if (cp->cp_ecx & CPUID_INTC_ECX_F16C)
1427                                         add_x86_feature(featureset,
1428                                             X86FSET_F16C);
1429 
1430                                 if (cp->cp_ecx & CPUID_INTC_ECX_FMA)
1431                                         add_x86_feature(featureset,
1432                                             X86FSET_FMA);
1433 
1434                                 if (cpi->cpi_std[7].cp_ebx &
1435                                     CPUID_INTC_EBX_7_0_BMI1)
1436                                         add_x86_feature(featureset,
1437                                             X86FSET_BMI1);
1438 
1439                                 if (cpi->cpi_std[7].cp_ebx &
1440                                     CPUID_INTC_EBX_7_0_BMI2)
1441                                         add_x86_feature(featureset,
1442                                             X86FSET_BMI2);
1443 
1444                                 if (cpi->cpi_std[7].cp_ebx &
1445                                     CPUID_INTC_EBX_7_0_AVX2)
1446                                         add_x86_feature(featureset,
1447                                             X86FSET_AVX2);
1448                         }
1449 
1450                         if (cpi->cpi_std[7].cp_ebx &
1451                             CPUID_INTC_EBX_7_0_AVX512F) {
1452                                 add_x86_feature(featureset, X86FSET_AVX512F);
1453 
1454                                 if (cpi->cpi_std[7].cp_ebx &
1455                                     CPUID_INTC_EBX_7_0_AVX512DQ)
1456                                         add_x86_feature(featureset,
1457                                             X86FSET_AVX512DQ);
1458                                 if (cpi->cpi_std[7].cp_ebx &
1459                                     CPUID_INTC_EBX_7_0_AVX512IFMA)
1460                                         add_x86_feature(featureset,
1461                                             X86FSET_AVX512FMA);
1462                                 if (cpi->cpi_std[7].cp_ebx &
1463                                     CPUID_INTC_EBX_7_0_AVX512PF)
1464                                         add_x86_feature(featureset,
1465                                             X86FSET_AVX512PF);
1466                                 if (cpi->cpi_std[7].cp_ebx &
1467                                     CPUID_INTC_EBX_7_0_AVX512ER)
1468                                         add_x86_feature(featureset,
1469                                             X86FSET_AVX512ER);
1470                                 if (cpi->cpi_std[7].cp_ebx &
1471                                     CPUID_INTC_EBX_7_0_AVX512CD)
1472                                         add_x86_feature(featureset,
1473                                             X86FSET_AVX512CD);
1474                                 if (cpi->cpi_std[7].cp_ebx &
1475                                     CPUID_INTC_EBX_7_0_AVX512BW)
1476                                         add_x86_feature(featureset,
1477                                             X86FSET_AVX512BW);
1478                                 if (cpi->cpi_std[7].cp_ebx &
1479                                     CPUID_INTC_EBX_7_0_AVX512VL)
1480                                         add_x86_feature(featureset,
1481                                             X86FSET_AVX512VL);
1482 
1483                                 if (cpi->cpi_std[7].cp_ecx &
1484                                     CPUID_INTC_ECX_7_0_AVX512VBMI)
1485                                         add_x86_feature(featureset,
1486                                             X86FSET_AVX512VBMI);
1487                                 if (cpi->cpi_std[7].cp_ecx &
1488                                     CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
1489                                         add_x86_feature(featureset,
1490                                             X86FSET_AVX512VPOPCDQ);
1491 
1492                                 if (cpi->cpi_std[7].cp_edx &
1493                                     CPUID_INTC_EDX_7_0_AVX5124NNIW)
1494                                         add_x86_feature(featureset,
1495                                             X86FSET_AVX512NNIW);
1496                                 if (cpi->cpi_std[7].cp_edx &
1497                                     CPUID_INTC_EDX_7_0_AVX5124FMAPS)
1498                                         add_x86_feature(featureset,
1499                                             X86FSET_AVX512FMAPS);
1500                         }
1501                 }
1502         }
1503         if (cp->cp_ecx & CPUID_INTC_ECX_X2APIC) {
1504                 add_x86_feature(featureset, X86FSET_X2APIC);
1505         }
1506         if (cp->cp_edx & CPUID_INTC_EDX_DE) {
1507                 add_x86_feature(featureset, X86FSET_DE);
1508         }
1509 #if !defined(__xpv)
1510         if (cp->cp_ecx & CPUID_INTC_ECX_MON) {
1511 
1512                 /*
1513                  * We require the CLFLUSH instruction for erratum workaround
1514                  * to use MONITOR/MWAIT.
1515                  */
1516                 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1517                         cpi->cpi_mwait.support |= MWAIT_SUPPORT;
1518                         add_x86_feature(featureset, X86FSET_MWAIT);
1519                 } else {
1520                         extern int idle_cpu_assert_cflush_monitor;
1521 
1522                         /*
1523                          * All processors we are aware of which have
1524                          * MONITOR/MWAIT also have CLFLUSH.
1525                          */
1526                         if (idle_cpu_assert_cflush_monitor) {
1527                                 ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
1528                                     (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
1529                         }
1530                 }
1531         }
1532 #endif  /* __xpv */
1533 
1534         if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
1535                 add_x86_feature(featureset, X86FSET_VMX);
1536         }
1537 
1538         if (cp->cp_ecx & CPUID_INTC_ECX_RDRAND)
1539                 add_x86_feature(featureset, X86FSET_RDRAND);
1540 
1541         /*
1542          * Only need it first time, rest of the cpus would follow suit.
1543          * we only capture this for the bootcpu.
1544          */
1545         if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1546                 add_x86_feature(featureset, X86FSET_CLFSH);
1547                 x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
1548         }
1549         if (is_x86_feature(featureset, X86FSET_PAE))
1550                 cpi->cpi_pabits = 36;
1551 
1552         /*
1553          * Hyperthreading configuration is slightly tricky on Intel
1554          * and pure clones, and even trickier on AMD.
1555          *
1556          * (AMD chose to set the HTT bit on their CMP processors,
1557          * even though they're not actually hyperthreaded.  Thus it
1558          * takes a bit more work to figure out what's really going
1559          * on ... see the handling of the CMP_LGCY bit below)
1560          */
1561         if (cp->cp_edx & CPUID_INTC_EDX_HTT) {
1562                 cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
1563                 if (cpi->cpi_ncpu_per_chip > 1)
1564                         add_x86_feature(featureset, X86FSET_HTT);
1565         } else {
1566                 cpi->cpi_ncpu_per_chip = 1;
1567         }
1568 
1569         if (cpi->cpi_vendor == X86_VENDOR_Intel && cpi->cpi_maxeax >= 0xD &&
1570             !xsave_force_disable) {
1571                 struct cpuid_regs r, *ecp;
1572 
1573                 ecp = &r;
1574                 ecp->cp_eax = 0xD;
1575                 ecp->cp_ecx = 1;
1576                 ecp->cp_edx = ecp->cp_ebx = 0;
1577                 (void) __cpuid_insn(ecp);
1578 
1579                 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEOPT)
1580                         add_x86_feature(featureset, X86FSET_XSAVEOPT);
1581                 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEC)
1582                         add_x86_feature(featureset, X86FSET_XSAVEC);
1583                 if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVES)
1584                         add_x86_feature(featureset, X86FSET_XSAVES);
1585         }
1586 
1587         /*
1588          * Work on the "extended" feature information, doing
1589          * some basic initialization for cpuid_pass2()
1590          */
1591         xcpuid = 0;
1592         switch (cpi->cpi_vendor) {
1593         case X86_VENDOR_Intel:
1594                 /*
1595                  * On KVM we know we will have proper support for extended
1596                  * cpuid.
1597                  */
1598                 if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf ||
1599                     (get_hwenv() == HW_KVM && cpi->cpi_family == 6 &&
1600                     (cpi->cpi_model == 6 || cpi->cpi_model == 2)))
1601                         xcpuid++;
1602                 break;
1603         case X86_VENDOR_AMD:
1604                 if (cpi->cpi_family > 5 ||
1605                     (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
1606                         xcpuid++;
1607                 break;
1608         case X86_VENDOR_Cyrix:
1609                 /*
1610                  * Only these Cyrix CPUs are -known- to support
1611                  * extended cpuid operations.
1612                  */
1613                 if (x86_type == X86_TYPE_VIA_CYRIX_III ||
1614                     x86_type == X86_TYPE_CYRIX_GXm)
1615                         xcpuid++;
1616                 break;
1617         case X86_VENDOR_Centaur:
1618         case X86_VENDOR_TM:
1619         default:
1620                 xcpuid++;
1621                 break;
1622         }
1623 
1624         if (xcpuid) {
1625                 cp = &cpi->cpi_extd[0];
1626                 cp->cp_eax = 0x80000000;
1627                 cpi->cpi_xmaxeax = __cpuid_insn(cp);
1628         }
1629 
1630         if (cpi->cpi_xmaxeax & 0x80000000) {
1631 
1632                 if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
1633                         cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;
1634 
1635                 switch (cpi->cpi_vendor) {
1636                 case X86_VENDOR_Intel:
1637                 case X86_VENDOR_AMD:
1638                         if (cpi->cpi_xmaxeax < 0x80000001)
1639                                 break;
1640                         cp = &cpi->cpi_extd[1];
1641                         cp->cp_eax = 0x80000001;
1642                         (void) __cpuid_insn(cp);
1643 
1644                         if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1645                             cpi->cpi_family == 5 &&
1646                             cpi->cpi_model == 6 &&
1647                             cpi->cpi_step == 6) {
1648                                 /*
1649                                  * K6 model 6 uses bit 10 to indicate SYSC
1650                                  * Later models use bit 11. Fix it here.
1651                                  */
1652                                 if (cp->cp_edx & 0x400) {
1653                                         cp->cp_edx &= ~0x400;
1654                                         cp->cp_edx |= CPUID_AMD_EDX_SYSC;
1655                                 }
1656                         }
1657 
1658                         platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);
1659 
1660                         /*
1661                          * Compute the additions to the kernel's feature word.
1662                          */
1663                         if (cp->cp_edx & CPUID_AMD_EDX_NX) {
1664                                 add_x86_feature(featureset, X86FSET_NX);
1665                         }
1666 
1667                         /*
1668                          * Regardless whether or not we boot 64-bit,
1669                          * we should have a way to identify whether
1670                          * the CPU is capable of running 64-bit.
1671                          */
1672                         if (cp->cp_edx & CPUID_AMD_EDX_LM) {
1673                                 add_x86_feature(featureset, X86FSET_64);
1674                         }
1675 
1676 #if defined(__amd64)
1677                         /* 1 GB large page - enable only for 64 bit kernel */
1678                         if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
1679                                 add_x86_feature(featureset, X86FSET_1GPG);
1680                         }
1681 #endif
1682 
1683                         if ((cpi->cpi_vendor == X86_VENDOR_AMD) &&
1684                             (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
1685                             (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
1686                                 add_x86_feature(featureset, X86FSET_SSE4A);
1687                         }
1688 
1689                         /*
1690                          * If both the HTT and CMP_LGCY bits are set,
1691                          * then we're not actually HyperThreaded.  Read
1692                          * "AMD CPUID Specification" for more details.
1693                          */
1694                         if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1695                             is_x86_feature(featureset, X86FSET_HTT) &&
1696                             (cp->cp_ecx & CPUID_AMD_ECX_CMP_LGCY)) {
1697                                 remove_x86_feature(featureset, X86FSET_HTT);
1698                                 add_x86_feature(featureset, X86FSET_CMP);
1699                         }
1700 #if defined(__amd64)
1701                         /*
1702                          * It's really tricky to support syscall/sysret in
1703                          * the i386 kernel; we rely on sysenter/sysexit
1704                          * instead.  In the amd64 kernel, things are -way-
1705                          * better.
1706                          */
1707                         if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
1708                                 add_x86_feature(featureset, X86FSET_ASYSC);
1709                         }
1710 
1711                         /*
1712                          * While we're thinking about system calls, note
1713                          * that AMD processors don't support sysenter
1714                          * in long mode at all, so don't try to program them.
1715                          */
1716                         if (x86_vendor == X86_VENDOR_AMD) {
1717                                 remove_x86_feature(featureset, X86FSET_SEP);
1718                         }
1719 #endif
1720                         if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
1721                                 add_x86_feature(featureset, X86FSET_TSCP);
1722                         }
1723 
1724                         if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
1725                                 add_x86_feature(featureset, X86FSET_SVM);
1726                         }
1727 
1728                         if (cp->cp_ecx & CPUID_AMD_ECX_TOPOEXT) {
1729                                 add_x86_feature(featureset, X86FSET_TOPOEXT);
1730                         }
1731                         break;
1732                 default:
1733                         break;
1734                 }
1735 
1736                 /*
1737                  * Get CPUID data about processor cores and hyperthreads.
1738                  */
1739                 switch (cpi->cpi_vendor) {
1740                 case X86_VENDOR_Intel:
1741                         if (cpi->cpi_maxeax >= 4) {
1742                                 cp = &cpi->cpi_std[4];
1743                                 cp->cp_eax = 4;
1744                                 cp->cp_ecx = 0;
1745                                 (void) __cpuid_insn(cp);
1746                                 platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
1747                         }
1748                         /*FALLTHROUGH*/
1749                 case X86_VENDOR_AMD:
1750                         if (cpi->cpi_xmaxeax < 0x80000008)
1751                                 break;
1752                         cp = &cpi->cpi_extd[8];
1753                         cp->cp_eax = 0x80000008;
1754                         (void) __cpuid_insn(cp);
1755                         platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, cp);
1756 
1757                         /*
1758                          * Virtual and physical address limits from
1759                          * cpuid override previously guessed values.
1760                          */
1761                         cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
1762                         cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
1763                         break;
1764                 default:
1765                         break;
1766                 }
1767 
1768                 /*
1769                  * Derive the number of cores per chip
1770                  */
1771                 switch (cpi->cpi_vendor) {
1772                 case X86_VENDOR_Intel:
1773                         if (cpi->cpi_maxeax < 4) {
1774                                 cpi->cpi_ncore_per_chip = 1;
1775                                 break;
1776                         } else {
1777                                 cpi->cpi_ncore_per_chip =
1778                                     BITX((cpi)->cpi_std[4].cp_eax, 31, 26) + 1;
1779                         }
1780                         break;
1781                 case X86_VENDOR_AMD:
1782                         if (cpi->cpi_xmaxeax < 0x80000008) {
1783                                 cpi->cpi_ncore_per_chip = 1;
1784                                 break;
1785                         } else {
1786                                 /*
1787                                  * On family 0xf cpuid fn 2 ECX[7:0] "NC" is
1788                                  * 1 less than the number of physical cores on
1789                                  * the chip.  In family 0x10 this value can
1790                                  * be affected by "downcoring" - it reflects
1791                                  * 1 less than the number of cores actually
1792                                  * enabled on this node.
1793                                  */
1794                                 cpi->cpi_ncore_per_chip =
1795                                     BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
1796                         }
1797                         break;
1798                 default:
1799                         cpi->cpi_ncore_per_chip = 1;
1800                         break;
1801                 }
1802 
1803                 /*
1804                  * Get CPUID data about TSC Invariance in Deep C-State.
1805                  */
1806                 switch (cpi->cpi_vendor) {
1807                 case X86_VENDOR_Intel:
1808                         if (cpi->cpi_maxeax >= 7) {
1809                                 cp = &cpi->cpi_extd[7];
1810                                 cp->cp_eax = 0x80000007;
1811                                 cp->cp_ecx = 0;
1812                                 (void) __cpuid_insn(cp);
1813                         }
1814                         break;
1815                 default:
1816                         break;
1817                 }
1818         } else {
1819                 cpi->cpi_ncore_per_chip = 1;
1820         }
1821 
1822         /*
1823          * If more than one core, then this processor is CMP.
1824          */
1825         if (cpi->cpi_ncore_per_chip > 1) {
1826                 add_x86_feature(featureset, X86FSET_CMP);
1827         }
1828 
1829         /*
1830          * If the number of cores is the same as the number
1831          * of CPUs, then we cannot have HyperThreading.
1832          */
1833         if (cpi->cpi_ncpu_per_chip == cpi->cpi_ncore_per_chip) {
1834                 remove_x86_feature(featureset, X86FSET_HTT);
1835         }
1836 
1837         cpi->cpi_apicid = CPI_APIC_ID(cpi);
1838         cpi->cpi_procnodes_per_pkg = 1;
1839         cpi->cpi_cores_per_compunit = 1;
1840         if (is_x86_feature(featureset, X86FSET_HTT) == B_FALSE &&
1841             is_x86_feature(featureset, X86FSET_CMP) == B_FALSE) {
1842                 /*
1843                  * Single-core single-threaded processors.
1844                  */
1845                 cpi->cpi_chipid = -1;
1846                 cpi->cpi_clogid = 0;
1847                 cpi->cpi_coreid = cpu->cpu_id;
1848                 cpi->cpi_pkgcoreid = 0;
1849                 if (cpi->cpi_vendor == X86_VENDOR_AMD)
1850                         cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
1851                 else
1852                         cpi->cpi_procnodeid = cpi->cpi_chipid;
1853         } else if (cpi->cpi_ncpu_per_chip > 1) {
1854                 if (cpi->cpi_vendor == X86_VENDOR_Intel)
1855                         cpuid_intel_getids(cpu, featureset);
1856                 else if (cpi->cpi_vendor == X86_VENDOR_AMD)
1857                         cpuid_amd_getids(cpu);
1858                 else {
1859                         /*
1860                          * All other processors are currently
1861                          * assumed to have single cores.
1862                          */
1863                         cpi->cpi_coreid = cpi->cpi_chipid;
1864                         cpi->cpi_pkgcoreid = 0;
1865                         cpi->cpi_procnodeid = cpi->cpi_chipid;
1866                         cpi->cpi_compunitid = cpi->cpi_chipid;
1867                 }
1868         }
1869 
1870         /*
1871          * Synthesize chip "revision" and socket type
1872          */
1873         cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
1874             cpi->cpi_model, cpi->cpi_step);
1875         cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
1876             cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
1877         cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
1878             cpi->cpi_model, cpi->cpi_step);
1879 
1880         /*
1881          * While we're here, check for the AMD "Error Pointer Zero/Restore"
1882          * feature. This can be used to setup the FP save handlers
1883          * appropriately.
1884          */
1885         if (cpi->cpi_vendor == X86_VENDOR_AMD) {
1886                 if (cpi->cpi_xmaxeax >= 0x80000008 &&
1887                     cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
1888                         /* Special handling for AMD FP not necessary. */
1889                         cpi->cpi_fp_amd_save = 0;
1890                 } else {
1891                         cpi->cpi_fp_amd_save = 1;
1892                 }
1893         }
1894 
1895 pass1_done:
1896         cpi->cpi_pass = 1;
1897 }
1898 
1899 /*
1900  * Make copies of the cpuid table entries we depend on, in
1901  * part for ease of parsing now, in part so that we have only
1902  * one place to correct any of it, in part for ease of
1903  * later export to userland, and in part so we can look at
1904  * this stuff in a crash dump.
1905  */
1906 
1907 /*ARGSUSED*/
1908 void
1909 cpuid_pass2(cpu_t *cpu)
1910 {
1911         uint_t n, nmax;
1912         int i;
1913         struct cpuid_regs *cp;
1914         uint8_t *dp;
1915         uint32_t *iptr;
1916         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
1917 
1918         ASSERT(cpi->cpi_pass == 1);
1919 
1920         if (cpi->cpi_maxeax < 1)
1921                 goto pass2_done;
1922 
1923         if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
1924                 nmax = NMAX_CPI_STD;
1925         /*
1926          * (We already handled n == 0 and n == 1 in pass 1)
1927          */
1928         for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
1929                 cp->cp_eax = n;
1930 
1931                 /*
1932                  * CPUID function 4 expects %ecx to be initialized
1933                  * with an index which indicates which cache to return
1934                  * information about. The OS is expected to call function 4
1935                  * with %ecx set to 0, 1, 2, ... until it returns with
1936                  * EAX[4:0] set to 0, which indicates there are no more
1937                  * caches.
1938                  *
1939                  * Here, populate cpi_std[4] with the information returned by
1940                  * function 4 when %ecx == 0, and do the rest in cpuid_pass3()
1941                  * when dynamic memory allocation becomes available.
1942                  *
1943                  * Note: we need to explicitly initialize %ecx here, since
1944                  * function 4 may have been previously invoked.
1945                  *
1946                  * The same is all true for CPUID function 7.
1947                  */
1948                 if (n == 4 || n == 7)
1949                         cp->cp_ecx = 0;
1950 
1951                 (void) __cpuid_insn(cp);
1952                 platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
1953                 switch (n) {
1954                 case 2:
1955                         /*
1956                          * "the lower 8 bits of the %eax register
1957                          * contain a value that identifies the number
1958                          * of times the cpuid [instruction] has to be
1959                          * executed to obtain a complete image of the
1960                          * processor's caching systems."
1961                          *
1962                          * How *do* they make this stuff up?
1963                          */
1964                         cpi->cpi_ncache = sizeof (*cp) *
1965                             BITX(cp->cp_eax, 7, 0);
1966                         if (cpi->cpi_ncache == 0)
1967                                 break;
1968                         cpi->cpi_ncache--;   /* skip count byte */
1969 
1970                         /*
1971                          * Well, for now, rather than attempt to implement
1972                          * this slightly dubious algorithm, we just look
1973                          * at the first 15 ..
1974                          */
1975                         if (cpi->cpi_ncache > (sizeof (*cp) - 1))
1976                                 cpi->cpi_ncache = sizeof (*cp) - 1;
1977 
1978                         dp = cpi->cpi_cacheinfo;
1979                         if (BITX(cp->cp_eax, 31, 31) == 0) {
1980                                 uint8_t *p = (void *)&cp->cp_eax;
1981                                 for (i = 1; i < 4; i++)
1982                                         if (p[i] != 0)
1983                                                 *dp++ = p[i];
1984                         }
1985                         if (BITX(cp->cp_ebx, 31, 31) == 0) {
1986                                 uint8_t *p = (void *)&cp->cp_ebx;
1987                                 for (i = 0; i < 4; i++)
1988                                         if (p[i] != 0)
1989                                                 *dp++ = p[i];
1990                         }
1991                         if (BITX(cp->cp_ecx, 31, 31) == 0) {
1992                                 uint8_t *p = (void *)&cp->cp_ecx;
1993                                 for (i = 0; i < 4; i++)
1994                                         if (p[i] != 0)
1995                                                 *dp++ = p[i];
1996                         }
1997                         if (BITX(cp->cp_edx, 31, 31) == 0) {
1998                                 uint8_t *p = (void *)&cp->cp_edx;
1999                                 for (i = 0; i < 4; i++)
2000                                         if (p[i] != 0)
2001                                                 *dp++ = p[i];
2002                         }
2003                         break;
2004 
2005                 case 3: /* Processor serial number, if PSN supported */
2006                         break;
2007 
2008                 case 4: /* Deterministic cache parameters */
2009                         break;
2010 
2011                 case 5: /* Monitor/Mwait parameters */
2012                 {
2013                         size_t mwait_size;
2014 
2015                         /*
2016                          * check cpi_mwait.support which was set in cpuid_pass1
2017                          */
2018                         if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
2019                                 break;
2020 
2021                         /*
2022                          * Protect ourself from insane mwait line size.
2023                          * Workaround for incomplete hardware emulator(s).
2024                          */
2025                         mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
2026                         if (mwait_size < sizeof (uint32_t) ||
2027                             !ISP2(mwait_size)) {
2028 #if DEBUG
2029                                 cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
2030                                     "size %ld", cpu->cpu_id, (long)mwait_size);
2031 #endif
2032                                 break;
2033                         }
2034 
2035                         cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
2036                         cpi->cpi_mwait.mon_max = mwait_size;
2037                         if (MWAIT_EXTENSION(cpi)) {
2038                                 cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
2039                                 if (MWAIT_INT_ENABLE(cpi))
2040                                         cpi->cpi_mwait.support |=
2041                                             MWAIT_ECX_INT_ENABLE;
2042                         }
2043                         break;
2044                 }
2045                 default:
2046                         break;
2047                 }
2048         }
2049 
2050         if (cpi->cpi_maxeax >= 0xB && cpi->cpi_vendor == X86_VENDOR_Intel) {
2051                 struct cpuid_regs regs;
2052 
2053                 cp = &regs;
2054                 cp->cp_eax = 0xB;
2055                 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2056 
2057                 (void) __cpuid_insn(cp);
2058 
2059                 /*
2060                  * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
2061                  * indicates that the extended topology enumeration leaf is
2062                  * available.
2063                  */
2064                 if (cp->cp_ebx) {
2065                         uint32_t x2apic_id;
2066                         uint_t coreid_shift = 0;
2067                         uint_t ncpu_per_core = 1;
2068                         uint_t chipid_shift = 0;
2069                         uint_t ncpu_per_chip = 1;
2070                         uint_t i;
2071                         uint_t level;
2072 
2073                         for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
2074                                 cp->cp_eax = 0xB;
2075                                 cp->cp_ecx = i;
2076 
2077                                 (void) __cpuid_insn(cp);
2078                                 level = CPI_CPU_LEVEL_TYPE(cp);
2079 
2080                                 if (level == 1) {
2081                                         x2apic_id = cp->cp_edx;
2082                                         coreid_shift = BITX(cp->cp_eax, 4, 0);
2083                                         ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
2084                                 } else if (level == 2) {
2085                                         x2apic_id = cp->cp_edx;
2086                                         chipid_shift = BITX(cp->cp_eax, 4, 0);
2087                                         ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
2088                                 }
2089                         }
2090 
2091                         cpi->cpi_apicid = x2apic_id;
2092                         cpi->cpi_ncpu_per_chip = ncpu_per_chip;
2093                         cpi->cpi_ncore_per_chip = ncpu_per_chip /
2094                             ncpu_per_core;
2095                         cpi->cpi_chipid = x2apic_id >> chipid_shift;
2096                         cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
2097                         cpi->cpi_coreid = x2apic_id >> coreid_shift;
2098                         cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
2099                 }
2100 
2101                 /* Make cp NULL so that we don't stumble on others */
2102                 cp = NULL;
2103         }
2104 
2105         /*
2106          * XSAVE enumeration
2107          */
2108         if (cpi->cpi_maxeax >= 0xD) {
2109                 struct cpuid_regs regs;
2110                 boolean_t cpuid_d_valid = B_TRUE;
2111 
2112                 cp = &regs;
2113                 cp->cp_eax = 0xD;
2114                 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
2115 
2116                 (void) __cpuid_insn(cp);
2117 
2118                 /*
2119                  * Sanity checks for debug
2120                  */
2121                 if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
2122                     (cp->cp_eax & XFEATURE_SSE) == 0) {
2123                         cpuid_d_valid = B_FALSE;
2124                 }
2125 
2126                 cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
2127                 cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
2128                 cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;
2129 
2130                 /*
2131                  * If the hw supports AVX, get the size and offset in the save
2132                  * area for the ymm state.
2133                  */
2134                 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
2135                         cp->cp_eax = 0xD;
2136                         cp->cp_ecx = 2;
2137                         cp->cp_edx = cp->cp_ebx = 0;
2138 
2139                         (void) __cpuid_insn(cp);
2140 
2141                         if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
2142                             cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
2143                                 cpuid_d_valid = B_FALSE;
2144                         }
2145 
2146                         cpi->cpi_xsave.ymm_size = cp->cp_eax;
2147                         cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
2148                 }
2149 
2150                 /*
2151                  * If the hw supports MPX, get the size and offset in the
2152                  * save area for BNDREGS and BNDCSR.
2153                  */
2154                 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_MPX) {
2155                         cp->cp_eax = 0xD;
2156                         cp->cp_ecx = 3;
2157                         cp->cp_edx = cp->cp_ebx = 0;
2158 
2159                         (void) __cpuid_insn(cp);
2160 
2161                         cpi->cpi_xsave.bndregs_size = cp->cp_eax;
2162                         cpi->cpi_xsave.bndregs_offset = cp->cp_ebx;
2163 
2164                         cp->cp_eax = 0xD;
2165                         cp->cp_ecx = 4;
2166                         cp->cp_edx = cp->cp_ebx = 0;
2167 
2168                         (void) __cpuid_insn(cp);
2169 
2170                         cpi->cpi_xsave.bndcsr_size = cp->cp_eax;
2171                         cpi->cpi_xsave.bndcsr_offset = cp->cp_ebx;
2172                 }
2173 
2174                 /*
2175                  * If the hw supports AVX512, get the size and offset in the
2176                  * save area for the opmask registers and zmm state.
2177                  */
2178                 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX512) {
2179                         cp->cp_eax = 0xD;
2180                         cp->cp_ecx = 5;
2181                         cp->cp_edx = cp->cp_ebx = 0;
2182 
2183                         (void) __cpuid_insn(cp);
2184 
2185                         cpi->cpi_xsave.opmask_size = cp->cp_eax;
2186                         cpi->cpi_xsave.opmask_offset = cp->cp_ebx;
2187 
2188                         cp->cp_eax = 0xD;
2189                         cp->cp_ecx = 6;
2190                         cp->cp_edx = cp->cp_ebx = 0;
2191 
2192                         (void) __cpuid_insn(cp);
2193 
2194                         cpi->cpi_xsave.zmmlo_size = cp->cp_eax;
2195                         cpi->cpi_xsave.zmmlo_offset = cp->cp_ebx;
2196 
2197                         cp->cp_eax = 0xD;
2198                         cp->cp_ecx = 7;
2199                         cp->cp_edx = cp->cp_ebx = 0;
2200 
2201                         (void) __cpuid_insn(cp);
2202 
2203                         cpi->cpi_xsave.zmmhi_size = cp->cp_eax;
2204                         cpi->cpi_xsave.zmmhi_offset = cp->cp_ebx;
2205                 }
2206 
2207                 if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
2208                         xsave_state_size = 0;
2209                 } else if (cpuid_d_valid) {
2210                         xsave_state_size = cpi->cpi_xsave.xsav_max_size;
2211                 } else {
2212                         /* Broken CPUID 0xD, probably in HVM */
2213                         cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
2214                             "value: hw_low = %d, hw_high = %d, xsave_size = %d"
2215                             ", ymm_size = %d, ymm_offset = %d\n",
2216                             cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
2217                             cpi->cpi_xsave.xsav_hw_features_high,
2218                             (int)cpi->cpi_xsave.xsav_max_size,
2219                             (int)cpi->cpi_xsave.ymm_size,
2220                             (int)cpi->cpi_xsave.ymm_offset);
2221 
2222                         if (xsave_state_size != 0) {
2223                                 /*
2224                                  * This must be a non-boot CPU. We cannot
2225                                  * continue, because boot cpu has already
2226                                  * enabled XSAVE.
2227                                  */
2228                                 ASSERT(cpu->cpu_id != 0);
2229                                 cmn_err(CE_PANIC, "cpu%d: we have already "
2230                                     "enabled XSAVE on boot cpu, cannot "
2231                                     "continue.", cpu->cpu_id);
2232                         } else {
2233                                 /*
2234                                  * If we reached here on the boot CPU, it's also
2235                                  * almost certain that we'll reach here on the
2236                                  * non-boot CPUs. When we're here on a boot CPU
2237                                  * we should disable the feature, on a non-boot
2238                                  * CPU we need to confirm that we have.
2239                                  */
2240                                 if (cpu->cpu_id == 0) {
2241                                         remove_x86_feature(x86_featureset,
2242                                             X86FSET_XSAVE);
2243                                         remove_x86_feature(x86_featureset,
2244                                             X86FSET_AVX);
2245                                         remove_x86_feature(x86_featureset,
2246                                             X86FSET_F16C);
2247                                         remove_x86_feature(x86_featureset,
2248                                             X86FSET_BMI1);
2249                                         remove_x86_feature(x86_featureset,
2250                                             X86FSET_BMI2);
2251                                         remove_x86_feature(x86_featureset,
2252                                             X86FSET_FMA);
2253                                         remove_x86_feature(x86_featureset,
2254                                             X86FSET_AVX2);
2255                                         remove_x86_feature(x86_featureset,
2256                                             X86FSET_MPX);
2257                                         remove_x86_feature(x86_featureset,
2258                                             X86FSET_AVX512F);
2259                                         remove_x86_feature(x86_featureset,
2260                                             X86FSET_AVX512DQ);
2261                                         remove_x86_feature(x86_featureset,
2262                                             X86FSET_AVX512PF);
2263                                         remove_x86_feature(x86_featureset,
2264                                             X86FSET_AVX512ER);
2265                                         remove_x86_feature(x86_featureset,
2266                                             X86FSET_AVX512CD);
2267                                         remove_x86_feature(x86_featureset,
2268                                             X86FSET_AVX512BW);
2269                                         remove_x86_feature(x86_featureset,
2270                                             X86FSET_AVX512VL);
2271                                         remove_x86_feature(x86_featureset,
2272                                             X86FSET_AVX512FMA);
2273                                         remove_x86_feature(x86_featureset,
2274                                             X86FSET_AVX512VBMI);
2275                                         remove_x86_feature(x86_featureset,
2276                                             X86FSET_AVX512VPOPCDQ);
2277                                         remove_x86_feature(x86_featureset,
2278                                             X86FSET_AVX512NNIW);
2279                                         remove_x86_feature(x86_featureset,
2280                                             X86FSET_AVX512FMAPS);
2281 
2282                                         CPI_FEATURES_ECX(cpi) &=
2283                                             ~CPUID_INTC_ECX_XSAVE;
2284                                         CPI_FEATURES_ECX(cpi) &=
2285                                             ~CPUID_INTC_ECX_AVX;
2286                                         CPI_FEATURES_ECX(cpi) &=
2287                                             ~CPUID_INTC_ECX_F16C;
2288                                         CPI_FEATURES_ECX(cpi) &=
2289                                             ~CPUID_INTC_ECX_FMA;
2290                                         CPI_FEATURES_7_0_EBX(cpi) &=
2291                                             ~CPUID_INTC_EBX_7_0_BMI1;
2292                                         CPI_FEATURES_7_0_EBX(cpi) &=
2293                                             ~CPUID_INTC_EBX_7_0_BMI2;
2294                                         CPI_FEATURES_7_0_EBX(cpi) &=
2295                                             ~CPUID_INTC_EBX_7_0_AVX2;
2296                                         CPI_FEATURES_7_0_EBX(cpi) &=
2297                                             ~CPUID_INTC_EBX_7_0_MPX;
2298                                         CPI_FEATURES_7_0_EBX(cpi) &=
2299                                             ~CPUID_INTC_EBX_7_0_ALL_AVX512;
2300 
2301                                         CPI_FEATURES_7_0_ECX(cpi) &=
2302                                             ~CPUID_INTC_ECX_7_0_ALL_AVX512;
2303 
2304                                         CPI_FEATURES_7_0_EDX(cpi) &=
2305                                             ~CPUID_INTC_EDX_7_0_ALL_AVX512;
2306 
2307                                         xsave_force_disable = B_TRUE;
2308                                 } else {
2309                                         VERIFY(is_x86_feature(x86_featureset,
2310                                             X86FSET_XSAVE) == B_FALSE);
2311                                 }
2312                         }
2313                 }
2314         }
2315 
2316 
2317         if ((cpi->cpi_xmaxeax & 0x80000000) == 0)
2318                 goto pass2_done;
2319 
2320         if ((nmax = cpi->cpi_xmaxeax - 0x80000000 + 1) > NMAX_CPI_EXTD)
2321                 nmax = NMAX_CPI_EXTD;
2322         /*
2323          * Copy the extended properties, fixing them as we go.
2324          * (We already handled n == 0 and n == 1 in pass 1)
2325          */
2326         iptr = (void *)cpi->cpi_brandstr;
2327         for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
2328                 cp->cp_eax = 0x80000000 + n;
2329                 (void) __cpuid_insn(cp);
2330                 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000000 + n, cp);
2331                 switch (n) {
2332                 case 2:
2333                 case 3:
2334                 case 4:
2335                         /*
2336                          * Extract the brand string
2337                          */
2338                         *iptr++ = cp->cp_eax;
2339                         *iptr++ = cp->cp_ebx;
2340                         *iptr++ = cp->cp_ecx;
2341                         *iptr++ = cp->cp_edx;
2342                         break;
2343                 case 5:
2344                         switch (cpi->cpi_vendor) {
2345                         case X86_VENDOR_AMD:
2346                                 /*
2347                                  * The Athlon and Duron were the first
2348                                  * parts to report the sizes of the
2349                                  * TLB for large pages. Before then,
2350                                  * we don't trust the data.
2351                                  */
2352                                 if (cpi->cpi_family < 6 ||
2353                                     (cpi->cpi_family == 6 &&
2354                                     cpi->cpi_model < 1))
2355                                         cp->cp_eax = 0;
2356                                 break;
2357                         default:
2358                                 break;
2359                         }
2360                         break;
2361                 case 6:
2362                         switch (cpi->cpi_vendor) {
2363                         case X86_VENDOR_AMD:
2364                                 /*
2365                                  * The Athlon and Duron were the first
2366                                  * AMD parts with L2 TLB's.
2367                                  * Before then, don't trust the data.
2368                                  */
2369                                 if (cpi->cpi_family < 6 ||
2370                                     cpi->cpi_family == 6 &&
2371                                     cpi->cpi_model < 1)
2372                                         cp->cp_eax = cp->cp_ebx = 0;
2373                                 /*
2374                                  * AMD Duron rev A0 reports L2
2375                                  * cache size incorrectly as 1K
2376                                  * when it is really 64K
2377                                  */
2378                                 if (cpi->cpi_family == 6 &&
2379                                     cpi->cpi_model == 3 &&
2380                                     cpi->cpi_step == 0) {
2381                                         cp->cp_ecx &= 0xffff;
2382                                         cp->cp_ecx |= 0x400000;
2383                                 }
2384                                 break;
2385                         case X86_VENDOR_Cyrix:  /* VIA C3 */
2386                                 /*
2387                                  * VIA C3 processors are a bit messed
2388                                  * up w.r.t. encoding cache sizes in %ecx
2389                                  */
2390                                 if (cpi->cpi_family != 6)
2391                                         break;
2392                                 /*
2393                                  * model 7 and 8 were incorrectly encoded
2394                                  *
2395                                  * xxx is model 8 really broken?
2396                                  */
2397                                 if (cpi->cpi_model == 7 ||
2398                                     cpi->cpi_model == 8)
2399                                         cp->cp_ecx =
2400                                             BITX(cp->cp_ecx, 31, 24) << 16 |
2401                                             BITX(cp->cp_ecx, 23, 16) << 12 |
2402                                             BITX(cp->cp_ecx, 15, 8) << 8 |
2403                                             BITX(cp->cp_ecx, 7, 0);
2404                                 /*
2405                                  * model 9 stepping 1 has wrong associativity
2406                                  */
2407                                 if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
2408                                         cp->cp_ecx |= 8 << 12;
2409                                 break;
2410                         case X86_VENDOR_Intel:
2411                                 /*
2412                                  * Extended L2 Cache features function.
2413                                  * First appeared on Prescott.
2414                                  */
2415                         default:
2416                                 break;
2417                         }
2418                         break;
2419                 default:
2420                         break;
2421                 }
2422         }
2423 
2424 pass2_done:
2425         cpi->cpi_pass = 2;
2426 }
2427 
2428 static const char *
2429 intel_cpubrand(const struct cpuid_info *cpi)
2430 {
2431         int i;
2432 
2433         if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2434             cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2435                 return ("i486");
2436 
2437         switch (cpi->cpi_family) {
2438         case 5:
2439                 return ("Intel Pentium(r)");
2440         case 6:
2441                 switch (cpi->cpi_model) {
2442                         uint_t celeron, xeon;
2443                         const struct cpuid_regs *cp;
2444                 case 0:
2445                 case 1:
2446                 case 2:
2447                         return ("Intel Pentium(r) Pro");
2448                 case 3:
2449                 case 4:
2450                         return ("Intel Pentium(r) II");
2451                 case 6:
2452                         return ("Intel Celeron(r)");
2453                 case 5:
2454                 case 7:
2455                         celeron = xeon = 0;
2456                         cp = &cpi->cpi_std[2];   /* cache info */
2457 
2458                         for (i = 1; i < 4; i++) {
2459                                 uint_t tmp;
2460 
2461                                 tmp = (cp->cp_eax >> (8 * i)) & 0xff;
2462                                 if (tmp == 0x40)
2463                                         celeron++;
2464                                 if (tmp >= 0x44 && tmp <= 0x45)
2465                                         xeon++;
2466                         }
2467 
2468                         for (i = 0; i < 2; i++) {
2469                                 uint_t tmp;
2470 
2471                                 tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
2472                                 if (tmp == 0x40)
2473                                         celeron++;
2474                                 else if (tmp >= 0x44 && tmp <= 0x45)
2475                                         xeon++;
2476                         }
2477 
2478                         for (i = 0; i < 4; i++) {
2479                                 uint_t tmp;
2480 
2481                                 tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
2482                                 if (tmp == 0x40)
2483                                         celeron++;
2484                                 else if (tmp >= 0x44 && tmp <= 0x45)
2485                                         xeon++;
2486                         }
2487 
2488                         for (i = 0; i < 4; i++) {
2489                                 uint_t tmp;
2490 
2491                                 tmp = (cp->cp_edx >> (8 * i)) & 0xff;
2492                                 if (tmp == 0x40)
2493                                         celeron++;
2494                                 else if (tmp >= 0x44 && tmp <= 0x45)
2495                                         xeon++;
2496                         }
2497 
2498                         if (celeron)
2499                                 return ("Intel Celeron(r)");
2500                         if (xeon)
2501                                 return (cpi->cpi_model == 5 ?
2502                                     "Intel Pentium(r) II Xeon(tm)" :
2503                                     "Intel Pentium(r) III Xeon(tm)");
2504                         return (cpi->cpi_model == 5 ?
2505                             "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
2506                             "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
2507                 default:
2508                         break;
2509                 }
2510         default:
2511                 break;
2512         }
2513 
2514         /* BrandID is present if the field is nonzero */
2515         if (cpi->cpi_brandid != 0) {
2516                 static const struct {
2517                         uint_t bt_bid;
2518                         const char *bt_str;
2519                 } brand_tbl[] = {
2520                         { 0x1,  "Intel(r) Celeron(r)" },
2521                         { 0x2,  "Intel(r) Pentium(r) III" },
2522                         { 0x3,  "Intel(r) Pentium(r) III Xeon(tm)" },
2523                         { 0x4,  "Intel(r) Pentium(r) III" },
2524                         { 0x6,  "Mobile Intel(r) Pentium(r) III" },
2525                         { 0x7,  "Mobile Intel(r) Celeron(r)" },
2526                         { 0x8,  "Intel(r) Pentium(r) 4" },
2527                         { 0x9,  "Intel(r) Pentium(r) 4" },
2528                         { 0xa,  "Intel(r) Celeron(r)" },
2529                         { 0xb,  "Intel(r) Xeon(tm)" },
2530                         { 0xc,  "Intel(r) Xeon(tm) MP" },
2531                         { 0xe,  "Mobile Intel(r) Pentium(r) 4" },
2532                         { 0xf,  "Mobile Intel(r) Celeron(r)" },
2533                         { 0x11, "Mobile Genuine Intel(r)" },
2534                         { 0x12, "Intel(r) Celeron(r) M" },
2535                         { 0x13, "Mobile Intel(r) Celeron(r)" },
2536                         { 0x14, "Intel(r) Celeron(r)" },
2537                         { 0x15, "Mobile Genuine Intel(r)" },
2538                         { 0x16, "Intel(r) Pentium(r) M" },
2539                         { 0x17, "Mobile Intel(r) Celeron(r)" }
2540                 };
2541                 uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
2542                 uint_t sgn;
2543 
2544                 sgn = (cpi->cpi_family << 8) |
2545                     (cpi->cpi_model << 4) | cpi->cpi_step;
2546 
2547                 for (i = 0; i < btblmax; i++)
2548                         if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
2549                                 break;
2550                 if (i < btblmax) {
2551                         if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
2552                                 return ("Intel(r) Celeron(r)");
2553                         if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
2554                                 return ("Intel(r) Xeon(tm) MP");
2555                         if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
2556                                 return ("Intel(r) Xeon(tm)");
2557                         return (brand_tbl[i].bt_str);
2558                 }
2559         }
2560 
2561         return (NULL);
2562 }
2563 
2564 static const char *
2565 amd_cpubrand(const struct cpuid_info *cpi)
2566 {
2567         if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2568             cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2569                 return ("i486 compatible");
2570 
2571         switch (cpi->cpi_family) {
2572         case 5:
2573                 switch (cpi->cpi_model) {
2574                 case 0:
2575                 case 1:
2576                 case 2:
2577                 case 3:
2578                 case 4:
2579                 case 5:
2580                         return ("AMD-K5(r)");
2581                 case 6:
2582                 case 7:
2583                         return ("AMD-K6(r)");
2584                 case 8:
2585                         return ("AMD-K6(r)-2");
2586                 case 9:
2587                         return ("AMD-K6(r)-III");
2588                 default:
2589                         return ("AMD (family 5)");
2590                 }
2591         case 6:
2592                 switch (cpi->cpi_model) {
2593                 case 1:
2594                         return ("AMD-K7(tm)");
2595                 case 0:
2596                 case 2:
2597                 case 4:
2598                         return ("AMD Athlon(tm)");
2599                 case 3:
2600                 case 7:
2601                         return ("AMD Duron(tm)");
2602                 case 6:
2603                 case 8:
2604                 case 10:
2605                         /*
2606                          * Use the L2 cache size to distinguish
2607                          */
2608                         return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
2609                             "AMD Athlon(tm)" : "AMD Duron(tm)");
2610                 default:
2611                         return ("AMD (family 6)");
2612                 }
2613         default:
2614                 break;
2615         }
2616 
2617         if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
2618             cpi->cpi_brandid != 0) {
2619                 switch (BITX(cpi->cpi_brandid, 7, 5)) {
2620                 case 3:
2621                         return ("AMD Opteron(tm) UP 1xx");
2622                 case 4:
2623                         return ("AMD Opteron(tm) DP 2xx");
2624                 case 5:
2625                         return ("AMD Opteron(tm) MP 8xx");
2626                 default:
2627                         return ("AMD Opteron(tm)");
2628                 }
2629         }
2630 
2631         return (NULL);
2632 }
2633 
2634 static const char *
2635 cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
2636 {
2637         if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2638             cpi->cpi_maxeax < 1 || cpi->cpi_family < 5 ||
2639             type == X86_TYPE_CYRIX_486)
2640                 return ("i486 compatible");
2641 
2642         switch (type) {
2643         case X86_TYPE_CYRIX_6x86:
2644                 return ("Cyrix 6x86");
2645         case X86_TYPE_CYRIX_6x86L:
2646                 return ("Cyrix 6x86L");
2647         case X86_TYPE_CYRIX_6x86MX:
2648                 return ("Cyrix 6x86MX");
2649         case X86_TYPE_CYRIX_GXm:
2650                 return ("Cyrix GXm");
2651         case X86_TYPE_CYRIX_MediaGX:
2652                 return ("Cyrix MediaGX");
2653         case X86_TYPE_CYRIX_MII:
2654                 return ("Cyrix M2");
2655         case X86_TYPE_VIA_CYRIX_III:
2656                 return ("VIA Cyrix M3");
2657         default:
2658                 /*
2659                  * Have another wild guess ..
2660                  */
2661                 if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
2662                         return ("Cyrix 5x86");
2663                 else if (cpi->cpi_family == 5) {
2664                         switch (cpi->cpi_model) {
2665                         case 2:
2666                                 return ("Cyrix 6x86");  /* Cyrix M1 */
2667                         case 4:
2668                                 return ("Cyrix MediaGX");
2669                         default:
2670                                 break;
2671                         }
2672                 } else if (cpi->cpi_family == 6) {
2673                         switch (cpi->cpi_model) {
2674                         case 0:
2675                                 return ("Cyrix 6x86MX"); /* Cyrix M2? */
2676                         case 5:
2677                         case 6:
2678                         case 7:
2679                         case 8:
2680                         case 9:
2681                                 return ("VIA C3");
2682                         default:
2683                                 break;
2684                         }
2685                 }
2686                 break;
2687         }
2688         return (NULL);
2689 }
2690 
2691 /*
2692  * This only gets called in the case that the CPU extended
2693  * feature brand string (0x80000002, 0x80000003, 0x80000004)
2694  * aren't available, or contain null bytes for some reason.
2695  */
2696 static void
2697 fabricate_brandstr(struct cpuid_info *cpi)
2698 {
2699         const char *brand = NULL;
2700 
2701         switch (cpi->cpi_vendor) {
2702         case X86_VENDOR_Intel:
2703                 brand = intel_cpubrand(cpi);
2704                 break;
2705         case X86_VENDOR_AMD:
2706                 brand = amd_cpubrand(cpi);
2707                 break;
2708         case X86_VENDOR_Cyrix:
2709                 brand = cyrix_cpubrand(cpi, x86_type);
2710                 break;
2711         case X86_VENDOR_NexGen:
2712                 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2713                         brand = "NexGen Nx586";
2714                 break;
2715         case X86_VENDOR_Centaur:
2716                 if (cpi->cpi_family == 5)
2717                         switch (cpi->cpi_model) {
2718                         case 4:
2719                                 brand = "Centaur C6";
2720                                 break;
2721                         case 8:
2722                                 brand = "Centaur C2";
2723                                 break;
2724                         case 9:
2725                                 brand = "Centaur C3";
2726                                 break;
2727                         default:
2728                                 break;
2729                         }
2730                 break;
2731         case X86_VENDOR_Rise:
2732                 if (cpi->cpi_family == 5 &&
2733                     (cpi->cpi_model == 0 || cpi->cpi_model == 2))
2734                         brand = "Rise mP6";
2735                 break;
2736         case X86_VENDOR_SiS:
2737                 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2738                         brand = "SiS 55x";
2739                 break;
2740         case X86_VENDOR_TM:
2741                 if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
2742                         brand = "Transmeta Crusoe TM3x00 or TM5x00";
2743                 break;
2744         case X86_VENDOR_NSC:
2745         case X86_VENDOR_UMC:
2746         default:
2747                 break;
2748         }
2749         if (brand) {
2750                 (void) strcpy((char *)cpi->cpi_brandstr, brand);
2751                 return;
2752         }
2753 
2754         /*
2755          * If all else fails ...
2756          */
2757         (void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
2758             "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
2759             cpi->cpi_model, cpi->cpi_step);
2760 }
2761 
2762 /*
2763  * This routine is called just after kernel memory allocation
2764  * becomes available on cpu0, and as part of mp_startup() on
2765  * the other cpus.
2766  *
2767  * Fixup the brand string, and collect any information from cpuid
2768  * that requires dynamically allocated storage to represent.
2769  */
2770 /*ARGSUSED*/
2771 void
2772 cpuid_pass3(cpu_t *cpu)
2773 {
2774         int     i, max, shft, level, size;
2775         struct cpuid_regs regs;
2776         struct cpuid_regs *cp;
2777         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2778 
2779         ASSERT(cpi->cpi_pass == 2);
2780 
2781         /*
2782          * Function 4: Deterministic cache parameters
2783          *
2784          * Take this opportunity to detect the number of threads
2785          * sharing the last level cache, and construct a corresponding
2786          * cache id. The respective cpuid_info members are initialized
2787          * to the default case of "no last level cache sharing".
2788          */
2789         cpi->cpi_ncpu_shr_last_cache = 1;
2790         cpi->cpi_last_lvl_cacheid = cpu->cpu_id;
2791 
2792         if (cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) {
2793 
2794                 /*
2795                  * Find the # of elements (size) returned by fn 4, and along
2796                  * the way detect last level cache sharing details.
2797                  */
2798                 bzero(&regs, sizeof (regs));
2799                 cp = &regs;
2800                 for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
2801                         cp->cp_eax = 4;
2802                         cp->cp_ecx = i;
2803 
2804                         (void) __cpuid_insn(cp);
2805 
2806                         if (CPI_CACHE_TYPE(cp) == 0)
2807                                 break;
2808                         level = CPI_CACHE_LVL(cp);
2809                         if (level > max) {
2810                                 max = level;
2811                                 cpi->cpi_ncpu_shr_last_cache =
2812                                     CPI_NTHR_SHR_CACHE(cp) + 1;
2813                         }
2814                 }
2815                 cpi->cpi_std_4_size = size = i;
2816 
2817                 /*
2818                  * Allocate the cpi_std_4 array. The first element
2819                  * references the regs for fn 4, %ecx == 0, which
2820                  * cpuid_pass2() stashed in cpi->cpi_std[4].
2821                  */
2822                 if (size > 0) {
2823                         cpi->cpi_std_4 =
2824                             kmem_alloc(size * sizeof (cp), KM_SLEEP);
2825                         cpi->cpi_std_4[0] = &cpi->cpi_std[4];
2826 
2827                         /*
2828                          * Allocate storage to hold the additional regs
2829                          * for function 4, %ecx == 1 .. cpi_std_4_size.
2830                          *
2831                          * The regs for fn 4, %ecx == 0 has already
2832                          * been allocated as indicated above.
2833                          */
2834                         for (i = 1; i < size; i++) {
2835                                 cp = cpi->cpi_std_4[i] =
2836                                     kmem_zalloc(sizeof (regs), KM_SLEEP);
2837                                 cp->cp_eax = 4;
2838                                 cp->cp_ecx = i;
2839 
2840                                 (void) __cpuid_insn(cp);
2841                         }
2842                 }
2843                 /*
2844                  * Determine the number of bits needed to represent
2845                  * the number of CPUs sharing the last level cache.
2846                  *
2847                  * Shift off that number of bits from the APIC id to
2848                  * derive the cache id.
2849                  */
2850                 shft = 0;
2851                 for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
2852                         shft++;
2853                 cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
2854         }
2855 
2856         /*
2857          * Now fixup the brand string
2858          */
2859         if ((cpi->cpi_xmaxeax & 0x80000000) == 0) {
2860                 fabricate_brandstr(cpi);
2861         } else {
2862 
2863                 /*
2864                  * If we successfully extracted a brand string from the cpuid
2865                  * instruction, clean it up by removing leading spaces and
2866                  * similar junk.
2867                  */
2868                 if (cpi->cpi_brandstr[0]) {
2869                         size_t maxlen = sizeof (cpi->cpi_brandstr);
2870                         char *src, *dst;
2871 
2872                         dst = src = (char *)cpi->cpi_brandstr;
2873                         src[maxlen - 1] = '\0';
2874                         /*
2875                          * strip leading spaces
2876                          */
2877                         while (*src == ' ')
2878                                 src++;
2879                         /*
2880                          * Remove any 'Genuine' or "Authentic" prefixes
2881                          */
2882                         if (strncmp(src, "Genuine ", 8) == 0)
2883                                 src += 8;
2884                         if (strncmp(src, "Authentic ", 10) == 0)
2885                                 src += 10;
2886 
2887                         /*
2888                          * Now do an in-place copy.
2889                          * Map (R) to (r) and (TM) to (tm).
2890                          * The era of teletypes is long gone, and there's
2891                          * -really- no need to shout.
2892                          */
2893                         while (*src != '\0') {
2894                                 if (src[0] == '(') {
2895                                         if (strncmp(src + 1, "R)", 2) == 0) {
2896                                                 (void) strncpy(dst, "(r)", 3);
2897                                                 src += 3;
2898                                                 dst += 3;
2899                                                 continue;
2900                                         }
2901                                         if (strncmp(src + 1, "TM)", 3) == 0) {
2902                                                 (void) strncpy(dst, "(tm)", 4);
2903                                                 src += 4;
2904                                                 dst += 4;
2905                                                 continue;
2906                                         }
2907                                 }
2908                                 *dst++ = *src++;
2909                         }
2910                         *dst = '\0';
2911 
2912                         /*
2913                          * Finally, remove any trailing spaces
2914                          */
2915                         while (--dst > cpi->cpi_brandstr)
2916                                 if (*dst == ' ')
2917                                         *dst = '\0';
2918                                 else
2919                                         break;
2920                 } else
2921                         fabricate_brandstr(cpi);
2922         }
2923         cpi->cpi_pass = 3;
2924 }
2925 
2926 /*
2927  * This routine is called out of bind_hwcap() much later in the life
2928  * of the kernel (post_startup()).  The job of this routine is to resolve
2929  * the hardware feature support and kernel support for those features into
2930  * what we're actually going to tell applications via the aux vector.
2931  */
2932 void
2933 cpuid_pass4(cpu_t *cpu, uint_t *hwcap_out)
2934 {
2935         struct cpuid_info *cpi;
2936         uint_t hwcap_flags = 0, hwcap_flags_2 = 0;
2937 
2938         if (cpu == NULL)
2939                 cpu = CPU;
2940         cpi = cpu->cpu_m.mcpu_cpi;
2941 
2942         ASSERT(cpi->cpi_pass == 3);
2943 
2944         if (cpi->cpi_maxeax >= 1) {
2945                 uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
2946                 uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];
2947                 uint32_t *ebx = &cpi->cpi_support[STD_EBX_FEATURES];
2948 
2949                 *edx = CPI_FEATURES_EDX(cpi);
2950                 *ecx = CPI_FEATURES_ECX(cpi);
2951                 *ebx = CPI_FEATURES_7_0_EBX(cpi);
2952 
2953                 /*
2954                  * [these require explicit kernel support]
2955                  */
2956                 if (!is_x86_feature(x86_featureset, X86FSET_SEP))
2957                         *edx &= ~CPUID_INTC_EDX_SEP;
2958 
2959                 if (!is_x86_feature(x86_featureset, X86FSET_SSE))
2960                         *edx &= ~(CPUID_INTC_EDX_FXSR|CPUID_INTC_EDX_SSE);
2961                 if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
2962                         *edx &= ~CPUID_INTC_EDX_SSE2;
2963 
2964                 if (!is_x86_feature(x86_featureset, X86FSET_HTT))
2965                         *edx &= ~CPUID_INTC_EDX_HTT;
2966 
2967                 if (!is_x86_feature(x86_featureset, X86FSET_SSE3))
2968                         *ecx &= ~CPUID_INTC_ECX_SSE3;
2969 
2970                 if (!is_x86_feature(x86_featureset, X86FSET_SSSE3))
2971                         *ecx &= ~CPUID_INTC_ECX_SSSE3;
2972                 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_1))
2973                         *ecx &= ~CPUID_INTC_ECX_SSE4_1;
2974                 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2))
2975                         *ecx &= ~CPUID_INTC_ECX_SSE4_2;
2976                 if (!is_x86_feature(x86_featureset, X86FSET_AES))
2977                         *ecx &= ~CPUID_INTC_ECX_AES;
2978                 if (!is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ))
2979                         *ecx &= ~CPUID_INTC_ECX_PCLMULQDQ;
2980                 if (!is_x86_feature(x86_featureset, X86FSET_XSAVE))
2981                         *ecx &= ~(CPUID_INTC_ECX_XSAVE |
2982                             CPUID_INTC_ECX_OSXSAVE);
2983                 if (!is_x86_feature(x86_featureset, X86FSET_AVX))
2984                         *ecx &= ~CPUID_INTC_ECX_AVX;
2985                 if (!is_x86_feature(x86_featureset, X86FSET_F16C))
2986                         *ecx &= ~CPUID_INTC_ECX_F16C;
2987                 if (!is_x86_feature(x86_featureset, X86FSET_FMA))
2988                         *ecx &= ~CPUID_INTC_ECX_FMA;
2989                 if (!is_x86_feature(x86_featureset, X86FSET_BMI1))
2990                         *ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
2991                 if (!is_x86_feature(x86_featureset, X86FSET_BMI2))
2992                         *ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
2993                 if (!is_x86_feature(x86_featureset, X86FSET_AVX2))
2994                         *ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
2995                 if (!is_x86_feature(x86_featureset, X86FSET_RDSEED))
2996                         *ebx &= ~CPUID_INTC_EBX_7_0_RDSEED;
2997                 if (!is_x86_feature(x86_featureset, X86FSET_ADX))
2998                         *ebx &= ~CPUID_INTC_EBX_7_0_ADX;
2999 
3000                 /*
3001                  * [no explicit support required beyond x87 fp context]
3002                  */
3003                 if (!fpu_exists)
3004                         *edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);
3005 
3006                 /*
3007                  * Now map the supported feature vector to things that we
3008                  * think userland will care about.
3009                  */
3010                 if (*edx & CPUID_INTC_EDX_SEP)
3011                         hwcap_flags |= AV_386_SEP;
3012                 if (*edx & CPUID_INTC_EDX_SSE)
3013                         hwcap_flags |= AV_386_FXSR | AV_386_SSE;
3014                 if (*edx & CPUID_INTC_EDX_SSE2)
3015                         hwcap_flags |= AV_386_SSE2;
3016                 if (*ecx & CPUID_INTC_ECX_SSE3)
3017                         hwcap_flags |= AV_386_SSE3;
3018                 if (*ecx & CPUID_INTC_ECX_SSSE3)
3019                         hwcap_flags |= AV_386_SSSE3;
3020                 if (*ecx & CPUID_INTC_ECX_SSE4_1)
3021                         hwcap_flags |= AV_386_SSE4_1;
3022                 if (*ecx & CPUID_INTC_ECX_SSE4_2)
3023                         hwcap_flags |= AV_386_SSE4_2;
3024                 if (*ecx & CPUID_INTC_ECX_MOVBE)
3025                         hwcap_flags |= AV_386_MOVBE;
3026                 if (*ecx & CPUID_INTC_ECX_AES)
3027                         hwcap_flags |= AV_386_AES;
3028                 if (*ecx & CPUID_INTC_ECX_PCLMULQDQ)
3029                         hwcap_flags |= AV_386_PCLMULQDQ;
3030                 if ((*ecx & CPUID_INTC_ECX_XSAVE) &&
3031                     (*ecx & CPUID_INTC_ECX_OSXSAVE)) {
3032                         hwcap_flags |= AV_386_XSAVE;
3033 
3034                         if (*ecx & CPUID_INTC_ECX_AVX) {
3035                                 uint32_t *ecx_7 = &CPI_FEATURES_7_0_ECX(cpi);
3036                                 uint32_t *edx_7 = &CPI_FEATURES_7_0_EDX(cpi);
3037 
3038                                 hwcap_flags |= AV_386_AVX;
3039                                 if (*ecx & CPUID_INTC_ECX_F16C)
3040                                         hwcap_flags_2 |= AV_386_2_F16C;
3041                                 if (*ecx & CPUID_INTC_ECX_FMA)
3042                                         hwcap_flags_2 |= AV_386_2_FMA;
3043 
3044                                 if (*ebx & CPUID_INTC_EBX_7_0_BMI1)
3045                                         hwcap_flags_2 |= AV_386_2_BMI1;
3046                                 if (*ebx & CPUID_INTC_EBX_7_0_BMI2)
3047                                         hwcap_flags_2 |= AV_386_2_BMI2;
3048                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX2)
3049                                         hwcap_flags_2 |= AV_386_2_AVX2;
3050                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512F)
3051                                         hwcap_flags_2 |= AV_386_2_AVX512F;
3052                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512DQ)
3053                                         hwcap_flags_2 |= AV_386_2_AVX512DQ;
3054                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512IFMA)
3055                                         hwcap_flags_2 |= AV_386_2_AVX512IFMA;
3056                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512PF)
3057                                         hwcap_flags_2 |= AV_386_2_AVX512PF;
3058                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512ER)
3059                                         hwcap_flags_2 |= AV_386_2_AVX512ER;
3060                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512CD)
3061                                         hwcap_flags_2 |= AV_386_2_AVX512CD;
3062                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512BW)
3063                                         hwcap_flags_2 |= AV_386_2_AVX512BW;
3064                                 if (*ebx & CPUID_INTC_EBX_7_0_AVX512VL)
3065                                         hwcap_flags_2 |= AV_386_2_AVX512VL;
3066 
3067                                 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VBMI)
3068                                         hwcap_flags_2 |= AV_386_2_AVX512VBMI;
3069                                 if (*ecx_7 & CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
3070                                         hwcap_flags_2 |= AV_386_2_AVX512VPOPCDQ;
3071 
3072                                 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124NNIW)
3073                                         hwcap_flags_2 |= AV_386_2_AVX512_4NNIW;
3074                                 if (*edx_7 & CPUID_INTC_EDX_7_0_AVX5124FMAPS)
3075                                         hwcap_flags_2 |= AV_386_2_AVX512_4FMAPS;
3076                         }
3077                 }
3078                 if (*ecx & CPUID_INTC_ECX_VMX)
3079                         hwcap_flags |= AV_386_VMX;
3080                 if (*ecx & CPUID_INTC_ECX_POPCNT)
3081                         hwcap_flags |= AV_386_POPCNT;
3082                 if (*edx & CPUID_INTC_EDX_FPU)
3083                         hwcap_flags |= AV_386_FPU;
3084                 if (*edx & CPUID_INTC_EDX_MMX)
3085                         hwcap_flags |= AV_386_MMX;
3086 
3087                 if (*edx & CPUID_INTC_EDX_TSC)
3088                         hwcap_flags |= AV_386_TSC;
3089                 if (*edx & CPUID_INTC_EDX_CX8)
3090                         hwcap_flags |= AV_386_CX8;
3091                 if (*edx & CPUID_INTC_EDX_CMOV)
3092                         hwcap_flags |= AV_386_CMOV;
3093                 if (*ecx & CPUID_INTC_ECX_CX16)
3094                         hwcap_flags |= AV_386_CX16;
3095 
3096                 if (*ecx & CPUID_INTC_ECX_RDRAND)
3097                         hwcap_flags_2 |= AV_386_2_RDRAND;
3098                 if (*ebx & CPUID_INTC_EBX_7_0_ADX)
3099                         hwcap_flags_2 |= AV_386_2_ADX;
3100                 if (*ebx & CPUID_INTC_EBX_7_0_RDSEED)
3101                         hwcap_flags_2 |= AV_386_2_RDSEED;
3102 
3103         }
3104 
3105         if (cpi->cpi_xmaxeax < 0x80000001)
3106                 goto pass4_done;
3107 
3108         switch (cpi->cpi_vendor) {
3109                 struct cpuid_regs cp;
3110                 uint32_t *edx, *ecx;
3111 
3112         case X86_VENDOR_Intel:
3113                 /*
3114                  * Seems like Intel duplicated what we necessary
3115                  * here to make the initial crop of 64-bit OS's work.
3116                  * Hopefully, those are the only "extended" bits
3117                  * they'll add.
3118                  */
3119                 /*FALLTHROUGH*/
3120 
3121         case X86_VENDOR_AMD:
3122                 edx = &cpi->cpi_support[AMD_EDX_FEATURES];
3123                 ecx = &cpi->cpi_support[AMD_ECX_FEATURES];
3124 
3125                 *edx = CPI_FEATURES_XTD_EDX(cpi);
3126                 *ecx = CPI_FEATURES_XTD_ECX(cpi);
3127 
3128                 /*
3129                  * [these features require explicit kernel support]
3130                  */
3131                 switch (cpi->cpi_vendor) {
3132                 case X86_VENDOR_Intel:
3133                         if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3134                                 *edx &= ~CPUID_AMD_EDX_TSCP;
3135                         break;
3136 
3137                 case X86_VENDOR_AMD:
3138                         if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
3139                                 *edx &= ~CPUID_AMD_EDX_TSCP;
3140                         if (!is_x86_feature(x86_featureset, X86FSET_SSE4A))
3141                                 *ecx &= ~CPUID_AMD_ECX_SSE4A;
3142                         break;
3143 
3144                 default:
3145                         break;
3146                 }
3147 
3148                 /*
3149                  * [no explicit support required beyond
3150                  * x87 fp context and exception handlers]
3151                  */
3152                 if (!fpu_exists)
3153                         *edx &= ~(CPUID_AMD_EDX_MMXamd |
3154                             CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);
3155 
3156                 if (!is_x86_feature(x86_featureset, X86FSET_NX))
3157                         *edx &= ~CPUID_AMD_EDX_NX;
3158 #if !defined(__amd64)
3159                 *edx &= ~CPUID_AMD_EDX_LM;
3160 #endif
3161                 /*
3162                  * Now map the supported feature vector to
3163                  * things that we think userland will care about.
3164                  */
3165 #if defined(__amd64)
3166                 if (*edx & CPUID_AMD_EDX_SYSC)
3167                         hwcap_flags |= AV_386_AMD_SYSC;
3168 #endif
3169                 if (*edx & CPUID_AMD_EDX_MMXamd)
3170                         hwcap_flags |= AV_386_AMD_MMX;
3171                 if (*edx & CPUID_AMD_EDX_3DNow)
3172                         hwcap_flags |= AV_386_AMD_3DNow;
3173                 if (*edx & CPUID_AMD_EDX_3DNowx)
3174                         hwcap_flags |= AV_386_AMD_3DNowx;
3175                 if (*ecx & CPUID_AMD_ECX_SVM)
3176                         hwcap_flags |= AV_386_AMD_SVM;
3177 
3178                 switch (cpi->cpi_vendor) {
3179                 case X86_VENDOR_AMD:
3180                         if (*edx & CPUID_AMD_EDX_TSCP)
3181                                 hwcap_flags |= AV_386_TSCP;
3182                         if (*ecx & CPUID_AMD_ECX_AHF64)
3183                                 hwcap_flags |= AV_386_AHF;
3184                         if (*ecx & CPUID_AMD_ECX_SSE4A)
3185                                 hwcap_flags |= AV_386_AMD_SSE4A;
3186                         if (*ecx & CPUID_AMD_ECX_LZCNT)
3187                                 hwcap_flags |= AV_386_AMD_LZCNT;
3188                         break;
3189 
3190                 case X86_VENDOR_Intel:
3191                         if (*edx & CPUID_AMD_EDX_TSCP)
3192                                 hwcap_flags |= AV_386_TSCP;
3193                         /*
3194                          * Aarrgh.
3195                          * Intel uses a different bit in the same word.
3196                          */
3197                         if (*ecx & CPUID_INTC_ECX_AHF64)
3198                                 hwcap_flags |= AV_386_AHF;
3199                         break;
3200 
3201                 default:
3202                         break;
3203                 }
3204                 break;
3205 
3206         case X86_VENDOR_TM:
3207                 cp.cp_eax = 0x80860001;
3208                 (void) __cpuid_insn(&cp);
3209                 cpi->cpi_support[TM_EDX_FEATURES] = cp.cp_edx;
3210                 break;
3211 
3212         default:
3213                 break;
3214         }
3215 
3216 pass4_done:
3217         cpi->cpi_pass = 4;
3218         if (hwcap_out != NULL) {
3219                 hwcap_out[0] = hwcap_flags;
3220                 hwcap_out[1] = hwcap_flags_2;
3221         }
3222 }
3223 
3224 
3225 /*
3226  * Simulate the cpuid instruction using the data we previously
3227  * captured about this CPU.  We try our best to return the truth
3228  * about the hardware, independently of kernel support.
3229  */
3230 uint32_t
3231 cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
3232 {
3233         struct cpuid_info *cpi;
3234         struct cpuid_regs *xcp;
3235 
3236         if (cpu == NULL)
3237                 cpu = CPU;
3238         cpi = cpu->cpu_m.mcpu_cpi;
3239 
3240         ASSERT(cpuid_checkpass(cpu, 3));
3241 
3242         /*
3243          * CPUID data is cached in two separate places: cpi_std for standard
3244          * CPUID functions, and cpi_extd for extended CPUID functions.
3245          */
3246         if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD)
3247                 xcp = &cpi->cpi_std[cp->cp_eax];
3248         else if (cp->cp_eax >= 0x80000000 && cp->cp_eax <= cpi->cpi_xmaxeax &&
3249             cp->cp_eax < 0x80000000 + NMAX_CPI_EXTD)
3250                 xcp = &cpi->cpi_extd[cp->cp_eax - 0x80000000];
3251         else
3252                 /*
3253                  * The caller is asking for data from an input parameter which
3254                  * the kernel has not cached.  In this case we go fetch from
3255                  * the hardware and return the data directly to the user.
3256                  */
3257                 return (__cpuid_insn(cp));
3258 
3259         cp->cp_eax = xcp->cp_eax;
3260         cp->cp_ebx = xcp->cp_ebx;
3261         cp->cp_ecx = xcp->cp_ecx;
3262         cp->cp_edx = xcp->cp_edx;
3263         return (cp->cp_eax);
3264 }
3265 
3266 int
3267 cpuid_checkpass(cpu_t *cpu, int pass)
3268 {
3269         return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
3270             cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
3271 }
3272 
3273 int
3274 cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
3275 {
3276         ASSERT(cpuid_checkpass(cpu, 3));
3277 
3278         return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
3279 }
3280 
3281 int
3282 cpuid_is_cmt(cpu_t *cpu)
3283 {
3284         if (cpu == NULL)
3285                 cpu = CPU;
3286 
3287         ASSERT(cpuid_checkpass(cpu, 1));
3288 
3289         return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
3290 }
3291 
3292 /*
3293  * AMD and Intel both implement the 64-bit variant of the syscall
3294  * instruction (syscallq), so if there's -any- support for syscall,
3295  * cpuid currently says "yes, we support this".
3296  *
3297  * However, Intel decided to -not- implement the 32-bit variant of the
3298  * syscall instruction, so we provide a predicate to allow our caller
3299  * to test that subtlety here.
3300  *
3301  * XXPV Currently, 32-bit syscall instructions don't work via the hypervisor,
3302  *      even in the case where the hardware would in fact support it.
3303  */
3304 /*ARGSUSED*/
3305 int
3306 cpuid_syscall32_insn(cpu_t *cpu)
3307 {
3308         ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), 1));
3309 
3310 #if !defined(__xpv)
3311         if (cpu == NULL)
3312                 cpu = CPU;
3313 
3314         /*CSTYLED*/
3315         {
3316                 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3317 
3318                 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
3319                     cpi->cpi_xmaxeax >= 0x80000001 &&
3320                     (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
3321                         return (1);
3322         }
3323 #endif
3324         return (0);
3325 }
3326 
3327 int
3328 cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
3329 {
3330         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3331 
3332         static const char fmt[] =
3333             "x86 (%s %X family %d model %d step %d clock %d MHz)";
3334         static const char fmt_ht[] =
3335             "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";
3336 
3337         ASSERT(cpuid_checkpass(cpu, 1));
3338 
3339         if (cpuid_is_cmt(cpu))
3340                 return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
3341                     cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3342                     cpi->cpi_family, cpi->cpi_model,
3343                     cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3344         return (snprintf(s, n, fmt,
3345             cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
3346             cpi->cpi_family, cpi->cpi_model,
3347             cpi->cpi_step, cpu->cpu_type_info.pi_clock));
3348 }
3349 
3350 const char *
3351 cpuid_getvendorstr(cpu_t *cpu)
3352 {
3353         ASSERT(cpuid_checkpass(cpu, 1));
3354         return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
3355 }
3356 
3357 uint_t
3358 cpuid_getvendor(cpu_t *cpu)
3359 {
3360         ASSERT(cpuid_checkpass(cpu, 1));
3361         return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
3362 }
3363 
3364 uint_t
3365 cpuid_getfamily(cpu_t *cpu)
3366 {
3367         ASSERT(cpuid_checkpass(cpu, 1));
3368         return (cpu->cpu_m.mcpu_cpi->cpi_family);
3369 }
3370 
3371 uint_t
3372 cpuid_getmodel(cpu_t *cpu)
3373 {
3374         ASSERT(cpuid_checkpass(cpu, 1));
3375         return (cpu->cpu_m.mcpu_cpi->cpi_model);
3376 }
3377 
3378 uint_t
3379 cpuid_get_ncpu_per_chip(cpu_t *cpu)
3380 {
3381         ASSERT(cpuid_checkpass(cpu, 1));
3382         return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
3383 }
3384 
3385 uint_t
3386 cpuid_get_ncore_per_chip(cpu_t *cpu)
3387 {
3388         ASSERT(cpuid_checkpass(cpu, 1));
3389         return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
3390 }
3391 
3392 uint_t
3393 cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
3394 {
3395         ASSERT(cpuid_checkpass(cpu, 2));
3396         return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
3397 }
3398 
3399 id_t
3400 cpuid_get_last_lvl_cacheid(cpu_t *cpu)
3401 {
3402         ASSERT(cpuid_checkpass(cpu, 2));
3403         return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3404 }
3405 
3406 uint_t
3407 cpuid_getstep(cpu_t *cpu)
3408 {
3409         ASSERT(cpuid_checkpass(cpu, 1));
3410         return (cpu->cpu_m.mcpu_cpi->cpi_step);
3411 }
3412 
3413 uint_t
3414 cpuid_getsig(struct cpu *cpu)
3415 {
3416         ASSERT(cpuid_checkpass(cpu, 1));
3417         return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
3418 }
3419 
3420 uint32_t
3421 cpuid_getchiprev(struct cpu *cpu)
3422 {
3423         ASSERT(cpuid_checkpass(cpu, 1));
3424         return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
3425 }
3426 
3427 const char *
3428 cpuid_getchiprevstr(struct cpu *cpu)
3429 {
3430         ASSERT(cpuid_checkpass(cpu, 1));
3431         return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
3432 }
3433 
3434 uint32_t
3435 cpuid_getsockettype(struct cpu *cpu)
3436 {
3437         ASSERT(cpuid_checkpass(cpu, 1));
3438         return (cpu->cpu_m.mcpu_cpi->cpi_socket);
3439 }
3440 
3441 const char *
3442 cpuid_getsocketstr(cpu_t *cpu)
3443 {
3444         static const char *socketstr = NULL;
3445         struct cpuid_info *cpi;
3446 
3447         ASSERT(cpuid_checkpass(cpu, 1));
3448         cpi = cpu->cpu_m.mcpu_cpi;
3449 
3450         /* Assume that socket types are the same across the system */
3451         if (socketstr == NULL)
3452                 socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
3453                     cpi->cpi_model, cpi->cpi_step);
3454 
3455 
3456         return (socketstr);
3457 }
3458 
3459 int
3460 cpuid_get_chipid(cpu_t *cpu)
3461 {
3462         ASSERT(cpuid_checkpass(cpu, 1));
3463 
3464         if (cpuid_is_cmt(cpu))
3465                 return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
3466         return (cpu->cpu_id);
3467 }
3468 
3469 id_t
3470 cpuid_get_coreid(cpu_t *cpu)
3471 {
3472         ASSERT(cpuid_checkpass(cpu, 1));
3473         return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
3474 }
3475 
3476 int
3477 cpuid_get_pkgcoreid(cpu_t *cpu)
3478 {
3479         ASSERT(cpuid_checkpass(cpu, 1));
3480         return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
3481 }
3482 
3483 int
3484 cpuid_get_clogid(cpu_t *cpu)
3485 {
3486         ASSERT(cpuid_checkpass(cpu, 1));
3487         return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
3488 }
3489 
3490 int
3491 cpuid_get_cacheid(cpu_t *cpu)
3492 {
3493         ASSERT(cpuid_checkpass(cpu, 1));
3494         return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3495 }
3496 
3497 uint_t
3498 cpuid_get_procnodeid(cpu_t *cpu)
3499 {
3500         ASSERT(cpuid_checkpass(cpu, 1));
3501         return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
3502 }
3503 
3504 uint_t
3505 cpuid_get_procnodes_per_pkg(cpu_t *cpu)
3506 {
3507         ASSERT(cpuid_checkpass(cpu, 1));
3508         return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
3509 }
3510 
3511 uint_t
3512 cpuid_get_compunitid(cpu_t *cpu)
3513 {
3514         ASSERT(cpuid_checkpass(cpu, 1));
3515         return (cpu->cpu_m.mcpu_cpi->cpi_compunitid);
3516 }
3517 
3518 uint_t
3519 cpuid_get_cores_per_compunit(cpu_t *cpu)
3520 {
3521         ASSERT(cpuid_checkpass(cpu, 1));
3522         return (cpu->cpu_m.mcpu_cpi->cpi_cores_per_compunit);
3523 }
3524 
3525 /*ARGSUSED*/
3526 int
3527 cpuid_have_cr8access(cpu_t *cpu)
3528 {
3529 #if defined(__amd64)
3530         return (1);
3531 #else
3532         struct cpuid_info *cpi;
3533 
3534         ASSERT(cpu != NULL);
3535         cpi = cpu->cpu_m.mcpu_cpi;
3536         if (cpi->cpi_vendor == X86_VENDOR_AMD && cpi->cpi_maxeax >= 1 &&
3537             (CPI_FEATURES_XTD_ECX(cpi) & CPUID_AMD_ECX_CR8D) != 0)
3538                 return (1);
3539         return (0);
3540 #endif
3541 }
3542 
3543 uint32_t
3544 cpuid_get_apicid(cpu_t *cpu)
3545 {
3546         ASSERT(cpuid_checkpass(cpu, 1));
3547         if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
3548                 return (UINT32_MAX);
3549         } else {
3550                 return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
3551         }
3552 }
3553 
3554 void
3555 cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
3556 {
3557         struct cpuid_info *cpi;
3558 
3559         if (cpu == NULL)
3560                 cpu = CPU;
3561         cpi = cpu->cpu_m.mcpu_cpi;
3562 
3563         ASSERT(cpuid_checkpass(cpu, 1));
3564 
3565         if (pabits)
3566                 *pabits = cpi->cpi_pabits;
3567         if (vabits)
3568                 *vabits = cpi->cpi_vabits;
3569 }
3570 
3571 size_t
3572 cpuid_get_xsave_size()
3573 {
3574         return (MAX(cpuid_info0.cpi_xsave.xsav_max_size,
3575             sizeof (struct xsave_state)));
3576 }
3577 
3578 /*
3579  * Return true if the CPUs on this system require 'pointer clearing' for the
3580  * floating point error pointer exception handling. In the past, this has been
3581  * true for all AMD K7 & K8 CPUs, although newer AMD CPUs have been changed to
3582  * behave the same as Intel. This is checked via the CPUID_AMD_EBX_ERR_PTR_ZERO
3583  * feature bit and is reflected in the cpi_fp_amd_save member. Once this has
3584  * been confirmed on hardware which supports that feature, this test should be
3585  * narrowed. In the meantime, we always follow the existing behavior on any AMD
3586  * CPU.
3587  */
3588 boolean_t
3589 cpuid_need_fp_excp_handling()
3590 {
3591         return (cpuid_info0.cpi_vendor == X86_VENDOR_AMD);
3592 }
3593 
3594 /*
3595  * Returns the number of data TLB entries for a corresponding
3596  * pagesize.  If it can't be computed, or isn't known, the
3597  * routine returns zero.  If you ask about an architecturally
3598  * impossible pagesize, the routine will panic (so that the
3599  * hat implementor knows that things are inconsistent.)
3600  */
3601 uint_t
3602 cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
3603 {
3604         struct cpuid_info *cpi;
3605         uint_t dtlb_nent = 0;
3606 
3607         if (cpu == NULL)
3608                 cpu = CPU;
3609         cpi = cpu->cpu_m.mcpu_cpi;
3610 
3611         ASSERT(cpuid_checkpass(cpu, 1));
3612 
3613         /*
3614          * Check the L2 TLB info
3615          */
3616         if (cpi->cpi_xmaxeax >= 0x80000006) {
3617                 struct cpuid_regs *cp = &cpi->cpi_extd[6];
3618 
3619                 switch (pagesize) {
3620 
3621                 case 4 * 1024:
3622                         /*
3623                          * All zero in the top 16 bits of the register
3624                          * indicates a unified TLB. Size is in low 16 bits.
3625                          */
3626                         if ((cp->cp_ebx & 0xffff0000) == 0)
3627                                 dtlb_nent = cp->cp_ebx & 0x0000ffff;
3628                         else
3629                                 dtlb_nent = BITX(cp->cp_ebx, 27, 16);
3630                         break;
3631 
3632                 case 2 * 1024 * 1024:
3633                         if ((cp->cp_eax & 0xffff0000) == 0)
3634                                 dtlb_nent = cp->cp_eax & 0x0000ffff;
3635                         else
3636                                 dtlb_nent = BITX(cp->cp_eax, 27, 16);
3637                         break;
3638 
3639                 default:
3640                         panic("unknown L2 pagesize");
3641                         /*NOTREACHED*/
3642                 }
3643         }
3644 
3645         if (dtlb_nent != 0)
3646                 return (dtlb_nent);
3647 
3648         /*
3649          * No L2 TLB support for this size, try L1.
3650          */
3651         if (cpi->cpi_xmaxeax >= 0x80000005) {
3652                 struct cpuid_regs *cp = &cpi->cpi_extd[5];
3653 
3654                 switch (pagesize) {
3655                 case 4 * 1024:
3656                         dtlb_nent = BITX(cp->cp_ebx, 23, 16);
3657                         break;
3658                 case 2 * 1024 * 1024:
3659                         dtlb_nent = BITX(cp->cp_eax, 23, 16);
3660                         break;
3661                 default:
3662                         panic("unknown L1 d-TLB pagesize");
3663                         /*NOTREACHED*/
3664                 }
3665         }
3666 
3667         return (dtlb_nent);
3668 }
3669 
3670 /*
3671  * Return 0 if the erratum is not present or not applicable, positive
3672  * if it is, and negative if the status of the erratum is unknown.
3673  *
3674  * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
3675  * Processors" #25759, Rev 3.57, August 2005
3676  */
3677 int
3678 cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
3679 {
3680         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3681         uint_t eax;
3682 
3683         /*
3684          * Bail out if this CPU isn't an AMD CPU, or if it's
3685          * a legacy (32-bit) AMD CPU.
3686          */
3687         if (cpi->cpi_vendor != X86_VENDOR_AMD ||
3688             cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
3689             cpi->cpi_family == 6)
3690 
3691                 return (0);
3692 
3693         eax = cpi->cpi_std[1].cp_eax;
3694 
3695 #define SH_B0(eax)      (eax == 0xf40 || eax == 0xf50)
3696 #define SH_B3(eax)      (eax == 0xf51)
3697 #define B(eax)          (SH_B0(eax) || SH_B3(eax))
3698 
3699 #define SH_C0(eax)      (eax == 0xf48 || eax == 0xf58)
3700 
3701 #define SH_CG(eax)      (eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
3702 #define DH_CG(eax)      (eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
3703 #define CH_CG(eax)      (eax == 0xf82 || eax == 0xfb2)
3704 #define CG(eax)         (SH_CG(eax) || DH_CG(eax) || CH_CG(eax))
3705 
3706 #define SH_D0(eax)      (eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
3707 #define DH_D0(eax)      (eax == 0x10fc0 || eax == 0x10ff0)
3708 #define CH_D0(eax)      (eax == 0x10f80 || eax == 0x10fb0)
3709 #define D0(eax)         (SH_D0(eax) || DH_D0(eax) || CH_D0(eax))
3710 
3711 #define SH_E0(eax)      (eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
3712 #define JH_E1(eax)      (eax == 0x20f10)        /* JH8_E0 had 0x20f30 */
3713 #define DH_E3(eax)      (eax == 0x20fc0 || eax == 0x20ff0)
3714 #define SH_E4(eax)      (eax == 0x20f51 || eax == 0x20f71)
3715 #define BH_E4(eax)      (eax == 0x20fb1)
3716 #define SH_E5(eax)      (eax == 0x20f42)
3717 #define DH_E6(eax)      (eax == 0x20ff2 || eax == 0x20fc2)
3718 #define JH_E6(eax)      (eax == 0x20f12 || eax == 0x20f32)
3719 #define EX(eax)         (SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
3720                             SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
3721                             DH_E6(eax) || JH_E6(eax))
3722 
3723 #define DR_AX(eax)      (eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
3724 #define DR_B0(eax)      (eax == 0x100f20)
3725 #define DR_B1(eax)      (eax == 0x100f21)
3726 #define DR_BA(eax)      (eax == 0x100f2a)
3727 #define DR_B2(eax)      (eax == 0x100f22)
3728 #define DR_B3(eax)      (eax == 0x100f23)
3729 #define RB_C0(eax)      (eax == 0x100f40)
3730 
3731         switch (erratum) {
3732         case 1:
3733                 return (cpi->cpi_family < 0x10);
3734         case 51:        /* what does the asterisk mean? */
3735                 return (B(eax) || SH_C0(eax) || CG(eax));
3736         case 52:
3737                 return (B(eax));
3738         case 57:
3739                 return (cpi->cpi_family <= 0x11);
3740         case 58:
3741                 return (B(eax));
3742         case 60:
3743                 return (cpi->cpi_family <= 0x11);
3744         case 61:
3745         case 62:
3746         case 63:
3747         case 64:
3748         case 65:
3749         case 66:
3750         case 68:
3751         case 69:
3752         case 70:
3753         case 71:
3754                 return (B(eax));
3755         case 72:
3756                 return (SH_B0(eax));
3757         case 74:
3758                 return (B(eax));
3759         case 75:
3760                 return (cpi->cpi_family < 0x10);
3761         case 76:
3762                 return (B(eax));
3763         case 77:
3764                 return (cpi->cpi_family <= 0x11);
3765         case 78:
3766                 return (B(eax) || SH_C0(eax));
3767         case 79:
3768                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3769         case 80:
3770         case 81:
3771         case 82:
3772                 return (B(eax));
3773         case 83:
3774                 return (B(eax) || SH_C0(eax) || CG(eax));
3775         case 85:
3776                 return (cpi->cpi_family < 0x10);
3777         case 86:
3778                 return (SH_C0(eax) || CG(eax));
3779         case 88:
3780 #if !defined(__amd64)
3781                 return (0);
3782 #else
3783                 return (B(eax) || SH_C0(eax));
3784 #endif
3785         case 89:
3786                 return (cpi->cpi_family < 0x10);
3787         case 90:
3788                 return (B(eax) || SH_C0(eax) || CG(eax));
3789         case 91:
3790         case 92:
3791                 return (B(eax) || SH_C0(eax));
3792         case 93:
3793                 return (SH_C0(eax));
3794         case 94:
3795                 return (B(eax) || SH_C0(eax) || CG(eax));
3796         case 95:
3797 #if !defined(__amd64)
3798                 return (0);
3799 #else
3800                 return (B(eax) || SH_C0(eax));
3801 #endif
3802         case 96:
3803                 return (B(eax) || SH_C0(eax) || CG(eax));
3804         case 97:
3805         case 98:
3806                 return (SH_C0(eax) || CG(eax));
3807         case 99:
3808                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3809         case 100:
3810                 return (B(eax) || SH_C0(eax));
3811         case 101:
3812         case 103:
3813                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3814         case 104:
3815                 return (SH_C0(eax) || CG(eax) || D0(eax));
3816         case 105:
3817         case 106:
3818         case 107:
3819                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3820         case 108:
3821                 return (DH_CG(eax));
3822         case 109:
3823                 return (SH_C0(eax) || CG(eax) || D0(eax));
3824         case 110:
3825                 return (D0(eax) || EX(eax));
3826         case 111:
3827                 return (CG(eax));
3828         case 112:
3829                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3830         case 113:
3831                 return (eax == 0x20fc0);
3832         case 114:
3833                 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3834         case 115:
3835                 return (SH_E0(eax) || JH_E1(eax));
3836         case 116:
3837                 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3838         case 117:
3839                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3840         case 118:
3841                 return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
3842                     JH_E6(eax));
3843         case 121:
3844                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3845         case 122:
3846                 return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
3847         case 123:
3848                 return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
3849         case 131:
3850                 return (cpi->cpi_family < 0x10);
3851         case 6336786:
3852                 /*
3853                  * Test for AdvPowerMgmtInfo.TscPStateInvariant
3854                  * if this is a K8 family or newer processor
3855                  */
3856                 if (CPI_FAMILY(cpi) == 0xf) {
3857                         struct cpuid_regs regs;
3858                         regs.cp_eax = 0x80000007;
3859                         (void) __cpuid_insn(&regs);
3860                         return (!(regs.cp_edx & 0x100));
3861                 }
3862                 return (0);
3863         case 6323525:
3864                 return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
3865                     (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);
3866 
3867         case 6671130:
3868                 /*
3869                  * check for processors (pre-Shanghai) that do not provide
3870                  * optimal management of 1gb ptes in its tlb.
3871                  */
3872                 return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);
3873 
3874         case 298:
3875                 return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
3876                     DR_B2(eax) || RB_C0(eax));
3877 
3878         case 721:
3879 #if defined(__amd64)
3880                 return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);
3881 #else
3882                 return (0);
3883 #endif
3884 
3885         default:
3886                 return (-1);
3887 
3888         }
3889 }
3890 
3891 /*
3892  * Determine if specified erratum is present via OSVW (OS Visible Workaround).
3893  * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
3894  */
3895 int
3896 osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
3897 {
3898         struct cpuid_info       *cpi;
3899         uint_t                  osvwid;
3900         static int              osvwfeature = -1;
3901         uint64_t                osvwlength;
3902 
3903 
3904         cpi = cpu->cpu_m.mcpu_cpi;
3905 
3906         /* confirm OSVW supported */
3907         if (osvwfeature == -1) {
3908                 osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
3909         } else {
3910                 /* assert that osvw feature setting is consistent on all cpus */
3911                 ASSERT(osvwfeature ==
3912                     (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
3913         }
3914         if (!osvwfeature)
3915                 return (-1);
3916 
3917         osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;
3918 
3919         switch (erratum) {
3920         case 298:       /* osvwid is 0 */
3921                 osvwid = 0;
3922                 if (osvwlength <= (uint64_t)osvwid) {
3923                         /* osvwid 0 is unknown */
3924                         return (-1);
3925                 }
3926 
3927                 /*
3928                  * Check the OSVW STATUS MSR to determine the state
3929                  * of the erratum where:
3930                  *   0 - fixed by HW
3931                  *   1 - BIOS has applied the workaround when BIOS
3932                  *   workaround is available. (Or for other errata,
3933                  *   OS workaround is required.)
3934                  * For a value of 1, caller will confirm that the
3935                  * erratum 298 workaround has indeed been applied by BIOS.
3936                  *
3937                  * A 1 may be set in cpus that have a HW fix
3938                  * in a mixed cpu system. Regarding erratum 298:
3939                  *   In a multiprocessor platform, the workaround above
3940                  *   should be applied to all processors regardless of
3941                  *   silicon revision when an affected processor is
3942                  *   present.
3943                  */
3944 
3945                 return (rdmsr(MSR_AMD_OSVW_STATUS +
3946                     (osvwid / OSVW_ID_CNT_PER_MSR)) &
3947                     (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));
3948 
3949         default:
3950                 return (-1);
3951         }
3952 }
3953 
3954 static const char assoc_str[] = "associativity";
3955 static const char line_str[] = "line-size";
3956 static const char size_str[] = "size";
3957 
3958 static void
3959 add_cache_prop(dev_info_t *devi, const char *label, const char *type,
3960     uint32_t val)
3961 {
3962         char buf[128];
3963 
3964         /*
3965          * ndi_prop_update_int() is used because it is desirable for
3966          * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
3967          */
3968         if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
3969                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
3970 }
3971 
3972 /*
3973  * Intel-style cache/tlb description
3974  *
3975  * Standard cpuid level 2 gives a randomly ordered
3976  * selection of tags that index into a table that describes
3977  * cache and tlb properties.
3978  */
3979 
3980 static const char l1_icache_str[] = "l1-icache";
3981 static const char l1_dcache_str[] = "l1-dcache";
3982 static const char l2_cache_str[] = "l2-cache";
3983 static const char l3_cache_str[] = "l3-cache";
3984 static const char itlb4k_str[] = "itlb-4K";
3985 static const char dtlb4k_str[] = "dtlb-4K";
3986 static const char itlb2M_str[] = "itlb-2M";
3987 static const char itlb4M_str[] = "itlb-4M";
3988 static const char dtlb4M_str[] = "dtlb-4M";
3989 static const char dtlb24_str[] = "dtlb0-2M-4M";
3990 static const char itlb424_str[] = "itlb-4K-2M-4M";
3991 static const char itlb24_str[] = "itlb-2M-4M";
3992 static const char dtlb44_str[] = "dtlb-4K-4M";
3993 static const char sl1_dcache_str[] = "sectored-l1-dcache";
3994 static const char sl2_cache_str[] = "sectored-l2-cache";
3995 static const char itrace_str[] = "itrace-cache";
3996 static const char sl3_cache_str[] = "sectored-l3-cache";
3997 static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";
3998 
3999 static const struct cachetab {
4000         uint8_t         ct_code;
4001         uint8_t         ct_assoc;
4002         uint16_t        ct_line_size;
4003         size_t          ct_size;
4004         const char      *ct_label;
4005 } intel_ctab[] = {
4006         /*
4007          * maintain descending order!
4008          *
4009          * Codes ignored - Reason
4010          * ----------------------
4011          * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
4012          * f0H/f1H - Currently we do not interpret prefetch size by design
4013          */
4014         { 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
4015         { 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
4016         { 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
4017         { 0xde, 12, 64, 6*1024*1024, l3_cache_str},
4018         { 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
4019         { 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
4020         { 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
4021         { 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
4022         { 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
4023         { 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
4024         { 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
4025         { 0xd0, 4, 64, 512*1024, l3_cache_str},
4026         { 0xca, 4, 0, 512, sh_l2_tlb4k_str},
4027         { 0xc0, 4, 0, 8, dtlb44_str },
4028         { 0xba, 4, 0, 64, dtlb4k_str },
4029         { 0xb4, 4, 0, 256, dtlb4k_str },
4030         { 0xb3, 4, 0, 128, dtlb4k_str },
4031         { 0xb2, 4, 0, 64, itlb4k_str },
4032         { 0xb0, 4, 0, 128, itlb4k_str },
4033         { 0x87, 8, 64, 1024*1024, l2_cache_str},
4034         { 0x86, 4, 64, 512*1024, l2_cache_str},
4035         { 0x85, 8, 32, 2*1024*1024, l2_cache_str},
4036         { 0x84, 8, 32, 1024*1024, l2_cache_str},
4037         { 0x83, 8, 32, 512*1024, l2_cache_str},
4038         { 0x82, 8, 32, 256*1024, l2_cache_str},
4039         { 0x80, 8, 64, 512*1024, l2_cache_str},
4040         { 0x7f, 2, 64, 512*1024, l2_cache_str},
4041         { 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
4042         { 0x7c, 8, 64, 1024*1024, sl2_cache_str},
4043         { 0x7b, 8, 64, 512*1024, sl2_cache_str},
4044         { 0x7a, 8, 64, 256*1024, sl2_cache_str},
4045         { 0x79, 8, 64, 128*1024, sl2_cache_str},
4046         { 0x78, 8, 64, 1024*1024, l2_cache_str},
4047         { 0x73, 8, 0, 64*1024, itrace_str},
4048         { 0x72, 8, 0, 32*1024, itrace_str},
4049         { 0x71, 8, 0, 16*1024, itrace_str},
4050         { 0x70, 8, 0, 12*1024, itrace_str},
4051         { 0x68, 4, 64, 32*1024, sl1_dcache_str},
4052         { 0x67, 4, 64, 16*1024, sl1_dcache_str},
4053         { 0x66, 4, 64, 8*1024, sl1_dcache_str},
4054         { 0x60, 8, 64, 16*1024, sl1_dcache_str},
4055         { 0x5d, 0, 0, 256, dtlb44_str},
4056         { 0x5c, 0, 0, 128, dtlb44_str},
4057         { 0x5b, 0, 0, 64, dtlb44_str},
4058         { 0x5a, 4, 0, 32, dtlb24_str},
4059         { 0x59, 0, 0, 16, dtlb4k_str},
4060         { 0x57, 4, 0, 16, dtlb4k_str},
4061         { 0x56, 4, 0, 16, dtlb4M_str},
4062         { 0x55, 0, 0, 7, itlb24_str},
4063         { 0x52, 0, 0, 256, itlb424_str},
4064         { 0x51, 0, 0, 128, itlb424_str},
4065         { 0x50, 0, 0, 64, itlb424_str},
4066         { 0x4f, 0, 0, 32, itlb4k_str},
4067         { 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
4068         { 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
4069         { 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
4070         { 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
4071         { 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
4072         { 0x49, 16, 64, 4*1024*1024, l3_cache_str},
4073         { 0x48, 12, 64, 3*1024*1024, l2_cache_str},
4074         { 0x47, 8, 64, 8*1024*1024, l3_cache_str},
4075         { 0x46, 4, 64, 4*1024*1024, l3_cache_str},
4076         { 0x45, 4, 32, 2*1024*1024, l2_cache_str},
4077         { 0x44, 4, 32, 1024*1024, l2_cache_str},
4078         { 0x43, 4, 32, 512*1024, l2_cache_str},
4079         { 0x42, 4, 32, 256*1024, l2_cache_str},
4080         { 0x41, 4, 32, 128*1024, l2_cache_str},
4081         { 0x3e, 4, 64, 512*1024, sl2_cache_str},
4082         { 0x3d, 6, 64, 384*1024, sl2_cache_str},
4083         { 0x3c, 4, 64, 256*1024, sl2_cache_str},
4084         { 0x3b, 2, 64, 128*1024, sl2_cache_str},
4085         { 0x3a, 6, 64, 192*1024, sl2_cache_str},
4086         { 0x39, 4, 64, 128*1024, sl2_cache_str},
4087         { 0x30, 8, 64, 32*1024, l1_icache_str},
4088         { 0x2c, 8, 64, 32*1024, l1_dcache_str},
4089         { 0x29, 8, 64, 4096*1024, sl3_cache_str},
4090         { 0x25, 8, 64, 2048*1024, sl3_cache_str},
4091         { 0x23, 8, 64, 1024*1024, sl3_cache_str},
4092         { 0x22, 4, 64, 512*1024, sl3_cache_str},
4093         { 0x0e, 6, 64, 24*1024, l1_dcache_str},
4094         { 0x0d, 4, 32, 16*1024, l1_dcache_str},
4095         { 0x0c, 4, 32, 16*1024, l1_dcache_str},
4096         { 0x0b, 4, 0, 4, itlb4M_str},
4097         { 0x0a, 2, 32, 8*1024, l1_dcache_str},
4098         { 0x08, 4, 32, 16*1024, l1_icache_str},
4099         { 0x06, 4, 32, 8*1024, l1_icache_str},
4100         { 0x05, 4, 0, 32, dtlb4M_str},
4101         { 0x04, 4, 0, 8, dtlb4M_str},
4102         { 0x03, 4, 0, 64, dtlb4k_str},
4103         { 0x02, 4, 0, 2, itlb4M_str},
4104         { 0x01, 4, 0, 32, itlb4k_str},
4105         { 0 }
4106 };
4107 
4108 static const struct cachetab cyrix_ctab[] = {
4109         { 0x70, 4, 0, 32, "tlb-4K" },
4110         { 0x80, 4, 16, 16*1024, "l1-cache" },
4111         { 0 }
4112 };
4113 
4114 /*
4115  * Search a cache table for a matching entry
4116  */
4117 static const struct cachetab *
4118 find_cacheent(const struct cachetab *ct, uint_t code)
4119 {
4120         if (code != 0) {
4121                 for (; ct->ct_code != 0; ct++)
4122                         if (ct->ct_code <= code)
4123                                 break;
4124                 if (ct->ct_code == code)
4125                         return (ct);
4126         }
4127         return (NULL);
4128 }
4129 
4130 /*
4131  * Populate cachetab entry with L2 or L3 cache-information using
4132  * cpuid function 4. This function is called from intel_walk_cacheinfo()
4133  * when descriptor 0x49 is encountered. It returns 0 if no such cache
4134  * information is found.
4135  */
4136 static int
4137 intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
4138 {
4139         uint32_t level, i;
4140         int ret = 0;
4141 
4142         for (i = 0; i < cpi->cpi_std_4_size; i++) {
4143                 level = CPI_CACHE_LVL(cpi->cpi_std_4[i]);
4144 
4145                 if (level == 2 || level == 3) {
4146                         ct->ct_assoc = CPI_CACHE_WAYS(cpi->cpi_std_4[i]) + 1;
4147                         ct->ct_line_size =
4148                             CPI_CACHE_COH_LN_SZ(cpi->cpi_std_4[i]) + 1;
4149                         ct->ct_size = ct->ct_assoc *
4150                             (CPI_CACHE_PARTS(cpi->cpi_std_4[i]) + 1) *
4151                             ct->ct_line_size *
4152                             (cpi->cpi_std_4[i]->cp_ecx + 1);
4153 
4154                         if (level == 2) {
4155                                 ct->ct_label = l2_cache_str;
4156                         } else if (level == 3) {
4157                                 ct->ct_label = l3_cache_str;
4158                         }
4159                         ret = 1;
4160                 }
4161         }
4162 
4163         return (ret);
4164 }
4165 
4166 /*
4167  * Walk the cacheinfo descriptor, applying 'func' to every valid element
4168  * The walk is terminated if the walker returns non-zero.
4169  */
4170 static void
4171 intel_walk_cacheinfo(struct cpuid_info *cpi,
4172     void *arg, int (*func)(void *, const struct cachetab *))
4173 {
4174         const struct cachetab *ct;
4175         struct cachetab des_49_ct, des_b1_ct;
4176         uint8_t *dp;
4177         int i;
4178 
4179         if ((dp = cpi->cpi_cacheinfo) == NULL)
4180                 return;
4181         for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4182                 /*
4183                  * For overloaded descriptor 0x49 we use cpuid function 4
4184                  * if supported by the current processor, to create
4185                  * cache information.
4186                  * For overloaded descriptor 0xb1 we use X86_PAE flag
4187                  * to disambiguate the cache information.
4188                  */
4189                 if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
4190                     intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
4191                                 ct = &des_49_ct;
4192                 } else if (*dp == 0xb1) {
4193                         des_b1_ct.ct_code = 0xb1;
4194                         des_b1_ct.ct_assoc = 4;
4195                         des_b1_ct.ct_line_size = 0;
4196                         if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
4197                                 des_b1_ct.ct_size = 8;
4198                                 des_b1_ct.ct_label = itlb2M_str;
4199                         } else {
4200                                 des_b1_ct.ct_size = 4;
4201                                 des_b1_ct.ct_label = itlb4M_str;
4202                         }
4203                         ct = &des_b1_ct;
4204                 } else {
4205                         if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
4206                                 continue;
4207                         }
4208                 }
4209 
4210                 if (func(arg, ct) != 0) {
4211                         break;
4212                 }
4213         }
4214 }
4215 
4216 /*
4217  * (Like the Intel one, except for Cyrix CPUs)
4218  */
4219 static void
4220 cyrix_walk_cacheinfo(struct cpuid_info *cpi,
4221     void *arg, int (*func)(void *, const struct cachetab *))
4222 {
4223         const struct cachetab *ct;
4224         uint8_t *dp;
4225         int i;
4226 
4227         if ((dp = cpi->cpi_cacheinfo) == NULL)
4228                 return;
4229         for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
4230                 /*
4231                  * Search Cyrix-specific descriptor table first ..
4232                  */
4233                 if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
4234                         if (func(arg, ct) != 0)
4235                                 break;
4236                         continue;
4237                 }
4238                 /*
4239                  * .. else fall back to the Intel one
4240                  */
4241                 if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
4242                         if (func(arg, ct) != 0)
4243                                 break;
4244                         continue;
4245                 }
4246         }
4247 }
4248 
4249 /*
4250  * A cacheinfo walker that adds associativity, line-size, and size properties
4251  * to the devinfo node it is passed as an argument.
4252  */
4253 static int
4254 add_cacheent_props(void *arg, const struct cachetab *ct)
4255 {
4256         dev_info_t *devi = arg;
4257 
4258         add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
4259         if (ct->ct_line_size != 0)
4260                 add_cache_prop(devi, ct->ct_label, line_str,
4261                     ct->ct_line_size);
4262         add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
4263         return (0);
4264 }
4265 
4266 
4267 static const char fully_assoc[] = "fully-associative?";
4268 
4269 /*
4270  * AMD style cache/tlb description
4271  *
4272  * Extended functions 5 and 6 directly describe properties of
4273  * tlbs and various cache levels.
4274  */
4275 static void
4276 add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4277 {
4278         switch (assoc) {
4279         case 0: /* reserved; ignore */
4280                 break;
4281         default:
4282                 add_cache_prop(devi, label, assoc_str, assoc);
4283                 break;
4284         case 0xff:
4285                 add_cache_prop(devi, label, fully_assoc, 1);
4286                 break;
4287         }
4288 }
4289 
4290 static void
4291 add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4292 {
4293         if (size == 0)
4294                 return;
4295         add_cache_prop(devi, label, size_str, size);
4296         add_amd_assoc(devi, label, assoc);
4297 }
4298 
4299 static void
4300 add_amd_cache(dev_info_t *devi, const char *label,
4301     uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4302 {
4303         if (size == 0 || line_size == 0)
4304                 return;
4305         add_amd_assoc(devi, label, assoc);
4306         /*
4307          * Most AMD parts have a sectored cache. Multiple cache lines are
4308          * associated with each tag. A sector consists of all cache lines
4309          * associated with a tag. For example, the AMD K6-III has a sector
4310          * size of 2 cache lines per tag.
4311          */
4312         if (lines_per_tag != 0)
4313                 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4314         add_cache_prop(devi, label, line_str, line_size);
4315         add_cache_prop(devi, label, size_str, size * 1024);
4316 }
4317 
4318 static void
4319 add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
4320 {
4321         switch (assoc) {
4322         case 0: /* off */
4323                 break;
4324         case 1:
4325         case 2:
4326         case 4:
4327                 add_cache_prop(devi, label, assoc_str, assoc);
4328                 break;
4329         case 6:
4330                 add_cache_prop(devi, label, assoc_str, 8);
4331                 break;
4332         case 8:
4333                 add_cache_prop(devi, label, assoc_str, 16);
4334                 break;
4335         case 0xf:
4336                 add_cache_prop(devi, label, fully_assoc, 1);
4337                 break;
4338         default: /* reserved; ignore */
4339                 break;
4340         }
4341 }
4342 
4343 static void
4344 add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
4345 {
4346         if (size == 0 || assoc == 0)
4347                 return;
4348         add_amd_l2_assoc(devi, label, assoc);
4349         add_cache_prop(devi, label, size_str, size);
4350 }
4351 
4352 static void
4353 add_amd_l2_cache(dev_info_t *devi, const char *label,
4354     uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
4355 {
4356         if (size == 0 || assoc == 0 || line_size == 0)
4357                 return;
4358         add_amd_l2_assoc(devi, label, assoc);
4359         if (lines_per_tag != 0)
4360                 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
4361         add_cache_prop(devi, label, line_str, line_size);
4362         add_cache_prop(devi, label, size_str, size * 1024);
4363 }
4364 
4365 static void
4366 amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
4367 {
4368         struct cpuid_regs *cp;
4369 
4370         if (cpi->cpi_xmaxeax < 0x80000005)
4371                 return;
4372         cp = &cpi->cpi_extd[5];
4373 
4374         /*
4375          * 4M/2M L1 TLB configuration
4376          *
4377          * We report the size for 2M pages because AMD uses two
4378          * TLB entries for one 4M page.
4379          */
4380         add_amd_tlb(devi, "dtlb-2M",
4381             BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
4382         add_amd_tlb(devi, "itlb-2M",
4383             BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));
4384 
4385         /*
4386          * 4K L1 TLB configuration
4387          */
4388 
4389         switch (cpi->cpi_vendor) {
4390                 uint_t nentries;
4391         case X86_VENDOR_TM:
4392                 if (cpi->cpi_family >= 5) {
4393                         /*
4394                          * Crusoe processors have 256 TLB entries, but
4395                          * cpuid data format constrains them to only
4396                          * reporting 255 of them.
4397                          */
4398                         if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
4399                                 nentries = 256;
4400                         /*
4401                          * Crusoe processors also have a unified TLB
4402                          */
4403                         add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
4404                             nentries);
4405                         break;
4406                 }
4407                 /*FALLTHROUGH*/
4408         default:
4409                 add_amd_tlb(devi, itlb4k_str,
4410                     BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
4411                 add_amd_tlb(devi, dtlb4k_str,
4412                     BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
4413                 break;
4414         }
4415 
4416         /*
4417          * data L1 cache configuration
4418          */
4419 
4420         add_amd_cache(devi, l1_dcache_str,
4421             BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
4422             BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));
4423 
4424         /*
4425          * code L1 cache configuration
4426          */
4427 
4428         add_amd_cache(devi, l1_icache_str,
4429             BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
4430             BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));
4431 
4432         if (cpi->cpi_xmaxeax < 0x80000006)
4433                 return;
4434         cp = &cpi->cpi_extd[6];
4435 
4436         /* Check for a unified L2 TLB for large pages */
4437 
4438         if (BITX(cp->cp_eax, 31, 16) == 0)
4439                 add_amd_l2_tlb(devi, "l2-tlb-2M",
4440                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4441         else {
4442                 add_amd_l2_tlb(devi, "l2-dtlb-2M",
4443                     BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4444                 add_amd_l2_tlb(devi, "l2-itlb-2M",
4445                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4446         }
4447 
4448         /* Check for a unified L2 TLB for 4K pages */
4449 
4450         if (BITX(cp->cp_ebx, 31, 16) == 0) {
4451                 add_amd_l2_tlb(devi, "l2-tlb-4K",
4452                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4453         } else {
4454                 add_amd_l2_tlb(devi, "l2-dtlb-4K",
4455                     BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
4456                 add_amd_l2_tlb(devi, "l2-itlb-4K",
4457                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
4458         }
4459 
4460         add_amd_l2_cache(devi, l2_cache_str,
4461             BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
4462             BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
4463 }
4464 
4465 /*
4466  * There are two basic ways that the x86 world describes it cache
4467  * and tlb architecture - Intel's way and AMD's way.
4468  *
4469  * Return which flavor of cache architecture we should use
4470  */
4471 static int
4472 x86_which_cacheinfo(struct cpuid_info *cpi)
4473 {
4474         switch (cpi->cpi_vendor) {
4475         case X86_VENDOR_Intel:
4476                 if (cpi->cpi_maxeax >= 2)
4477                         return (X86_VENDOR_Intel);
4478                 break;
4479         case X86_VENDOR_AMD:
4480                 /*
4481                  * The K5 model 1 was the first part from AMD that reported
4482                  * cache sizes via extended cpuid functions.
4483                  */
4484                 if (cpi->cpi_family > 5 ||
4485                     (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
4486                         return (X86_VENDOR_AMD);
4487                 break;
4488         case X86_VENDOR_TM:
4489                 if (cpi->cpi_family >= 5)
4490                         return (X86_VENDOR_AMD);
4491                 /*FALLTHROUGH*/
4492         default:
4493                 /*
4494                  * If they have extended CPU data for 0x80000005
4495                  * then we assume they have AMD-format cache
4496                  * information.
4497                  *
4498                  * If not, and the vendor happens to be Cyrix,
4499                  * then try our-Cyrix specific handler.
4500                  *
4501                  * If we're not Cyrix, then assume we're using Intel's
4502                  * table-driven format instead.
4503                  */
4504                 if (cpi->cpi_xmaxeax >= 0x80000005)
4505                         return (X86_VENDOR_AMD);
4506                 else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
4507                         return (X86_VENDOR_Cyrix);
4508                 else if (cpi->cpi_maxeax >= 2)
4509                         return (X86_VENDOR_Intel);
4510                 break;
4511         }
4512         return (-1);
4513 }
4514 
4515 void
4516 cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
4517     struct cpuid_info *cpi)
4518 {
4519         dev_info_t *cpu_devi;
4520         int create;
4521 
4522         cpu_devi = (dev_info_t *)dip;
4523 
4524         /* device_type */
4525         (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4526             "device_type", "cpu");
4527 
4528         /* reg */
4529         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4530             "reg", cpu_id);
4531 
4532         /* cpu-mhz, and clock-frequency */
4533         if (cpu_freq > 0) {
4534                 long long mul;
4535 
4536                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4537                     "cpu-mhz", cpu_freq);
4538                 if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
4539                         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4540                             "clock-frequency", (int)mul);
4541         }
4542 
4543         if (!is_x86_feature(x86_featureset, X86FSET_CPUID)) {
4544                 return;
4545         }
4546 
4547         /* vendor-id */
4548         (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4549             "vendor-id", cpi->cpi_vendorstr);
4550 
4551         if (cpi->cpi_maxeax == 0) {
4552                 return;
4553         }
4554 
4555         /*
4556          * family, model, and step
4557          */
4558         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4559             "family", CPI_FAMILY(cpi));
4560         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4561             "cpu-model", CPI_MODEL(cpi));
4562         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4563             "stepping-id", CPI_STEP(cpi));
4564 
4565         /* type */
4566         switch (cpi->cpi_vendor) {
4567         case X86_VENDOR_Intel:
4568                 create = 1;
4569                 break;
4570         default:
4571                 create = 0;
4572                 break;
4573         }
4574         if (create)
4575                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4576                     "type", CPI_TYPE(cpi));
4577 
4578         /* ext-family */
4579         switch (cpi->cpi_vendor) {
4580         case X86_VENDOR_Intel:
4581         case X86_VENDOR_AMD:
4582                 create = cpi->cpi_family >= 0xf;
4583                 break;
4584         default:
4585                 create = 0;
4586                 break;
4587         }
4588         if (create)
4589                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4590                     "ext-family", CPI_FAMILY_XTD(cpi));
4591 
4592         /* ext-model */
4593         switch (cpi->cpi_vendor) {
4594         case X86_VENDOR_Intel:
4595                 create = IS_EXTENDED_MODEL_INTEL(cpi);
4596                 break;
4597         case X86_VENDOR_AMD:
4598                 create = CPI_FAMILY(cpi) == 0xf;
4599                 break;
4600         default:
4601                 create = 0;
4602                 break;
4603         }
4604         if (create)
4605                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4606                     "ext-model", CPI_MODEL_XTD(cpi));
4607 
4608         /* generation */
4609         switch (cpi->cpi_vendor) {
4610         case X86_VENDOR_AMD:
4611                 /*
4612                  * AMD K5 model 1 was the first part to support this
4613                  */
4614                 create = cpi->cpi_xmaxeax >= 0x80000001;
4615                 break;
4616         default:
4617                 create = 0;
4618                 break;
4619         }
4620         if (create)
4621                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4622                     "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));
4623 
4624         /* brand-id */
4625         switch (cpi->cpi_vendor) {
4626         case X86_VENDOR_Intel:
4627                 /*
4628                  * brand id first appeared on Pentium III Xeon model 8,
4629                  * and Celeron model 8 processors and Opteron
4630                  */
4631                 create = cpi->cpi_family > 6 ||
4632                     (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
4633                 break;
4634         case X86_VENDOR_AMD:
4635                 create = cpi->cpi_family >= 0xf;
4636                 break;
4637         default:
4638                 create = 0;
4639                 break;
4640         }
4641         if (create && cpi->cpi_brandid != 0) {
4642                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4643                     "brand-id", cpi->cpi_brandid);
4644         }
4645 
4646         /* chunks, and apic-id */
4647         switch (cpi->cpi_vendor) {
4648                 /*
4649                  * first available on Pentium IV and Opteron (K8)
4650                  */
4651         case X86_VENDOR_Intel:
4652                 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4653                 break;
4654         case X86_VENDOR_AMD:
4655                 create = cpi->cpi_family >= 0xf;
4656                 break;
4657         default:
4658                 create = 0;
4659                 break;
4660         }
4661         if (create) {
4662                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4663                     "chunks", CPI_CHUNKS(cpi));
4664                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4665                     "apic-id", cpi->cpi_apicid);
4666                 if (cpi->cpi_chipid >= 0) {
4667                         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4668                             "chip#", cpi->cpi_chipid);
4669                         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4670                             "clog#", cpi->cpi_clogid);
4671                 }
4672         }
4673 
4674         /* cpuid-features */
4675         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4676             "cpuid-features", CPI_FEATURES_EDX(cpi));
4677 
4678 
4679         /* cpuid-features-ecx */
4680         switch (cpi->cpi_vendor) {
4681         case X86_VENDOR_Intel:
4682                 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4683                 break;
4684         case X86_VENDOR_AMD:
4685                 create = cpi->cpi_family >= 0xf;
4686                 break;
4687         default:
4688                 create = 0;
4689                 break;
4690         }
4691         if (create)
4692                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4693                     "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));
4694 
4695         /* ext-cpuid-features */
4696         switch (cpi->cpi_vendor) {
4697         case X86_VENDOR_Intel:
4698         case X86_VENDOR_AMD:
4699         case X86_VENDOR_Cyrix:
4700         case X86_VENDOR_TM:
4701         case X86_VENDOR_Centaur:
4702                 create = cpi->cpi_xmaxeax >= 0x80000001;
4703                 break;
4704         default:
4705                 create = 0;
4706                 break;
4707         }
4708         if (create) {
4709                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4710                     "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
4711                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4712                     "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
4713         }
4714 
4715         /*
4716          * Brand String first appeared in Intel Pentium IV, AMD K5
4717          * model 1, and Cyrix GXm.  On earlier models we try and
4718          * simulate something similar .. so this string should always
4719          * same -something- about the processor, however lame.
4720          */
4721         (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4722             "brand-string", cpi->cpi_brandstr);
4723 
4724         /*
4725          * Finally, cache and tlb information
4726          */
4727         switch (x86_which_cacheinfo(cpi)) {
4728         case X86_VENDOR_Intel:
4729                 intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4730                 break;
4731         case X86_VENDOR_Cyrix:
4732                 cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4733                 break;
4734         case X86_VENDOR_AMD:
4735                 amd_cache_info(cpi, cpu_devi);
4736                 break;
4737         default:
4738                 break;
4739         }
4740 }
4741 
4742 struct l2info {
4743         int *l2i_csz;
4744         int *l2i_lsz;
4745         int *l2i_assoc;
4746         int l2i_ret;
4747 };
4748 
4749 /*
4750  * A cacheinfo walker that fetches the size, line-size and associativity
4751  * of the L2 cache
4752  */
4753 static int
4754 intel_l2cinfo(void *arg, const struct cachetab *ct)
4755 {
4756         struct l2info *l2i = arg;
4757         int *ip;
4758 
4759         if (ct->ct_label != l2_cache_str &&
4760             ct->ct_label != sl2_cache_str)
4761                 return (0);     /* not an L2 -- keep walking */
4762 
4763         if ((ip = l2i->l2i_csz) != NULL)
4764                 *ip = ct->ct_size;
4765         if ((ip = l2i->l2i_lsz) != NULL)
4766                 *ip = ct->ct_line_size;
4767         if ((ip = l2i->l2i_assoc) != NULL)
4768                 *ip = ct->ct_assoc;
4769         l2i->l2i_ret = ct->ct_size;
4770         return (1);             /* was an L2 -- terminate walk */
4771 }
4772 
4773 /*
4774  * AMD L2/L3 Cache and TLB Associativity Field Definition:
4775  *
4776  *      Unlike the associativity for the L1 cache and tlb where the 8 bit
4777  *      value is the associativity, the associativity for the L2 cache and
4778  *      tlb is encoded in the following table. The 4 bit L2 value serves as
4779  *      an index into the amd_afd[] array to determine the associativity.
4780  *      -1 is undefined. 0 is fully associative.
4781  */
4782 
4783 static int amd_afd[] =
4784         {-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};
4785 
4786 static void
4787 amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
4788 {
4789         struct cpuid_regs *cp;
4790         uint_t size, assoc;
4791         int i;
4792         int *ip;
4793 
4794         if (cpi->cpi_xmaxeax < 0x80000006)
4795                 return;
4796         cp = &cpi->cpi_extd[6];
4797 
4798         if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
4799             (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
4800                 uint_t cachesz = size * 1024;
4801                 assoc = amd_afd[i];
4802 
4803                 ASSERT(assoc != -1);
4804 
4805                 if ((ip = l2i->l2i_csz) != NULL)
4806                         *ip = cachesz;
4807                 if ((ip = l2i->l2i_lsz) != NULL)
4808                         *ip = BITX(cp->cp_ecx, 7, 0);
4809                 if ((ip = l2i->l2i_assoc) != NULL)
4810                         *ip = assoc;
4811                 l2i->l2i_ret = cachesz;
4812         }
4813 }
4814 
4815 int
4816 getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
4817 {
4818         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
4819         struct l2info __l2info, *l2i = &__l2info;
4820 
4821         l2i->l2i_csz = csz;
4822         l2i->l2i_lsz = lsz;
4823         l2i->l2i_assoc = assoc;
4824         l2i->l2i_ret = -1;
4825 
4826         switch (x86_which_cacheinfo(cpi)) {
4827         case X86_VENDOR_Intel:
4828                 intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4829                 break;
4830         case X86_VENDOR_Cyrix:
4831                 cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4832                 break;
4833         case X86_VENDOR_AMD:
4834                 amd_l2cacheinfo(cpi, l2i);
4835                 break;
4836         default:
4837                 break;
4838         }
4839         return (l2i->l2i_ret);
4840 }
4841 
4842 #if !defined(__xpv)
4843 
4844 uint32_t *
4845 cpuid_mwait_alloc(cpu_t *cpu)
4846 {
4847         uint32_t        *ret;
4848         size_t          mwait_size;
4849 
4850         ASSERT(cpuid_checkpass(CPU, 2));
4851 
4852         mwait_size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
4853         if (mwait_size == 0)
4854                 return (NULL);
4855 
4856         /*
4857          * kmem_alloc() returns cache line size aligned data for mwait_size
4858          * allocations.  mwait_size is currently cache line sized.  Neither
4859          * of these implementation details are guarantied to be true in the
4860          * future.
4861          *
4862          * First try allocating mwait_size as kmem_alloc() currently returns
4863          * correctly aligned memory.  If kmem_alloc() does not return
4864          * mwait_size aligned memory, then use mwait_size ROUNDUP.
4865          *
4866          * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
4867          * decide to free this memory.
4868          */
4869         ret = kmem_zalloc(mwait_size, KM_SLEEP);
4870         if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) {
4871                 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4872                 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size;
4873                 *ret = MWAIT_RUNNING;
4874                 return (ret);
4875         } else {
4876                 kmem_free(ret, mwait_size);
4877                 ret = kmem_zalloc(mwait_size * 2, KM_SLEEP);
4878                 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4879                 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size * 2;
4880                 ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size);
4881                 *ret = MWAIT_RUNNING;
4882                 return (ret);
4883         }
4884 }
4885 
4886 void
4887 cpuid_mwait_free(cpu_t *cpu)
4888 {
4889         if (cpu->cpu_m.mcpu_cpi == NULL) {
4890                 return;
4891         }
4892 
4893         if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
4894             cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
4895                 kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
4896                     cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
4897         }
4898 
4899         cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
4900         cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
4901 }
4902 
4903 void
4904 patch_tsc_read(int flag)
4905 {
4906         size_t cnt;
4907 
4908         switch (flag) {
4909         case TSC_NONE:
4910                 cnt = &_no_rdtsc_end - &_no_rdtsc_start;
4911                 (void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
4912                 break;
4913         case TSC_RDTSC_MFENCE:
4914                 cnt = &_tsc_mfence_end - &_tsc_mfence_start;
4915                 (void) memcpy((void *)tsc_read,
4916                     (void *)&_tsc_mfence_start, cnt);
4917                 break;
4918         case TSC_RDTSC_LFENCE:
4919                 cnt = &_tsc_lfence_end - &_tsc_lfence_start;
4920                 (void) memcpy((void *)tsc_read,
4921                     (void *)&_tsc_lfence_start, cnt);
4922                 break;
4923         case TSC_TSCP:
4924                 cnt = &_tscp_end - &_tscp_start;
4925                 (void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
4926                 break;
4927         default:
4928                 /* Bail for unexpected TSC types. (TSC_NONE covers 0) */
4929                 cmn_err(CE_PANIC, "Unrecogized TSC type: %d", flag);
4930                 break;
4931         }
4932         tsc_type = flag;
4933 }
4934 
4935 int
4936 cpuid_deep_cstates_supported(void)
4937 {
4938         struct cpuid_info *cpi;
4939         struct cpuid_regs regs;
4940 
4941         ASSERT(cpuid_checkpass(CPU, 1));
4942 
4943         cpi = CPU->cpu_m.mcpu_cpi;
4944 
4945         if (!is_x86_feature(x86_featureset, X86FSET_CPUID))
4946                 return (0);
4947 
4948         switch (cpi->cpi_vendor) {
4949         case X86_VENDOR_Intel:
4950                 if (cpi->cpi_xmaxeax < 0x80000007)
4951                         return (0);
4952 
4953                 /*
4954                  * TSC run at a constant rate in all ACPI C-states?
4955                  */
4956                 regs.cp_eax = 0x80000007;
4957                 (void) __cpuid_insn(&regs);
4958                 return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);
4959 
4960         default:
4961                 return (0);
4962         }
4963 }
4964 
4965 #endif  /* !__xpv */
4966 
4967 void
4968 post_startup_cpu_fixups(void)
4969 {
4970 #ifndef __xpv
4971         /*
4972          * Some AMD processors support C1E state. Entering this state will
4973          * cause the local APIC timer to stop, which we can't deal with at
4974          * this time.
4975          */
4976         if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
4977                 on_trap_data_t otd;
4978                 uint64_t reg;
4979 
4980                 if (!on_trap(&otd, OT_DATA_ACCESS)) {
4981                         reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
4982                         /* Disable C1E state if it is enabled by BIOS */
4983                         if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
4984                             AMD_ACTONCMPHALT_MASK) {
4985                                 reg &= ~(AMD_ACTONCMPHALT_MASK <<
4986                                     AMD_ACTONCMPHALT_SHIFT);
4987                                 wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
4988                         }
4989                 }
4990                 no_trap();
4991         }
4992 #endif  /* !__xpv */
4993 }
4994 
4995 /*
4996  * Setup necessary registers to enable XSAVE feature on this processor.
4997  * This function needs to be called early enough, so that no xsave/xrstor
4998  * ops will execute on the processor before the MSRs are properly set up.
4999  *
5000  * Current implementation has the following assumption:
5001  * - cpuid_pass1() is done, so that X86 features are known.
5002  * - fpu_probe() is done, so that fp_save_mech is chosen.
5003  */
5004 void
5005 xsave_setup_msr(cpu_t *cpu)
5006 {
5007         ASSERT(fp_save_mech == FP_XSAVE);
5008         ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
5009 
5010         /* Enable OSXSAVE in CR4. */
5011         setcr4(getcr4() | CR4_OSXSAVE);
5012         /*
5013          * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
5014          * correct value.
5015          */
5016         cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
5017         setup_xfem();
5018 }
5019 
5020 /*
5021  * Starting with the Westmere processor the local
5022  * APIC timer will continue running in all C-states,
5023  * including the deepest C-states.
5024  */
5025 int
5026 cpuid_arat_supported(void)
5027 {
5028         struct cpuid_info *cpi;
5029         struct cpuid_regs regs;
5030 
5031         ASSERT(cpuid_checkpass(CPU, 1));
5032         ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5033 
5034         cpi = CPU->cpu_m.mcpu_cpi;
5035 
5036         switch (cpi->cpi_vendor) {
5037         case X86_VENDOR_Intel:
5038                 /*
5039                  * Always-running Local APIC Timer is
5040                  * indicated by CPUID.6.EAX[2].
5041                  */
5042                 if (cpi->cpi_maxeax >= 6) {
5043                         regs.cp_eax = 6;
5044                         (void) cpuid_insn(NULL, &regs);
5045                         return (regs.cp_eax & CPUID_CSTATE_ARAT);
5046                 } else {
5047                         return (0);
5048                 }
5049         default:
5050                 return (0);
5051         }
5052 }
5053 
5054 /*
5055  * Check support for Intel ENERGY_PERF_BIAS feature
5056  */
5057 int
5058 cpuid_iepb_supported(struct cpu *cp)
5059 {
5060         struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
5061         struct cpuid_regs regs;
5062 
5063         ASSERT(cpuid_checkpass(cp, 1));
5064 
5065         if (!(is_x86_feature(x86_featureset, X86FSET_CPUID)) ||
5066             !(is_x86_feature(x86_featureset, X86FSET_MSR))) {
5067                 return (0);
5068         }
5069 
5070         /*
5071          * Intel ENERGY_PERF_BIAS MSR is indicated by
5072          * capability bit CPUID.6.ECX.3
5073          */
5074         if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
5075                 return (0);
5076 
5077         regs.cp_eax = 0x6;
5078         (void) cpuid_insn(NULL, &regs);
5079         return (regs.cp_ecx & CPUID_EPB_SUPPORT);
5080 }
5081 
5082 /*
5083  * Check support for TSC deadline timer
5084  *
5085  * TSC deadline timer provides a superior software programming
5086  * model over local APIC timer that eliminates "time drifts".
5087  * Instead of specifying a relative time, software specifies an
5088  * absolute time as the target at which the processor should
5089  * generate a timer event.
5090  */
5091 int
5092 cpuid_deadline_tsc_supported(void)
5093 {
5094         struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
5095         struct cpuid_regs regs;
5096 
5097         ASSERT(cpuid_checkpass(CPU, 1));
5098         ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
5099 
5100         switch (cpi->cpi_vendor) {
5101         case X86_VENDOR_Intel:
5102                 if (cpi->cpi_maxeax >= 1) {
5103                         regs.cp_eax = 1;
5104                         (void) cpuid_insn(NULL, &regs);
5105                         return (regs.cp_ecx & CPUID_DEADLINE_TSC);
5106                 } else {
5107                         return (0);
5108                 }
5109         default:
5110                 return (0);
5111         }
5112 }
5113 
5114 #if defined(__amd64) && !defined(__xpv)
5115 /*
5116  * Patch in versions of bcopy for high performance Intel Nhm processors
5117  * and later...
5118  */
5119 void
5120 patch_memops(uint_t vendor)
5121 {
5122         size_t cnt, i;
5123         caddr_t to, from;
5124 
5125         if ((vendor == X86_VENDOR_Intel) &&
5126             is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
5127                 cnt = &bcopy_patch_end - &bcopy_patch_start;
5128                 to = &bcopy_ck_size;
5129                 from = &bcopy_patch_start;
5130                 for (i = 0; i < cnt; i++) {
5131                         *to++ = *from++;
5132                 }
5133         }
5134 }
5135 #endif  /* __amd64 && !__xpv */
5136 
5137 /*
5138  * This function finds the number of bits to represent the number of cores per
5139  * chip and the number of strands per core for the Intel platforms.
5140  * It re-uses the x2APIC cpuid code of the cpuid_pass2().
5141  */
5142 void
5143 cpuid_get_ext_topo(uint_t vendor, uint_t *core_nbits, uint_t *strand_nbits)
5144 {
5145         struct cpuid_regs regs;
5146         struct cpuid_regs *cp = &regs;
5147 
5148         if (vendor != X86_VENDOR_Intel) {
5149                 return;
5150         }
5151 
5152         /* if the cpuid level is 0xB, extended topo is available. */
5153         cp->cp_eax = 0;
5154         if (__cpuid_insn(cp) >= 0xB) {
5155 
5156                 cp->cp_eax = 0xB;
5157                 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
5158                 (void) __cpuid_insn(cp);
5159 
5160                 /*
5161                  * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
5162                  * indicates that the extended topology enumeration leaf is
5163                  * available.
5164                  */
5165                 if (cp->cp_ebx) {
5166                         uint_t coreid_shift = 0;
5167                         uint_t chipid_shift = 0;
5168                         uint_t i;
5169                         uint_t level;
5170 
5171                         for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
5172                                 cp->cp_eax = 0xB;
5173                                 cp->cp_ecx = i;
5174 
5175                                 (void) __cpuid_insn(cp);
5176                                 level = CPI_CPU_LEVEL_TYPE(cp);
5177 
5178                                 if (level == 1) {
5179                                         /*
5180                                          * Thread level processor topology
5181                                          * Number of bits shift right APIC ID
5182                                          * to get the coreid.
5183                                          */
5184                                         coreid_shift = BITX(cp->cp_eax, 4, 0);
5185                                 } else if (level == 2) {
5186                                         /*
5187                                          * Core level processor topology
5188                                          * Number of bits shift right APIC ID
5189                                          * to get the chipid.
5190                                          */
5191                                         chipid_shift = BITX(cp->cp_eax, 4, 0);
5192                                 }
5193                         }
5194 
5195                         if (coreid_shift > 0 && chipid_shift > coreid_shift) {
5196                                 *strand_nbits = coreid_shift;
5197                                 *core_nbits = chipid_shift - coreid_shift;
5198                         }
5199                 }
5200         }
5201 }