1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 /*
  26  * Copyright (c) 2010, Intel Corporation.
  27  * All rights reserved.
  28  */
  29 /*
  30  * Copyright 2019 Joyent, Inc.
  31  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  32  * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  33  */
  34 
  35 #include <sys/types.h>
  36 #include <sys/thread.h>
  37 #include <sys/cpuvar.h>
  38 #include <sys/cpu.h>
  39 #include <sys/t_lock.h>
  40 #include <sys/param.h>
  41 #include <sys/proc.h>
  42 #include <sys/disp.h>
  43 #include <sys/class.h>
  44 #include <sys/cmn_err.h>
  45 #include <sys/debug.h>
  46 #include <sys/note.h>
  47 #include <sys/asm_linkage.h>
  48 #include <sys/x_call.h>
  49 #include <sys/systm.h>
  50 #include <sys/var.h>
  51 #include <sys/vtrace.h>
  52 #include <vm/hat.h>
  53 #include <vm/as.h>
  54 #include <vm/seg_kmem.h>
  55 #include <vm/seg_kp.h>
  56 #include <sys/segments.h>
  57 #include <sys/kmem.h>
  58 #include <sys/stack.h>
  59 #include <sys/smp_impldefs.h>
  60 #include <sys/x86_archext.h>
  61 #include <sys/machsystm.h>
  62 #include <sys/traptrace.h>
  63 #include <sys/clock.h>
  64 #include <sys/cpc_impl.h>
  65 #include <sys/pg.h>
  66 #include <sys/cmt.h>
  67 #include <sys/dtrace.h>
  68 #include <sys/archsystm.h>
  69 #include <sys/fp.h>
  70 #include <sys/reboot.h>
  71 #include <sys/kdi_machimpl.h>
  72 #include <vm/hat_i86.h>
  73 #include <vm/vm_dep.h>
  74 #include <sys/memnode.h>
  75 #include <sys/pci_cfgspace.h>
  76 #include <sys/mach_mmu.h>
  77 #include <sys/sysmacros.h>
  78 #if defined(__xpv)
  79 #include <sys/hypervisor.h>
  80 #endif
  81 #include <sys/cpu_module.h>
  82 #include <sys/ontrap.h>
  83 
  84 struct cpu      cpus[1] __aligned(MMU_PAGESIZE);
  85 struct cpu      *cpu[NCPU] = {&cpus[0]};
  86 struct cpu      *cpu_free_list;
  87 cpu_core_t      cpu_core[NCPU];
  88 
  89 #define cpu_next_free   cpu_prev
  90 
  91 /*
  92  * Useful for disabling MP bring-up on a MP capable system.
  93  */
  94 int use_mp = 1;
  95 
  96 /*
  97  * to be set by a PSM to indicate what cpus
  98  * are sitting around on the system.
  99  */
 100 cpuset_t mp_cpus;
 101 
 102 /*
 103  * This variable is used by the hat layer to decide whether or not
 104  * critical sections are needed to prevent race conditions.  For sun4m,
 105  * this variable is set once enough MP initialization has been done in
 106  * order to allow cross calls.
 107  */
 108 int flushes_require_xcalls;
 109 
 110 cpuset_t cpu_ready_set;         /* initialized in startup() */
 111 
 112 static void mp_startup_boot(void);
 113 static void mp_startup_hotplug(void);
 114 
 115 static void cpu_sep_enable(void);
 116 static void cpu_sep_disable(void);
 117 static void cpu_asysc_enable(void);
 118 static void cpu_asysc_disable(void);
 119 
 120 /*
 121  * Init CPU info - get CPU type info for processor_info system call.
 122  */
 123 void
 124 init_cpu_info(struct cpu *cp)
 125 {
 126         processor_info_t *pi = &cp->cpu_type_info;
 127 
 128         /*
 129          * Get clock-frequency property for the CPU.
 130          */
 131         pi->pi_clock = cpu_freq;
 132 
 133         /*
 134          * Current frequency in Hz.
 135          */
 136         cp->cpu_curr_clock = cpu_freq_hz;
 137 
 138         /*
 139          * Supported frequencies.
 140          */
 141         if (cp->cpu_supp_freqs == NULL) {
 142                 cpu_set_supp_freqs(cp, NULL);
 143         }
 144 
 145         (void) strcpy(pi->pi_processor_type, "i386");
 146         if (fpu_exists)
 147                 (void) strcpy(pi->pi_fputypes, "i387 compatible");
 148 
 149         cp->cpu_idstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
 150         cp->cpu_brandstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
 151 
 152         /*
 153          * If called for the BSP, cp is equal to current CPU.
 154          * For non-BSPs, cpuid info of cp is not ready yet, so use cpuid info
 155          * of current CPU as default values for cpu_idstr and cpu_brandstr.
 156          * They will be corrected in mp_startup_common() after cpuid_pass1()
 157          * has been invoked on target CPU.
 158          */
 159         (void) cpuid_getidstr(CPU, cp->cpu_idstr, CPU_IDSTRLEN);
 160         (void) cpuid_getbrandstr(CPU, cp->cpu_brandstr, CPU_IDSTRLEN);
 161 }
 162 
 163 /*
 164  * Configure syscall support on this CPU.
 165  */
 166 /*ARGSUSED*/
 167 void
 168 init_cpu_syscall(struct cpu *cp)
 169 {
 170         kpreempt_disable();
 171 
 172         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
 173             is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
 174                 uint64_t flags;
 175 
 176 #if !defined(__xpv)
 177                 /*
 178                  * The syscall instruction imposes a certain ordering on
 179                  * segment selectors, so we double-check that ordering
 180                  * here.
 181                  */
 182                 CTASSERT(KDS_SEL == KCS_SEL + 8);
 183                 CTASSERT(UDS_SEL == U32CS_SEL + 8);
 184                 CTASSERT(UCS_SEL == U32CS_SEL + 16);
 185 #endif
 186 
 187                 /*
 188                  * Turn syscall/sysret extensions on.
 189                  */
 190                 cpu_asysc_enable();
 191 
 192                 /*
 193                  * Program the magic registers ..
 194                  */
 195                 wrmsr(MSR_AMD_STAR,
 196                     ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);
 197                 if (kpti_enable == 1) {
 198                         wrmsr(MSR_AMD_LSTAR,
 199                             (uint64_t)(uintptr_t)tr_sys_syscall);
 200                         wrmsr(MSR_AMD_CSTAR,
 201                             (uint64_t)(uintptr_t)tr_sys_syscall32);
 202                 } else {
 203                         wrmsr(MSR_AMD_LSTAR,
 204                             (uint64_t)(uintptr_t)sys_syscall);
 205                         wrmsr(MSR_AMD_CSTAR,
 206                             (uint64_t)(uintptr_t)sys_syscall32);
 207                 }
 208 
 209                 /*
 210                  * This list of flags is masked off the incoming
 211                  * %rfl when we enter the kernel.
 212                  */
 213                 flags = PS_IE | PS_T;
 214                 if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_TRUE)
 215                         flags |= PS_ACHK;
 216                 wrmsr(MSR_AMD_SFMASK, flags);
 217         }
 218 
 219         /*
 220          * On 64-bit kernels on Nocona machines, the 32-bit syscall
 221          * variant isn't available to 32-bit applications, but sysenter is.
 222          */
 223         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
 224             is_x86_feature(x86_featureset, X86FSET_SEP)) {
 225 
 226 #if !defined(__xpv)
 227                 /*
 228                  * The sysenter instruction imposes a certain ordering on
 229                  * segment selectors, so we double-check that ordering
 230                  * here. See "sysenter" in Intel document 245471-012, "IA-32
 231                  * Intel Architecture Software Developer's Manual Volume 2:
 232                  * Instruction Set Reference"
 233                  */
 234                 CTASSERT(KDS_SEL == KCS_SEL + 8);
 235 
 236                 CTASSERT(U32CS_SEL == ((KCS_SEL + 16) | 3));
 237                 CTASSERT(UDS_SEL == U32CS_SEL + 8);
 238 #endif
 239 
 240                 cpu_sep_enable();
 241 
 242                 /*
 243                  * resume() sets this value to the base of the threads stack
 244                  * via a context handler.
 245                  */
 246                 wrmsr(MSR_INTC_SEP_ESP, 0);
 247 
 248                 if (kpti_enable == 1) {
 249                         wrmsr(MSR_INTC_SEP_EIP,
 250                             (uint64_t)(uintptr_t)tr_sys_sysenter);
 251                 } else {
 252                         wrmsr(MSR_INTC_SEP_EIP,
 253                             (uint64_t)(uintptr_t)sys_sysenter);
 254                 }
 255         }
 256 
 257         kpreempt_enable();
 258 }
 259 
 260 #if !defined(__xpv)
 261 /*
 262  * Configure per-cpu ID GDT
 263  */
 264 static void
 265 init_cpu_id_gdt(struct cpu *cp)
 266 {
 267         /* Write cpu_id into limit field of GDT for usermode retrieval */
 268 #if defined(__amd64)
 269         set_usegd(&cp->cpu_gdt[GDT_CPUID], SDP_SHORT, NULL, cp->cpu_id,
 270             SDT_MEMRODA, SEL_UPL, SDP_BYTES, SDP_OP32);
 271 #elif defined(__i386)
 272         set_usegd(&cp->cpu_gdt[GDT_CPUID], NULL, cp->cpu_id, SDT_MEMRODA,
 273             SEL_UPL, SDP_BYTES, SDP_OP32);
 274 #endif
 275 }
 276 #endif /* !defined(__xpv) */
 277 
 278 /*
 279  * Multiprocessor initialization.
 280  *
 281  * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the
 282  * startup and idle threads for the specified CPU.
 283  * Parameter boot is true for boot time operations and is false for CPU
 284  * DR operations.
 285  */
 286 static struct cpu *
 287 mp_cpu_configure_common(int cpun, boolean_t boot)
 288 {
 289         struct cpu *cp;
 290         kthread_id_t tp;
 291         caddr_t sp;
 292         proc_t *procp;
 293 #if !defined(__xpv)
 294         extern int idle_cpu_prefer_mwait;
 295         extern void cpu_idle_mwait();
 296 #endif
 297         extern void idle();
 298         extern void cpu_idle();
 299 
 300 #ifdef TRAPTRACE
 301         trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun];
 302 #endif
 303 
 304         ASSERT(MUTEX_HELD(&cpu_lock));
 305         ASSERT(cpun < NCPU && cpu[cpun] == NULL);
 306 
 307         if (cpu_free_list == NULL) {
 308                 cp = kmem_zalloc(sizeof (*cp), KM_SLEEP);
 309         } else {
 310                 cp = cpu_free_list;
 311                 cpu_free_list = cp->cpu_next_free;
 312         }
 313 
 314         cp->cpu_m.mcpu_istamp = cpun << 16;
 315 
 316         /* Create per CPU specific threads in the process p0. */
 317         procp = &p0;
 318 
 319         /*
 320          * Initialize the dispatcher first.
 321          */
 322         disp_cpu_init(cp);
 323 
 324         cpu_vm_data_init(cp);
 325 
 326         /*
 327          * Allocate and initialize the startup thread for this CPU.
 328          * Interrupt and process switch stacks get allocated later
 329          * when the CPU starts running.
 330          */
 331         tp = thread_create(NULL, 0, NULL, NULL, 0, procp,
 332             TS_STOPPED, maxclsyspri);
 333 
 334         /*
 335          * Set state to TS_ONPROC since this thread will start running
 336          * as soon as the CPU comes online.
 337          *
 338          * All the other fields of the thread structure are setup by
 339          * thread_create().
 340          */
 341         THREAD_ONPROC(tp, cp);
 342         tp->t_preempt = 1;
 343         tp->t_bound_cpu = cp;
 344         tp->t_affinitycnt = 1;
 345         tp->t_cpu = cp;
 346         tp->t_disp_queue = cp->cpu_disp;
 347 
 348         /*
 349          * Setup thread to start in mp_startup_common.
 350          */
 351         sp = tp->t_stk;
 352         tp->t_sp = (uintptr_t)(sp - MINFRAME);
 353 #if defined(__amd64)
 354         tp->t_sp -= STACK_ENTRY_ALIGN;               /* fake a call */
 355 #endif
 356         /*
 357          * Setup thread start entry point for boot or hotplug.
 358          */
 359         if (boot) {
 360                 tp->t_pc = (uintptr_t)mp_startup_boot;
 361         } else {
 362                 tp->t_pc = (uintptr_t)mp_startup_hotplug;
 363         }
 364 
 365         cp->cpu_id = cpun;
 366         cp->cpu_self = cp;
 367         cp->cpu_thread = tp;
 368         cp->cpu_lwp = NULL;
 369         cp->cpu_dispthread = tp;
 370         cp->cpu_dispatch_pri = DISP_PRIO(tp);
 371 
 372         /*
 373          * cpu_base_spl must be set explicitly here to prevent any blocking
 374          * operations in mp_startup_common from causing the spl of the cpu
 375          * to drop to 0 (allowing device interrupts before we're ready) in
 376          * resume().
 377          * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY.
 378          * As an extra bit of security on DEBUG kernels, this is enforced with
 379          * an assertion in mp_startup_common() -- before cpu_base_spl is set
 380          * to its proper value.
 381          */
 382         cp->cpu_base_spl = ipltospl(LOCK_LEVEL);
 383 
 384         /*
 385          * Now, initialize per-CPU idle thread for this CPU.
 386          */
 387         tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1);
 388 
 389         cp->cpu_idle_thread = tp;
 390 
 391         tp->t_preempt = 1;
 392         tp->t_bound_cpu = cp;
 393         tp->t_affinitycnt = 1;
 394         tp->t_cpu = cp;
 395         tp->t_disp_queue = cp->cpu_disp;
 396 
 397         /*
 398          * Bootstrap the CPU's PG data
 399          */
 400         pg_cpu_bootstrap(cp);
 401 
 402         /*
 403          * Perform CPC initialization on the new CPU.
 404          */
 405         kcpc_hw_init(cp);
 406 
 407         /*
 408          * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2
 409          * for each CPU.
 410          */
 411         setup_vaddr_for_ppcopy(cp);
 412 
 413         /*
 414          * Allocate page for new GDT and initialize from current GDT.
 415          */
 416 #if !defined(__lint)
 417         ASSERT((sizeof (*cp->cpu_gdt) * NGDT) <= PAGESIZE);
 418 #endif
 419         cp->cpu_gdt = kmem_zalloc(PAGESIZE, KM_SLEEP);
 420         bcopy(CPU->cpu_gdt, cp->cpu_gdt, (sizeof (*cp->cpu_gdt) * NGDT));
 421 
 422 #if defined(__i386)
 423         /*
 424          * setup kernel %gs.
 425          */
 426         set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA,
 427             SEL_KPL, 0, 1);
 428 #endif
 429 
 430         /*
 431          * Allocate pages for the CPU LDT.
 432          */
 433         cp->cpu_m.mcpu_ldt = kmem_zalloc(LDT_CPU_SIZE, KM_SLEEP);
 434         cp->cpu_m.mcpu_ldt_len = 0;
 435 
 436         /*
 437          * Allocate a per-CPU IDT and initialize the new IDT to the currently
 438          * runing CPU.
 439          */
 440 #if !defined(__lint)
 441         ASSERT((sizeof (*CPU->cpu_idt) * NIDT) <= PAGESIZE);
 442 #endif
 443         cp->cpu_idt = kmem_alloc(PAGESIZE, KM_SLEEP);
 444         bcopy(CPU->cpu_idt, cp->cpu_idt, PAGESIZE);
 445 
 446         /*
 447          * alloc space for cpuid info
 448          */
 449         cpuid_alloc_space(cp);
 450 #if !defined(__xpv)
 451         if (is_x86_feature(x86_featureset, X86FSET_MWAIT) &&
 452             idle_cpu_prefer_mwait) {
 453                 cp->cpu_m.mcpu_mwait = cpuid_mwait_alloc(cp);
 454                 cp->cpu_m.mcpu_idle_cpu = cpu_idle_mwait;
 455         } else
 456 #endif
 457                 cp->cpu_m.mcpu_idle_cpu = cpu_idle;
 458 
 459         init_cpu_info(cp);
 460 
 461 #if !defined(__xpv)
 462         init_cpu_id_gdt(cp);
 463 #endif
 464 
 465         /*
 466          * alloc space for ucode_info
 467          */
 468         ucode_alloc_space(cp);
 469         xc_init_cpu(cp);
 470         hat_cpu_online(cp);
 471 
 472 #ifdef TRAPTRACE
 473         /*
 474          * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers
 475          */
 476         ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP);
 477         ttc->ttc_next = ttc->ttc_first;
 478         ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize;
 479 #endif
 480 
 481         /*
 482          * Record that we have another CPU.
 483          */
 484         /*
 485          * Initialize the interrupt threads for this CPU
 486          */
 487         cpu_intr_alloc(cp, NINTR_THREADS);
 488 
 489         cp->cpu_flags = CPU_OFFLINE | CPU_QUIESCED | CPU_POWEROFF;
 490         cpu_set_state(cp);
 491 
 492         /*
 493          * Add CPU to list of available CPUs.  It'll be on the active list
 494          * after mp_startup_common().
 495          */
 496         cpu_add_unit(cp);
 497 
 498         return (cp);
 499 }
 500 
 501 /*
 502  * Undo what was done in mp_cpu_configure_common
 503  */
 504 static void
 505 mp_cpu_unconfigure_common(struct cpu *cp, int error)
 506 {
 507         ASSERT(MUTEX_HELD(&cpu_lock));
 508 
 509         /*
 510          * Remove the CPU from the list of available CPUs.
 511          */
 512         cpu_del_unit(cp->cpu_id);
 513 
 514         if (error == ETIMEDOUT) {
 515                 /*
 516                  * The cpu was started, but never *seemed* to run any
 517                  * code in the kernel; it's probably off spinning in its
 518                  * own private world, though with potential references to
 519                  * our kmem-allocated IDTs and GDTs (for example).
 520                  *
 521                  * Worse still, it may actually wake up some time later,
 522                  * so rather than guess what it might or might not do, we
 523                  * leave the fundamental data structures intact.
 524                  */
 525                 cp->cpu_flags = 0;
 526                 return;
 527         }
 528 
 529         /*
 530          * At this point, the only threads bound to this CPU should
 531          * special per-cpu threads: it's idle thread, it's pause threads,
 532          * and it's interrupt threads.  Clean these up.
 533          */
 534         cpu_destroy_bound_threads(cp);
 535         cp->cpu_idle_thread = NULL;
 536 
 537         /*
 538          * Free the interrupt stack.
 539          */
 540         segkp_release(segkp,
 541             cp->cpu_intr_stack - (INTR_STACK_SIZE - SA(MINFRAME)));
 542         cp->cpu_intr_stack = NULL;
 543 
 544 #ifdef TRAPTRACE
 545         /*
 546          * Discard the trap trace buffer
 547          */
 548         {
 549                 trap_trace_ctl_t *ttc = &trap_trace_ctl[cp->cpu_id];
 550 
 551                 kmem_free((void *)ttc->ttc_first, trap_trace_bufsize);
 552                 ttc->ttc_first = (uintptr_t)NULL;
 553         }
 554 #endif
 555 
 556         hat_cpu_offline(cp);
 557 
 558         ucode_free_space(cp);
 559 
 560         /* Free CPU ID string and brand string. */
 561         if (cp->cpu_idstr) {
 562                 kmem_free(cp->cpu_idstr, CPU_IDSTRLEN);
 563                 cp->cpu_idstr = NULL;
 564         }
 565         if (cp->cpu_brandstr) {
 566                 kmem_free(cp->cpu_brandstr, CPU_IDSTRLEN);
 567                 cp->cpu_brandstr = NULL;
 568         }
 569 
 570 #if !defined(__xpv)
 571         if (cp->cpu_m.mcpu_mwait != NULL) {
 572                 cpuid_mwait_free(cp);
 573                 cp->cpu_m.mcpu_mwait = NULL;
 574         }
 575 #endif
 576         cpuid_free_space(cp);
 577 
 578         if (cp->cpu_idt != CPU->cpu_idt)
 579                 kmem_free(cp->cpu_idt, PAGESIZE);
 580         cp->cpu_idt = NULL;
 581 
 582         kmem_free(cp->cpu_m.mcpu_ldt, LDT_CPU_SIZE);
 583         cp->cpu_m.mcpu_ldt = NULL;
 584         cp->cpu_m.mcpu_ldt_len = 0;
 585 
 586         kmem_free(cp->cpu_gdt, PAGESIZE);
 587         cp->cpu_gdt = NULL;
 588 
 589         if (cp->cpu_supp_freqs != NULL) {
 590                 size_t len = strlen(cp->cpu_supp_freqs) + 1;
 591                 kmem_free(cp->cpu_supp_freqs, len);
 592                 cp->cpu_supp_freqs = NULL;
 593         }
 594 
 595         teardown_vaddr_for_ppcopy(cp);
 596 
 597         kcpc_hw_fini(cp);
 598 
 599         cp->cpu_dispthread = NULL;
 600         cp->cpu_thread = NULL;       /* discarded by cpu_destroy_bound_threads() */
 601 
 602         cpu_vm_data_destroy(cp);
 603 
 604         xc_fini_cpu(cp);
 605         disp_cpu_fini(cp);
 606 
 607         ASSERT(cp != CPU0);
 608         bzero(cp, sizeof (*cp));
 609         cp->cpu_next_free = cpu_free_list;
 610         cpu_free_list = cp;
 611 }
 612 
 613 /*
 614  * Apply workarounds for known errata, and warn about those that are absent.
 615  *
 616  * System vendors occasionally create configurations which contain different
 617  * revisions of the CPUs that are almost but not exactly the same.  At the
 618  * time of writing, this meant that their clock rates were the same, their
 619  * feature sets were the same, but the required workaround were -not-
 620  * necessarily the same.  So, this routine is invoked on -every- CPU soon
 621  * after starting to make sure that the resulting system contains the most
 622  * pessimal set of workarounds needed to cope with *any* of the CPUs in the
 623  * system.
 624  *
 625  * workaround_errata is invoked early in mlsetup() for CPU 0, and in
 626  * mp_startup_common() for all slave CPUs. Slaves process workaround_errata
 627  * prior to acknowledging their readiness to the master, so this routine will
 628  * never be executed by multiple CPUs in parallel, thus making updates to
 629  * global data safe.
 630  *
 631  * These workarounds are based on Rev 3.57 of the Revision Guide for
 632  * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005.
 633  */
 634 
 635 #if defined(OPTERON_ERRATUM_88)
 636 int opteron_erratum_88;         /* if non-zero -> at least one cpu has it */
 637 #endif
 638 
 639 #if defined(OPTERON_ERRATUM_91)
 640 int opteron_erratum_91;         /* if non-zero -> at least one cpu has it */
 641 #endif
 642 
 643 #if defined(OPTERON_ERRATUM_93)
 644 int opteron_erratum_93;         /* if non-zero -> at least one cpu has it */
 645 #endif
 646 
 647 #if defined(OPTERON_ERRATUM_95)
 648 int opteron_erratum_95;         /* if non-zero -> at least one cpu has it */
 649 #endif
 650 
 651 #if defined(OPTERON_ERRATUM_100)
 652 int opteron_erratum_100;        /* if non-zero -> at least one cpu has it */
 653 #endif
 654 
 655 #if defined(OPTERON_ERRATUM_108)
 656 int opteron_erratum_108;        /* if non-zero -> at least one cpu has it */
 657 #endif
 658 
 659 #if defined(OPTERON_ERRATUM_109)
 660 int opteron_erratum_109;        /* if non-zero -> at least one cpu has it */
 661 #endif
 662 
 663 #if defined(OPTERON_ERRATUM_121)
 664 int opteron_erratum_121;        /* if non-zero -> at least one cpu has it */
 665 #endif
 666 
 667 #if defined(OPTERON_ERRATUM_122)
 668 int opteron_erratum_122;        /* if non-zero -> at least one cpu has it */
 669 #endif
 670 
 671 #if defined(OPTERON_ERRATUM_123)
 672 int opteron_erratum_123;        /* if non-zero -> at least one cpu has it */
 673 #endif
 674 
 675 #if defined(OPTERON_ERRATUM_131)
 676 int opteron_erratum_131;        /* if non-zero -> at least one cpu has it */
 677 #endif
 678 
 679 #if defined(OPTERON_WORKAROUND_6336786)
 680 int opteron_workaround_6336786; /* non-zero -> WA relevant and applied */
 681 int opteron_workaround_6336786_UP = 0;  /* Not needed for UP */
 682 #endif
 683 
 684 #if defined(OPTERON_WORKAROUND_6323525)
 685 int opteron_workaround_6323525; /* if non-zero -> at least one cpu has it */
 686 #endif
 687 
 688 #if defined(OPTERON_ERRATUM_298)
 689 int opteron_erratum_298;
 690 #endif
 691 
 692 #if defined(OPTERON_ERRATUM_721)
 693 int opteron_erratum_721;
 694 #endif
 695 
 696 static void
 697 workaround_warning(cpu_t *cp, uint_t erratum)
 698 {
 699         cmn_err(CE_WARN, "cpu%d: no workaround for erratum %u",
 700             cp->cpu_id, erratum);
 701 }
 702 
 703 static void
 704 workaround_applied(uint_t erratum)
 705 {
 706         if (erratum > 1000000)
 707                 cmn_err(CE_CONT, "?workaround applied for cpu issue #%d\n",
 708                     erratum);
 709         else
 710                 cmn_err(CE_CONT, "?workaround applied for cpu erratum #%d\n",
 711                     erratum);
 712 }
 713 
 714 static void
 715 msr_warning(cpu_t *cp, const char *rw, uint_t msr, int error)
 716 {
 717         cmn_err(CE_WARN, "cpu%d: couldn't %smsr 0x%x, error %d",
 718             cp->cpu_id, rw, msr, error);
 719 }
 720 
 721 /*
 722  * Determine the number of nodes in a Hammer / Greyhound / Griffin family
 723  * system.
 724  */
 725 static uint_t
 726 opteron_get_nnodes(void)
 727 {
 728         static uint_t nnodes = 0;
 729 
 730         if (nnodes == 0) {
 731 #ifdef  DEBUG
 732                 uint_t family;
 733 
 734                 /*
 735                  * This routine uses a PCI config space based mechanism
 736                  * for retrieving the number of nodes in the system.
 737                  * Device 24, function 0, offset 0x60 as used here is not
 738                  * AMD processor architectural, and may not work on processor
 739                  * families other than those listed below.
 740                  *
 741                  * Callers of this routine must ensure that we're running on
 742                  * a processor which supports this mechanism.
 743                  * The assertion below is meant to catch calls on unsupported
 744                  * processors.
 745                  */
 746                 family = cpuid_getfamily(CPU);
 747                 ASSERT(family == 0xf || family == 0x10 || family == 0x11);
 748 #endif  /* DEBUG */
 749 
 750                 /*
 751                  * Obtain the number of nodes in the system from
 752                  * bits [6:4] of the Node ID register on node 0.
 753                  *
 754                  * The actual node count is NodeID[6:4] + 1
 755                  *
 756                  * The Node ID register is accessed via function 0,
 757                  * offset 0x60. Node 0 is device 24.
 758                  */
 759                 nnodes = ((pci_getl_func(0, 24, 0, 0x60) & 0x70) >> 4) + 1;
 760         }
 761         return (nnodes);
 762 }
 763 
 764 uint_t
 765 do_erratum_298(struct cpu *cpu)
 766 {
 767         static int      osvwrc = -3;
 768         extern int      osvw_opteron_erratum(cpu_t *, uint_t);
 769 
 770         /*
 771          * L2 Eviction May Occur During Processor Operation To Set
 772          * Accessed or Dirty Bit.
 773          */
 774         if (osvwrc == -3) {
 775                 osvwrc = osvw_opteron_erratum(cpu, 298);
 776         } else {
 777                 /* osvw return codes should be consistent for all cpus */
 778                 ASSERT(osvwrc == osvw_opteron_erratum(cpu, 298));
 779         }
 780 
 781         switch (osvwrc) {
 782         case 0:         /* erratum is not present: do nothing */
 783                 break;
 784         case 1:         /* erratum is present: BIOS workaround applied */
 785                 /*
 786                  * check if workaround is actually in place and issue warning
 787                  * if not.
 788                  */
 789                 if (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
 790                     ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0)) {
 791 #if defined(OPTERON_ERRATUM_298)
 792                         opteron_erratum_298++;
 793 #else
 794                         workaround_warning(cpu, 298);
 795                         return (1);
 796 #endif
 797                 }
 798                 break;
 799         case -1:        /* cannot determine via osvw: check cpuid */
 800                 if ((cpuid_opteron_erratum(cpu, 298) > 0) &&
 801                     (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
 802                     ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0))) {
 803 #if defined(OPTERON_ERRATUM_298)
 804                         opteron_erratum_298++;
 805 #else
 806                         workaround_warning(cpu, 298);
 807                         return (1);
 808 #endif
 809                 }
 810                 break;
 811         }
 812         return (0);
 813 }
 814 
 815 uint_t
 816 workaround_errata(struct cpu *cpu)
 817 {
 818         uint_t missing = 0;
 819 
 820         ASSERT(cpu == CPU);
 821 
 822         /*LINTED*/
 823         if (cpuid_opteron_erratum(cpu, 88) > 0) {
 824                 /*
 825                  * SWAPGS May Fail To Read Correct GS Base
 826                  */
 827 #if defined(OPTERON_ERRATUM_88)
 828                 /*
 829                  * The workaround is an mfence in the relevant assembler code
 830                  */
 831                 opteron_erratum_88++;
 832 #else
 833                 workaround_warning(cpu, 88);
 834                 missing++;
 835 #endif
 836         }
 837 
 838         if (cpuid_opteron_erratum(cpu, 91) > 0) {
 839                 /*
 840                  * Software Prefetches May Report A Page Fault
 841                  */
 842 #if defined(OPTERON_ERRATUM_91)
 843                 /*
 844                  * fix is in trap.c
 845                  */
 846                 opteron_erratum_91++;
 847 #else
 848                 workaround_warning(cpu, 91);
 849                 missing++;
 850 #endif
 851         }
 852 
 853         if (cpuid_opteron_erratum(cpu, 93) > 0) {
 854                 /*
 855                  * RSM Auto-Halt Restart Returns to Incorrect RIP
 856                  */
 857 #if defined(OPTERON_ERRATUM_93)
 858                 /*
 859                  * fix is in trap.c
 860                  */
 861                 opteron_erratum_93++;
 862 #else
 863                 workaround_warning(cpu, 93);
 864                 missing++;
 865 #endif
 866         }
 867 
 868         /*LINTED*/
 869         if (cpuid_opteron_erratum(cpu, 95) > 0) {
 870                 /*
 871                  * RET Instruction May Return to Incorrect EIP
 872                  */
 873 #if defined(OPTERON_ERRATUM_95)
 874 #if defined(_LP64)
 875                 /*
 876                  * Workaround this by ensuring that 32-bit user code and
 877                  * 64-bit kernel code never occupy the same address
 878                  * range mod 4G.
 879                  */
 880                 if (_userlimit32 > 0xc0000000ul)
 881                         *(uintptr_t *)&_userlimit32 = 0xc0000000ul;
 882 
 883                 /*LINTED*/
 884                 ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u);
 885                 opteron_erratum_95++;
 886 #endif  /* _LP64 */
 887 #else
 888                 workaround_warning(cpu, 95);
 889                 missing++;
 890 #endif
 891         }
 892 
 893         if (cpuid_opteron_erratum(cpu, 100) > 0) {
 894                 /*
 895                  * Compatibility Mode Branches Transfer to Illegal Address
 896                  */
 897 #if defined(OPTERON_ERRATUM_100)
 898                 /*
 899                  * fix is in trap.c
 900                  */
 901                 opteron_erratum_100++;
 902 #else
 903                 workaround_warning(cpu, 100);
 904                 missing++;
 905 #endif
 906         }
 907 
 908         /*LINTED*/
 909         if (cpuid_opteron_erratum(cpu, 108) > 0) {
 910                 /*
 911                  * CPUID Instruction May Return Incorrect Model Number In
 912                  * Some Processors
 913                  */
 914 #if defined(OPTERON_ERRATUM_108)
 915                 /*
 916                  * (Our cpuid-handling code corrects the model number on
 917                  * those processors)
 918                  */
 919 #else
 920                 workaround_warning(cpu, 108);
 921                 missing++;
 922 #endif
 923         }
 924 
 925         /*LINTED*/
 926         if (cpuid_opteron_erratum(cpu, 109) > 0) do {
 927                 /*
 928                  * Certain Reverse REP MOVS May Produce Unpredictable Behavior
 929                  */
 930 #if defined(OPTERON_ERRATUM_109)
 931                 /*
 932                  * The "workaround" is to print a warning to upgrade the BIOS
 933                  */
 934                 uint64_t value;
 935                 const uint_t msr = MSR_AMD_PATCHLEVEL;
 936                 int err;
 937 
 938                 if ((err = checked_rdmsr(msr, &value)) != 0) {
 939                         msr_warning(cpu, "rd", msr, err);
 940                         workaround_warning(cpu, 109);
 941                         missing++;
 942                 }
 943                 if (value == 0)
 944                         opteron_erratum_109++;
 945 #else
 946                 workaround_warning(cpu, 109);
 947                 missing++;
 948 #endif
 949         /*CONSTANTCONDITION*/
 950         } while (0);
 951 
 952         /*LINTED*/
 953         if (cpuid_opteron_erratum(cpu, 121) > 0) {
 954                 /*
 955                  * Sequential Execution Across Non_Canonical Boundary Caused
 956                  * Processor Hang
 957                  */
 958 #if defined(OPTERON_ERRATUM_121)
 959 #if defined(_LP64)
 960                 /*
 961                  * Erratum 121 is only present in long (64 bit) mode.
 962                  * Workaround is to include the page immediately before the
 963                  * va hole to eliminate the possibility of system hangs due to
 964                  * sequential execution across the va hole boundary.
 965                  */
 966                 if (opteron_erratum_121)
 967                         opteron_erratum_121++;
 968                 else {
 969                         if (hole_start) {
 970                                 hole_start -= PAGESIZE;
 971                         } else {
 972                                 /*
 973                                  * hole_start not yet initialized by
 974                                  * mmu_init. Initialize hole_start
 975                                  * with value to be subtracted.
 976                                  */
 977                                 hole_start = PAGESIZE;
 978                         }
 979                         opteron_erratum_121++;
 980                 }
 981 #endif  /* _LP64 */
 982 #else
 983                 workaround_warning(cpu, 121);
 984                 missing++;
 985 #endif
 986         }
 987 
 988         /*LINTED*/
 989         if (cpuid_opteron_erratum(cpu, 122) > 0) do {
 990                 /*
 991                  * TLB Flush Filter May Cause Coherency Problem in
 992                  * Multiprocessor Systems
 993                  */
 994 #if defined(OPTERON_ERRATUM_122)
 995                 uint64_t value;
 996                 const uint_t msr = MSR_AMD_HWCR;
 997                 int error;
 998 
 999                 /*
1000                  * Erratum 122 is only present in MP configurations (multi-core
1001                  * or multi-processor).
1002                  */
1003 #if defined(__xpv)
1004                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1005                         break;
1006                 if (!opteron_erratum_122 && xpv_nr_phys_cpus() == 1)
1007                         break;
1008 #else
1009                 if (!opteron_erratum_122 && opteron_get_nnodes() == 1 &&
1010                     cpuid_get_ncpu_per_chip(cpu) == 1)
1011                         break;
1012 #endif
1013                 /* disable TLB Flush Filter */
1014 
1015                 if ((error = checked_rdmsr(msr, &value)) != 0) {
1016                         msr_warning(cpu, "rd", msr, error);
1017                         workaround_warning(cpu, 122);
1018                         missing++;
1019                 } else {
1020                         value |= (uint64_t)AMD_HWCR_FFDIS;
1021                         if ((error = checked_wrmsr(msr, value)) != 0) {
1022                                 msr_warning(cpu, "wr", msr, error);
1023                                 workaround_warning(cpu, 122);
1024                                 missing++;
1025                         }
1026                 }
1027                 opteron_erratum_122++;
1028 #else
1029                 workaround_warning(cpu, 122);
1030                 missing++;
1031 #endif
1032         /*CONSTANTCONDITION*/
1033         } while (0);
1034 
1035         /*LINTED*/
1036         if (cpuid_opteron_erratum(cpu, 123) > 0) do {
1037                 /*
1038                  * Bypassed Reads May Cause Data Corruption of System Hang in
1039                  * Dual Core Processors
1040                  */
1041 #if defined(OPTERON_ERRATUM_123)
1042                 uint64_t value;
1043                 const uint_t msr = MSR_AMD_PATCHLEVEL;
1044                 int err;
1045 
1046                 /*
1047                  * Erratum 123 applies only to multi-core cpus.
1048                  */
1049                 if (cpuid_get_ncpu_per_chip(cpu) < 2)
1050                         break;
1051 #if defined(__xpv)
1052                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1053                         break;
1054 #endif
1055                 /*
1056                  * The "workaround" is to print a warning to upgrade the BIOS
1057                  */
1058                 if ((err = checked_rdmsr(msr, &value)) != 0) {
1059                         msr_warning(cpu, "rd", msr, err);
1060                         workaround_warning(cpu, 123);
1061                         missing++;
1062                 }
1063                 if (value == 0)
1064                         opteron_erratum_123++;
1065 #else
1066                 workaround_warning(cpu, 123);
1067                 missing++;
1068 
1069 #endif
1070         /*CONSTANTCONDITION*/
1071         } while (0);
1072 
1073         /*LINTED*/
1074         if (cpuid_opteron_erratum(cpu, 131) > 0) do {
1075                 /*
1076                  * Multiprocessor Systems with Four or More Cores May Deadlock
1077                  * Waiting for a Probe Response
1078                  */
1079 #if defined(OPTERON_ERRATUM_131)
1080                 uint64_t nbcfg;
1081                 const uint_t msr = MSR_AMD_NB_CFG;
1082                 const uint64_t wabits =
1083                     AMD_NB_CFG_SRQ_HEARTBEAT | AMD_NB_CFG_SRQ_SPR;
1084                 int error;
1085 
1086                 /*
1087                  * Erratum 131 applies to any system with four or more cores.
1088                  */
1089                 if (opteron_erratum_131)
1090                         break;
1091 #if defined(__xpv)
1092                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1093                         break;
1094                 if (xpv_nr_phys_cpus() < 4)
1095                         break;
1096 #else
1097                 if (opteron_get_nnodes() * cpuid_get_ncpu_per_chip(cpu) < 4)
1098                         break;
1099 #endif
1100                 /*
1101                  * Print a warning if neither of the workarounds for
1102                  * erratum 131 is present.
1103                  */
1104                 if ((error = checked_rdmsr(msr, &nbcfg)) != 0) {
1105                         msr_warning(cpu, "rd", msr, error);
1106                         workaround_warning(cpu, 131);
1107                         missing++;
1108                 } else if ((nbcfg & wabits) == 0) {
1109                         opteron_erratum_131++;
1110                 } else {
1111                         /* cannot have both workarounds set */
1112                         ASSERT((nbcfg & wabits) != wabits);
1113                 }
1114 #else
1115                 workaround_warning(cpu, 131);
1116                 missing++;
1117 #endif
1118         /*CONSTANTCONDITION*/
1119         } while (0);
1120 
1121         /*
1122          * This isn't really an erratum, but for convenience the
1123          * detection/workaround code lives here and in cpuid_opteron_erratum.
1124          * Note, the technique only is valid on families before 12h and
1125          * certainly doesn't work when we're virtualized. This is checked for in
1126          * the erratum workaround.
1127          */
1128         if (cpuid_opteron_erratum(cpu, 6336786) > 0) {
1129 #if defined(OPTERON_WORKAROUND_6336786)
1130                 /*
1131                  * Disable C1-Clock ramping on multi-core/multi-processor
1132                  * K8 platforms to guard against TSC drift.
1133                  */
1134                 if (opteron_workaround_6336786) {
1135                         opteron_workaround_6336786++;
1136 #if defined(__xpv)
1137                 } else if ((DOMAIN_IS_INITDOMAIN(xen_info) &&
1138                     xpv_nr_phys_cpus() > 1) ||
1139                     opteron_workaround_6336786_UP) {
1140                         /*
1141                          * XXPV Hmm.  We can't walk the Northbridges on
1142                          *      the hypervisor; so just complain and drive
1143                          *      on.  This probably needs to be fixed in
1144                          *      the hypervisor itself.
1145                          */
1146                         opteron_workaround_6336786++;
1147                         workaround_warning(cpu, 6336786);
1148 #else   /* __xpv */
1149                 } else if ((opteron_get_nnodes() *
1150                     cpuid_get_ncpu_per_chip(cpu) > 1) ||
1151                     opteron_workaround_6336786_UP) {
1152 
1153                         uint_t  node, nnodes;
1154                         uint8_t data;
1155 
1156                         nnodes = opteron_get_nnodes();
1157                         for (node = 0; node < nnodes; node++) {
1158                                 /*
1159                                  * Clear PMM7[1:0] (function 3, offset 0x87)
1160                                  * Northbridge device is the node id + 24.
1161                                  */
1162                                 data = pci_getb_func(0, node + 24, 3, 0x87);
1163                                 data &= 0xFC;
1164                                 pci_putb_func(0, node + 24, 3, 0x87, data);
1165                         }
1166                         opteron_workaround_6336786++;
1167 #endif  /* __xpv */
1168                 }
1169 #else
1170                 workaround_warning(cpu, 6336786);
1171                 missing++;
1172 #endif
1173         }
1174 
1175         /*LINTED*/
1176         /*
1177          * Mutex primitives don't work as expected. This is erratum #147 from
1178          * 'Revision Guide for AMD Athlon 64 and AMD Opteron Processors'
1179          * document 25759.
1180          */
1181         if (cpuid_opteron_erratum(cpu, 6323525) > 0) {
1182 #if defined(OPTERON_WORKAROUND_6323525)
1183                 /*
1184                  * This problem only occurs with 2 or more cores. If bit in
1185                  * MSR_AMD_BU_CFG set, then not applicable. The workaround
1186                  * is to patch the semaphone routines with the lfence
1187                  * instruction to provide necessary load memory barrier with
1188                  * possible subsequent read-modify-write ops.
1189                  *
1190                  * It is too early in boot to call the patch routine so
1191                  * set erratum variable to be done in startup_end().
1192                  */
1193                 if (opteron_workaround_6323525) {
1194                         opteron_workaround_6323525++;
1195 #if defined(__xpv)
1196                 } else if (is_x86_feature(x86_featureset, X86FSET_SSE2)) {
1197                         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1198                                 /*
1199                                  * XXPV Use dom0_msr here when extended
1200                                  *      operations are supported?
1201                                  */
1202                                 if (xpv_nr_phys_cpus() > 1)
1203                                         opteron_workaround_6323525++;
1204                         } else {
1205                                 /*
1206                                  * We have no way to tell how many physical
1207                                  * cpus there are, or even if this processor
1208                                  * has the problem, so enable the workaround
1209                                  * unconditionally (at some performance cost).
1210                                  */
1211                                 opteron_workaround_6323525++;
1212                         }
1213 #else   /* __xpv */
1214                 } else if (is_x86_feature(x86_featureset, X86FSET_SSE2) &&
1215                     ((opteron_get_nnodes() *
1216                     cpuid_get_ncpu_per_chip(cpu)) > 1)) {
1217                         if ((xrdmsr(MSR_AMD_BU_CFG) & (UINT64_C(1) << 33)) == 0)
1218                                 opteron_workaround_6323525++;
1219 #endif  /* __xpv */
1220                 }
1221 #else
1222                 workaround_warning(cpu, 6323525);
1223                 missing++;
1224 #endif
1225         }
1226 
1227         missing += do_erratum_298(cpu);
1228 
1229         if (cpuid_opteron_erratum(cpu, 721) > 0) {
1230 #if defined(OPTERON_ERRATUM_721)
1231                 on_trap_data_t otd;
1232 
1233                 if (!on_trap(&otd, OT_DATA_ACCESS))
1234                         wrmsr(MSR_AMD_DE_CFG,
1235                             rdmsr(MSR_AMD_DE_CFG) | AMD_DE_CFG_E721);
1236                 no_trap();
1237 
1238                 opteron_erratum_721++;
1239 #else
1240                 workaround_warning(cpu, 721);
1241                 missing++;
1242 #endif
1243         }
1244 
1245 #ifdef __xpv
1246         return (0);
1247 #else
1248         return (missing);
1249 #endif
1250 }
1251 
1252 void
1253 workaround_errata_end()
1254 {
1255 #if defined(OPTERON_ERRATUM_88)
1256         if (opteron_erratum_88)
1257                 workaround_applied(88);
1258 #endif
1259 #if defined(OPTERON_ERRATUM_91)
1260         if (opteron_erratum_91)
1261                 workaround_applied(91);
1262 #endif
1263 #if defined(OPTERON_ERRATUM_93)
1264         if (opteron_erratum_93)
1265                 workaround_applied(93);
1266 #endif
1267 #if defined(OPTERON_ERRATUM_95)
1268         if (opteron_erratum_95)
1269                 workaround_applied(95);
1270 #endif
1271 #if defined(OPTERON_ERRATUM_100)
1272         if (opteron_erratum_100)
1273                 workaround_applied(100);
1274 #endif
1275 #if defined(OPTERON_ERRATUM_108)
1276         if (opteron_erratum_108)
1277                 workaround_applied(108);
1278 #endif
1279 #if defined(OPTERON_ERRATUM_109)
1280         if (opteron_erratum_109) {
1281                 cmn_err(CE_WARN,
1282                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1283                     " processor\nerratum 109 was not detected; updating your"
1284                     " system's BIOS to a version\ncontaining this"
1285                     " microcode patch is HIGHLY recommended or erroneous"
1286                     " system\noperation may occur.\n");
1287         }
1288 #endif
1289 #if defined(OPTERON_ERRATUM_121)
1290         if (opteron_erratum_121)
1291                 workaround_applied(121);
1292 #endif
1293 #if defined(OPTERON_ERRATUM_122)
1294         if (opteron_erratum_122)
1295                 workaround_applied(122);
1296 #endif
1297 #if defined(OPTERON_ERRATUM_123)
1298         if (opteron_erratum_123) {
1299                 cmn_err(CE_WARN,
1300                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1301                     " processor\nerratum 123 was not detected; updating your"
1302                     " system's BIOS to a version\ncontaining this"
1303                     " microcode patch is HIGHLY recommended or erroneous"
1304                     " system\noperation may occur.\n");
1305         }
1306 #endif
1307 #if defined(OPTERON_ERRATUM_131)
1308         if (opteron_erratum_131) {
1309                 cmn_err(CE_WARN,
1310                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1311                     " processor\nerratum 131 was not detected; updating your"
1312                     " system's BIOS to a version\ncontaining this"
1313                     " microcode patch is HIGHLY recommended or erroneous"
1314                     " system\noperation may occur.\n");
1315         }
1316 #endif
1317 #if defined(OPTERON_WORKAROUND_6336786)
1318         if (opteron_workaround_6336786)
1319                 workaround_applied(6336786);
1320 #endif
1321 #if defined(OPTERON_WORKAROUND_6323525)
1322         if (opteron_workaround_6323525)
1323                 workaround_applied(6323525);
1324 #endif
1325 #if defined(OPTERON_ERRATUM_298)
1326         if (opteron_erratum_298) {
1327                 cmn_err(CE_WARN,
1328                     "BIOS microcode patch for AMD 64/Opteron(tm)"
1329                     " processor\nerratum 298 was not detected; updating your"
1330                     " system's BIOS to a version\ncontaining this"
1331                     " microcode patch is HIGHLY recommended or erroneous"
1332                     " system\noperation may occur.\n");
1333         }
1334 #endif
1335 #if defined(OPTERON_ERRATUM_721)
1336         if (opteron_erratum_721)
1337                 workaround_applied(721);
1338 #endif
1339 }
1340 
1341 /*
1342  * The procset_slave and procset_master are used to synchronize
1343  * between the control CPU and the target CPU when starting CPUs.
1344  */
1345 static cpuset_t procset_slave, procset_master;
1346 
1347 static void
1348 mp_startup_wait(cpuset_t *sp, processorid_t cpuid)
1349 {
1350         cpuset_t tempset;
1351 
1352         for (tempset = *sp; !CPU_IN_SET(tempset, cpuid);
1353             tempset = *(volatile cpuset_t *)sp) {
1354                 SMT_PAUSE();
1355         }
1356         CPUSET_ATOMIC_DEL(*(cpuset_t *)sp, cpuid);
1357 }
1358 
1359 static void
1360 mp_startup_signal(cpuset_t *sp, processorid_t cpuid)
1361 {
1362         cpuset_t tempset;
1363 
1364         CPUSET_ATOMIC_ADD(*(cpuset_t *)sp, cpuid);
1365         for (tempset = *sp; CPU_IN_SET(tempset, cpuid);
1366             tempset = *(volatile cpuset_t *)sp) {
1367                 SMT_PAUSE();
1368         }
1369 }
1370 
1371 int
1372 mp_start_cpu_common(cpu_t *cp, boolean_t boot)
1373 {
1374         _NOTE(ARGUNUSED(boot));
1375 
1376         void *ctx;
1377         int delays;
1378         int error = 0;
1379         cpuset_t tempset;
1380         processorid_t cpuid;
1381 #ifndef __xpv
1382         extern void cpupm_init(cpu_t *);
1383 #endif
1384 
1385         ASSERT(cp != NULL);
1386         cpuid = cp->cpu_id;
1387         ctx = mach_cpucontext_alloc(cp);
1388         if (ctx == NULL) {
1389                 cmn_err(CE_WARN,
1390                     "cpu%d: failed to allocate context", cp->cpu_id);
1391                 return (EAGAIN);
1392         }
1393         error = mach_cpu_start(cp, ctx);
1394         if (error != 0) {
1395                 cmn_err(CE_WARN,
1396                     "cpu%d: failed to start, error %d", cp->cpu_id, error);
1397                 mach_cpucontext_free(cp, ctx, error);
1398                 return (error);
1399         }
1400 
1401         for (delays = 0, tempset = procset_slave; !CPU_IN_SET(tempset, cpuid);
1402             delays++) {
1403                 if (delays == 500) {
1404                         /*
1405                          * After five seconds, things are probably looking
1406                          * a bit bleak - explain the hang.
1407                          */
1408                         cmn_err(CE_NOTE, "cpu%d: started, "
1409                             "but not running in the kernel yet", cpuid);
1410                 } else if (delays > 2000) {
1411                         /*
1412                          * We waited at least 20 seconds, bail ..
1413                          */
1414                         error = ETIMEDOUT;
1415                         cmn_err(CE_WARN, "cpu%d: timed out", cpuid);
1416                         mach_cpucontext_free(cp, ctx, error);
1417                         return (error);
1418                 }
1419 
1420                 /*
1421                  * wait at least 10ms, then check again..
1422                  */
1423                 delay(USEC_TO_TICK_ROUNDUP(10000));
1424                 tempset = *((volatile cpuset_t *)&procset_slave);
1425         }
1426         CPUSET_ATOMIC_DEL(procset_slave, cpuid);
1427 
1428         mach_cpucontext_free(cp, ctx, 0);
1429 
1430 #ifndef __xpv
1431         if (tsc_gethrtime_enable)
1432                 tsc_sync_master(cpuid);
1433 #endif
1434 
1435         if (dtrace_cpu_init != NULL) {
1436                 (*dtrace_cpu_init)(cpuid);
1437         }
1438 
1439         /*
1440          * During CPU DR operations, the cpu_lock is held by current
1441          * (the control) thread. We can't release the cpu_lock here
1442          * because that will break the CPU DR logic.
1443          * On the other hand, CPUPM and processor group initialization
1444          * routines need to access the cpu_lock. So we invoke those
1445          * routines here on behalf of mp_startup_common().
1446          *
1447          * CPUPM and processor group initialization routines depend
1448          * on the cpuid probing results. Wait for mp_startup_common()
1449          * to signal that cpuid probing is done.
1450          */
1451         mp_startup_wait(&procset_slave, cpuid);
1452 #ifndef __xpv
1453         cpupm_init(cp);
1454 #endif
1455         (void) pg_cpu_init(cp, B_FALSE);
1456         cpu_set_state(cp);
1457         mp_startup_signal(&procset_master, cpuid);
1458 
1459         return (0);
1460 }
1461 
1462 /*
1463  * Start a single cpu, assuming that the kernel context is available
1464  * to successfully start another cpu.
1465  *
1466  * (For example, real mode code is mapped into the right place
1467  * in memory and is ready to be run.)
1468  */
1469 int
1470 start_cpu(processorid_t who)
1471 {
1472         cpu_t *cp;
1473         int error = 0;
1474         cpuset_t tempset;
1475 
1476         ASSERT(who != 0);
1477 
1478         /*
1479          * Check if there's at least a Mbyte of kmem available
1480          * before attempting to start the cpu.
1481          */
1482         if (kmem_avail() < 1024 * 1024) {
1483                 /*
1484                  * Kick off a reap in case that helps us with
1485                  * later attempts ..
1486                  */
1487                 kmem_reap();
1488                 return (ENOMEM);
1489         }
1490 
1491         /*
1492          * First configure cpu.
1493          */
1494         cp = mp_cpu_configure_common(who, B_TRUE);
1495         ASSERT(cp != NULL);
1496 
1497         /*
1498          * Then start cpu.
1499          */
1500         error = mp_start_cpu_common(cp, B_TRUE);
1501         if (error != 0) {
1502                 mp_cpu_unconfigure_common(cp, error);
1503                 return (error);
1504         }
1505 
1506         mutex_exit(&cpu_lock);
1507         tempset = cpu_ready_set;
1508         while (!CPU_IN_SET(tempset, who)) {
1509                 drv_usecwait(1);
1510                 tempset = *((volatile cpuset_t *)&cpu_ready_set);
1511         }
1512         mutex_enter(&cpu_lock);
1513 
1514         return (0);
1515 }
1516 
1517 void
1518 start_other_cpus(int cprboot)
1519 {
1520         _NOTE(ARGUNUSED(cprboot));
1521 
1522         uint_t who;
1523         uint_t bootcpuid = 0;
1524 
1525         /*
1526          * Initialize our own cpu_info.
1527          */
1528         init_cpu_info(CPU);
1529 
1530 #if !defined(__xpv)
1531         init_cpu_id_gdt(CPU);
1532 #endif
1533 
1534         cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_idstr);
1535         cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_brandstr);
1536 
1537         /*
1538          * KPTI initialisation happens very early in boot, before logging is
1539          * set up. Output a status message now as the boot CPU comes online.
1540          */
1541         cmn_err(CE_CONT, "?KPTI %s (PCID %s, INVPCID %s)\n",
1542             kpti_enable ? "enabled" : "disabled",
1543             x86_use_pcid == 1 ? "in use" :
1544             (is_x86_feature(x86_featureset, X86FSET_PCID) ? "disabled" :
1545             "not supported"),
1546             x86_use_pcid == 1 && x86_use_invpcid == 1 ? "in use" :
1547             (is_x86_feature(x86_featureset, X86FSET_INVPCID) ? "disabled" :
1548             "not supported"));
1549 
1550         /*
1551          * Initialize our syscall handlers
1552          */
1553         init_cpu_syscall(CPU);
1554 
1555         /*
1556          * Take the boot cpu out of the mp_cpus set because we know
1557          * it's already running.  Add it to the cpu_ready_set for
1558          * precisely the same reason.
1559          */
1560         CPUSET_DEL(mp_cpus, bootcpuid);
1561         CPUSET_ADD(cpu_ready_set, bootcpuid);
1562 
1563         /*
1564          * skip the rest of this if
1565          * . only 1 cpu dectected and system isn't hotplug-capable
1566          * . not using MP
1567          */
1568         if ((CPUSET_ISNULL(mp_cpus) && plat_dr_support_cpu() == 0) ||
1569             use_mp == 0) {
1570                 if (use_mp == 0)
1571                         cmn_err(CE_CONT, "?***** Not in MP mode\n");
1572                 goto done;
1573         }
1574 
1575         /*
1576          * perform such initialization as is needed
1577          * to be able to take CPUs on- and off-line.
1578          */
1579         cpu_pause_init();
1580 
1581         xc_init_cpu(CPU);               /* initialize processor crosscalls */
1582 
1583         if (mach_cpucontext_init() != 0)
1584                 goto done;
1585 
1586         flushes_require_xcalls = 1;
1587 
1588         /*
1589          * We lock our affinity to the master CPU to ensure that all slave CPUs
1590          * do their TSC syncs with the same CPU.
1591          */
1592         affinity_set(CPU_CURRENT);
1593 
1594         for (who = 0; who < NCPU; who++) {
1595                 if (!CPU_IN_SET(mp_cpus, who))
1596                         continue;
1597                 ASSERT(who != bootcpuid);
1598 
1599                 mutex_enter(&cpu_lock);
1600                 if (start_cpu(who) != 0)
1601                         CPUSET_DEL(mp_cpus, who);
1602                 cpu_state_change_notify(who, CPU_SETUP);
1603                 mutex_exit(&cpu_lock);
1604         }
1605 
1606         /* Free the space allocated to hold the microcode file */
1607         ucode_cleanup();
1608 
1609         affinity_clear();
1610 
1611         mach_cpucontext_fini();
1612 
1613 done:
1614         if (get_hwenv() == HW_NATIVE)
1615                 workaround_errata_end();
1616         cmi_post_mpstartup();
1617 
1618         if (use_mp && ncpus != boot_max_ncpus) {
1619                 cmn_err(CE_NOTE,
1620                     "System detected %d cpus, but "
1621                     "only %d cpu(s) were enabled during boot.",
1622                     boot_max_ncpus, ncpus);
1623                 cmn_err(CE_NOTE,
1624                     "Use \"boot-ncpus\" parameter to enable more CPU(s). "
1625                     "See eeprom(1M).");
1626         }
1627 }
1628 
1629 int
1630 mp_cpu_configure(int cpuid)
1631 {
1632         cpu_t *cp;
1633 
1634         if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1635                 return (ENOTSUP);
1636         }
1637 
1638         cp = cpu_get(cpuid);
1639         if (cp != NULL) {
1640                 return (EALREADY);
1641         }
1642 
1643         /*
1644          * Check if there's at least a Mbyte of kmem available
1645          * before attempting to start the cpu.
1646          */
1647         if (kmem_avail() < 1024 * 1024) {
1648                 /*
1649                  * Kick off a reap in case that helps us with
1650                  * later attempts ..
1651                  */
1652                 kmem_reap();
1653                 return (ENOMEM);
1654         }
1655 
1656         cp = mp_cpu_configure_common(cpuid, B_FALSE);
1657         ASSERT(cp != NULL && cpu_get(cpuid) == cp);
1658 
1659         return (cp != NULL ? 0 : EAGAIN);
1660 }
1661 
1662 int
1663 mp_cpu_unconfigure(int cpuid)
1664 {
1665         cpu_t *cp;
1666 
1667         if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1668                 return (ENOTSUP);
1669         } else if (cpuid < 0 || cpuid >= max_ncpus) {
1670                 return (EINVAL);
1671         }
1672 
1673         cp = cpu_get(cpuid);
1674         if (cp == NULL) {
1675                 return (ENODEV);
1676         }
1677         mp_cpu_unconfigure_common(cp, 0);
1678 
1679         return (0);
1680 }
1681 
1682 /*
1683  * Startup function for 'other' CPUs (besides boot cpu).
1684  * Called from real_mode_start.
1685  *
1686  * WARNING: until CPU_READY is set, mp_startup_common and routines called by
1687  * mp_startup_common should not call routines (e.g. kmem_free) that could call
1688  * hat_unload which requires CPU_READY to be set.
1689  */
1690 static void
1691 mp_startup_common(boolean_t boot)
1692 {
1693         cpu_t *cp = CPU;
1694         uchar_t new_x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
1695         extern void cpu_event_init_cpu(cpu_t *);
1696 
1697         /*
1698          * We need to get TSC on this proc synced (i.e., any delta
1699          * from cpu0 accounted for) as soon as we can, because many
1700          * many things use gethrtime/pc_gethrestime, including
1701          * interrupts, cmn_err, etc.  Before we can do that, we want to
1702          * clear TSC if we're on a buggy Sandy/Ivy Bridge CPU, so do that
1703          * right away.
1704          */
1705         bzero(new_x86_featureset, BT_SIZEOFMAP(NUM_X86_FEATURES));
1706         cpuid_pass1(cp, new_x86_featureset);
1707 
1708         if (boot && get_hwenv() == HW_NATIVE &&
1709             cpuid_getvendor(CPU) == X86_VENDOR_Intel &&
1710             cpuid_getfamily(CPU) == 6 &&
1711             (cpuid_getmodel(CPU) == 0x2d || cpuid_getmodel(CPU) == 0x3e) &&
1712             is_x86_feature(new_x86_featureset, X86FSET_TSC)) {
1713                 (void) wrmsr(REG_TSC, 0UL);
1714         }
1715 
1716         /* Let the control CPU continue into tsc_sync_master() */
1717         mp_startup_signal(&procset_slave, cp->cpu_id);
1718 
1719 #ifndef __xpv
1720         if (tsc_gethrtime_enable)
1721                 tsc_sync_slave();
1722 #endif
1723 
1724         /*
1725          * Once this was done from assembly, but it's safer here; if
1726          * it blocks, we need to be able to swtch() to and from, and
1727          * since we get here by calling t_pc, we need to do that call
1728          * before swtch() overwrites it.
1729          */
1730         (void) (*ap_mlsetup)();
1731 
1732 #ifndef __xpv
1733         /*
1734          * Program this cpu's PAT
1735          */
1736         pat_sync();
1737 #endif
1738 
1739         /*
1740          * Set up TSC_AUX to contain the cpuid for this processor
1741          * for the rdtscp instruction.
1742          */
1743         if (is_x86_feature(x86_featureset, X86FSET_TSCP))
1744                 (void) wrmsr(MSR_AMD_TSCAUX, cp->cpu_id);
1745 
1746         /*
1747          * Initialize this CPU's syscall handlers
1748          */
1749         init_cpu_syscall(cp);
1750 
1751         /*
1752          * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the
1753          * highest level at which a routine is permitted to block on
1754          * an adaptive mutex (allows for cpu poke interrupt in case
1755          * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks
1756          * device interrupts that may end up in the hat layer issuing cross
1757          * calls before CPU_READY is set.
1758          */
1759         splx(ipltospl(LOCK_LEVEL));
1760         sti();
1761 
1762         /*
1763          * There exists a small subset of systems which expose differing
1764          * MWAIT/MONITOR support between CPUs.  If MWAIT support is absent from
1765          * the boot CPU, but is found on a later CPU, the system continues to
1766          * operate as if no MWAIT support is available.
1767          *
1768          * The reverse case, where MWAIT is available on the boot CPU but not
1769          * on a subsequently initialized CPU, is not presently allowed and will
1770          * result in a panic.
1771          */
1772         if (is_x86_feature(x86_featureset, X86FSET_MWAIT) !=
1773             is_x86_feature(new_x86_featureset, X86FSET_MWAIT)) {
1774                 if (!is_x86_feature(x86_featureset, X86FSET_MWAIT)) {
1775                         remove_x86_feature(new_x86_featureset, X86FSET_MWAIT);
1776                 } else {
1777                         panic("unsupported mixed cpu mwait support detected");
1778                 }
1779         }
1780 
1781         /*
1782          * We could be more sophisticated here, and just mark the CPU
1783          * as "faulted" but at this point we'll opt for the easier
1784          * answer of dying horribly.  Provided the boot cpu is ok,
1785          * the system can be recovered by booting with use_mp set to zero.
1786          */
1787         if (workaround_errata(cp) != 0)
1788                 panic("critical workaround(s) missing for cpu%d", cp->cpu_id);
1789 
1790         /*
1791          * We can touch cpu_flags here without acquiring the cpu_lock here
1792          * because the cpu_lock is held by the control CPU which is running
1793          * mp_start_cpu_common().
1794          * Need to clear CPU_QUIESCED flag before calling any function which
1795          * may cause thread context switching, such as kmem_alloc() etc.
1796          * The idle thread checks for CPU_QUIESCED flag and loops for ever if
1797          * it's set. So the startup thread may have no chance to switch back
1798          * again if it's switched away with CPU_QUIESCED set.
1799          */
1800         cp->cpu_flags &= ~(CPU_POWEROFF | CPU_QUIESCED);
1801 
1802         enable_pcid();
1803 
1804         /*
1805          * Setup this processor for XSAVE.
1806          */
1807         if (fp_save_mech == FP_XSAVE) {
1808                 xsave_setup_msr(cp);
1809         }
1810 
1811         cpuid_pass2(cp);
1812         cpuid_pass3(cp);
1813         cpuid_pass4(cp, NULL);
1814 
1815         /*
1816          * Correct cpu_idstr and cpu_brandstr on target CPU after
1817          * cpuid_pass1() is done.
1818          */
1819         (void) cpuid_getidstr(cp, cp->cpu_idstr, CPU_IDSTRLEN);
1820         (void) cpuid_getbrandstr(cp, cp->cpu_brandstr, CPU_IDSTRLEN);
1821 
1822         cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS;
1823 
1824         post_startup_cpu_fixups();
1825 
1826         cpu_event_init_cpu(cp);
1827 
1828         /*
1829          * Enable preemption here so that contention for any locks acquired
1830          * later in mp_startup_common may be preempted if the thread owning
1831          * those locks is continuously executing on other CPUs (for example,
1832          * this CPU must be preemptible to allow other CPUs to pause it during
1833          * their startup phases).  It's safe to enable preemption here because
1834          * the CPU state is pretty-much fully constructed.
1835          */
1836         curthread->t_preempt = 0;
1837 
1838         /* The base spl should still be at LOCK LEVEL here */
1839         ASSERT(cp->cpu_base_spl == ipltospl(LOCK_LEVEL));
1840         set_base_spl();         /* Restore the spl to its proper value */
1841 
1842         pghw_physid_create(cp);
1843         /*
1844          * Delegate initialization tasks, which need to access the cpu_lock,
1845          * to mp_start_cpu_common() because we can't acquire the cpu_lock here
1846          * during CPU DR operations.
1847          */
1848         mp_startup_signal(&procset_slave, cp->cpu_id);
1849         mp_startup_wait(&procset_master, cp->cpu_id);
1850         pg_cmt_cpu_startup(cp);
1851 
1852         if (boot) {
1853                 mutex_enter(&cpu_lock);
1854                 cp->cpu_flags &= ~CPU_OFFLINE;
1855                 cpu_enable_intr(cp);
1856                 cpu_add_active(cp);
1857                 mutex_exit(&cpu_lock);
1858         }
1859 
1860         /* Enable interrupts */
1861         (void) spl0();
1862 
1863         /*
1864          * Fill out cpu_ucode_info.  Update microcode if necessary.
1865          */
1866         ucode_check(cp);
1867         cpuid_pass_ucode(cp, new_x86_featureset);
1868 
1869         /*
1870          * Do a sanity check to make sure this new CPU is a sane thing
1871          * to add to the collection of processors running this system.
1872          *
1873          * XXX  Clearly this needs to get more sophisticated, if x86
1874          * systems start to get built out of heterogenous CPUs; as is
1875          * likely to happen once the number of processors in a configuration
1876          * gets large enough.
1877          */
1878         if (compare_x86_featureset(x86_featureset, new_x86_featureset) ==
1879             B_FALSE) {
1880                 cmn_err(CE_CONT, "cpu%d: featureset\n", cp->cpu_id);
1881                 print_x86_featureset(new_x86_featureset);
1882                 cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id);
1883         }
1884 
1885 #ifndef __xpv
1886         {
1887                 /*
1888                  * Set up the CPU module for this CPU.  This can't be done
1889                  * before this CPU is made CPU_READY, because we may (in
1890                  * heterogeneous systems) need to go load another CPU module.
1891                  * The act of attempting to load a module may trigger a
1892                  * cross-call, which will ASSERT unless this cpu is CPU_READY.
1893                  */
1894                 cmi_hdl_t hdl;
1895 
1896                 if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
1897                     cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) {
1898                         if (is_x86_feature(x86_featureset, X86FSET_MCA))
1899                                 cmi_mca_init(hdl);
1900                         cp->cpu_m.mcpu_cmi_hdl = hdl;
1901                 }
1902         }
1903 #endif /* __xpv */
1904 
1905         if (boothowto & RB_DEBUG)
1906                 kdi_cpu_init();
1907 
1908         (void) mach_cpu_create_device_node(cp, NULL);
1909 
1910         /*
1911          * Setting the bit in cpu_ready_set must be the last operation in
1912          * processor initialization; the boot CPU will continue to boot once
1913          * it sees this bit set for all active CPUs.
1914          */
1915         CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id);
1916 
1917         cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr);
1918         cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr);
1919         cmn_err(CE_CONT, "?cpu%d initialization complete - online\n",
1920             cp->cpu_id);
1921 
1922         /*
1923          * Now we are done with the startup thread, so free it up.
1924          */
1925         thread_exit();
1926         panic("mp_startup: cannot return");
1927         /*NOTREACHED*/
1928 }
1929 
1930 /*
1931  * Startup function for 'other' CPUs at boot time (besides boot cpu).
1932  */
1933 static void
1934 mp_startup_boot(void)
1935 {
1936         mp_startup_common(B_TRUE);
1937 }
1938 
1939 /*
1940  * Startup function for hotplug CPUs at runtime.
1941  */
1942 void
1943 mp_startup_hotplug(void)
1944 {
1945         mp_startup_common(B_FALSE);
1946 }
1947 
1948 /*
1949  * Start CPU on user request.
1950  */
1951 /* ARGSUSED */
1952 int
1953 mp_cpu_start(struct cpu *cp)
1954 {
1955         ASSERT(MUTEX_HELD(&cpu_lock));
1956         return (0);
1957 }
1958 
1959 /*
1960  * Stop CPU on user request.
1961  */
1962 int
1963 mp_cpu_stop(struct cpu *cp)
1964 {
1965         extern int cbe_psm_timer_mode;
1966         ASSERT(MUTEX_HELD(&cpu_lock));
1967 
1968 #ifdef __xpv
1969         /*
1970          * We can't offline vcpu0.
1971          */
1972         if (cp->cpu_id == 0)
1973                 return (EBUSY);
1974 #endif
1975 
1976         /*
1977          * If TIMER_PERIODIC mode is used, CPU0 is the one running it;
1978          * can't stop it.  (This is true only for machines with no TSC.)
1979          */
1980 
1981         if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0))
1982                 return (EBUSY);
1983 
1984         return (0);
1985 }
1986 
1987 /*
1988  * Take the specified CPU out of participation in interrupts.
1989  *
1990  * Usually, we hold cpu_lock. But we cannot assert as such due to the
1991  * exception - i_cpr_save_context() - where we have mutual exclusion via a
1992  * separate mechanism.
1993  */
1994 int
1995 cpu_disable_intr(struct cpu *cp)
1996 {
1997         if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS)
1998                 return (EBUSY);
1999 
2000         cp->cpu_flags &= ~CPU_ENABLE;
2001         ncpus_intr_enabled--;
2002         return (0);
2003 }
2004 
2005 /*
2006  * Allow the specified CPU to participate in interrupts.
2007  */
2008 void
2009 cpu_enable_intr(struct cpu *cp)
2010 {
2011         ASSERT(MUTEX_HELD(&cpu_lock));
2012         cp->cpu_flags |= CPU_ENABLE;
2013         ncpus_intr_enabled++;
2014         psm_enable_intr(cp->cpu_id);
2015 }
2016 
2017 void
2018 mp_cpu_faulted_enter(struct cpu *cp)
2019 {
2020 #ifdef __xpv
2021         _NOTE(ARGUNUSED(cp));
2022 #else
2023         cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
2024 
2025         if (hdl != NULL) {
2026                 cmi_hdl_hold(hdl);
2027         } else {
2028                 hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
2029                     cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
2030         }
2031         if (hdl != NULL) {
2032                 cmi_faulted_enter(hdl);
2033                 cmi_hdl_rele(hdl);
2034         }
2035 #endif
2036 }
2037 
2038 void
2039 mp_cpu_faulted_exit(struct cpu *cp)
2040 {
2041 #ifdef __xpv
2042         _NOTE(ARGUNUSED(cp));
2043 #else
2044         cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
2045 
2046         if (hdl != NULL) {
2047                 cmi_hdl_hold(hdl);
2048         } else {
2049                 hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
2050                     cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
2051         }
2052         if (hdl != NULL) {
2053                 cmi_faulted_exit(hdl);
2054                 cmi_hdl_rele(hdl);
2055         }
2056 #endif
2057 }
2058 
2059 /*
2060  * The following two routines are used as context operators on threads belonging
2061  * to processes with a private LDT (see sysi86).  Due to the rarity of such
2062  * processes, these routines are currently written for best code readability and
2063  * organization rather than speed.  We could avoid checking x86_featureset at
2064  * every context switch by installing different context ops, depending on
2065  * x86_featureset, at LDT creation time -- one for each combination of fast
2066  * syscall features.
2067  */
2068 
2069 void
2070 cpu_fast_syscall_disable(void)
2071 {
2072         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2073             is_x86_feature(x86_featureset, X86FSET_SEP))
2074                 cpu_sep_disable();
2075         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2076             is_x86_feature(x86_featureset, X86FSET_ASYSC))
2077                 cpu_asysc_disable();
2078 }
2079 
2080 void
2081 cpu_fast_syscall_enable(void)
2082 {
2083         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2084             is_x86_feature(x86_featureset, X86FSET_SEP))
2085                 cpu_sep_enable();
2086         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2087             is_x86_feature(x86_featureset, X86FSET_ASYSC))
2088                 cpu_asysc_enable();
2089 }
2090 
2091 static void
2092 cpu_sep_enable(void)
2093 {
2094         ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2095         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2096 
2097         wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL);
2098 }
2099 
2100 static void
2101 cpu_sep_disable(void)
2102 {
2103         ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2104         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2105 
2106         /*
2107          * Setting the SYSENTER_CS_MSR register to 0 causes software executing
2108          * the sysenter or sysexit instruction to trigger a #gp fault.
2109          */
2110         wrmsr(MSR_INTC_SEP_CS, 0);
2111 }
2112 
2113 static void
2114 cpu_asysc_enable(void)
2115 {
2116         ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2117         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2118 
2119         wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) |
2120             (uint64_t)(uintptr_t)AMD_EFER_SCE);
2121 }
2122 
2123 static void
2124 cpu_asysc_disable(void)
2125 {
2126         ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2127         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2128 
2129         /*
2130          * Turn off the SCE (syscall enable) bit in the EFER register. Software
2131          * executing syscall or sysret with this bit off will incur a #ud trap.
2132          */
2133         wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) &
2134             ~((uint64_t)(uintptr_t)AMD_EFER_SCE));
2135 }