1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 /*
  26  * Copyright (c) 2010, Intel Corporation.
  27  * All rights reserved.
  28  */
  29 /*
  30  * Copyright 2018 Joyent, Inc.
  31  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  32  */
  33 
  34 #include <sys/types.h>
  35 #include <sys/thread.h>
  36 #include <sys/cpuvar.h>
  37 #include <sys/cpu.h>
  38 #include <sys/t_lock.h>
  39 #include <sys/param.h>
  40 #include <sys/proc.h>
  41 #include <sys/disp.h>
  42 #include <sys/class.h>
  43 #include <sys/cmn_err.h>
  44 #include <sys/debug.h>
  45 #include <sys/note.h>
  46 #include <sys/asm_linkage.h>
  47 #include <sys/x_call.h>
  48 #include <sys/systm.h>
  49 #include <sys/var.h>
  50 #include <sys/vtrace.h>
  51 #include <vm/hat.h>
  52 #include <vm/as.h>
  53 #include <vm/seg_kmem.h>
  54 #include <vm/seg_kp.h>
  55 #include <sys/segments.h>
  56 #include <sys/kmem.h>
  57 #include <sys/stack.h>
  58 #include <sys/smp_impldefs.h>
  59 #include <sys/x86_archext.h>
  60 #include <sys/machsystm.h>
  61 #include <sys/traptrace.h>
  62 #include <sys/clock.h>
  63 #include <sys/cpc_impl.h>
  64 #include <sys/pg.h>
  65 #include <sys/cmt.h>
  66 #include <sys/dtrace.h>
  67 #include <sys/archsystm.h>
  68 #include <sys/fp.h>
  69 #include <sys/reboot.h>
  70 #include <sys/kdi_machimpl.h>
  71 #include <vm/hat_i86.h>
  72 #include <vm/vm_dep.h>
  73 #include <sys/memnode.h>
  74 #include <sys/pci_cfgspace.h>
  75 #include <sys/mach_mmu.h>
  76 #include <sys/sysmacros.h>
  77 #if defined(__xpv)
  78 #include <sys/hypervisor.h>
  79 #endif
  80 #include <sys/cpu_module.h>
  81 #include <sys/ontrap.h>
  82 
  83 struct cpu      cpus[1] __aligned(MMU_PAGESIZE);
  84 struct cpu      *cpu[NCPU] = {&cpus[0]};
  85 struct cpu      *cpu_free_list;
  86 cpu_core_t      cpu_core[NCPU];
  87 
  88 #define cpu_next_free   cpu_prev
  89 
  90 /*
  91  * Useful for disabling MP bring-up on a MP capable system.
  92  */
  93 int use_mp = 1;
  94 
  95 /*
  96  * to be set by a PSM to indicate what cpus
  97  * are sitting around on the system.
  98  */
  99 cpuset_t mp_cpus;
 100 
 101 /*
 102  * This variable is used by the hat layer to decide whether or not
 103  * critical sections are needed to prevent race conditions.  For sun4m,
 104  * this variable is set once enough MP initialization has been done in
 105  * order to allow cross calls.
 106  */
 107 int flushes_require_xcalls;
 108 
 109 cpuset_t cpu_ready_set;         /* initialized in startup() */
 110 
 111 static void mp_startup_boot(void);
 112 static void mp_startup_hotplug(void);
 113 
 114 static void cpu_sep_enable(void);
 115 static void cpu_sep_disable(void);
 116 static void cpu_asysc_enable(void);
 117 static void cpu_asysc_disable(void);
 118 
 119 /*
 120  * Init CPU info - get CPU type info for processor_info system call.
 121  */
 122 void
 123 init_cpu_info(struct cpu *cp)
 124 {
 125         processor_info_t *pi = &cp->cpu_type_info;
 126 
 127         /*
 128          * Get clock-frequency property for the CPU.
 129          */
 130         pi->pi_clock = cpu_freq;
 131 
 132         /*
 133          * Current frequency in Hz.
 134          */
 135         cp->cpu_curr_clock = cpu_freq_hz;
 136 
 137         /*
 138          * Supported frequencies.
 139          */
 140         if (cp->cpu_supp_freqs == NULL) {
 141                 cpu_set_supp_freqs(cp, NULL);
 142         }
 143 
 144         (void) strcpy(pi->pi_processor_type, "i386");
 145         if (fpu_exists)
 146                 (void) strcpy(pi->pi_fputypes, "i387 compatible");
 147 
 148         cp->cpu_idstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
 149         cp->cpu_brandstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
 150 
 151         /*
 152          * If called for the BSP, cp is equal to current CPU.
 153          * For non-BSPs, cpuid info of cp is not ready yet, so use cpuid info
 154          * of current CPU as default values for cpu_idstr and cpu_brandstr.
 155          * They will be corrected in mp_startup_common() after cpuid_pass1()
 156          * has been invoked on target CPU.
 157          */
 158         (void) cpuid_getidstr(CPU, cp->cpu_idstr, CPU_IDSTRLEN);
 159         (void) cpuid_getbrandstr(CPU, cp->cpu_brandstr, CPU_IDSTRLEN);
 160 }
 161 
 162 /*
 163  * Configure syscall support on this CPU.
 164  */
 165 /*ARGSUSED*/
 166 void
 167 init_cpu_syscall(struct cpu *cp)
 168 {
 169         kpreempt_disable();
 170 
 171         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
 172             is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
 173                 uint64_t flags;
 174 
 175 #if !defined(__xpv)
 176                 /*
 177                  * The syscall instruction imposes a certain ordering on
 178                  * segment selectors, so we double-check that ordering
 179                  * here.
 180                  */
 181                 CTASSERT(KDS_SEL == KCS_SEL + 8);
 182                 CTASSERT(UDS_SEL == U32CS_SEL + 8);
 183                 CTASSERT(UCS_SEL == U32CS_SEL + 16);
 184 #endif
 185 
 186                 /*
 187                  * Turn syscall/sysret extensions on.
 188                  */
 189                 cpu_asysc_enable();
 190 
 191                 /*
 192                  * Program the magic registers ..
 193                  */
 194                 wrmsr(MSR_AMD_STAR,
 195                     ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);
 196                 if (kpti_enable == 1) {
 197                         wrmsr(MSR_AMD_LSTAR,
 198                             (uint64_t)(uintptr_t)tr_sys_syscall);
 199                         wrmsr(MSR_AMD_CSTAR,
 200                             (uint64_t)(uintptr_t)tr_sys_syscall32);
 201                 } else {
 202                         wrmsr(MSR_AMD_LSTAR,
 203                             (uint64_t)(uintptr_t)sys_syscall);
 204                         wrmsr(MSR_AMD_CSTAR,
 205                             (uint64_t)(uintptr_t)sys_syscall32);
 206                 }
 207 
 208                 /*
 209                  * This list of flags is masked off the incoming
 210                  * %rfl when we enter the kernel.
 211                  */
 212                 flags = PS_IE | PS_T;
 213                 if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_TRUE)
 214                         flags |= PS_ACHK;
 215                 wrmsr(MSR_AMD_SFMASK, flags);
 216         }
 217 
 218         /*
 219          * On 64-bit kernels on Nocona machines, the 32-bit syscall
 220          * variant isn't available to 32-bit applications, but sysenter is.
 221          */
 222         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
 223             is_x86_feature(x86_featureset, X86FSET_SEP)) {
 224 
 225 #if !defined(__xpv)
 226                 /*
 227                  * The sysenter instruction imposes a certain ordering on
 228                  * segment selectors, so we double-check that ordering
 229                  * here. See "sysenter" in Intel document 245471-012, "IA-32
 230                  * Intel Architecture Software Developer's Manual Volume 2:
 231                  * Instruction Set Reference"
 232                  */
 233                 CTASSERT(KDS_SEL == KCS_SEL + 8);
 234 
 235                 CTASSERT(U32CS_SEL == ((KCS_SEL + 16) | 3));
 236                 CTASSERT(UDS_SEL == U32CS_SEL + 8);
 237 #endif
 238 
 239                 cpu_sep_enable();
 240 
 241                 /*
 242                  * resume() sets this value to the base of the threads stack
 243                  * via a context handler.
 244                  */
 245                 wrmsr(MSR_INTC_SEP_ESP, 0);
 246 
 247                 if (kpti_enable == 1) {
 248                         wrmsr(MSR_INTC_SEP_EIP,
 249                             (uint64_t)(uintptr_t)tr_sys_sysenter);
 250                 } else {
 251                         wrmsr(MSR_INTC_SEP_EIP,
 252                             (uint64_t)(uintptr_t)sys_sysenter);
 253                 }
 254         }
 255 
 256         kpreempt_enable();
 257 }
 258 
 259 #if !defined(__xpv)
 260 /*
 261  * Configure per-cpu ID GDT
 262  */
 263 static void
 264 init_cpu_id_gdt(struct cpu *cp)
 265 {
 266         /* Write cpu_id into limit field of GDT for usermode retrieval */
 267 #if defined(__amd64)
 268         set_usegd(&cp->cpu_gdt[GDT_CPUID], SDP_SHORT, NULL, cp->cpu_id,
 269             SDT_MEMRODA, SEL_UPL, SDP_BYTES, SDP_OP32);
 270 #elif defined(__i386)
 271         set_usegd(&cp->cpu_gdt[GDT_CPUID], NULL, cp->cpu_id, SDT_MEMRODA,
 272             SEL_UPL, SDP_BYTES, SDP_OP32);
 273 #endif
 274 }
 275 #endif /* !defined(__xpv) */
 276 
 277 /*
 278  * Multiprocessor initialization.
 279  *
 280  * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the
 281  * startup and idle threads for the specified CPU.
 282  * Parameter boot is true for boot time operations and is false for CPU
 283  * DR operations.
 284  */
 285 static struct cpu *
 286 mp_cpu_configure_common(int cpun, boolean_t boot)
 287 {
 288         struct cpu *cp;
 289         kthread_id_t tp;
 290         caddr_t sp;
 291         proc_t *procp;
 292 #if !defined(__xpv)
 293         extern int idle_cpu_prefer_mwait;
 294         extern void cpu_idle_mwait();
 295 #endif
 296         extern void idle();
 297         extern void cpu_idle();
 298 
 299 #ifdef TRAPTRACE
 300         trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun];
 301 #endif
 302 
 303         ASSERT(MUTEX_HELD(&cpu_lock));
 304         ASSERT(cpun < NCPU && cpu[cpun] == NULL);
 305 
 306         if (cpu_free_list == NULL) {
 307                 cp = kmem_zalloc(sizeof (*cp), KM_SLEEP);
 308         } else {
 309                 cp = cpu_free_list;
 310                 cpu_free_list = cp->cpu_next_free;
 311         }
 312 
 313         cp->cpu_m.mcpu_istamp = cpun << 16;
 314 
 315         /* Create per CPU specific threads in the process p0. */
 316         procp = &p0;
 317 
 318         /*
 319          * Initialize the dispatcher first.
 320          */
 321         disp_cpu_init(cp);
 322 
 323         cpu_vm_data_init(cp);
 324 
 325         /*
 326          * Allocate and initialize the startup thread for this CPU.
 327          * Interrupt and process switch stacks get allocated later
 328          * when the CPU starts running.
 329          */
 330         tp = thread_create(NULL, 0, NULL, NULL, 0, procp,
 331             TS_STOPPED, maxclsyspri);
 332 
 333         /*
 334          * Set state to TS_ONPROC since this thread will start running
 335          * as soon as the CPU comes online.
 336          *
 337          * All the other fields of the thread structure are setup by
 338          * thread_create().
 339          */
 340         THREAD_ONPROC(tp, cp);
 341         tp->t_preempt = 1;
 342         tp->t_bound_cpu = cp;
 343         tp->t_affinitycnt = 1;
 344         tp->t_cpu = cp;
 345         tp->t_disp_queue = cp->cpu_disp;
 346 
 347         /*
 348          * Setup thread to start in mp_startup_common.
 349          */
 350         sp = tp->t_stk;
 351         tp->t_sp = (uintptr_t)(sp - MINFRAME);
 352 #if defined(__amd64)
 353         tp->t_sp -= STACK_ENTRY_ALIGN;               /* fake a call */
 354 #endif
 355         /*
 356          * Setup thread start entry point for boot or hotplug.
 357          */
 358         if (boot) {
 359                 tp->t_pc = (uintptr_t)mp_startup_boot;
 360         } else {
 361                 tp->t_pc = (uintptr_t)mp_startup_hotplug;
 362         }
 363 
 364         cp->cpu_id = cpun;
 365         cp->cpu_self = cp;
 366         cp->cpu_thread = tp;
 367         cp->cpu_lwp = NULL;
 368         cp->cpu_dispthread = tp;
 369         cp->cpu_dispatch_pri = DISP_PRIO(tp);
 370 
 371         /*
 372          * cpu_base_spl must be set explicitly here to prevent any blocking
 373          * operations in mp_startup_common from causing the spl of the cpu
 374          * to drop to 0 (allowing device interrupts before we're ready) in
 375          * resume().
 376          * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY.
 377          * As an extra bit of security on DEBUG kernels, this is enforced with
 378          * an assertion in mp_startup_common() -- before cpu_base_spl is set
 379          * to its proper value.
 380          */
 381         cp->cpu_base_spl = ipltospl(LOCK_LEVEL);
 382 
 383         /*
 384          * Now, initialize per-CPU idle thread for this CPU.
 385          */
 386         tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1);
 387 
 388         cp->cpu_idle_thread = tp;
 389 
 390         tp->t_preempt = 1;
 391         tp->t_bound_cpu = cp;
 392         tp->t_affinitycnt = 1;
 393         tp->t_cpu = cp;
 394         tp->t_disp_queue = cp->cpu_disp;
 395 
 396         /*
 397          * Bootstrap the CPU's PG data
 398          */
 399         pg_cpu_bootstrap(cp);
 400 
 401         /*
 402          * Perform CPC initialization on the new CPU.
 403          */
 404         kcpc_hw_init(cp);
 405 
 406         /*
 407          * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2
 408          * for each CPU.
 409          */
 410         setup_vaddr_for_ppcopy(cp);
 411 
 412         /*
 413          * Allocate page for new GDT and initialize from current GDT.
 414          */
 415 #if !defined(__lint)
 416         ASSERT((sizeof (*cp->cpu_gdt) * NGDT) <= PAGESIZE);
 417 #endif
 418         cp->cpu_gdt = kmem_zalloc(PAGESIZE, KM_SLEEP);
 419         bcopy(CPU->cpu_gdt, cp->cpu_gdt, (sizeof (*cp->cpu_gdt) * NGDT));
 420 
 421 #if defined(__i386)
 422         /*
 423          * setup kernel %gs.
 424          */
 425         set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA,
 426             SEL_KPL, 0, 1);
 427 #endif
 428 
 429         /*
 430          * Allocate pages for the CPU LDT.
 431          */
 432         cp->cpu_m.mcpu_ldt = kmem_zalloc(LDT_CPU_SIZE, KM_SLEEP);
 433         cp->cpu_m.mcpu_ldt_len = 0;
 434 
 435         /*
 436          * Allocate a per-CPU IDT and initialize the new IDT to the currently
 437          * runing CPU.
 438          */
 439 #if !defined(__lint)
 440         ASSERT((sizeof (*CPU->cpu_idt) * NIDT) <= PAGESIZE);
 441 #endif
 442         cp->cpu_idt = kmem_alloc(PAGESIZE, KM_SLEEP);
 443         bcopy(CPU->cpu_idt, cp->cpu_idt, PAGESIZE);
 444 
 445         /*
 446          * alloc space for cpuid info
 447          */
 448         cpuid_alloc_space(cp);
 449 #if !defined(__xpv)
 450         if (is_x86_feature(x86_featureset, X86FSET_MWAIT) &&
 451             idle_cpu_prefer_mwait) {
 452                 cp->cpu_m.mcpu_mwait = cpuid_mwait_alloc(cp);
 453                 cp->cpu_m.mcpu_idle_cpu = cpu_idle_mwait;
 454         } else
 455 #endif
 456                 cp->cpu_m.mcpu_idle_cpu = cpu_idle;
 457 
 458         init_cpu_info(cp);
 459 
 460 #if !defined(__xpv)
 461         init_cpu_id_gdt(cp);
 462 #endif
 463 
 464         /*
 465          * alloc space for ucode_info
 466          */
 467         ucode_alloc_space(cp);
 468         xc_init_cpu(cp);
 469         hat_cpu_online(cp);
 470 
 471 #ifdef TRAPTRACE
 472         /*
 473          * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers
 474          */
 475         ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP);
 476         ttc->ttc_next = ttc->ttc_first;
 477         ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize;
 478 #endif
 479 
 480         /*
 481          * Record that we have another CPU.
 482          */
 483         /*
 484          * Initialize the interrupt threads for this CPU
 485          */
 486         cpu_intr_alloc(cp, NINTR_THREADS);
 487 
 488         cp->cpu_flags = CPU_OFFLINE | CPU_QUIESCED | CPU_POWEROFF;
 489         cpu_set_state(cp);
 490 
 491         /*
 492          * Add CPU to list of available CPUs.  It'll be on the active list
 493          * after mp_startup_common().
 494          */
 495         cpu_add_unit(cp);
 496 
 497         return (cp);
 498 }
 499 
 500 /*
 501  * Undo what was done in mp_cpu_configure_common
 502  */
 503 static void
 504 mp_cpu_unconfigure_common(struct cpu *cp, int error)
 505 {
 506         ASSERT(MUTEX_HELD(&cpu_lock));
 507 
 508         /*
 509          * Remove the CPU from the list of available CPUs.
 510          */
 511         cpu_del_unit(cp->cpu_id);
 512 
 513         if (error == ETIMEDOUT) {
 514                 /*
 515                  * The cpu was started, but never *seemed* to run any
 516                  * code in the kernel; it's probably off spinning in its
 517                  * own private world, though with potential references to
 518                  * our kmem-allocated IDTs and GDTs (for example).
 519                  *
 520                  * Worse still, it may actually wake up some time later,
 521                  * so rather than guess what it might or might not do, we
 522                  * leave the fundamental data structures intact.
 523                  */
 524                 cp->cpu_flags = 0;
 525                 return;
 526         }
 527 
 528         /*
 529          * At this point, the only threads bound to this CPU should
 530          * special per-cpu threads: it's idle thread, it's pause threads,
 531          * and it's interrupt threads.  Clean these up.
 532          */
 533         cpu_destroy_bound_threads(cp);
 534         cp->cpu_idle_thread = NULL;
 535 
 536         /*
 537          * Free the interrupt stack.
 538          */
 539         segkp_release(segkp,
 540             cp->cpu_intr_stack - (INTR_STACK_SIZE - SA(MINFRAME)));
 541         cp->cpu_intr_stack = NULL;
 542 
 543 #ifdef TRAPTRACE
 544         /*
 545          * Discard the trap trace buffer
 546          */
 547         {
 548                 trap_trace_ctl_t *ttc = &trap_trace_ctl[cp->cpu_id];
 549 
 550                 kmem_free((void *)ttc->ttc_first, trap_trace_bufsize);
 551                 ttc->ttc_first = NULL;
 552         }
 553 #endif
 554 
 555         hat_cpu_offline(cp);
 556 
 557         ucode_free_space(cp);
 558 
 559         /* Free CPU ID string and brand string. */
 560         if (cp->cpu_idstr) {
 561                 kmem_free(cp->cpu_idstr, CPU_IDSTRLEN);
 562                 cp->cpu_idstr = NULL;
 563         }
 564         if (cp->cpu_brandstr) {
 565                 kmem_free(cp->cpu_brandstr, CPU_IDSTRLEN);
 566                 cp->cpu_brandstr = NULL;
 567         }
 568 
 569 #if !defined(__xpv)
 570         if (cp->cpu_m.mcpu_mwait != NULL) {
 571                 cpuid_mwait_free(cp);
 572                 cp->cpu_m.mcpu_mwait = NULL;
 573         }
 574 #endif
 575         cpuid_free_space(cp);
 576 
 577         if (cp->cpu_idt != CPU->cpu_idt)
 578                 kmem_free(cp->cpu_idt, PAGESIZE);
 579         cp->cpu_idt = NULL;
 580 
 581         kmem_free(cp->cpu_m.mcpu_ldt, LDT_CPU_SIZE);
 582         cp->cpu_m.mcpu_ldt = NULL;
 583         cp->cpu_m.mcpu_ldt_len = 0;
 584 
 585         kmem_free(cp->cpu_gdt, PAGESIZE);
 586         cp->cpu_gdt = NULL;
 587 
 588         if (cp->cpu_supp_freqs != NULL) {
 589                 size_t len = strlen(cp->cpu_supp_freqs) + 1;
 590                 kmem_free(cp->cpu_supp_freqs, len);
 591                 cp->cpu_supp_freqs = NULL;
 592         }
 593 
 594         teardown_vaddr_for_ppcopy(cp);
 595 
 596         kcpc_hw_fini(cp);
 597 
 598         cp->cpu_dispthread = NULL;
 599         cp->cpu_thread = NULL;       /* discarded by cpu_destroy_bound_threads() */
 600 
 601         cpu_vm_data_destroy(cp);
 602 
 603         xc_fini_cpu(cp);
 604         disp_cpu_fini(cp);
 605 
 606         ASSERT(cp != CPU0);
 607         bzero(cp, sizeof (*cp));
 608         cp->cpu_next_free = cpu_free_list;
 609         cpu_free_list = cp;
 610 }
 611 
 612 /*
 613  * Apply workarounds for known errata, and warn about those that are absent.
 614  *
 615  * System vendors occasionally create configurations which contain different
 616  * revisions of the CPUs that are almost but not exactly the same.  At the
 617  * time of writing, this meant that their clock rates were the same, their
 618  * feature sets were the same, but the required workaround were -not-
 619  * necessarily the same.  So, this routine is invoked on -every- CPU soon
 620  * after starting to make sure that the resulting system contains the most
 621  * pessimal set of workarounds needed to cope with *any* of the CPUs in the
 622  * system.
 623  *
 624  * workaround_errata is invoked early in mlsetup() for CPU 0, and in
 625  * mp_startup_common() for all slave CPUs. Slaves process workaround_errata
 626  * prior to acknowledging their readiness to the master, so this routine will
 627  * never be executed by multiple CPUs in parallel, thus making updates to
 628  * global data safe.
 629  *
 630  * These workarounds are based on Rev 3.57 of the Revision Guide for
 631  * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005.
 632  */
 633 
 634 #if defined(OPTERON_ERRATUM_88)
 635 int opteron_erratum_88;         /* if non-zero -> at least one cpu has it */
 636 #endif
 637 
 638 #if defined(OPTERON_ERRATUM_91)
 639 int opteron_erratum_91;         /* if non-zero -> at least one cpu has it */
 640 #endif
 641 
 642 #if defined(OPTERON_ERRATUM_93)
 643 int opteron_erratum_93;         /* if non-zero -> at least one cpu has it */
 644 #endif
 645 
 646 #if defined(OPTERON_ERRATUM_95)
 647 int opteron_erratum_95;         /* if non-zero -> at least one cpu has it */
 648 #endif
 649 
 650 #if defined(OPTERON_ERRATUM_100)
 651 int opteron_erratum_100;        /* if non-zero -> at least one cpu has it */
 652 #endif
 653 
 654 #if defined(OPTERON_ERRATUM_108)
 655 int opteron_erratum_108;        /* if non-zero -> at least one cpu has it */
 656 #endif
 657 
 658 #if defined(OPTERON_ERRATUM_109)
 659 int opteron_erratum_109;        /* if non-zero -> at least one cpu has it */
 660 #endif
 661 
 662 #if defined(OPTERON_ERRATUM_121)
 663 int opteron_erratum_121;        /* if non-zero -> at least one cpu has it */
 664 #endif
 665 
 666 #if defined(OPTERON_ERRATUM_122)
 667 int opteron_erratum_122;        /* if non-zero -> at least one cpu has it */
 668 #endif
 669 
 670 #if defined(OPTERON_ERRATUM_123)
 671 int opteron_erratum_123;        /* if non-zero -> at least one cpu has it */
 672 #endif
 673 
 674 #if defined(OPTERON_ERRATUM_131)
 675 int opteron_erratum_131;        /* if non-zero -> at least one cpu has it */
 676 #endif
 677 
 678 #if defined(OPTERON_WORKAROUND_6336786)
 679 int opteron_workaround_6336786; /* non-zero -> WA relevant and applied */
 680 int opteron_workaround_6336786_UP = 0;  /* Not needed for UP */
 681 #endif
 682 
 683 #if defined(OPTERON_WORKAROUND_6323525)
 684 int opteron_workaround_6323525; /* if non-zero -> at least one cpu has it */
 685 #endif
 686 
 687 #if defined(OPTERON_ERRATUM_298)
 688 int opteron_erratum_298;
 689 #endif
 690 
 691 #if defined(OPTERON_ERRATUM_721)
 692 int opteron_erratum_721;
 693 #endif
 694 
 695 static void
 696 workaround_warning(cpu_t *cp, uint_t erratum)
 697 {
 698         cmn_err(CE_WARN, "cpu%d: no workaround for erratum %u",
 699             cp->cpu_id, erratum);
 700 }
 701 
 702 static void
 703 workaround_applied(uint_t erratum)
 704 {
 705         if (erratum > 1000000)
 706                 cmn_err(CE_CONT, "?workaround applied for cpu issue #%d\n",
 707                     erratum);
 708         else
 709                 cmn_err(CE_CONT, "?workaround applied for cpu erratum #%d\n",
 710                     erratum);
 711 }
 712 
 713 static void
 714 msr_warning(cpu_t *cp, const char *rw, uint_t msr, int error)
 715 {
 716         cmn_err(CE_WARN, "cpu%d: couldn't %smsr 0x%x, error %d",
 717             cp->cpu_id, rw, msr, error);
 718 }
 719 
 720 /*
 721  * Determine the number of nodes in a Hammer / Greyhound / Griffin family
 722  * system.
 723  */
 724 static uint_t
 725 opteron_get_nnodes(void)
 726 {
 727         static uint_t nnodes = 0;
 728 
 729         if (nnodes == 0) {
 730 #ifdef  DEBUG
 731                 uint_t family;
 732 
 733                 /*
 734                  * This routine uses a PCI config space based mechanism
 735                  * for retrieving the number of nodes in the system.
 736                  * Device 24, function 0, offset 0x60 as used here is not
 737                  * AMD processor architectural, and may not work on processor
 738                  * families other than those listed below.
 739                  *
 740                  * Callers of this routine must ensure that we're running on
 741                  * a processor which supports this mechanism.
 742                  * The assertion below is meant to catch calls on unsupported
 743                  * processors.
 744                  */
 745                 family = cpuid_getfamily(CPU);
 746                 ASSERT(family == 0xf || family == 0x10 || family == 0x11);
 747 #endif  /* DEBUG */
 748 
 749                 /*
 750                  * Obtain the number of nodes in the system from
 751                  * bits [6:4] of the Node ID register on node 0.
 752                  *
 753                  * The actual node count is NodeID[6:4] + 1
 754                  *
 755                  * The Node ID register is accessed via function 0,
 756                  * offset 0x60. Node 0 is device 24.
 757                  */
 758                 nnodes = ((pci_getl_func(0, 24, 0, 0x60) & 0x70) >> 4) + 1;
 759         }
 760         return (nnodes);
 761 }
 762 
 763 uint_t
 764 do_erratum_298(struct cpu *cpu)
 765 {
 766         static int      osvwrc = -3;
 767         extern int      osvw_opteron_erratum(cpu_t *, uint_t);
 768 
 769         /*
 770          * L2 Eviction May Occur During Processor Operation To Set
 771          * Accessed or Dirty Bit.
 772          */
 773         if (osvwrc == -3) {
 774                 osvwrc = osvw_opteron_erratum(cpu, 298);
 775         } else {
 776                 /* osvw return codes should be consistent for all cpus */
 777                 ASSERT(osvwrc == osvw_opteron_erratum(cpu, 298));
 778         }
 779 
 780         switch (osvwrc) {
 781         case 0:         /* erratum is not present: do nothing */
 782                 break;
 783         case 1:         /* erratum is present: BIOS workaround applied */
 784                 /*
 785                  * check if workaround is actually in place and issue warning
 786                  * if not.
 787                  */
 788                 if (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
 789                     ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0)) {
 790 #if defined(OPTERON_ERRATUM_298)
 791                         opteron_erratum_298++;
 792 #else
 793                         workaround_warning(cpu, 298);
 794                         return (1);
 795 #endif
 796                 }
 797                 break;
 798         case -1:        /* cannot determine via osvw: check cpuid */
 799                 if ((cpuid_opteron_erratum(cpu, 298) > 0) &&
 800                     (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
 801                     ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0))) {
 802 #if defined(OPTERON_ERRATUM_298)
 803                         opteron_erratum_298++;
 804 #else
 805                         workaround_warning(cpu, 298);
 806                         return (1);
 807 #endif
 808                 }
 809                 break;
 810         }
 811         return (0);
 812 }
 813 
 814 uint_t
 815 workaround_errata(struct cpu *cpu)
 816 {
 817         uint_t missing = 0;
 818 
 819         ASSERT(cpu == CPU);
 820 
 821         /*LINTED*/
 822         if (cpuid_opteron_erratum(cpu, 88) > 0) {
 823                 /*
 824                  * SWAPGS May Fail To Read Correct GS Base
 825                  */
 826 #if defined(OPTERON_ERRATUM_88)
 827                 /*
 828                  * The workaround is an mfence in the relevant assembler code
 829                  */
 830                 opteron_erratum_88++;
 831 #else
 832                 workaround_warning(cpu, 88);
 833                 missing++;
 834 #endif
 835         }
 836 
 837         if (cpuid_opteron_erratum(cpu, 91) > 0) {
 838                 /*
 839                  * Software Prefetches May Report A Page Fault
 840                  */
 841 #if defined(OPTERON_ERRATUM_91)
 842                 /*
 843                  * fix is in trap.c
 844                  */
 845                 opteron_erratum_91++;
 846 #else
 847                 workaround_warning(cpu, 91);
 848                 missing++;
 849 #endif
 850         }
 851 
 852         if (cpuid_opteron_erratum(cpu, 93) > 0) {
 853                 /*
 854                  * RSM Auto-Halt Restart Returns to Incorrect RIP
 855                  */
 856 #if defined(OPTERON_ERRATUM_93)
 857                 /*
 858                  * fix is in trap.c
 859                  */
 860                 opteron_erratum_93++;
 861 #else
 862                 workaround_warning(cpu, 93);
 863                 missing++;
 864 #endif
 865         }
 866 
 867         /*LINTED*/
 868         if (cpuid_opteron_erratum(cpu, 95) > 0) {
 869                 /*
 870                  * RET Instruction May Return to Incorrect EIP
 871                  */
 872 #if defined(OPTERON_ERRATUM_95)
 873 #if defined(_LP64)
 874                 /*
 875                  * Workaround this by ensuring that 32-bit user code and
 876                  * 64-bit kernel code never occupy the same address
 877                  * range mod 4G.
 878                  */
 879                 if (_userlimit32 > 0xc0000000ul)
 880                         *(uintptr_t *)&_userlimit32 = 0xc0000000ul;
 881 
 882                 /*LINTED*/
 883                 ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u);
 884                 opteron_erratum_95++;
 885 #endif  /* _LP64 */
 886 #else
 887                 workaround_warning(cpu, 95);
 888                 missing++;
 889 #endif
 890         }
 891 
 892         if (cpuid_opteron_erratum(cpu, 100) > 0) {
 893                 /*
 894                  * Compatibility Mode Branches Transfer to Illegal Address
 895                  */
 896 #if defined(OPTERON_ERRATUM_100)
 897                 /*
 898                  * fix is in trap.c
 899                  */
 900                 opteron_erratum_100++;
 901 #else
 902                 workaround_warning(cpu, 100);
 903                 missing++;
 904 #endif
 905         }
 906 
 907         /*LINTED*/
 908         if (cpuid_opteron_erratum(cpu, 108) > 0) {
 909                 /*
 910                  * CPUID Instruction May Return Incorrect Model Number In
 911                  * Some Processors
 912                  */
 913 #if defined(OPTERON_ERRATUM_108)
 914                 /*
 915                  * (Our cpuid-handling code corrects the model number on
 916                  * those processors)
 917                  */
 918 #else
 919                 workaround_warning(cpu, 108);
 920                 missing++;
 921 #endif
 922         }
 923 
 924         /*LINTED*/
 925         if (cpuid_opteron_erratum(cpu, 109) > 0) do {
 926                 /*
 927                  * Certain Reverse REP MOVS May Produce Unpredictable Behavior
 928                  */
 929 #if defined(OPTERON_ERRATUM_109)
 930                 /*
 931                  * The "workaround" is to print a warning to upgrade the BIOS
 932                  */
 933                 uint64_t value;
 934                 const uint_t msr = MSR_AMD_PATCHLEVEL;
 935                 int err;
 936 
 937                 if ((err = checked_rdmsr(msr, &value)) != 0) {
 938                         msr_warning(cpu, "rd", msr, err);
 939                         workaround_warning(cpu, 109);
 940                         missing++;
 941                 }
 942                 if (value == 0)
 943                         opteron_erratum_109++;
 944 #else
 945                 workaround_warning(cpu, 109);
 946                 missing++;
 947 #endif
 948         /*CONSTANTCONDITION*/
 949         } while (0);
 950 
 951         /*LINTED*/
 952         if (cpuid_opteron_erratum(cpu, 121) > 0) {
 953                 /*
 954                  * Sequential Execution Across Non_Canonical Boundary Caused
 955                  * Processor Hang
 956                  */
 957 #if defined(OPTERON_ERRATUM_121)
 958 #if defined(_LP64)
 959                 /*
 960                  * Erratum 121 is only present in long (64 bit) mode.
 961                  * Workaround is to include the page immediately before the
 962                  * va hole to eliminate the possibility of system hangs due to
 963                  * sequential execution across the va hole boundary.
 964                  */
 965                 if (opteron_erratum_121)
 966                         opteron_erratum_121++;
 967                 else {
 968                         if (hole_start) {
 969                                 hole_start -= PAGESIZE;
 970                         } else {
 971                                 /*
 972                                  * hole_start not yet initialized by
 973                                  * mmu_init. Initialize hole_start
 974                                  * with value to be subtracted.
 975                                  */
 976                                 hole_start = PAGESIZE;
 977                         }
 978                         opteron_erratum_121++;
 979                 }
 980 #endif  /* _LP64 */
 981 #else
 982                 workaround_warning(cpu, 121);
 983                 missing++;
 984 #endif
 985         }
 986 
 987         /*LINTED*/
 988         if (cpuid_opteron_erratum(cpu, 122) > 0) do {
 989                 /*
 990                  * TLB Flush Filter May Cause Coherency Problem in
 991                  * Multiprocessor Systems
 992                  */
 993 #if defined(OPTERON_ERRATUM_122)
 994                 uint64_t value;
 995                 const uint_t msr = MSR_AMD_HWCR;
 996                 int error;
 997 
 998                 /*
 999                  * Erratum 122 is only present in MP configurations (multi-core
1000                  * or multi-processor).
1001                  */
1002 #if defined(__xpv)
1003                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1004                         break;
1005                 if (!opteron_erratum_122 && xpv_nr_phys_cpus() == 1)
1006                         break;
1007 #else
1008                 if (!opteron_erratum_122 && opteron_get_nnodes() == 1 &&
1009                     cpuid_get_ncpu_per_chip(cpu) == 1)
1010                         break;
1011 #endif
1012                 /* disable TLB Flush Filter */
1013 
1014                 if ((error = checked_rdmsr(msr, &value)) != 0) {
1015                         msr_warning(cpu, "rd", msr, error);
1016                         workaround_warning(cpu, 122);
1017                         missing++;
1018                 } else {
1019                         value |= (uint64_t)AMD_HWCR_FFDIS;
1020                         if ((error = checked_wrmsr(msr, value)) != 0) {
1021                                 msr_warning(cpu, "wr", msr, error);
1022                                 workaround_warning(cpu, 122);
1023                                 missing++;
1024                         }
1025                 }
1026                 opteron_erratum_122++;
1027 #else
1028                 workaround_warning(cpu, 122);
1029                 missing++;
1030 #endif
1031         /*CONSTANTCONDITION*/
1032         } while (0);
1033 
1034         /*LINTED*/
1035         if (cpuid_opteron_erratum(cpu, 123) > 0) do {
1036                 /*
1037                  * Bypassed Reads May Cause Data Corruption of System Hang in
1038                  * Dual Core Processors
1039                  */
1040 #if defined(OPTERON_ERRATUM_123)
1041                 uint64_t value;
1042                 const uint_t msr = MSR_AMD_PATCHLEVEL;
1043                 int err;
1044 
1045                 /*
1046                  * Erratum 123 applies only to multi-core cpus.
1047                  */
1048                 if (cpuid_get_ncpu_per_chip(cpu) < 2)
1049                         break;
1050 #if defined(__xpv)
1051                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1052                         break;
1053 #endif
1054                 /*
1055                  * The "workaround" is to print a warning to upgrade the BIOS
1056                  */
1057                 if ((err = checked_rdmsr(msr, &value)) != 0) {
1058                         msr_warning(cpu, "rd", msr, err);
1059                         workaround_warning(cpu, 123);
1060                         missing++;
1061                 }
1062                 if (value == 0)
1063                         opteron_erratum_123++;
1064 #else
1065                 workaround_warning(cpu, 123);
1066                 missing++;
1067 
1068 #endif
1069         /*CONSTANTCONDITION*/
1070         } while (0);
1071 
1072         /*LINTED*/
1073         if (cpuid_opteron_erratum(cpu, 131) > 0) do {
1074                 /*
1075                  * Multiprocessor Systems with Four or More Cores May Deadlock
1076                  * Waiting for a Probe Response
1077                  */
1078 #if defined(OPTERON_ERRATUM_131)
1079                 uint64_t nbcfg;
1080                 const uint_t msr = MSR_AMD_NB_CFG;
1081                 const uint64_t wabits =
1082                     AMD_NB_CFG_SRQ_HEARTBEAT | AMD_NB_CFG_SRQ_SPR;
1083                 int error;
1084 
1085                 /*
1086                  * Erratum 131 applies to any system with four or more cores.
1087                  */
1088                 if (opteron_erratum_131)
1089                         break;
1090 #if defined(__xpv)
1091                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1092                         break;
1093                 if (xpv_nr_phys_cpus() < 4)
1094                         break;
1095 #else
1096                 if (opteron_get_nnodes() * cpuid_get_ncpu_per_chip(cpu) < 4)
1097                         break;
1098 #endif
1099                 /*
1100                  * Print a warning if neither of the workarounds for
1101                  * erratum 131 is present.
1102                  */
1103                 if ((error = checked_rdmsr(msr, &nbcfg)) != 0) {
1104                         msr_warning(cpu, "rd", msr, error);
1105                         workaround_warning(cpu, 131);
1106                         missing++;
1107                 } else if ((nbcfg & wabits) == 0) {
1108                         opteron_erratum_131++;
1109                 } else {
1110                         /* cannot have both workarounds set */
1111                         ASSERT((nbcfg & wabits) != wabits);
1112                 }
1113 #else
1114                 workaround_warning(cpu, 131);
1115                 missing++;
1116 #endif
1117         /*CONSTANTCONDITION*/
1118         } while (0);
1119 
1120         /*
1121          * This isn't really an erratum, but for convenience the
1122          * detection/workaround code lives here and in cpuid_opteron_erratum.
1123          */
1124         if (cpuid_opteron_erratum(cpu, 6336786) > 0) {
1125 #if defined(OPTERON_WORKAROUND_6336786)
1126                 /*
1127                  * Disable C1-Clock ramping on multi-core/multi-processor
1128                  * K8 platforms to guard against TSC drift.
1129                  */
1130                 if (opteron_workaround_6336786) {
1131                         opteron_workaround_6336786++;
1132 #if defined(__xpv)
1133                 } else if ((DOMAIN_IS_INITDOMAIN(xen_info) &&
1134                     xpv_nr_phys_cpus() > 1) ||
1135                     opteron_workaround_6336786_UP) {
1136                         /*
1137                          * XXPV Hmm.  We can't walk the Northbridges on
1138                          *      the hypervisor; so just complain and drive
1139                          *      on.  This probably needs to be fixed in
1140                          *      the hypervisor itself.
1141                          */
1142                         opteron_workaround_6336786++;
1143                         workaround_warning(cpu, 6336786);
1144 #else   /* __xpv */
1145                 } else if ((opteron_get_nnodes() *
1146                     cpuid_get_ncpu_per_chip(cpu) > 1) ||
1147                     opteron_workaround_6336786_UP) {
1148 
1149                         uint_t  node, nnodes;
1150                         uint8_t data;
1151 
1152                         nnodes = opteron_get_nnodes();
1153                         for (node = 0; node < nnodes; node++) {
1154                                 /*
1155                                  * Clear PMM7[1:0] (function 3, offset 0x87)
1156                                  * Northbridge device is the node id + 24.
1157                                  */
1158                                 data = pci_getb_func(0, node + 24, 3, 0x87);
1159                                 data &= 0xFC;
1160                                 pci_putb_func(0, node + 24, 3, 0x87, data);
1161                         }
1162                         opteron_workaround_6336786++;
1163 #endif  /* __xpv */
1164                 }
1165 #else
1166                 workaround_warning(cpu, 6336786);
1167                 missing++;
1168 #endif
1169         }
1170 
1171         /*LINTED*/
1172         /*
1173          * Mutex primitives don't work as expected.
1174          */
1175         if (cpuid_opteron_erratum(cpu, 6323525) > 0) {
1176 #if defined(OPTERON_WORKAROUND_6323525)
1177                 /*
1178                  * This problem only occurs with 2 or more cores. If bit in
1179                  * MSR_AMD_BU_CFG set, then not applicable. The workaround
1180                  * is to patch the semaphone routines with the lfence
1181                  * instruction to provide necessary load memory barrier with
1182                  * possible subsequent read-modify-write ops.
1183                  *
1184                  * It is too early in boot to call the patch routine so
1185                  * set erratum variable to be done in startup_end().
1186                  */
1187                 if (opteron_workaround_6323525) {
1188                         opteron_workaround_6323525++;
1189 #if defined(__xpv)
1190                 } else if (is_x86_feature(x86_featureset, X86FSET_SSE2)) {
1191                         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1192                                 /*
1193                                  * XXPV Use dom0_msr here when extended
1194                                  *      operations are supported?
1195                                  */
1196                                 if (xpv_nr_phys_cpus() > 1)
1197                                         opteron_workaround_6323525++;
1198                         } else {
1199                                 /*
1200                                  * We have no way to tell how many physical
1201                                  * cpus there are, or even if this processor
1202                                  * has the problem, so enable the workaround
1203                                  * unconditionally (at some performance cost).
1204                                  */
1205                                 opteron_workaround_6323525++;
1206                         }
1207 #else   /* __xpv */
1208                 } else if (is_x86_feature(x86_featureset, X86FSET_SSE2) &&
1209                     ((opteron_get_nnodes() *
1210                     cpuid_get_ncpu_per_chip(cpu)) > 1)) {
1211                         if ((xrdmsr(MSR_AMD_BU_CFG) & (UINT64_C(1) << 33)) == 0)
1212                                 opteron_workaround_6323525++;
1213 #endif  /* __xpv */
1214                 }
1215 #else
1216                 workaround_warning(cpu, 6323525);
1217                 missing++;
1218 #endif
1219         }
1220 
1221         missing += do_erratum_298(cpu);
1222 
1223         if (cpuid_opteron_erratum(cpu, 721) > 0) {
1224 #if defined(OPTERON_ERRATUM_721)
1225                 on_trap_data_t otd;
1226 
1227                 if (!on_trap(&otd, OT_DATA_ACCESS))
1228                         wrmsr(MSR_AMD_DE_CFG,
1229                             rdmsr(MSR_AMD_DE_CFG) | AMD_DE_CFG_E721);
1230                 no_trap();
1231 
1232                 opteron_erratum_721++;
1233 #else
1234                 workaround_warning(cpu, 721);
1235                 missing++;
1236 #endif
1237         }
1238 
1239 #ifdef __xpv
1240         return (0);
1241 #else
1242         return (missing);
1243 #endif
1244 }
1245 
1246 void
1247 workaround_errata_end()
1248 {
1249 #if defined(OPTERON_ERRATUM_88)
1250         if (opteron_erratum_88)
1251                 workaround_applied(88);
1252 #endif
1253 #if defined(OPTERON_ERRATUM_91)
1254         if (opteron_erratum_91)
1255                 workaround_applied(91);
1256 #endif
1257 #if defined(OPTERON_ERRATUM_93)
1258         if (opteron_erratum_93)
1259                 workaround_applied(93);
1260 #endif
1261 #if defined(OPTERON_ERRATUM_95)
1262         if (opteron_erratum_95)
1263                 workaround_applied(95);
1264 #endif
1265 #if defined(OPTERON_ERRATUM_100)
1266         if (opteron_erratum_100)
1267                 workaround_applied(100);
1268 #endif
1269 #if defined(OPTERON_ERRATUM_108)
1270         if (opteron_erratum_108)
1271                 workaround_applied(108);
1272 #endif
1273 #if defined(OPTERON_ERRATUM_109)
1274         if (opteron_erratum_109) {
1275                 cmn_err(CE_WARN,
1276                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1277                     " processor\nerratum 109 was not detected; updating your"
1278                     " system's BIOS to a version\ncontaining this"
1279                     " microcode patch is HIGHLY recommended or erroneous"
1280                     " system\noperation may occur.\n");
1281         }
1282 #endif
1283 #if defined(OPTERON_ERRATUM_121)
1284         if (opteron_erratum_121)
1285                 workaround_applied(121);
1286 #endif
1287 #if defined(OPTERON_ERRATUM_122)
1288         if (opteron_erratum_122)
1289                 workaround_applied(122);
1290 #endif
1291 #if defined(OPTERON_ERRATUM_123)
1292         if (opteron_erratum_123) {
1293                 cmn_err(CE_WARN,
1294                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1295                     " processor\nerratum 123 was not detected; updating your"
1296                     " system's BIOS to a version\ncontaining this"
1297                     " microcode patch is HIGHLY recommended or erroneous"
1298                     " system\noperation may occur.\n");
1299         }
1300 #endif
1301 #if defined(OPTERON_ERRATUM_131)
1302         if (opteron_erratum_131) {
1303                 cmn_err(CE_WARN,
1304                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1305                     " processor\nerratum 131 was not detected; updating your"
1306                     " system's BIOS to a version\ncontaining this"
1307                     " microcode patch is HIGHLY recommended or erroneous"
1308                     " system\noperation may occur.\n");
1309         }
1310 #endif
1311 #if defined(OPTERON_WORKAROUND_6336786)
1312         if (opteron_workaround_6336786)
1313                 workaround_applied(6336786);
1314 #endif
1315 #if defined(OPTERON_WORKAROUND_6323525)
1316         if (opteron_workaround_6323525)
1317                 workaround_applied(6323525);
1318 #endif
1319 #if defined(OPTERON_ERRATUM_298)
1320         if (opteron_erratum_298) {
1321                 cmn_err(CE_WARN,
1322                     "BIOS microcode patch for AMD 64/Opteron(tm)"
1323                     " processor\nerratum 298 was not detected; updating your"
1324                     " system's BIOS to a version\ncontaining this"
1325                     " microcode patch is HIGHLY recommended or erroneous"
1326                     " system\noperation may occur.\n");
1327         }
1328 #endif
1329 #if defined(OPTERON_ERRATUM_721)
1330         if (opteron_erratum_721)
1331                 workaround_applied(721);
1332 #endif
1333 }
1334 
1335 /*
1336  * The procset_slave and procset_master are used to synchronize
1337  * between the control CPU and the target CPU when starting CPUs.
1338  */
1339 static cpuset_t procset_slave, procset_master;
1340 
1341 static void
1342 mp_startup_wait(cpuset_t *sp, processorid_t cpuid)
1343 {
1344         cpuset_t tempset;
1345 
1346         for (tempset = *sp; !CPU_IN_SET(tempset, cpuid);
1347             tempset = *(volatile cpuset_t *)sp) {
1348                 SMT_PAUSE();
1349         }
1350         CPUSET_ATOMIC_DEL(*(cpuset_t *)sp, cpuid);
1351 }
1352 
1353 static void
1354 mp_startup_signal(cpuset_t *sp, processorid_t cpuid)
1355 {
1356         cpuset_t tempset;
1357 
1358         CPUSET_ATOMIC_ADD(*(cpuset_t *)sp, cpuid);
1359         for (tempset = *sp; CPU_IN_SET(tempset, cpuid);
1360             tempset = *(volatile cpuset_t *)sp) {
1361                 SMT_PAUSE();
1362         }
1363 }
1364 
1365 int
1366 mp_start_cpu_common(cpu_t *cp, boolean_t boot)
1367 {
1368         _NOTE(ARGUNUSED(boot));
1369 
1370         void *ctx;
1371         int delays;
1372         int error = 0;
1373         cpuset_t tempset;
1374         processorid_t cpuid;
1375 #ifndef __xpv
1376         extern void cpupm_init(cpu_t *);
1377 #endif
1378 
1379         ASSERT(cp != NULL);
1380         cpuid = cp->cpu_id;
1381         ctx = mach_cpucontext_alloc(cp);
1382         if (ctx == NULL) {
1383                 cmn_err(CE_WARN,
1384                     "cpu%d: failed to allocate context", cp->cpu_id);
1385                 return (EAGAIN);
1386         }
1387         error = mach_cpu_start(cp, ctx);
1388         if (error != 0) {
1389                 cmn_err(CE_WARN,
1390                     "cpu%d: failed to start, error %d", cp->cpu_id, error);
1391                 mach_cpucontext_free(cp, ctx, error);
1392                 return (error);
1393         }
1394 
1395         for (delays = 0, tempset = procset_slave; !CPU_IN_SET(tempset, cpuid);
1396             delays++) {
1397                 if (delays == 500) {
1398                         /*
1399                          * After five seconds, things are probably looking
1400                          * a bit bleak - explain the hang.
1401                          */
1402                         cmn_err(CE_NOTE, "cpu%d: started, "
1403                             "but not running in the kernel yet", cpuid);
1404                 } else if (delays > 2000) {
1405                         /*
1406                          * We waited at least 20 seconds, bail ..
1407                          */
1408                         error = ETIMEDOUT;
1409                         cmn_err(CE_WARN, "cpu%d: timed out", cpuid);
1410                         mach_cpucontext_free(cp, ctx, error);
1411                         return (error);
1412                 }
1413 
1414                 /*
1415                  * wait at least 10ms, then check again..
1416                  */
1417                 delay(USEC_TO_TICK_ROUNDUP(10000));
1418                 tempset = *((volatile cpuset_t *)&procset_slave);
1419         }
1420         CPUSET_ATOMIC_DEL(procset_slave, cpuid);
1421 
1422         mach_cpucontext_free(cp, ctx, 0);
1423 
1424 #ifndef __xpv
1425         if (tsc_gethrtime_enable)
1426                 tsc_sync_master(cpuid);
1427 #endif
1428 
1429         if (dtrace_cpu_init != NULL) {
1430                 (*dtrace_cpu_init)(cpuid);
1431         }
1432 
1433         /*
1434          * During CPU DR operations, the cpu_lock is held by current
1435          * (the control) thread. We can't release the cpu_lock here
1436          * because that will break the CPU DR logic.
1437          * On the other hand, CPUPM and processor group initialization
1438          * routines need to access the cpu_lock. So we invoke those
1439          * routines here on behalf of mp_startup_common().
1440          *
1441          * CPUPM and processor group initialization routines depend
1442          * on the cpuid probing results. Wait for mp_startup_common()
1443          * to signal that cpuid probing is done.
1444          */
1445         mp_startup_wait(&procset_slave, cpuid);
1446 #ifndef __xpv
1447         cpupm_init(cp);
1448 #endif
1449         (void) pg_cpu_init(cp, B_FALSE);
1450         cpu_set_state(cp);
1451         mp_startup_signal(&procset_master, cpuid);
1452 
1453         return (0);
1454 }
1455 
1456 /*
1457  * Start a single cpu, assuming that the kernel context is available
1458  * to successfully start another cpu.
1459  *
1460  * (For example, real mode code is mapped into the right place
1461  * in memory and is ready to be run.)
1462  */
1463 int
1464 start_cpu(processorid_t who)
1465 {
1466         cpu_t *cp;
1467         int error = 0;
1468         cpuset_t tempset;
1469 
1470         ASSERT(who != 0);
1471 
1472         /*
1473          * Check if there's at least a Mbyte of kmem available
1474          * before attempting to start the cpu.
1475          */
1476         if (kmem_avail() < 1024 * 1024) {
1477                 /*
1478                  * Kick off a reap in case that helps us with
1479                  * later attempts ..
1480                  */
1481                 kmem_reap();
1482                 return (ENOMEM);
1483         }
1484 
1485         /*
1486          * First configure cpu.
1487          */
1488         cp = mp_cpu_configure_common(who, B_TRUE);
1489         ASSERT(cp != NULL);
1490 
1491         /*
1492          * Then start cpu.
1493          */
1494         error = mp_start_cpu_common(cp, B_TRUE);
1495         if (error != 0) {
1496                 mp_cpu_unconfigure_common(cp, error);
1497                 return (error);
1498         }
1499 
1500         mutex_exit(&cpu_lock);
1501         tempset = cpu_ready_set;
1502         while (!CPU_IN_SET(tempset, who)) {
1503                 drv_usecwait(1);
1504                 tempset = *((volatile cpuset_t *)&cpu_ready_set);
1505         }
1506         mutex_enter(&cpu_lock);
1507 
1508         return (0);
1509 }
1510 
1511 void
1512 start_other_cpus(int cprboot)
1513 {
1514         _NOTE(ARGUNUSED(cprboot));
1515 
1516         uint_t who;
1517         uint_t bootcpuid = 0;
1518 
1519         /*
1520          * Initialize our own cpu_info.
1521          */
1522         init_cpu_info(CPU);
1523 
1524 #if !defined(__xpv)
1525         init_cpu_id_gdt(CPU);
1526 #endif
1527 
1528         cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_idstr);
1529         cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_brandstr);
1530 
1531         /*
1532          * Initialize our syscall handlers
1533          */
1534         init_cpu_syscall(CPU);
1535 
1536         /*
1537          * Take the boot cpu out of the mp_cpus set because we know
1538          * it's already running.  Add it to the cpu_ready_set for
1539          * precisely the same reason.
1540          */
1541         CPUSET_DEL(mp_cpus, bootcpuid);
1542         CPUSET_ADD(cpu_ready_set, bootcpuid);
1543 
1544         /*
1545          * skip the rest of this if
1546          * . only 1 cpu dectected and system isn't hotplug-capable
1547          * . not using MP
1548          */
1549         if ((CPUSET_ISNULL(mp_cpus) && plat_dr_support_cpu() == 0) ||
1550             use_mp == 0) {
1551                 if (use_mp == 0)
1552                         cmn_err(CE_CONT, "?***** Not in MP mode\n");
1553                 goto done;
1554         }
1555 
1556         /*
1557          * perform such initialization as is needed
1558          * to be able to take CPUs on- and off-line.
1559          */
1560         cpu_pause_init();
1561 
1562         xc_init_cpu(CPU);               /* initialize processor crosscalls */
1563 
1564         if (mach_cpucontext_init() != 0)
1565                 goto done;
1566 
1567         flushes_require_xcalls = 1;
1568 
1569         /*
1570          * We lock our affinity to the master CPU to ensure that all slave CPUs
1571          * do their TSC syncs with the same CPU.
1572          */
1573         affinity_set(CPU_CURRENT);
1574 
1575         for (who = 0; who < NCPU; who++) {
1576                 if (!CPU_IN_SET(mp_cpus, who))
1577                         continue;
1578                 ASSERT(who != bootcpuid);
1579 
1580                 mutex_enter(&cpu_lock);
1581                 if (start_cpu(who) != 0)
1582                         CPUSET_DEL(mp_cpus, who);
1583                 cpu_state_change_notify(who, CPU_SETUP);
1584                 mutex_exit(&cpu_lock);
1585         }
1586 
1587         /* Free the space allocated to hold the microcode file */
1588         ucode_cleanup();
1589 
1590         affinity_clear();
1591 
1592         mach_cpucontext_fini();
1593 
1594 done:
1595         if (get_hwenv() == HW_NATIVE)
1596                 workaround_errata_end();
1597         cmi_post_mpstartup();
1598 
1599         if (use_mp && ncpus != boot_max_ncpus) {
1600                 cmn_err(CE_NOTE,
1601                     "System detected %d cpus, but "
1602                     "only %d cpu(s) were enabled during boot.",
1603                     boot_max_ncpus, ncpus);
1604                 cmn_err(CE_NOTE,
1605                     "Use \"boot-ncpus\" parameter to enable more CPU(s). "
1606                     "See eeprom(1M).");
1607         }
1608 }
1609 
1610 int
1611 mp_cpu_configure(int cpuid)
1612 {
1613         cpu_t *cp;
1614 
1615         if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1616                 return (ENOTSUP);
1617         }
1618 
1619         cp = cpu_get(cpuid);
1620         if (cp != NULL) {
1621                 return (EALREADY);
1622         }
1623 
1624         /*
1625          * Check if there's at least a Mbyte of kmem available
1626          * before attempting to start the cpu.
1627          */
1628         if (kmem_avail() < 1024 * 1024) {
1629                 /*
1630                  * Kick off a reap in case that helps us with
1631                  * later attempts ..
1632                  */
1633                 kmem_reap();
1634                 return (ENOMEM);
1635         }
1636 
1637         cp = mp_cpu_configure_common(cpuid, B_FALSE);
1638         ASSERT(cp != NULL && cpu_get(cpuid) == cp);
1639 
1640         return (cp != NULL ? 0 : EAGAIN);
1641 }
1642 
1643 int
1644 mp_cpu_unconfigure(int cpuid)
1645 {
1646         cpu_t *cp;
1647 
1648         if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1649                 return (ENOTSUP);
1650         } else if (cpuid < 0 || cpuid >= max_ncpus) {
1651                 return (EINVAL);
1652         }
1653 
1654         cp = cpu_get(cpuid);
1655         if (cp == NULL) {
1656                 return (ENODEV);
1657         }
1658         mp_cpu_unconfigure_common(cp, 0);
1659 
1660         return (0);
1661 }
1662 
1663 /*
1664  * Startup function for 'other' CPUs (besides boot cpu).
1665  * Called from real_mode_start.
1666  *
1667  * WARNING: until CPU_READY is set, mp_startup_common and routines called by
1668  * mp_startup_common should not call routines (e.g. kmem_free) that could call
1669  * hat_unload which requires CPU_READY to be set.
1670  */
1671 static void
1672 mp_startup_common(boolean_t boot)
1673 {
1674         cpu_t *cp = CPU;
1675         uchar_t new_x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
1676         extern void cpu_event_init_cpu(cpu_t *);
1677 
1678         /*
1679          * We need to get TSC on this proc synced (i.e., any delta
1680          * from cpu0 accounted for) as soon as we can, because many
1681          * many things use gethrtime/pc_gethrestime, including
1682          * interrupts, cmn_err, etc.  Before we can do that, we want to
1683          * clear TSC if we're on a buggy Sandy/Ivy Bridge CPU, so do that
1684          * right away.
1685          */
1686         bzero(new_x86_featureset, BT_SIZEOFMAP(NUM_X86_FEATURES));
1687         cpuid_pass1(cp, new_x86_featureset);
1688 
1689         if (boot && get_hwenv() == HW_NATIVE &&
1690             cpuid_getvendor(CPU) == X86_VENDOR_Intel &&
1691             cpuid_getfamily(CPU) == 6 &&
1692             (cpuid_getmodel(CPU) == 0x2d || cpuid_getmodel(CPU) == 0x3e) &&
1693             is_x86_feature(new_x86_featureset, X86FSET_TSC)) {
1694                 (void) wrmsr(REG_TSC, 0UL);
1695         }
1696 
1697         /* Let the control CPU continue into tsc_sync_master() */
1698         mp_startup_signal(&procset_slave, cp->cpu_id);
1699 
1700 #ifndef __xpv
1701         if (tsc_gethrtime_enable)
1702                 tsc_sync_slave();
1703 #endif
1704 
1705         /*
1706          * Once this was done from assembly, but it's safer here; if
1707          * it blocks, we need to be able to swtch() to and from, and
1708          * since we get here by calling t_pc, we need to do that call
1709          * before swtch() overwrites it.
1710          */
1711         (void) (*ap_mlsetup)();
1712 
1713 #ifndef __xpv
1714         /*
1715          * Program this cpu's PAT
1716          */
1717         pat_sync();
1718 #endif
1719 
1720         /*
1721          * Set up TSC_AUX to contain the cpuid for this processor
1722          * for the rdtscp instruction.
1723          */
1724         if (is_x86_feature(x86_featureset, X86FSET_TSCP))
1725                 (void) wrmsr(MSR_AMD_TSCAUX, cp->cpu_id);
1726 
1727         /*
1728          * Initialize this CPU's syscall handlers
1729          */
1730         init_cpu_syscall(cp);
1731 
1732         /*
1733          * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the
1734          * highest level at which a routine is permitted to block on
1735          * an adaptive mutex (allows for cpu poke interrupt in case
1736          * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks
1737          * device interrupts that may end up in the hat layer issuing cross
1738          * calls before CPU_READY is set.
1739          */
1740         splx(ipltospl(LOCK_LEVEL));
1741         sti();
1742 
1743         /*
1744          * Do a sanity check to make sure this new CPU is a sane thing
1745          * to add to the collection of processors running this system.
1746          *
1747          * XXX  Clearly this needs to get more sophisticated, if x86
1748          * systems start to get built out of heterogenous CPUs; as is
1749          * likely to happen once the number of processors in a configuration
1750          * gets large enough.
1751          */
1752         if (compare_x86_featureset(x86_featureset, new_x86_featureset) ==
1753             B_FALSE) {
1754                 cmn_err(CE_CONT, "cpu%d: featureset\n", cp->cpu_id);
1755                 print_x86_featureset(new_x86_featureset);
1756                 cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id);
1757         }
1758 
1759         /*
1760          * There exists a small subset of systems which expose differing
1761          * MWAIT/MONITOR support between CPUs.  If MWAIT support is absent from
1762          * the boot CPU, but is found on a later CPU, the system continues to
1763          * operate as if no MWAIT support is available.
1764          *
1765          * The reverse case, where MWAIT is available on the boot CPU but not
1766          * on a subsequently initialized CPU, is not presently allowed and will
1767          * result in a panic.
1768          */
1769         if (is_x86_feature(x86_featureset, X86FSET_MWAIT) !=
1770             is_x86_feature(new_x86_featureset, X86FSET_MWAIT)) {
1771                 if (!is_x86_feature(x86_featureset, X86FSET_MWAIT)) {
1772                         remove_x86_feature(new_x86_featureset, X86FSET_MWAIT);
1773                 } else {
1774                         panic("unsupported mixed cpu mwait support detected");
1775                 }
1776         }
1777 
1778         /*
1779          * We could be more sophisticated here, and just mark the CPU
1780          * as "faulted" but at this point we'll opt for the easier
1781          * answer of dying horribly.  Provided the boot cpu is ok,
1782          * the system can be recovered by booting with use_mp set to zero.
1783          */
1784         if (workaround_errata(cp) != 0)
1785                 panic("critical workaround(s) missing for cpu%d", cp->cpu_id);
1786 
1787         /*
1788          * We can touch cpu_flags here without acquiring the cpu_lock here
1789          * because the cpu_lock is held by the control CPU which is running
1790          * mp_start_cpu_common().
1791          * Need to clear CPU_QUIESCED flag before calling any function which
1792          * may cause thread context switching, such as kmem_alloc() etc.
1793          * The idle thread checks for CPU_QUIESCED flag and loops for ever if
1794          * it's set. So the startup thread may have no chance to switch back
1795          * again if it's switched away with CPU_QUIESCED set.
1796          */
1797         cp->cpu_flags &= ~(CPU_POWEROFF | CPU_QUIESCED);
1798 
1799         enable_pcid();
1800 
1801         /*
1802          * Setup this processor for XSAVE.
1803          */
1804         if (fp_save_mech == FP_XSAVE) {
1805                 xsave_setup_msr(cp);
1806         }
1807 
1808         cpuid_pass2(cp);
1809         cpuid_pass3(cp);
1810         cpuid_pass4(cp, NULL);
1811 
1812         /*
1813          * Correct cpu_idstr and cpu_brandstr on target CPU after
1814          * cpuid_pass1() is done.
1815          */
1816         (void) cpuid_getidstr(cp, cp->cpu_idstr, CPU_IDSTRLEN);
1817         (void) cpuid_getbrandstr(cp, cp->cpu_brandstr, CPU_IDSTRLEN);
1818 
1819         cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS;
1820 
1821         post_startup_cpu_fixups();
1822 
1823         cpu_event_init_cpu(cp);
1824 
1825         /*
1826          * Enable preemption here so that contention for any locks acquired
1827          * later in mp_startup_common may be preempted if the thread owning
1828          * those locks is continuously executing on other CPUs (for example,
1829          * this CPU must be preemptible to allow other CPUs to pause it during
1830          * their startup phases).  It's safe to enable preemption here because
1831          * the CPU state is pretty-much fully constructed.
1832          */
1833         curthread->t_preempt = 0;
1834 
1835         /* The base spl should still be at LOCK LEVEL here */
1836         ASSERT(cp->cpu_base_spl == ipltospl(LOCK_LEVEL));
1837         set_base_spl();         /* Restore the spl to its proper value */
1838 
1839         pghw_physid_create(cp);
1840         /*
1841          * Delegate initialization tasks, which need to access the cpu_lock,
1842          * to mp_start_cpu_common() because we can't acquire the cpu_lock here
1843          * during CPU DR operations.
1844          */
1845         mp_startup_signal(&procset_slave, cp->cpu_id);
1846         mp_startup_wait(&procset_master, cp->cpu_id);
1847         pg_cmt_cpu_startup(cp);
1848 
1849         if (boot) {
1850                 mutex_enter(&cpu_lock);
1851                 cp->cpu_flags &= ~CPU_OFFLINE;
1852                 cpu_enable_intr(cp);
1853                 cpu_add_active(cp);
1854                 mutex_exit(&cpu_lock);
1855         }
1856 
1857         /* Enable interrupts */
1858         (void) spl0();
1859 
1860         /*
1861          * Fill out cpu_ucode_info.  Update microcode if necessary.
1862          */
1863         ucode_check(cp);
1864 
1865 #ifndef __xpv
1866         {
1867                 /*
1868                  * Set up the CPU module for this CPU.  This can't be done
1869                  * before this CPU is made CPU_READY, because we may (in
1870                  * heterogeneous systems) need to go load another CPU module.
1871                  * The act of attempting to load a module may trigger a
1872                  * cross-call, which will ASSERT unless this cpu is CPU_READY.
1873                  */
1874                 cmi_hdl_t hdl;
1875 
1876                 if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
1877                     cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) {
1878                         if (is_x86_feature(x86_featureset, X86FSET_MCA))
1879                                 cmi_mca_init(hdl);
1880                         cp->cpu_m.mcpu_cmi_hdl = hdl;
1881                 }
1882         }
1883 #endif /* __xpv */
1884 
1885         if (boothowto & RB_DEBUG)
1886                 kdi_cpu_init();
1887 
1888         /*
1889          * Setting the bit in cpu_ready_set must be the last operation in
1890          * processor initialization; the boot CPU will continue to boot once
1891          * it sees this bit set for all active CPUs.
1892          */
1893         CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id);
1894 
1895         (void) mach_cpu_create_device_node(cp, NULL);
1896 
1897         cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr);
1898         cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr);
1899         cmn_err(CE_CONT, "?cpu%d initialization complete - online\n",
1900             cp->cpu_id);
1901 
1902         /*
1903          * Now we are done with the startup thread, so free it up.
1904          */
1905         thread_exit();
1906         panic("mp_startup: cannot return");
1907         /*NOTREACHED*/
1908 }
1909 
1910 /*
1911  * Startup function for 'other' CPUs at boot time (besides boot cpu).
1912  */
1913 static void
1914 mp_startup_boot(void)
1915 {
1916         mp_startup_common(B_TRUE);
1917 }
1918 
1919 /*
1920  * Startup function for hotplug CPUs at runtime.
1921  */
1922 void
1923 mp_startup_hotplug(void)
1924 {
1925         mp_startup_common(B_FALSE);
1926 }
1927 
1928 /*
1929  * Start CPU on user request.
1930  */
1931 /* ARGSUSED */
1932 int
1933 mp_cpu_start(struct cpu *cp)
1934 {
1935         ASSERT(MUTEX_HELD(&cpu_lock));
1936         return (0);
1937 }
1938 
1939 /*
1940  * Stop CPU on user request.
1941  */
1942 int
1943 mp_cpu_stop(struct cpu *cp)
1944 {
1945         extern int cbe_psm_timer_mode;
1946         ASSERT(MUTEX_HELD(&cpu_lock));
1947 
1948 #ifdef __xpv
1949         /*
1950          * We can't offline vcpu0.
1951          */
1952         if (cp->cpu_id == 0)
1953                 return (EBUSY);
1954 #endif
1955 
1956         /*
1957          * If TIMER_PERIODIC mode is used, CPU0 is the one running it;
1958          * can't stop it.  (This is true only for machines with no TSC.)
1959          */
1960 
1961         if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0))
1962                 return (EBUSY);
1963 
1964         return (0);
1965 }
1966 
1967 /*
1968  * Take the specified CPU out of participation in interrupts.
1969  */
1970 int
1971 cpu_disable_intr(struct cpu *cp)
1972 {
1973         if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS)
1974                 return (EBUSY);
1975 
1976         cp->cpu_flags &= ~CPU_ENABLE;
1977         return (0);
1978 }
1979 
1980 /*
1981  * Allow the specified CPU to participate in interrupts.
1982  */
1983 void
1984 cpu_enable_intr(struct cpu *cp)
1985 {
1986         ASSERT(MUTEX_HELD(&cpu_lock));
1987         cp->cpu_flags |= CPU_ENABLE;
1988         psm_enable_intr(cp->cpu_id);
1989 }
1990 
1991 void
1992 mp_cpu_faulted_enter(struct cpu *cp)
1993 {
1994 #ifdef __xpv
1995         _NOTE(ARGUNUSED(cp));
1996 #else
1997         cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
1998 
1999         if (hdl != NULL) {
2000                 cmi_hdl_hold(hdl);
2001         } else {
2002                 hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
2003                     cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
2004         }
2005         if (hdl != NULL) {
2006                 cmi_faulted_enter(hdl);
2007                 cmi_hdl_rele(hdl);
2008         }
2009 #endif
2010 }
2011 
2012 void
2013 mp_cpu_faulted_exit(struct cpu *cp)
2014 {
2015 #ifdef __xpv
2016         _NOTE(ARGUNUSED(cp));
2017 #else
2018         cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
2019 
2020         if (hdl != NULL) {
2021                 cmi_hdl_hold(hdl);
2022         } else {
2023                 hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
2024                     cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
2025         }
2026         if (hdl != NULL) {
2027                 cmi_faulted_exit(hdl);
2028                 cmi_hdl_rele(hdl);
2029         }
2030 #endif
2031 }
2032 
2033 /*
2034  * The following two routines are used as context operators on threads belonging
2035  * to processes with a private LDT (see sysi86).  Due to the rarity of such
2036  * processes, these routines are currently written for best code readability and
2037  * organization rather than speed.  We could avoid checking x86_featureset at
2038  * every context switch by installing different context ops, depending on
2039  * x86_featureset, at LDT creation time -- one for each combination of fast
2040  * syscall features.
2041  */
2042 
2043 /*ARGSUSED*/
2044 void
2045 cpu_fast_syscall_disable(void *arg)
2046 {
2047         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2048             is_x86_feature(x86_featureset, X86FSET_SEP))
2049                 cpu_sep_disable();
2050         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2051             is_x86_feature(x86_featureset, X86FSET_ASYSC))
2052                 cpu_asysc_disable();
2053 }
2054 
2055 /*ARGSUSED*/
2056 void
2057 cpu_fast_syscall_enable(void *arg)
2058 {
2059         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2060             is_x86_feature(x86_featureset, X86FSET_SEP))
2061                 cpu_sep_enable();
2062         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2063             is_x86_feature(x86_featureset, X86FSET_ASYSC))
2064                 cpu_asysc_enable();
2065 }
2066 
2067 static void
2068 cpu_sep_enable(void)
2069 {
2070         ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2071         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2072 
2073         wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL);
2074 }
2075 
2076 static void
2077 cpu_sep_disable(void)
2078 {
2079         ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2080         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2081 
2082         /*
2083          * Setting the SYSENTER_CS_MSR register to 0 causes software executing
2084          * the sysenter or sysexit instruction to trigger a #gp fault.
2085          */
2086         wrmsr(MSR_INTC_SEP_CS, 0);
2087 }
2088 
2089 static void
2090 cpu_asysc_enable(void)
2091 {
2092         ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2093         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2094 
2095         wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) |
2096             (uint64_t)(uintptr_t)AMD_EFER_SCE);
2097 }
2098 
2099 static void
2100 cpu_asysc_disable(void)
2101 {
2102         ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2103         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2104 
2105         /*
2106          * Turn off the SCE (syscall enable) bit in the EFER register. Software
2107          * executing syscall or sysret with this bit off will incur a #ud trap.
2108          */
2109         wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) &
2110             ~((uint64_t)(uintptr_t)AMD_EFER_SCE));
2111 }