1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 /*
  26  * Copyright (c) 2010, Intel Corporation.
  27  * All rights reserved.
  28  */
  29 /*
  30  * Copyright 2016 Joyent, Inc.
  31  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  32  */
  33 
  34 #include <sys/types.h>
  35 #include <sys/thread.h>
  36 #include <sys/cpuvar.h>
  37 #include <sys/cpu.h>
  38 #include <sys/t_lock.h>
  39 #include <sys/param.h>
  40 #include <sys/proc.h>
  41 #include <sys/disp.h>
  42 #include <sys/class.h>
  43 #include <sys/cmn_err.h>
  44 #include <sys/debug.h>
  45 #include <sys/note.h>
  46 #include <sys/asm_linkage.h>
  47 #include <sys/x_call.h>
  48 #include <sys/systm.h>
  49 #include <sys/var.h>
  50 #include <sys/vtrace.h>
  51 #include <vm/hat.h>
  52 #include <vm/as.h>
  53 #include <vm/seg_kmem.h>
  54 #include <vm/seg_kp.h>
  55 #include <sys/segments.h>
  56 #include <sys/kmem.h>
  57 #include <sys/stack.h>
  58 #include <sys/smp_impldefs.h>
  59 #include <sys/x86_archext.h>
  60 #include <sys/machsystm.h>
  61 #include <sys/traptrace.h>
  62 #include <sys/clock.h>
  63 #include <sys/cpc_impl.h>
  64 #include <sys/pg.h>
  65 #include <sys/cmt.h>
  66 #include <sys/dtrace.h>
  67 #include <sys/archsystm.h>
  68 #include <sys/fp.h>
  69 #include <sys/reboot.h>
  70 #include <sys/kdi_machimpl.h>
  71 #include <vm/hat_i86.h>
  72 #include <vm/vm_dep.h>
  73 #include <sys/memnode.h>
  74 #include <sys/pci_cfgspace.h>
  75 #include <sys/mach_mmu.h>
  76 #include <sys/sysmacros.h>
  77 #if defined(__xpv)
  78 #include <sys/hypervisor.h>
  79 #endif
  80 #include <sys/cpu_module.h>
  81 #include <sys/ontrap.h>
  82 
  83 struct cpu      cpus[1];                        /* CPU data */
  84 struct cpu      *cpu[NCPU] = {&cpus[0]};    /* pointers to all CPUs */
  85 struct cpu      *cpu_free_list;                 /* list for released CPUs */
  86 cpu_core_t      cpu_core[NCPU];                 /* cpu_core structures */
  87 
  88 #define cpu_next_free   cpu_prev
  89 
  90 /*
  91  * Useful for disabling MP bring-up on a MP capable system.
  92  */
  93 int use_mp = 1;
  94 
  95 /*
  96  * to be set by a PSM to indicate what cpus
  97  * are sitting around on the system.
  98  */
  99 cpuset_t mp_cpus;
 100 
 101 /*
 102  * This variable is used by the hat layer to decide whether or not
 103  * critical sections are needed to prevent race conditions.  For sun4m,
 104  * this variable is set once enough MP initialization has been done in
 105  * order to allow cross calls.
 106  */
 107 int flushes_require_xcalls;
 108 
 109 cpuset_t cpu_ready_set;         /* initialized in startup() */
 110 
 111 static void mp_startup_boot(void);
 112 static void mp_startup_hotplug(void);
 113 
 114 static void cpu_sep_enable(void);
 115 static void cpu_sep_disable(void);
 116 static void cpu_asysc_enable(void);
 117 static void cpu_asysc_disable(void);
 118 
 119 /*
 120  * Init CPU info - get CPU type info for processor_info system call.
 121  */
 122 void
 123 init_cpu_info(struct cpu *cp)
 124 {
 125         processor_info_t *pi = &cp->cpu_type_info;
 126 
 127         /*
 128          * Get clock-frequency property for the CPU.
 129          */
 130         pi->pi_clock = cpu_freq;
 131 
 132         /*
 133          * Current frequency in Hz.
 134          */
 135         cp->cpu_curr_clock = cpu_freq_hz;
 136 
 137         /*
 138          * Supported frequencies.
 139          */
 140         if (cp->cpu_supp_freqs == NULL) {
 141                 cpu_set_supp_freqs(cp, NULL);
 142         }
 143 
 144         (void) strcpy(pi->pi_processor_type, "i386");
 145         if (fpu_exists)
 146                 (void) strcpy(pi->pi_fputypes, "i387 compatible");
 147 
 148         cp->cpu_idstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
 149         cp->cpu_brandstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
 150 
 151         /*
 152          * If called for the BSP, cp is equal to current CPU.
 153          * For non-BSPs, cpuid info of cp is not ready yet, so use cpuid info
 154          * of current CPU as default values for cpu_idstr and cpu_brandstr.
 155          * They will be corrected in mp_startup_common() after cpuid_pass1()
 156          * has been invoked on target CPU.
 157          */
 158         (void) cpuid_getidstr(CPU, cp->cpu_idstr, CPU_IDSTRLEN);
 159         (void) cpuid_getbrandstr(CPU, cp->cpu_brandstr, CPU_IDSTRLEN);
 160 }
 161 
 162 /*
 163  * Configure syscall support on this CPU.
 164  */
 165 /*ARGSUSED*/
 166 void
 167 init_cpu_syscall(struct cpu *cp)
 168 {
 169         kpreempt_disable();
 170 
 171 #if defined(__amd64)
 172         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
 173             is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
 174                 uint64_t flags;
 175 
 176 #if !defined(__lint)
 177                 /*
 178                  * The syscall instruction imposes a certain ordering on
 179                  * segment selectors, so we double-check that ordering
 180                  * here.
 181                  */
 182                 ASSERT(KDS_SEL == KCS_SEL + 8);
 183                 ASSERT(UDS_SEL == U32CS_SEL + 8);
 184                 ASSERT(UCS_SEL == U32CS_SEL + 16);
 185 #endif
 186                 /*
 187                  * Turn syscall/sysret extensions on.
 188                  */
 189                 cpu_asysc_enable();
 190 
 191                 /*
 192                  * Program the magic registers ..
 193                  */
 194                 wrmsr(MSR_AMD_STAR,
 195                     ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);
 196                 wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall);
 197                 wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32);
 198 
 199                 /*
 200                  * This list of flags is masked off the incoming
 201                  * %rfl when we enter the kernel.
 202                  */
 203                 flags = PS_IE | PS_T;
 204                 if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_TRUE)
 205                         flags |= PS_ACHK;
 206                 wrmsr(MSR_AMD_SFMASK, flags);
 207         }
 208 #endif
 209 
 210         /*
 211          * On 32-bit kernels, we use sysenter/sysexit because it's too
 212          * hard to use syscall/sysret, and it is more portable anyway.
 213          *
 214          * On 64-bit kernels on Nocona machines, the 32-bit syscall
 215          * variant isn't available to 32-bit applications, but sysenter is.
 216          */
 217         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
 218             is_x86_feature(x86_featureset, X86FSET_SEP)) {
 219 
 220 #if !defined(__lint)
 221                 /*
 222                  * The sysenter instruction imposes a certain ordering on
 223                  * segment selectors, so we double-check that ordering
 224                  * here. See "sysenter" in Intel document 245471-012, "IA-32
 225                  * Intel Architecture Software Developer's Manual Volume 2:
 226                  * Instruction Set Reference"
 227                  */
 228                 ASSERT(KDS_SEL == KCS_SEL + 8);
 229 
 230                 ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3));
 231                 ASSERT32(UDS_SEL == UCS_SEL + 8);
 232 
 233                 ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3));
 234                 ASSERT64(UDS_SEL == U32CS_SEL + 8);
 235 #endif
 236 
 237                 cpu_sep_enable();
 238 
 239                 /*
 240                  * resume() sets this value to the base of the threads stack
 241                  * via a context handler.
 242                  */
 243                 wrmsr(MSR_INTC_SEP_ESP, 0);
 244                 wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter);
 245         }
 246 
 247         kpreempt_enable();
 248 }
 249 
 250 #if !defined(__xpv)
 251 /*
 252  * Configure per-cpu ID GDT
 253  */
 254 static void
 255 init_cpu_id_gdt(struct cpu *cp)
 256 {
 257         /* Write cpu_id into limit field of GDT for usermode retrieval */
 258 #if defined(__amd64)
 259         set_usegd(&cp->cpu_gdt[GDT_CPUID], SDP_SHORT, NULL, cp->cpu_id,
 260             SDT_MEMRODA, SEL_UPL, SDP_BYTES, SDP_OP32);
 261 #elif defined(__i386)
 262         set_usegd(&cp->cpu_gdt[GDT_CPUID], NULL, cp->cpu_id, SDT_MEMRODA,
 263             SEL_UPL, SDP_BYTES, SDP_OP32);
 264 #endif
 265 }
 266 #endif /* !defined(__xpv) */
 267 
 268 /*
 269  * Multiprocessor initialization.
 270  *
 271  * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the
 272  * startup and idle threads for the specified CPU.
 273  * Parameter boot is true for boot time operations and is false for CPU
 274  * DR operations.
 275  */
 276 static struct cpu *
 277 mp_cpu_configure_common(int cpun, boolean_t boot)
 278 {
 279         struct cpu *cp;
 280         kthread_id_t tp;
 281         caddr_t sp;
 282         proc_t *procp;
 283 #if !defined(__xpv)
 284         extern int idle_cpu_prefer_mwait;
 285         extern void cpu_idle_mwait();
 286 #endif
 287         extern void idle();
 288         extern void cpu_idle();
 289 
 290 #ifdef TRAPTRACE
 291         trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun];
 292 #endif
 293 
 294         ASSERT(MUTEX_HELD(&cpu_lock));
 295         ASSERT(cpun < NCPU && cpu[cpun] == NULL);
 296 
 297         if (cpu_free_list == NULL) {
 298                 cp = kmem_zalloc(sizeof (*cp), KM_SLEEP);
 299         } else {
 300                 cp = cpu_free_list;
 301                 cpu_free_list = cp->cpu_next_free;
 302         }
 303 
 304         cp->cpu_m.mcpu_istamp = cpun << 16;
 305 
 306         /* Create per CPU specific threads in the process p0. */
 307         procp = &p0;
 308 
 309         /*
 310          * Initialize the dispatcher first.
 311          */
 312         disp_cpu_init(cp);
 313 
 314         cpu_vm_data_init(cp);
 315 
 316         /*
 317          * Allocate and initialize the startup thread for this CPU.
 318          * Interrupt and process switch stacks get allocated later
 319          * when the CPU starts running.
 320          */
 321         tp = thread_create(NULL, 0, NULL, NULL, 0, procp,
 322             TS_STOPPED, maxclsyspri);
 323 
 324         /*
 325          * Set state to TS_ONPROC since this thread will start running
 326          * as soon as the CPU comes online.
 327          *
 328          * All the other fields of the thread structure are setup by
 329          * thread_create().
 330          */
 331         THREAD_ONPROC(tp, cp);
 332         tp->t_preempt = 1;
 333         tp->t_bound_cpu = cp;
 334         tp->t_affinitycnt = 1;
 335         tp->t_cpu = cp;
 336         tp->t_disp_queue = cp->cpu_disp;
 337 
 338         /*
 339          * Setup thread to start in mp_startup_common.
 340          */
 341         sp = tp->t_stk;
 342         tp->t_sp = (uintptr_t)(sp - MINFRAME);
 343 #if defined(__amd64)
 344         tp->t_sp -= STACK_ENTRY_ALIGN;               /* fake a call */
 345 #endif
 346         /*
 347          * Setup thread start entry point for boot or hotplug.
 348          */
 349         if (boot) {
 350                 tp->t_pc = (uintptr_t)mp_startup_boot;
 351         } else {
 352                 tp->t_pc = (uintptr_t)mp_startup_hotplug;
 353         }
 354 
 355         cp->cpu_id = cpun;
 356         cp->cpu_self = cp;
 357         cp->cpu_thread = tp;
 358         cp->cpu_lwp = NULL;
 359         cp->cpu_dispthread = tp;
 360         cp->cpu_dispatch_pri = DISP_PRIO(tp);
 361 
 362         /*
 363          * cpu_base_spl must be set explicitly here to prevent any blocking
 364          * operations in mp_startup_common from causing the spl of the cpu
 365          * to drop to 0 (allowing device interrupts before we're ready) in
 366          * resume().
 367          * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY.
 368          * As an extra bit of security on DEBUG kernels, this is enforced with
 369          * an assertion in mp_startup_common() -- before cpu_base_spl is set
 370          * to its proper value.
 371          */
 372         cp->cpu_base_spl = ipltospl(LOCK_LEVEL);
 373 
 374         /*
 375          * Now, initialize per-CPU idle thread for this CPU.
 376          */
 377         tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1);
 378 
 379         cp->cpu_idle_thread = tp;
 380 
 381         tp->t_preempt = 1;
 382         tp->t_bound_cpu = cp;
 383         tp->t_affinitycnt = 1;
 384         tp->t_cpu = cp;
 385         tp->t_disp_queue = cp->cpu_disp;
 386 
 387         /*
 388          * Bootstrap the CPU's PG data
 389          */
 390         pg_cpu_bootstrap(cp);
 391 
 392         /*
 393          * Perform CPC initialization on the new CPU.
 394          */
 395         kcpc_hw_init(cp);
 396 
 397         /*
 398          * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2
 399          * for each CPU.
 400          */
 401         setup_vaddr_for_ppcopy(cp);
 402 
 403         /*
 404          * Allocate page for new GDT and initialize from current GDT.
 405          */
 406 #if !defined(__lint)
 407         ASSERT((sizeof (*cp->cpu_gdt) * NGDT) <= PAGESIZE);
 408 #endif
 409         cp->cpu_gdt = kmem_zalloc(PAGESIZE, KM_SLEEP);
 410         bcopy(CPU->cpu_gdt, cp->cpu_gdt, (sizeof (*cp->cpu_gdt) * NGDT));
 411 
 412 #if defined(__i386)
 413         /*
 414          * setup kernel %gs.
 415          */
 416         set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA,
 417             SEL_KPL, 0, 1);
 418 #endif
 419 
 420         /*
 421          * If we have more than one node, each cpu gets a copy of IDT
 422          * local to its node. If this is a Pentium box, we use cpu 0's
 423          * IDT. cpu 0's IDT has been made read-only to workaround the
 424          * cmpxchgl register bug
 425          */
 426         if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) {
 427 #if !defined(__lint)
 428                 ASSERT((sizeof (*CPU->cpu_idt) * NIDT) <= PAGESIZE);
 429 #endif
 430                 cp->cpu_idt = kmem_zalloc(PAGESIZE, KM_SLEEP);
 431                 bcopy(CPU->cpu_idt, cp->cpu_idt, PAGESIZE);
 432         } else {
 433                 cp->cpu_idt = CPU->cpu_idt;
 434         }
 435 
 436         /*
 437          * alloc space for cpuid info
 438          */
 439         cpuid_alloc_space(cp);
 440 #if !defined(__xpv)
 441         if (is_x86_feature(x86_featureset, X86FSET_MWAIT) &&
 442             idle_cpu_prefer_mwait) {
 443                 cp->cpu_m.mcpu_mwait = cpuid_mwait_alloc(cp);
 444                 cp->cpu_m.mcpu_idle_cpu = cpu_idle_mwait;
 445         } else
 446 #endif
 447                 cp->cpu_m.mcpu_idle_cpu = cpu_idle;
 448 
 449         init_cpu_info(cp);
 450 
 451 #if !defined(__xpv)
 452         init_cpu_id_gdt(cp);
 453 #endif
 454 
 455         /*
 456          * alloc space for ucode_info
 457          */
 458         ucode_alloc_space(cp);
 459         xc_init_cpu(cp);
 460         hat_cpu_online(cp);
 461 
 462 #ifdef TRAPTRACE
 463         /*
 464          * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers
 465          */
 466         ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP);
 467         ttc->ttc_next = ttc->ttc_first;
 468         ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize;
 469 #endif
 470 
 471         /*
 472          * Record that we have another CPU.
 473          */
 474         /*
 475          * Initialize the interrupt threads for this CPU
 476          */
 477         cpu_intr_alloc(cp, NINTR_THREADS);
 478 
 479         cp->cpu_flags = CPU_OFFLINE | CPU_QUIESCED | CPU_POWEROFF;
 480         cpu_set_state(cp);
 481 
 482         /*
 483          * Add CPU to list of available CPUs.  It'll be on the active list
 484          * after mp_startup_common().
 485          */
 486         cpu_add_unit(cp);
 487 
 488         return (cp);
 489 }
 490 
 491 /*
 492  * Undo what was done in mp_cpu_configure_common
 493  */
 494 static void
 495 mp_cpu_unconfigure_common(struct cpu *cp, int error)
 496 {
 497         ASSERT(MUTEX_HELD(&cpu_lock));
 498 
 499         /*
 500          * Remove the CPU from the list of available CPUs.
 501          */
 502         cpu_del_unit(cp->cpu_id);
 503 
 504         if (error == ETIMEDOUT) {
 505                 /*
 506                  * The cpu was started, but never *seemed* to run any
 507                  * code in the kernel; it's probably off spinning in its
 508                  * own private world, though with potential references to
 509                  * our kmem-allocated IDTs and GDTs (for example).
 510                  *
 511                  * Worse still, it may actually wake up some time later,
 512                  * so rather than guess what it might or might not do, we
 513                  * leave the fundamental data structures intact.
 514                  */
 515                 cp->cpu_flags = 0;
 516                 return;
 517         }
 518 
 519         /*
 520          * At this point, the only threads bound to this CPU should
 521          * special per-cpu threads: it's idle thread, it's pause threads,
 522          * and it's interrupt threads.  Clean these up.
 523          */
 524         cpu_destroy_bound_threads(cp);
 525         cp->cpu_idle_thread = NULL;
 526 
 527         /*
 528          * Free the interrupt stack.
 529          */
 530         segkp_release(segkp,
 531             cp->cpu_intr_stack - (INTR_STACK_SIZE - SA(MINFRAME)));
 532         cp->cpu_intr_stack = NULL;
 533 
 534 #ifdef TRAPTRACE
 535         /*
 536          * Discard the trap trace buffer
 537          */
 538         {
 539                 trap_trace_ctl_t *ttc = &trap_trace_ctl[cp->cpu_id];
 540 
 541                 kmem_free((void *)ttc->ttc_first, trap_trace_bufsize);
 542                 ttc->ttc_first = NULL;
 543         }
 544 #endif
 545 
 546         hat_cpu_offline(cp);
 547 
 548         ucode_free_space(cp);
 549 
 550         /* Free CPU ID string and brand string. */
 551         if (cp->cpu_idstr) {
 552                 kmem_free(cp->cpu_idstr, CPU_IDSTRLEN);
 553                 cp->cpu_idstr = NULL;
 554         }
 555         if (cp->cpu_brandstr) {
 556                 kmem_free(cp->cpu_brandstr, CPU_IDSTRLEN);
 557                 cp->cpu_brandstr = NULL;
 558         }
 559 
 560 #if !defined(__xpv)
 561         if (cp->cpu_m.mcpu_mwait != NULL) {
 562                 cpuid_mwait_free(cp);
 563                 cp->cpu_m.mcpu_mwait = NULL;
 564         }
 565 #endif
 566         cpuid_free_space(cp);
 567 
 568         if (cp->cpu_idt != CPU->cpu_idt)
 569                 kmem_free(cp->cpu_idt, PAGESIZE);
 570         cp->cpu_idt = NULL;
 571 
 572         kmem_free(cp->cpu_gdt, PAGESIZE);
 573         cp->cpu_gdt = NULL;
 574 
 575         if (cp->cpu_supp_freqs != NULL) {
 576                 size_t len = strlen(cp->cpu_supp_freqs) + 1;
 577                 kmem_free(cp->cpu_supp_freqs, len);
 578                 cp->cpu_supp_freqs = NULL;
 579         }
 580 
 581         teardown_vaddr_for_ppcopy(cp);
 582 
 583         kcpc_hw_fini(cp);
 584 
 585         cp->cpu_dispthread = NULL;
 586         cp->cpu_thread = NULL;       /* discarded by cpu_destroy_bound_threads() */
 587 
 588         cpu_vm_data_destroy(cp);
 589 
 590         xc_fini_cpu(cp);
 591         disp_cpu_fini(cp);
 592 
 593         ASSERT(cp != CPU0);
 594         bzero(cp, sizeof (*cp));
 595         cp->cpu_next_free = cpu_free_list;
 596         cpu_free_list = cp;
 597 }
 598 
 599 /*
 600  * Apply workarounds for known errata, and warn about those that are absent.
 601  *
 602  * System vendors occasionally create configurations which contain different
 603  * revisions of the CPUs that are almost but not exactly the same.  At the
 604  * time of writing, this meant that their clock rates were the same, their
 605  * feature sets were the same, but the required workaround were -not-
 606  * necessarily the same.  So, this routine is invoked on -every- CPU soon
 607  * after starting to make sure that the resulting system contains the most
 608  * pessimal set of workarounds needed to cope with *any* of the CPUs in the
 609  * system.
 610  *
 611  * workaround_errata is invoked early in mlsetup() for CPU 0, and in
 612  * mp_startup_common() for all slave CPUs. Slaves process workaround_errata
 613  * prior to acknowledging their readiness to the master, so this routine will
 614  * never be executed by multiple CPUs in parallel, thus making updates to
 615  * global data safe.
 616  *
 617  * These workarounds are based on Rev 3.57 of the Revision Guide for
 618  * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005.
 619  */
 620 
 621 #if defined(OPTERON_ERRATUM_88)
 622 int opteron_erratum_88;         /* if non-zero -> at least one cpu has it */
 623 #endif
 624 
 625 #if defined(OPTERON_ERRATUM_91)
 626 int opteron_erratum_91;         /* if non-zero -> at least one cpu has it */
 627 #endif
 628 
 629 #if defined(OPTERON_ERRATUM_93)
 630 int opteron_erratum_93;         /* if non-zero -> at least one cpu has it */
 631 #endif
 632 
 633 #if defined(OPTERON_ERRATUM_95)
 634 int opteron_erratum_95;         /* if non-zero -> at least one cpu has it */
 635 #endif
 636 
 637 #if defined(OPTERON_ERRATUM_100)
 638 int opteron_erratum_100;        /* if non-zero -> at least one cpu has it */
 639 #endif
 640 
 641 #if defined(OPTERON_ERRATUM_108)
 642 int opteron_erratum_108;        /* if non-zero -> at least one cpu has it */
 643 #endif
 644 
 645 #if defined(OPTERON_ERRATUM_109)
 646 int opteron_erratum_109;        /* if non-zero -> at least one cpu has it */
 647 #endif
 648 
 649 #if defined(OPTERON_ERRATUM_121)
 650 int opteron_erratum_121;        /* if non-zero -> at least one cpu has it */
 651 #endif
 652 
 653 #if defined(OPTERON_ERRATUM_122)
 654 int opteron_erratum_122;        /* if non-zero -> at least one cpu has it */
 655 #endif
 656 
 657 #if defined(OPTERON_ERRATUM_123)
 658 int opteron_erratum_123;        /* if non-zero -> at least one cpu has it */
 659 #endif
 660 
 661 #if defined(OPTERON_ERRATUM_131)
 662 int opteron_erratum_131;        /* if non-zero -> at least one cpu has it */
 663 #endif
 664 
 665 #if defined(OPTERON_WORKAROUND_6336786)
 666 int opteron_workaround_6336786; /* non-zero -> WA relevant and applied */
 667 int opteron_workaround_6336786_UP = 0;  /* Not needed for UP */
 668 #endif
 669 
 670 #if defined(OPTERON_WORKAROUND_6323525)
 671 int opteron_workaround_6323525; /* if non-zero -> at least one cpu has it */
 672 #endif
 673 
 674 #if defined(OPTERON_ERRATUM_298)
 675 int opteron_erratum_298;
 676 #endif
 677 
 678 #if defined(OPTERON_ERRATUM_721)
 679 int opteron_erratum_721;
 680 #endif
 681 
 682 static void
 683 workaround_warning(cpu_t *cp, uint_t erratum)
 684 {
 685         cmn_err(CE_WARN, "cpu%d: no workaround for erratum %u",
 686             cp->cpu_id, erratum);
 687 }
 688 
 689 static void
 690 workaround_applied(uint_t erratum)
 691 {
 692         if (erratum > 1000000)
 693                 cmn_err(CE_CONT, "?workaround applied for cpu issue #%d\n",
 694                     erratum);
 695         else
 696                 cmn_err(CE_CONT, "?workaround applied for cpu erratum #%d\n",
 697                     erratum);
 698 }
 699 
 700 static void
 701 msr_warning(cpu_t *cp, const char *rw, uint_t msr, int error)
 702 {
 703         cmn_err(CE_WARN, "cpu%d: couldn't %smsr 0x%x, error %d",
 704             cp->cpu_id, rw, msr, error);
 705 }
 706 
 707 /*
 708  * Determine the number of nodes in a Hammer / Greyhound / Griffin family
 709  * system.
 710  */
 711 static uint_t
 712 opteron_get_nnodes(void)
 713 {
 714         static uint_t nnodes = 0;
 715 
 716         if (nnodes == 0) {
 717 #ifdef  DEBUG
 718                 uint_t family;
 719 
 720                 /*
 721                  * This routine uses a PCI config space based mechanism
 722                  * for retrieving the number of nodes in the system.
 723                  * Device 24, function 0, offset 0x60 as used here is not
 724                  * AMD processor architectural, and may not work on processor
 725                  * families other than those listed below.
 726                  *
 727                  * Callers of this routine must ensure that we're running on
 728                  * a processor which supports this mechanism.
 729                  * The assertion below is meant to catch calls on unsupported
 730                  * processors.
 731                  */
 732                 family = cpuid_getfamily(CPU);
 733                 ASSERT(family == 0xf || family == 0x10 || family == 0x11);
 734 #endif  /* DEBUG */
 735 
 736                 /*
 737                  * Obtain the number of nodes in the system from
 738                  * bits [6:4] of the Node ID register on node 0.
 739                  *
 740                  * The actual node count is NodeID[6:4] + 1
 741                  *
 742                  * The Node ID register is accessed via function 0,
 743                  * offset 0x60. Node 0 is device 24.
 744                  */
 745                 nnodes = ((pci_getl_func(0, 24, 0, 0x60) & 0x70) >> 4) + 1;
 746         }
 747         return (nnodes);
 748 }
 749 
 750 uint_t
 751 do_erratum_298(struct cpu *cpu)
 752 {
 753         static int      osvwrc = -3;
 754         extern int      osvw_opteron_erratum(cpu_t *, uint_t);
 755 
 756         /*
 757          * L2 Eviction May Occur During Processor Operation To Set
 758          * Accessed or Dirty Bit.
 759          */
 760         if (osvwrc == -3) {
 761                 osvwrc = osvw_opteron_erratum(cpu, 298);
 762         } else {
 763                 /* osvw return codes should be consistent for all cpus */
 764                 ASSERT(osvwrc == osvw_opteron_erratum(cpu, 298));
 765         }
 766 
 767         switch (osvwrc) {
 768         case 0:         /* erratum is not present: do nothing */
 769                 break;
 770         case 1:         /* erratum is present: BIOS workaround applied */
 771                 /*
 772                  * check if workaround is actually in place and issue warning
 773                  * if not.
 774                  */
 775                 if (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
 776                     ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0)) {
 777 #if defined(OPTERON_ERRATUM_298)
 778                         opteron_erratum_298++;
 779 #else
 780                         workaround_warning(cpu, 298);
 781                         return (1);
 782 #endif
 783                 }
 784                 break;
 785         case -1:        /* cannot determine via osvw: check cpuid */
 786                 if ((cpuid_opteron_erratum(cpu, 298) > 0) &&
 787                     (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
 788                     ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0))) {
 789 #if defined(OPTERON_ERRATUM_298)
 790                         opteron_erratum_298++;
 791 #else
 792                         workaround_warning(cpu, 298);
 793                         return (1);
 794 #endif
 795                 }
 796                 break;
 797         }
 798         return (0);
 799 }
 800 
 801 uint_t
 802 workaround_errata(struct cpu *cpu)
 803 {
 804         uint_t missing = 0;
 805 
 806         ASSERT(cpu == CPU);
 807 
 808         /*LINTED*/
 809         if (cpuid_opteron_erratum(cpu, 88) > 0) {
 810                 /*
 811                  * SWAPGS May Fail To Read Correct GS Base
 812                  */
 813 #if defined(OPTERON_ERRATUM_88)
 814                 /*
 815                  * The workaround is an mfence in the relevant assembler code
 816                  */
 817                 opteron_erratum_88++;
 818 #else
 819                 workaround_warning(cpu, 88);
 820                 missing++;
 821 #endif
 822         }
 823 
 824         if (cpuid_opteron_erratum(cpu, 91) > 0) {
 825                 /*
 826                  * Software Prefetches May Report A Page Fault
 827                  */
 828 #if defined(OPTERON_ERRATUM_91)
 829                 /*
 830                  * fix is in trap.c
 831                  */
 832                 opteron_erratum_91++;
 833 #else
 834                 workaround_warning(cpu, 91);
 835                 missing++;
 836 #endif
 837         }
 838 
 839         if (cpuid_opteron_erratum(cpu, 93) > 0) {
 840                 /*
 841                  * RSM Auto-Halt Restart Returns to Incorrect RIP
 842                  */
 843 #if defined(OPTERON_ERRATUM_93)
 844                 /*
 845                  * fix is in trap.c
 846                  */
 847                 opteron_erratum_93++;
 848 #else
 849                 workaround_warning(cpu, 93);
 850                 missing++;
 851 #endif
 852         }
 853 
 854         /*LINTED*/
 855         if (cpuid_opteron_erratum(cpu, 95) > 0) {
 856                 /*
 857                  * RET Instruction May Return to Incorrect EIP
 858                  */
 859 #if defined(OPTERON_ERRATUM_95)
 860 #if defined(_LP64)
 861                 /*
 862                  * Workaround this by ensuring that 32-bit user code and
 863                  * 64-bit kernel code never occupy the same address
 864                  * range mod 4G.
 865                  */
 866                 if (_userlimit32 > 0xc0000000ul)
 867                         *(uintptr_t *)&_userlimit32 = 0xc0000000ul;
 868 
 869                 /*LINTED*/
 870                 ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u);
 871                 opteron_erratum_95++;
 872 #endif  /* _LP64 */
 873 #else
 874                 workaround_warning(cpu, 95);
 875                 missing++;
 876 #endif
 877         }
 878 
 879         if (cpuid_opteron_erratum(cpu, 100) > 0) {
 880                 /*
 881                  * Compatibility Mode Branches Transfer to Illegal Address
 882                  */
 883 #if defined(OPTERON_ERRATUM_100)
 884                 /*
 885                  * fix is in trap.c
 886                  */
 887                 opteron_erratum_100++;
 888 #else
 889                 workaround_warning(cpu, 100);
 890                 missing++;
 891 #endif
 892         }
 893 
 894         /*LINTED*/
 895         if (cpuid_opteron_erratum(cpu, 108) > 0) {
 896                 /*
 897                  * CPUID Instruction May Return Incorrect Model Number In
 898                  * Some Processors
 899                  */
 900 #if defined(OPTERON_ERRATUM_108)
 901                 /*
 902                  * (Our cpuid-handling code corrects the model number on
 903                  * those processors)
 904                  */
 905 #else
 906                 workaround_warning(cpu, 108);
 907                 missing++;
 908 #endif
 909         }
 910 
 911         /*LINTED*/
 912         if (cpuid_opteron_erratum(cpu, 109) > 0) do {
 913                 /*
 914                  * Certain Reverse REP MOVS May Produce Unpredictable Behavior
 915                  */
 916 #if defined(OPTERON_ERRATUM_109)
 917                 /*
 918                  * The "workaround" is to print a warning to upgrade the BIOS
 919                  */
 920                 uint64_t value;
 921                 const uint_t msr = MSR_AMD_PATCHLEVEL;
 922                 int err;
 923 
 924                 if ((err = checked_rdmsr(msr, &value)) != 0) {
 925                         msr_warning(cpu, "rd", msr, err);
 926                         workaround_warning(cpu, 109);
 927                         missing++;
 928                 }
 929                 if (value == 0)
 930                         opteron_erratum_109++;
 931 #else
 932                 workaround_warning(cpu, 109);
 933                 missing++;
 934 #endif
 935         /*CONSTANTCONDITION*/
 936         } while (0);
 937 
 938         /*LINTED*/
 939         if (cpuid_opteron_erratum(cpu, 121) > 0) {
 940                 /*
 941                  * Sequential Execution Across Non_Canonical Boundary Caused
 942                  * Processor Hang
 943                  */
 944 #if defined(OPTERON_ERRATUM_121)
 945 #if defined(_LP64)
 946                 /*
 947                  * Erratum 121 is only present in long (64 bit) mode.
 948                  * Workaround is to include the page immediately before the
 949                  * va hole to eliminate the possibility of system hangs due to
 950                  * sequential execution across the va hole boundary.
 951                  */
 952                 if (opteron_erratum_121)
 953                         opteron_erratum_121++;
 954                 else {
 955                         if (hole_start) {
 956                                 hole_start -= PAGESIZE;
 957                         } else {
 958                                 /*
 959                                  * hole_start not yet initialized by
 960                                  * mmu_init. Initialize hole_start
 961                                  * with value to be subtracted.
 962                                  */
 963                                 hole_start = PAGESIZE;
 964                         }
 965                         opteron_erratum_121++;
 966                 }
 967 #endif  /* _LP64 */
 968 #else
 969                 workaround_warning(cpu, 121);
 970                 missing++;
 971 #endif
 972         }
 973 
 974         /*LINTED*/
 975         if (cpuid_opteron_erratum(cpu, 122) > 0) do {
 976                 /*
 977                  * TLB Flush Filter May Cause Coherency Problem in
 978                  * Multiprocessor Systems
 979                  */
 980 #if defined(OPTERON_ERRATUM_122)
 981                 uint64_t value;
 982                 const uint_t msr = MSR_AMD_HWCR;
 983                 int error;
 984 
 985                 /*
 986                  * Erratum 122 is only present in MP configurations (multi-core
 987                  * or multi-processor).
 988                  */
 989 #if defined(__xpv)
 990                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
 991                         break;
 992                 if (!opteron_erratum_122 && xpv_nr_phys_cpus() == 1)
 993                         break;
 994 #else
 995                 if (!opteron_erratum_122 && opteron_get_nnodes() == 1 &&
 996                     cpuid_get_ncpu_per_chip(cpu) == 1)
 997                         break;
 998 #endif
 999                 /* disable TLB Flush Filter */
1000 
1001                 if ((error = checked_rdmsr(msr, &value)) != 0) {
1002                         msr_warning(cpu, "rd", msr, error);
1003                         workaround_warning(cpu, 122);
1004                         missing++;
1005                 } else {
1006                         value |= (uint64_t)AMD_HWCR_FFDIS;
1007                         if ((error = checked_wrmsr(msr, value)) != 0) {
1008                                 msr_warning(cpu, "wr", msr, error);
1009                                 workaround_warning(cpu, 122);
1010                                 missing++;
1011                         }
1012                 }
1013                 opteron_erratum_122++;
1014 #else
1015                 workaround_warning(cpu, 122);
1016                 missing++;
1017 #endif
1018         /*CONSTANTCONDITION*/
1019         } while (0);
1020 
1021         /*LINTED*/
1022         if (cpuid_opteron_erratum(cpu, 123) > 0) do {
1023                 /*
1024                  * Bypassed Reads May Cause Data Corruption of System Hang in
1025                  * Dual Core Processors
1026                  */
1027 #if defined(OPTERON_ERRATUM_123)
1028                 uint64_t value;
1029                 const uint_t msr = MSR_AMD_PATCHLEVEL;
1030                 int err;
1031 
1032                 /*
1033                  * Erratum 123 applies only to multi-core cpus.
1034                  */
1035                 if (cpuid_get_ncpu_per_chip(cpu) < 2)
1036                         break;
1037 #if defined(__xpv)
1038                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1039                         break;
1040 #endif
1041                 /*
1042                  * The "workaround" is to print a warning to upgrade the BIOS
1043                  */
1044                 if ((err = checked_rdmsr(msr, &value)) != 0) {
1045                         msr_warning(cpu, "rd", msr, err);
1046                         workaround_warning(cpu, 123);
1047                         missing++;
1048                 }
1049                 if (value == 0)
1050                         opteron_erratum_123++;
1051 #else
1052                 workaround_warning(cpu, 123);
1053                 missing++;
1054 
1055 #endif
1056         /*CONSTANTCONDITION*/
1057         } while (0);
1058 
1059         /*LINTED*/
1060         if (cpuid_opteron_erratum(cpu, 131) > 0) do {
1061                 /*
1062                  * Multiprocessor Systems with Four or More Cores May Deadlock
1063                  * Waiting for a Probe Response
1064                  */
1065 #if defined(OPTERON_ERRATUM_131)
1066                 uint64_t nbcfg;
1067                 const uint_t msr = MSR_AMD_NB_CFG;
1068                 const uint64_t wabits =
1069                     AMD_NB_CFG_SRQ_HEARTBEAT | AMD_NB_CFG_SRQ_SPR;
1070                 int error;
1071 
1072                 /*
1073                  * Erratum 131 applies to any system with four or more cores.
1074                  */
1075                 if (opteron_erratum_131)
1076                         break;
1077 #if defined(__xpv)
1078                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1079                         break;
1080                 if (xpv_nr_phys_cpus() < 4)
1081                         break;
1082 #else
1083                 if (opteron_get_nnodes() * cpuid_get_ncpu_per_chip(cpu) < 4)
1084                         break;
1085 #endif
1086                 /*
1087                  * Print a warning if neither of the workarounds for
1088                  * erratum 131 is present.
1089                  */
1090                 if ((error = checked_rdmsr(msr, &nbcfg)) != 0) {
1091                         msr_warning(cpu, "rd", msr, error);
1092                         workaround_warning(cpu, 131);
1093                         missing++;
1094                 } else if ((nbcfg & wabits) == 0) {
1095                         opteron_erratum_131++;
1096                 } else {
1097                         /* cannot have both workarounds set */
1098                         ASSERT((nbcfg & wabits) != wabits);
1099                 }
1100 #else
1101                 workaround_warning(cpu, 131);
1102                 missing++;
1103 #endif
1104         /*CONSTANTCONDITION*/
1105         } while (0);
1106 
1107         /*
1108          * This isn't really an erratum, but for convenience the
1109          * detection/workaround code lives here and in cpuid_opteron_erratum.
1110          */
1111         if (cpuid_opteron_erratum(cpu, 6336786) > 0) {
1112 #if defined(OPTERON_WORKAROUND_6336786)
1113                 /*
1114                  * Disable C1-Clock ramping on multi-core/multi-processor
1115                  * K8 platforms to guard against TSC drift.
1116                  */
1117                 if (opteron_workaround_6336786) {
1118                         opteron_workaround_6336786++;
1119 #if defined(__xpv)
1120                 } else if ((DOMAIN_IS_INITDOMAIN(xen_info) &&
1121                     xpv_nr_phys_cpus() > 1) ||
1122                     opteron_workaround_6336786_UP) {
1123                         /*
1124                          * XXPV Hmm.  We can't walk the Northbridges on
1125                          *      the hypervisor; so just complain and drive
1126                          *      on.  This probably needs to be fixed in
1127                          *      the hypervisor itself.
1128                          */
1129                         opteron_workaround_6336786++;
1130                         workaround_warning(cpu, 6336786);
1131 #else   /* __xpv */
1132                 } else if ((opteron_get_nnodes() *
1133                     cpuid_get_ncpu_per_chip(cpu) > 1) ||
1134                     opteron_workaround_6336786_UP) {
1135 
1136                         uint_t  node, nnodes;
1137                         uint8_t data;
1138 
1139                         nnodes = opteron_get_nnodes();
1140                         for (node = 0; node < nnodes; node++) {
1141                                 /*
1142                                  * Clear PMM7[1:0] (function 3, offset 0x87)
1143                                  * Northbridge device is the node id + 24.
1144                                  */
1145                                 data = pci_getb_func(0, node + 24, 3, 0x87);
1146                                 data &= 0xFC;
1147                                 pci_putb_func(0, node + 24, 3, 0x87, data);
1148                         }
1149                         opteron_workaround_6336786++;
1150 #endif  /* __xpv */
1151                 }
1152 #else
1153                 workaround_warning(cpu, 6336786);
1154                 missing++;
1155 #endif
1156         }
1157 
1158         /*LINTED*/
1159         /*
1160          * Mutex primitives don't work as expected.
1161          */
1162         if (cpuid_opteron_erratum(cpu, 6323525) > 0) {
1163 #if defined(OPTERON_WORKAROUND_6323525)
1164                 /*
1165                  * This problem only occurs with 2 or more cores. If bit in
1166                  * MSR_AMD_BU_CFG set, then not applicable. The workaround
1167                  * is to patch the semaphone routines with the lfence
1168                  * instruction to provide necessary load memory barrier with
1169                  * possible subsequent read-modify-write ops.
1170                  *
1171                  * It is too early in boot to call the patch routine so
1172                  * set erratum variable to be done in startup_end().
1173                  */
1174                 if (opteron_workaround_6323525) {
1175                         opteron_workaround_6323525++;
1176 #if defined(__xpv)
1177                 } else if (is_x86_feature(x86_featureset, X86FSET_SSE2)) {
1178                         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1179                                 /*
1180                                  * XXPV Use dom0_msr here when extended
1181                                  *      operations are supported?
1182                                  */
1183                                 if (xpv_nr_phys_cpus() > 1)
1184                                         opteron_workaround_6323525++;
1185                         } else {
1186                                 /*
1187                                  * We have no way to tell how many physical
1188                                  * cpus there are, or even if this processor
1189                                  * has the problem, so enable the workaround
1190                                  * unconditionally (at some performance cost).
1191                                  */
1192                                 opteron_workaround_6323525++;
1193                         }
1194 #else   /* __xpv */
1195                 } else if (is_x86_feature(x86_featureset, X86FSET_SSE2) &&
1196                     ((opteron_get_nnodes() *
1197                     cpuid_get_ncpu_per_chip(cpu)) > 1)) {
1198                         if ((xrdmsr(MSR_AMD_BU_CFG) & (UINT64_C(1) << 33)) == 0)
1199                                 opteron_workaround_6323525++;
1200 #endif  /* __xpv */
1201                 }
1202 #else
1203                 workaround_warning(cpu, 6323525);
1204                 missing++;
1205 #endif
1206         }
1207 
1208         missing += do_erratum_298(cpu);
1209 
1210         if (cpuid_opteron_erratum(cpu, 721) > 0) {
1211 #if defined(OPTERON_ERRATUM_721)
1212                 on_trap_data_t otd;
1213 
1214                 if (!on_trap(&otd, OT_DATA_ACCESS))
1215                         wrmsr(MSR_AMD_DE_CFG,
1216                             rdmsr(MSR_AMD_DE_CFG) | AMD_DE_CFG_E721);
1217                 no_trap();
1218 
1219                 opteron_erratum_721++;
1220 #else
1221                 workaround_warning(cpu, 721);
1222                 missing++;
1223 #endif
1224         }
1225 
1226 #ifdef __xpv
1227         return (0);
1228 #else
1229         return (missing);
1230 #endif
1231 }
1232 
1233 void
1234 workaround_errata_end()
1235 {
1236 #if defined(OPTERON_ERRATUM_88)
1237         if (opteron_erratum_88)
1238                 workaround_applied(88);
1239 #endif
1240 #if defined(OPTERON_ERRATUM_91)
1241         if (opteron_erratum_91)
1242                 workaround_applied(91);
1243 #endif
1244 #if defined(OPTERON_ERRATUM_93)
1245         if (opteron_erratum_93)
1246                 workaround_applied(93);
1247 #endif
1248 #if defined(OPTERON_ERRATUM_95)
1249         if (opteron_erratum_95)
1250                 workaround_applied(95);
1251 #endif
1252 #if defined(OPTERON_ERRATUM_100)
1253         if (opteron_erratum_100)
1254                 workaround_applied(100);
1255 #endif
1256 #if defined(OPTERON_ERRATUM_108)
1257         if (opteron_erratum_108)
1258                 workaround_applied(108);
1259 #endif
1260 #if defined(OPTERON_ERRATUM_109)
1261         if (opteron_erratum_109) {
1262                 cmn_err(CE_WARN,
1263                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1264                     " processor\nerratum 109 was not detected; updating your"
1265                     " system's BIOS to a version\ncontaining this"
1266                     " microcode patch is HIGHLY recommended or erroneous"
1267                     " system\noperation may occur.\n");
1268         }
1269 #endif
1270 #if defined(OPTERON_ERRATUM_121)
1271         if (opteron_erratum_121)
1272                 workaround_applied(121);
1273 #endif
1274 #if defined(OPTERON_ERRATUM_122)
1275         if (opteron_erratum_122)
1276                 workaround_applied(122);
1277 #endif
1278 #if defined(OPTERON_ERRATUM_123)
1279         if (opteron_erratum_123) {
1280                 cmn_err(CE_WARN,
1281                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1282                     " processor\nerratum 123 was not detected; updating your"
1283                     " system's BIOS to a version\ncontaining this"
1284                     " microcode patch is HIGHLY recommended or erroneous"
1285                     " system\noperation may occur.\n");
1286         }
1287 #endif
1288 #if defined(OPTERON_ERRATUM_131)
1289         if (opteron_erratum_131) {
1290                 cmn_err(CE_WARN,
1291                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1292                     " processor\nerratum 131 was not detected; updating your"
1293                     " system's BIOS to a version\ncontaining this"
1294                     " microcode patch is HIGHLY recommended or erroneous"
1295                     " system\noperation may occur.\n");
1296         }
1297 #endif
1298 #if defined(OPTERON_WORKAROUND_6336786)
1299         if (opteron_workaround_6336786)
1300                 workaround_applied(6336786);
1301 #endif
1302 #if defined(OPTERON_WORKAROUND_6323525)
1303         if (opteron_workaround_6323525)
1304                 workaround_applied(6323525);
1305 #endif
1306 #if defined(OPTERON_ERRATUM_298)
1307         if (opteron_erratum_298) {
1308                 cmn_err(CE_WARN,
1309                     "BIOS microcode patch for AMD 64/Opteron(tm)"
1310                     " processor\nerratum 298 was not detected; updating your"
1311                     " system's BIOS to a version\ncontaining this"
1312                     " microcode patch is HIGHLY recommended or erroneous"
1313                     " system\noperation may occur.\n");
1314         }
1315 #endif
1316 #if defined(OPTERON_ERRATUM_721)
1317         if (opteron_erratum_721)
1318                 workaround_applied(721);
1319 #endif
1320 }
1321 
1322 /*
1323  * The procset_slave and procset_master are used to synchronize
1324  * between the control CPU and the target CPU when starting CPUs.
1325  */
1326 static cpuset_t procset_slave, procset_master;
1327 
1328 static void
1329 mp_startup_wait(cpuset_t *sp, processorid_t cpuid)
1330 {
1331         cpuset_t tempset;
1332 
1333         for (tempset = *sp; !CPU_IN_SET(tempset, cpuid);
1334             tempset = *(volatile cpuset_t *)sp) {
1335                 SMT_PAUSE();
1336         }
1337         CPUSET_ATOMIC_DEL(*(cpuset_t *)sp, cpuid);
1338 }
1339 
1340 static void
1341 mp_startup_signal(cpuset_t *sp, processorid_t cpuid)
1342 {
1343         cpuset_t tempset;
1344 
1345         CPUSET_ATOMIC_ADD(*(cpuset_t *)sp, cpuid);
1346         for (tempset = *sp; CPU_IN_SET(tempset, cpuid);
1347             tempset = *(volatile cpuset_t *)sp) {
1348                 SMT_PAUSE();
1349         }
1350 }
1351 
1352 int
1353 mp_start_cpu_common(cpu_t *cp, boolean_t boot)
1354 {
1355         _NOTE(ARGUNUSED(boot));
1356 
1357         void *ctx;
1358         int delays;
1359         int error = 0;
1360         cpuset_t tempset;
1361         processorid_t cpuid;
1362 #ifndef __xpv
1363         extern void cpupm_init(cpu_t *);
1364 #endif
1365 
1366         ASSERT(cp != NULL);
1367         cpuid = cp->cpu_id;
1368         ctx = mach_cpucontext_alloc(cp);
1369         if (ctx == NULL) {
1370                 cmn_err(CE_WARN,
1371                     "cpu%d: failed to allocate context", cp->cpu_id);
1372                 return (EAGAIN);
1373         }
1374         error = mach_cpu_start(cp, ctx);
1375         if (error != 0) {
1376                 cmn_err(CE_WARN,
1377                     "cpu%d: failed to start, error %d", cp->cpu_id, error);
1378                 mach_cpucontext_free(cp, ctx, error);
1379                 return (error);
1380         }
1381 
1382         for (delays = 0, tempset = procset_slave; !CPU_IN_SET(tempset, cpuid);
1383             delays++) {
1384                 if (delays == 500) {
1385                         /*
1386                          * After five seconds, things are probably looking
1387                          * a bit bleak - explain the hang.
1388                          */
1389                         cmn_err(CE_NOTE, "cpu%d: started, "
1390                             "but not running in the kernel yet", cpuid);
1391                 } else if (delays > 2000) {
1392                         /*
1393                          * We waited at least 20 seconds, bail ..
1394                          */
1395                         error = ETIMEDOUT;
1396                         cmn_err(CE_WARN, "cpu%d: timed out", cpuid);
1397                         mach_cpucontext_free(cp, ctx, error);
1398                         return (error);
1399                 }
1400 
1401                 /*
1402                  * wait at least 10ms, then check again..
1403                  */
1404                 delay(USEC_TO_TICK_ROUNDUP(10000));
1405                 tempset = *((volatile cpuset_t *)&procset_slave);
1406         }
1407         CPUSET_ATOMIC_DEL(procset_slave, cpuid);
1408 
1409         mach_cpucontext_free(cp, ctx, 0);
1410 
1411 #ifndef __xpv
1412         if (tsc_gethrtime_enable)
1413                 tsc_sync_master(cpuid);
1414 #endif
1415 
1416         if (dtrace_cpu_init != NULL) {
1417                 (*dtrace_cpu_init)(cpuid);
1418         }
1419 
1420         /*
1421          * During CPU DR operations, the cpu_lock is held by current
1422          * (the control) thread. We can't release the cpu_lock here
1423          * because that will break the CPU DR logic.
1424          * On the other hand, CPUPM and processor group initialization
1425          * routines need to access the cpu_lock. So we invoke those
1426          * routines here on behalf of mp_startup_common().
1427          *
1428          * CPUPM and processor group initialization routines depend
1429          * on the cpuid probing results. Wait for mp_startup_common()
1430          * to signal that cpuid probing is done.
1431          */
1432         mp_startup_wait(&procset_slave, cpuid);
1433 #ifndef __xpv
1434         cpupm_init(cp);
1435 #endif
1436         (void) pg_cpu_init(cp, B_FALSE);
1437         cpu_set_state(cp);
1438         mp_startup_signal(&procset_master, cpuid);
1439 
1440         return (0);
1441 }
1442 
1443 /*
1444  * Start a single cpu, assuming that the kernel context is available
1445  * to successfully start another cpu.
1446  *
1447  * (For example, real mode code is mapped into the right place
1448  * in memory and is ready to be run.)
1449  */
1450 int
1451 start_cpu(processorid_t who)
1452 {
1453         cpu_t *cp;
1454         int error = 0;
1455         cpuset_t tempset;
1456 
1457         ASSERT(who != 0);
1458 
1459         /*
1460          * Check if there's at least a Mbyte of kmem available
1461          * before attempting to start the cpu.
1462          */
1463         if (kmem_avail() < 1024 * 1024) {
1464                 /*
1465                  * Kick off a reap in case that helps us with
1466                  * later attempts ..
1467                  */
1468                 kmem_reap();
1469                 return (ENOMEM);
1470         }
1471 
1472         /*
1473          * First configure cpu.
1474          */
1475         cp = mp_cpu_configure_common(who, B_TRUE);
1476         ASSERT(cp != NULL);
1477 
1478         /*
1479          * Then start cpu.
1480          */
1481         error = mp_start_cpu_common(cp, B_TRUE);
1482         if (error != 0) {
1483                 mp_cpu_unconfigure_common(cp, error);
1484                 return (error);
1485         }
1486 
1487         mutex_exit(&cpu_lock);
1488         tempset = cpu_ready_set;
1489         while (!CPU_IN_SET(tempset, who)) {
1490                 drv_usecwait(1);
1491                 tempset = *((volatile cpuset_t *)&cpu_ready_set);
1492         }
1493         mutex_enter(&cpu_lock);
1494 
1495         return (0);
1496 }
1497 
1498 void
1499 start_other_cpus(int cprboot)
1500 {
1501         _NOTE(ARGUNUSED(cprboot));
1502 
1503         uint_t who;
1504         uint_t bootcpuid = 0;
1505 
1506         /*
1507          * Initialize our own cpu_info.
1508          */
1509         init_cpu_info(CPU);
1510 
1511 #if !defined(__xpv)
1512         init_cpu_id_gdt(CPU);
1513 #endif
1514 
1515         cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_idstr);
1516         cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_brandstr);
1517 
1518         /*
1519          * Initialize our syscall handlers
1520          */
1521         init_cpu_syscall(CPU);
1522 
1523         /*
1524          * Take the boot cpu out of the mp_cpus set because we know
1525          * it's already running.  Add it to the cpu_ready_set for
1526          * precisely the same reason.
1527          */
1528         CPUSET_DEL(mp_cpus, bootcpuid);
1529         CPUSET_ADD(cpu_ready_set, bootcpuid);
1530 
1531         /*
1532          * skip the rest of this if
1533          * . only 1 cpu dectected and system isn't hotplug-capable
1534          * . not using MP
1535          */
1536         if ((CPUSET_ISNULL(mp_cpus) && plat_dr_support_cpu() == 0) ||
1537             use_mp == 0) {
1538                 if (use_mp == 0)
1539                         cmn_err(CE_CONT, "?***** Not in MP mode\n");
1540                 goto done;
1541         }
1542 
1543         /*
1544          * perform such initialization as is needed
1545          * to be able to take CPUs on- and off-line.
1546          */
1547         cpu_pause_init();
1548 
1549         xc_init_cpu(CPU);               /* initialize processor crosscalls */
1550 
1551         if (mach_cpucontext_init() != 0)
1552                 goto done;
1553 
1554         flushes_require_xcalls = 1;
1555 
1556         /*
1557          * We lock our affinity to the master CPU to ensure that all slave CPUs
1558          * do their TSC syncs with the same CPU.
1559          */
1560         affinity_set(CPU_CURRENT);
1561 
1562         for (who = 0; who < NCPU; who++) {
1563                 if (!CPU_IN_SET(mp_cpus, who))
1564                         continue;
1565                 ASSERT(who != bootcpuid);
1566 
1567                 mutex_enter(&cpu_lock);
1568                 if (start_cpu(who) != 0)
1569                         CPUSET_DEL(mp_cpus, who);
1570                 cpu_state_change_notify(who, CPU_SETUP);
1571                 mutex_exit(&cpu_lock);
1572         }
1573 
1574         /* Free the space allocated to hold the microcode file */
1575         ucode_cleanup();
1576 
1577         affinity_clear();
1578 
1579         mach_cpucontext_fini();
1580 
1581 done:
1582         if (get_hwenv() == HW_NATIVE)
1583                 workaround_errata_end();
1584         cmi_post_mpstartup();
1585 
1586         if (use_mp && ncpus != boot_max_ncpus) {
1587                 cmn_err(CE_NOTE,
1588                     "System detected %d cpus, but "
1589                     "only %d cpu(s) were enabled during boot.",
1590                     boot_max_ncpus, ncpus);
1591                 cmn_err(CE_NOTE,
1592                     "Use \"boot-ncpus\" parameter to enable more CPU(s). "
1593                     "See eeprom(1M).");
1594         }
1595 }
1596 
1597 int
1598 mp_cpu_configure(int cpuid)
1599 {
1600         cpu_t *cp;
1601 
1602         if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1603                 return (ENOTSUP);
1604         }
1605 
1606         cp = cpu_get(cpuid);
1607         if (cp != NULL) {
1608                 return (EALREADY);
1609         }
1610 
1611         /*
1612          * Check if there's at least a Mbyte of kmem available
1613          * before attempting to start the cpu.
1614          */
1615         if (kmem_avail() < 1024 * 1024) {
1616                 /*
1617                  * Kick off a reap in case that helps us with
1618                  * later attempts ..
1619                  */
1620                 kmem_reap();
1621                 return (ENOMEM);
1622         }
1623 
1624         cp = mp_cpu_configure_common(cpuid, B_FALSE);
1625         ASSERT(cp != NULL && cpu_get(cpuid) == cp);
1626 
1627         return (cp != NULL ? 0 : EAGAIN);
1628 }
1629 
1630 int
1631 mp_cpu_unconfigure(int cpuid)
1632 {
1633         cpu_t *cp;
1634 
1635         if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1636                 return (ENOTSUP);
1637         } else if (cpuid < 0 || cpuid >= max_ncpus) {
1638                 return (EINVAL);
1639         }
1640 
1641         cp = cpu_get(cpuid);
1642         if (cp == NULL) {
1643                 return (ENODEV);
1644         }
1645         mp_cpu_unconfigure_common(cp, 0);
1646 
1647         return (0);
1648 }
1649 
1650 /*
1651  * Startup function for 'other' CPUs (besides boot cpu).
1652  * Called from real_mode_start.
1653  *
1654  * WARNING: until CPU_READY is set, mp_startup_common and routines called by
1655  * mp_startup_common should not call routines (e.g. kmem_free) that could call
1656  * hat_unload which requires CPU_READY to be set.
1657  */
1658 static void
1659 mp_startup_common(boolean_t boot)
1660 {
1661         cpu_t *cp = CPU;
1662         uchar_t new_x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
1663         extern void cpu_event_init_cpu(cpu_t *);
1664 
1665         /*
1666          * We need to get TSC on this proc synced (i.e., any delta
1667          * from cpu0 accounted for) as soon as we can, because many
1668          * many things use gethrtime/pc_gethrestime, including
1669          * interrupts, cmn_err, etc.  Before we can do that, we want to
1670          * clear TSC if we're on a buggy Sandy/Ivy Bridge CPU, so do that
1671          * right away.
1672          */
1673         bzero(new_x86_featureset, BT_SIZEOFMAP(NUM_X86_FEATURES));
1674         cpuid_pass1(cp, new_x86_featureset);
1675 
1676         if (boot && get_hwenv() == HW_NATIVE &&
1677             cpuid_getvendor(CPU) == X86_VENDOR_Intel &&
1678             cpuid_getfamily(CPU) == 6 &&
1679             (cpuid_getmodel(CPU) == 0x2d || cpuid_getmodel(CPU) == 0x3e) &&
1680             is_x86_feature(new_x86_featureset, X86FSET_TSC)) {
1681                 (void) wrmsr(REG_TSC, 0UL);
1682         }
1683 
1684         /* Let the control CPU continue into tsc_sync_master() */
1685         mp_startup_signal(&procset_slave, cp->cpu_id);
1686 
1687 #ifndef __xpv
1688         if (tsc_gethrtime_enable)
1689                 tsc_sync_slave();
1690 #endif
1691 
1692         /*
1693          * Once this was done from assembly, but it's safer here; if
1694          * it blocks, we need to be able to swtch() to and from, and
1695          * since we get here by calling t_pc, we need to do that call
1696          * before swtch() overwrites it.
1697          */
1698         (void) (*ap_mlsetup)();
1699 
1700 #ifndef __xpv
1701         /*
1702          * Program this cpu's PAT
1703          */
1704         pat_sync();
1705 #endif
1706 
1707         /*
1708          * Set up TSC_AUX to contain the cpuid for this processor
1709          * for the rdtscp instruction.
1710          */
1711         if (is_x86_feature(x86_featureset, X86FSET_TSCP))
1712                 (void) wrmsr(MSR_AMD_TSCAUX, cp->cpu_id);
1713 
1714         /*
1715          * Initialize this CPU's syscall handlers
1716          */
1717         init_cpu_syscall(cp);
1718 
1719         /*
1720          * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the
1721          * highest level at which a routine is permitted to block on
1722          * an adaptive mutex (allows for cpu poke interrupt in case
1723          * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks
1724          * device interrupts that may end up in the hat layer issuing cross
1725          * calls before CPU_READY is set.
1726          */
1727         splx(ipltospl(LOCK_LEVEL));
1728         sti();
1729 
1730         /*
1731          * Do a sanity check to make sure this new CPU is a sane thing
1732          * to add to the collection of processors running this system.
1733          *
1734          * XXX  Clearly this needs to get more sophisticated, if x86
1735          * systems start to get built out of heterogenous CPUs; as is
1736          * likely to happen once the number of processors in a configuration
1737          * gets large enough.
1738          */
1739         if (compare_x86_featureset(x86_featureset, new_x86_featureset) ==
1740             B_FALSE) {
1741                 cmn_err(CE_CONT, "cpu%d: featureset\n", cp->cpu_id);
1742                 print_x86_featureset(new_x86_featureset);
1743                 cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id);
1744         }
1745 
1746         /*
1747          * There exists a small subset of systems which expose differing
1748          * MWAIT/MONITOR support between CPUs.  If MWAIT support is absent from
1749          * the boot CPU, but is found on a later CPU, the system continues to
1750          * operate as if no MWAIT support is available.
1751          *
1752          * The reverse case, where MWAIT is available on the boot CPU but not
1753          * on a subsequently initialized CPU, is not presently allowed and will
1754          * result in a panic.
1755          */
1756         if (is_x86_feature(x86_featureset, X86FSET_MWAIT) !=
1757             is_x86_feature(new_x86_featureset, X86FSET_MWAIT)) {
1758                 if (!is_x86_feature(x86_featureset, X86FSET_MWAIT)) {
1759                         remove_x86_feature(new_x86_featureset, X86FSET_MWAIT);
1760                 } else {
1761                         panic("unsupported mixed cpu mwait support detected");
1762                 }
1763         }
1764 
1765         /*
1766          * We could be more sophisticated here, and just mark the CPU
1767          * as "faulted" but at this point we'll opt for the easier
1768          * answer of dying horribly.  Provided the boot cpu is ok,
1769          * the system can be recovered by booting with use_mp set to zero.
1770          */
1771         if (workaround_errata(cp) != 0)
1772                 panic("critical workaround(s) missing for cpu%d", cp->cpu_id);
1773 
1774         /*
1775          * We can touch cpu_flags here without acquiring the cpu_lock here
1776          * because the cpu_lock is held by the control CPU which is running
1777          * mp_start_cpu_common().
1778          * Need to clear CPU_QUIESCED flag before calling any function which
1779          * may cause thread context switching, such as kmem_alloc() etc.
1780          * The idle thread checks for CPU_QUIESCED flag and loops for ever if
1781          * it's set. So the startup thread may have no chance to switch back
1782          * again if it's switched away with CPU_QUIESCED set.
1783          */
1784         cp->cpu_flags &= ~(CPU_POWEROFF | CPU_QUIESCED);
1785 
1786         /*
1787          * Setup this processor for XSAVE.
1788          */
1789         if (fp_save_mech == FP_XSAVE) {
1790                 xsave_setup_msr(cp);
1791         }
1792 
1793         cpuid_pass2(cp);
1794         cpuid_pass3(cp);
1795         cpuid_pass4(cp, NULL);
1796 
1797         /*
1798          * Correct cpu_idstr and cpu_brandstr on target CPU after
1799          * cpuid_pass1() is done.
1800          */
1801         (void) cpuid_getidstr(cp, cp->cpu_idstr, CPU_IDSTRLEN);
1802         (void) cpuid_getbrandstr(cp, cp->cpu_brandstr, CPU_IDSTRLEN);
1803 
1804         cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS;
1805 
1806         post_startup_cpu_fixups();
1807 
1808         cpu_event_init_cpu(cp);
1809 
1810         /*
1811          * Enable preemption here so that contention for any locks acquired
1812          * later in mp_startup_common may be preempted if the thread owning
1813          * those locks is continuously executing on other CPUs (for example,
1814          * this CPU must be preemptible to allow other CPUs to pause it during
1815          * their startup phases).  It's safe to enable preemption here because
1816          * the CPU state is pretty-much fully constructed.
1817          */
1818         curthread->t_preempt = 0;
1819 
1820         /* The base spl should still be at LOCK LEVEL here */
1821         ASSERT(cp->cpu_base_spl == ipltospl(LOCK_LEVEL));
1822         set_base_spl();         /* Restore the spl to its proper value */
1823 
1824         pghw_physid_create(cp);
1825         /*
1826          * Delegate initialization tasks, which need to access the cpu_lock,
1827          * to mp_start_cpu_common() because we can't acquire the cpu_lock here
1828          * during CPU DR operations.
1829          */
1830         mp_startup_signal(&procset_slave, cp->cpu_id);
1831         mp_startup_wait(&procset_master, cp->cpu_id);
1832         pg_cmt_cpu_startup(cp);
1833 
1834         if (boot) {
1835                 mutex_enter(&cpu_lock);
1836                 cp->cpu_flags &= ~CPU_OFFLINE;
1837                 cpu_enable_intr(cp);
1838                 cpu_add_active(cp);
1839                 mutex_exit(&cpu_lock);
1840         }
1841 
1842         /* Enable interrupts */
1843         (void) spl0();
1844 
1845         /*
1846          * Fill out cpu_ucode_info.  Update microcode if necessary.
1847          */
1848         ucode_check(cp);
1849 
1850 #ifndef __xpv
1851         {
1852                 /*
1853                  * Set up the CPU module for this CPU.  This can't be done
1854                  * before this CPU is made CPU_READY, because we may (in
1855                  * heterogeneous systems) need to go load another CPU module.
1856                  * The act of attempting to load a module may trigger a
1857                  * cross-call, which will ASSERT unless this cpu is CPU_READY.
1858                  */
1859                 cmi_hdl_t hdl;
1860 
1861                 if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
1862                     cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) {
1863                         if (is_x86_feature(x86_featureset, X86FSET_MCA))
1864                                 cmi_mca_init(hdl);
1865                         cp->cpu_m.mcpu_cmi_hdl = hdl;
1866                 }
1867         }
1868 #endif /* __xpv */
1869 
1870         if (boothowto & RB_DEBUG)
1871                 kdi_cpu_init();
1872 
1873         /*
1874          * Setting the bit in cpu_ready_set must be the last operation in
1875          * processor initialization; the boot CPU will continue to boot once
1876          * it sees this bit set for all active CPUs.
1877          */
1878         CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id);
1879 
1880         (void) mach_cpu_create_device_node(cp, NULL);
1881 
1882         cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr);
1883         cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr);
1884         cmn_err(CE_CONT, "?cpu%d initialization complete - online\n",
1885             cp->cpu_id);
1886 
1887         /*
1888          * Now we are done with the startup thread, so free it up.
1889          */
1890         thread_exit();
1891         panic("mp_startup: cannot return");
1892         /*NOTREACHED*/
1893 }
1894 
1895 /*
1896  * Startup function for 'other' CPUs at boot time (besides boot cpu).
1897  */
1898 static void
1899 mp_startup_boot(void)
1900 {
1901         mp_startup_common(B_TRUE);
1902 }
1903 
1904 /*
1905  * Startup function for hotplug CPUs at runtime.
1906  */
1907 void
1908 mp_startup_hotplug(void)
1909 {
1910         mp_startup_common(B_FALSE);
1911 }
1912 
1913 /*
1914  * Start CPU on user request.
1915  */
1916 /* ARGSUSED */
1917 int
1918 mp_cpu_start(struct cpu *cp)
1919 {
1920         ASSERT(MUTEX_HELD(&cpu_lock));
1921         return (0);
1922 }
1923 
1924 /*
1925  * Stop CPU on user request.
1926  */
1927 int
1928 mp_cpu_stop(struct cpu *cp)
1929 {
1930         extern int cbe_psm_timer_mode;
1931         ASSERT(MUTEX_HELD(&cpu_lock));
1932 
1933 #ifdef __xpv
1934         /*
1935          * We can't offline vcpu0.
1936          */
1937         if (cp->cpu_id == 0)
1938                 return (EBUSY);
1939 #endif
1940 
1941         /*
1942          * If TIMER_PERIODIC mode is used, CPU0 is the one running it;
1943          * can't stop it.  (This is true only for machines with no TSC.)
1944          */
1945 
1946         if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0))
1947                 return (EBUSY);
1948 
1949         return (0);
1950 }
1951 
1952 /*
1953  * Take the specified CPU out of participation in interrupts.
1954  */
1955 int
1956 cpu_disable_intr(struct cpu *cp)
1957 {
1958         if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS)
1959                 return (EBUSY);
1960 
1961         cp->cpu_flags &= ~CPU_ENABLE;
1962         return (0);
1963 }
1964 
1965 /*
1966  * Allow the specified CPU to participate in interrupts.
1967  */
1968 void
1969 cpu_enable_intr(struct cpu *cp)
1970 {
1971         ASSERT(MUTEX_HELD(&cpu_lock));
1972         cp->cpu_flags |= CPU_ENABLE;
1973         psm_enable_intr(cp->cpu_id);
1974 }
1975 
1976 void
1977 mp_cpu_faulted_enter(struct cpu *cp)
1978 {
1979 #ifdef __xpv
1980         _NOTE(ARGUNUSED(cp));
1981 #else
1982         cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
1983 
1984         if (hdl != NULL) {
1985                 cmi_hdl_hold(hdl);
1986         } else {
1987                 hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
1988                     cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
1989         }
1990         if (hdl != NULL) {
1991                 cmi_faulted_enter(hdl);
1992                 cmi_hdl_rele(hdl);
1993         }
1994 #endif
1995 }
1996 
1997 void
1998 mp_cpu_faulted_exit(struct cpu *cp)
1999 {
2000 #ifdef __xpv
2001         _NOTE(ARGUNUSED(cp));
2002 #else
2003         cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
2004 
2005         if (hdl != NULL) {
2006                 cmi_hdl_hold(hdl);
2007         } else {
2008                 hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
2009                     cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
2010         }
2011         if (hdl != NULL) {
2012                 cmi_faulted_exit(hdl);
2013                 cmi_hdl_rele(hdl);
2014         }
2015 #endif
2016 }
2017 
2018 /*
2019  * The following two routines are used as context operators on threads belonging
2020  * to processes with a private LDT (see sysi86).  Due to the rarity of such
2021  * processes, these routines are currently written for best code readability and
2022  * organization rather than speed.  We could avoid checking x86_featureset at
2023  * every context switch by installing different context ops, depending on
2024  * x86_featureset, at LDT creation time -- one for each combination of fast
2025  * syscall features.
2026  */
2027 
2028 /*ARGSUSED*/
2029 void
2030 cpu_fast_syscall_disable(void *arg)
2031 {
2032         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2033             is_x86_feature(x86_featureset, X86FSET_SEP))
2034                 cpu_sep_disable();
2035         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2036             is_x86_feature(x86_featureset, X86FSET_ASYSC))
2037                 cpu_asysc_disable();
2038 }
2039 
2040 /*ARGSUSED*/
2041 void
2042 cpu_fast_syscall_enable(void *arg)
2043 {
2044         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2045             is_x86_feature(x86_featureset, X86FSET_SEP))
2046                 cpu_sep_enable();
2047         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2048             is_x86_feature(x86_featureset, X86FSET_ASYSC))
2049                 cpu_asysc_enable();
2050 }
2051 
2052 static void
2053 cpu_sep_enable(void)
2054 {
2055         ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2056         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2057 
2058         wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL);
2059 }
2060 
2061 static void
2062 cpu_sep_disable(void)
2063 {
2064         ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2065         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2066 
2067         /*
2068          * Setting the SYSENTER_CS_MSR register to 0 causes software executing
2069          * the sysenter or sysexit instruction to trigger a #gp fault.
2070          */
2071         wrmsr(MSR_INTC_SEP_CS, 0);
2072 }
2073 
2074 static void
2075 cpu_asysc_enable(void)
2076 {
2077         ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2078         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2079 
2080         wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) |
2081             (uint64_t)(uintptr_t)AMD_EFER_SCE);
2082 }
2083 
2084 static void
2085 cpu_asysc_disable(void)
2086 {
2087         ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2088         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2089 
2090         /*
2091          * Turn off the SCE (syscall enable) bit in the EFER register. Software
2092          * executing syscall or sysret with this bit off will incur a #ud trap.
2093          */
2094         wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) &
2095             ~((uint64_t)(uintptr_t)AMD_EFER_SCE));
2096 }