1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 /*
  26  * Copyright (c) 2010, Intel Corporation.
  27  * All rights reserved.
  28  */
  29 /*
  30  * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
  31  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  32  */
  33 
  34 #include <sys/types.h>
  35 #include <sys/thread.h>
  36 #include <sys/cpuvar.h>
  37 #include <sys/cpu.h>
  38 #include <sys/t_lock.h>
  39 #include <sys/param.h>
  40 #include <sys/proc.h>
  41 #include <sys/disp.h>
  42 #include <sys/class.h>
  43 #include <sys/cmn_err.h>
  44 #include <sys/debug.h>
  45 #include <sys/note.h>
  46 #include <sys/asm_linkage.h>
  47 #include <sys/x_call.h>
  48 #include <sys/systm.h>
  49 #include <sys/var.h>
  50 #include <sys/vtrace.h>
  51 #include <vm/hat.h>
  52 #include <vm/as.h>
  53 #include <vm/seg_kmem.h>
  54 #include <vm/seg_kp.h>
  55 #include <sys/segments.h>
  56 #include <sys/kmem.h>
  57 #include <sys/stack.h>
  58 #include <sys/smp_impldefs.h>
  59 #include <sys/x86_archext.h>
  60 #include <sys/machsystm.h>
  61 #include <sys/traptrace.h>
  62 #include <sys/clock.h>
  63 #include <sys/cpc_impl.h>
  64 #include <sys/pg.h>
  65 #include <sys/cmt.h>
  66 #include <sys/dtrace.h>
  67 #include <sys/archsystm.h>
  68 #include <sys/fp.h>
  69 #include <sys/reboot.h>
  70 #include <sys/kdi_machimpl.h>
  71 #include <vm/hat_i86.h>
  72 #include <vm/vm_dep.h>
  73 #include <sys/memnode.h>
  74 #include <sys/pci_cfgspace.h>
  75 #include <sys/mach_mmu.h>
  76 #include <sys/sysmacros.h>
  77 #if defined(__xpv)
  78 #include <sys/hypervisor.h>
  79 #endif
  80 #include <sys/cpu_module.h>
  81 #include <sys/ontrap.h>
  82 
  83 struct cpu      cpus[1];                        /* CPU data */
  84 struct cpu      *cpu[NCPU] = {&cpus[0]};    /* pointers to all CPUs */
  85 struct cpu      *cpu_free_list;                 /* list for released CPUs */
  86 cpu_core_t      cpu_core[NCPU];                 /* cpu_core structures */
  87 
  88 #define cpu_next_free   cpu_prev
  89 
  90 /*
  91  * Useful for disabling MP bring-up on a MP capable system.
  92  */
  93 int use_mp = 1;
  94 
  95 /*
  96  * to be set by a PSM to indicate what cpus
  97  * are sitting around on the system.
  98  */
  99 cpuset_t mp_cpus;
 100 
 101 /*
 102  * This variable is used by the hat layer to decide whether or not
 103  * critical sections are needed to prevent race conditions.  For sun4m,
 104  * this variable is set once enough MP initialization has been done in
 105  * order to allow cross calls.
 106  */
 107 int flushes_require_xcalls;
 108 
 109 cpuset_t cpu_ready_set;         /* initialized in startup() */
 110 
 111 static void mp_startup_boot(void);
 112 static void mp_startup_hotplug(void);
 113 
 114 static void cpu_sep_enable(void);
 115 static void cpu_sep_disable(void);
 116 static void cpu_asysc_enable(void);
 117 static void cpu_asysc_disable(void);
 118 
 119 /*
 120  * Init CPU info - get CPU type info for processor_info system call.
 121  */
 122 void
 123 init_cpu_info(struct cpu *cp)
 124 {
 125         processor_info_t *pi = &cp->cpu_type_info;
 126 
 127         /*
 128          * Get clock-frequency property for the CPU.
 129          */
 130         pi->pi_clock = cpu_freq;
 131 
 132         /*
 133          * Current frequency in Hz.
 134          */
 135         cp->cpu_curr_clock = cpu_freq_hz;
 136 
 137         /*
 138          * Supported frequencies.
 139          */
 140         if (cp->cpu_supp_freqs == NULL) {
 141                 cpu_set_supp_freqs(cp, NULL);
 142         }
 143 
 144         (void) strcpy(pi->pi_processor_type, "i386");
 145         if (fpu_exists)
 146                 (void) strcpy(pi->pi_fputypes, "i387 compatible");
 147 
 148         cp->cpu_idstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
 149         cp->cpu_brandstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
 150 
 151         /*
 152          * If called for the BSP, cp is equal to current CPU.
 153          * For non-BSPs, cpuid info of cp is not ready yet, so use cpuid info
 154          * of current CPU as default values for cpu_idstr and cpu_brandstr.
 155          * They will be corrected in mp_startup_common() after cpuid_pass1()
 156          * has been invoked on target CPU.
 157          */
 158         (void) cpuid_getidstr(CPU, cp->cpu_idstr, CPU_IDSTRLEN);
 159         (void) cpuid_getbrandstr(CPU, cp->cpu_brandstr, CPU_IDSTRLEN);
 160 }
 161 
 162 /*
 163  * Configure syscall support on this CPU.
 164  */
 165 /*ARGSUSED*/
 166 void
 167 init_cpu_syscall(struct cpu *cp)
 168 {
 169         kpreempt_disable();
 170 
 171 #if defined(__amd64)
 172         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
 173             is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
 174 
 175 #if !defined(__lint)
 176                 /*
 177                  * The syscall instruction imposes a certain ordering on
 178                  * segment selectors, so we double-check that ordering
 179                  * here.
 180                  */
 181                 ASSERT(KDS_SEL == KCS_SEL + 8);
 182                 ASSERT(UDS_SEL == U32CS_SEL + 8);
 183                 ASSERT(UCS_SEL == U32CS_SEL + 16);
 184 #endif
 185                 /*
 186                  * Turn syscall/sysret extensions on.
 187                  */
 188                 cpu_asysc_enable();
 189 
 190                 /*
 191                  * Program the magic registers ..
 192                  */
 193                 wrmsr(MSR_AMD_STAR,
 194                     ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);
 195                 wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall);
 196                 wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32);
 197 
 198                 /*
 199                  * This list of flags is masked off the incoming
 200                  * %rfl when we enter the kernel.
 201                  */
 202                 wrmsr(MSR_AMD_SFMASK, (uint64_t)(uintptr_t)(PS_IE | PS_T));
 203         }
 204 #endif
 205 
 206         /*
 207          * On 32-bit kernels, we use sysenter/sysexit because it's too
 208          * hard to use syscall/sysret, and it is more portable anyway.
 209          *
 210          * On 64-bit kernels on Nocona machines, the 32-bit syscall
 211          * variant isn't available to 32-bit applications, but sysenter is.
 212          */
 213         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
 214             is_x86_feature(x86_featureset, X86FSET_SEP)) {
 215 
 216 #if !defined(__lint)
 217                 /*
 218                  * The sysenter instruction imposes a certain ordering on
 219                  * segment selectors, so we double-check that ordering
 220                  * here. See "sysenter" in Intel document 245471-012, "IA-32
 221                  * Intel Architecture Software Developer's Manual Volume 2:
 222                  * Instruction Set Reference"
 223                  */
 224                 ASSERT(KDS_SEL == KCS_SEL + 8);
 225 
 226                 ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3));
 227                 ASSERT32(UDS_SEL == UCS_SEL + 8);
 228 
 229                 ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3));
 230                 ASSERT64(UDS_SEL == U32CS_SEL + 8);
 231 #endif
 232 
 233                 cpu_sep_enable();
 234 
 235                 /*
 236                  * resume() sets this value to the base of the threads stack
 237                  * via a context handler.
 238                  */
 239                 wrmsr(MSR_INTC_SEP_ESP, 0);
 240                 wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter);
 241         }
 242 
 243         kpreempt_enable();
 244 }
 245 
 246 /*
 247  * Multiprocessor initialization.
 248  *
 249  * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the
 250  * startup and idle threads for the specified CPU.
 251  * Parameter boot is true for boot time operations and is false for CPU
 252  * DR operations.
 253  */
 254 static struct cpu *
 255 mp_cpu_configure_common(int cpun, boolean_t boot)
 256 {
 257         struct cpu *cp;
 258         kthread_id_t tp;
 259         caddr_t sp;
 260         proc_t *procp;
 261 #if !defined(__xpv)
 262         extern int idle_cpu_prefer_mwait;
 263         extern void cpu_idle_mwait();
 264 #endif
 265         extern void idle();
 266         extern void cpu_idle();
 267 
 268 #ifdef TRAPTRACE
 269         trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun];
 270 #endif
 271 
 272         ASSERT(MUTEX_HELD(&cpu_lock));
 273         ASSERT(cpun < NCPU && cpu[cpun] == NULL);
 274 
 275         if (cpu_free_list == NULL) {
 276                 cp = kmem_zalloc(sizeof (*cp), KM_SLEEP);
 277         } else {
 278                 cp = cpu_free_list;
 279                 cpu_free_list = cp->cpu_next_free;
 280         }
 281 
 282         cp->cpu_m.mcpu_istamp = cpun << 16;
 283 
 284         /* Create per CPU specific threads in the process p0. */
 285         procp = &p0;
 286 
 287         /*
 288          * Initialize the dispatcher first.
 289          */
 290         disp_cpu_init(cp);
 291 
 292         cpu_vm_data_init(cp);
 293 
 294         /*
 295          * Allocate and initialize the startup thread for this CPU.
 296          * Interrupt and process switch stacks get allocated later
 297          * when the CPU starts running.
 298          */
 299         tp = thread_create(NULL, 0, NULL, NULL, 0, procp,
 300             TS_STOPPED, maxclsyspri);
 301 
 302         /*
 303          * Set state to TS_ONPROC since this thread will start running
 304          * as soon as the CPU comes online.
 305          *
 306          * All the other fields of the thread structure are setup by
 307          * thread_create().
 308          */
 309         THREAD_ONPROC(tp, cp);
 310         tp->t_preempt = 1;
 311         tp->t_bound_cpu = cp;
 312         tp->t_affinitycnt = 1;
 313         tp->t_cpu = cp;
 314         tp->t_disp_queue = cp->cpu_disp;
 315 
 316         /*
 317          * Setup thread to start in mp_startup_common.
 318          */
 319         sp = tp->t_stk;
 320         tp->t_sp = (uintptr_t)(sp - MINFRAME);
 321 #if defined(__amd64)
 322         tp->t_sp -= STACK_ENTRY_ALIGN;               /* fake a call */
 323 #endif
 324         /*
 325          * Setup thread start entry point for boot or hotplug.
 326          */
 327         if (boot) {
 328                 tp->t_pc = (uintptr_t)mp_startup_boot;
 329         } else {
 330                 tp->t_pc = (uintptr_t)mp_startup_hotplug;
 331         }
 332 
 333         cp->cpu_id = cpun;
 334         cp->cpu_self = cp;
 335         cp->cpu_thread = tp;
 336         cp->cpu_lwp = NULL;
 337         cp->cpu_dispthread = tp;
 338         cp->cpu_dispatch_pri = DISP_PRIO(tp);
 339 
 340         /*
 341          * cpu_base_spl must be set explicitly here to prevent any blocking
 342          * operations in mp_startup_common from causing the spl of the cpu
 343          * to drop to 0 (allowing device interrupts before we're ready) in
 344          * resume().
 345          * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY.
 346          * As an extra bit of security on DEBUG kernels, this is enforced with
 347          * an assertion in mp_startup_common() -- before cpu_base_spl is set
 348          * to its proper value.
 349          */
 350         cp->cpu_base_spl = ipltospl(LOCK_LEVEL);
 351 
 352         /*
 353          * Now, initialize per-CPU idle thread for this CPU.
 354          */
 355         tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1);
 356 
 357         cp->cpu_idle_thread = tp;
 358 
 359         tp->t_preempt = 1;
 360         tp->t_bound_cpu = cp;
 361         tp->t_affinitycnt = 1;
 362         tp->t_cpu = cp;
 363         tp->t_disp_queue = cp->cpu_disp;
 364 
 365         /*
 366          * Bootstrap the CPU's PG data
 367          */
 368         pg_cpu_bootstrap(cp);
 369 
 370         /*
 371          * Perform CPC initialization on the new CPU.
 372          */
 373         kcpc_hw_init(cp);
 374 
 375         /*
 376          * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2
 377          * for each CPU.
 378          */
 379         setup_vaddr_for_ppcopy(cp);
 380 
 381         /*
 382          * Allocate page for new GDT and initialize from current GDT.
 383          */
 384 #if !defined(__lint)
 385         ASSERT((sizeof (*cp->cpu_gdt) * NGDT) <= PAGESIZE);
 386 #endif
 387         cp->cpu_gdt = kmem_zalloc(PAGESIZE, KM_SLEEP);
 388         bcopy(CPU->cpu_gdt, cp->cpu_gdt, (sizeof (*cp->cpu_gdt) * NGDT));
 389 
 390 #if defined(__i386)
 391         /*
 392          * setup kernel %gs.
 393          */
 394         set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA,
 395             SEL_KPL, 0, 1);
 396 #endif
 397 
 398         /*
 399          * If we have more than one node, each cpu gets a copy of IDT
 400          * local to its node. If this is a Pentium box, we use cpu 0's
 401          * IDT. cpu 0's IDT has been made read-only to workaround the
 402          * cmpxchgl register bug
 403          */
 404         if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) {
 405 #if !defined(__lint)
 406                 ASSERT((sizeof (*CPU->cpu_idt) * NIDT) <= PAGESIZE);
 407 #endif
 408                 cp->cpu_idt = kmem_zalloc(PAGESIZE, KM_SLEEP);
 409                 bcopy(CPU->cpu_idt, cp->cpu_idt, PAGESIZE);
 410         } else {
 411                 cp->cpu_idt = CPU->cpu_idt;
 412         }
 413 
 414         /*
 415          * Get interrupt priority data from cpu 0.
 416          */
 417         cp->cpu_pri_data = CPU->cpu_pri_data;
 418 
 419         /*
 420          * alloc space for cpuid info
 421          */
 422         cpuid_alloc_space(cp);
 423 #if !defined(__xpv)
 424         if (is_x86_feature(x86_featureset, X86FSET_MWAIT) &&
 425             idle_cpu_prefer_mwait) {
 426                 cp->cpu_m.mcpu_mwait = cpuid_mwait_alloc(cp);
 427                 cp->cpu_m.mcpu_idle_cpu = cpu_idle_mwait;
 428         } else
 429 #endif
 430                 cp->cpu_m.mcpu_idle_cpu = cpu_idle;
 431 
 432         init_cpu_info(cp);
 433 
 434         /*
 435          * alloc space for ucode_info
 436          */
 437         ucode_alloc_space(cp);
 438         xc_init_cpu(cp);
 439         hat_cpu_online(cp);
 440 
 441 #ifdef TRAPTRACE
 442         /*
 443          * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers
 444          */
 445         ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP);
 446         ttc->ttc_next = ttc->ttc_first;
 447         ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize;
 448 #endif
 449 
 450         /*
 451          * Record that we have another CPU.
 452          */
 453         /*
 454          * Initialize the interrupt threads for this CPU
 455          */
 456         cpu_intr_alloc(cp, NINTR_THREADS);
 457 
 458         cp->cpu_flags = CPU_OFFLINE | CPU_QUIESCED | CPU_POWEROFF;
 459         cpu_set_state(cp);
 460 
 461         /*
 462          * Add CPU to list of available CPUs.  It'll be on the active list
 463          * after mp_startup_common().
 464          */
 465         cpu_add_unit(cp);
 466 
 467         return (cp);
 468 }
 469 
 470 /*
 471  * Undo what was done in mp_cpu_configure_common
 472  */
 473 static void
 474 mp_cpu_unconfigure_common(struct cpu *cp, int error)
 475 {
 476         ASSERT(MUTEX_HELD(&cpu_lock));
 477 
 478         /*
 479          * Remove the CPU from the list of available CPUs.
 480          */
 481         cpu_del_unit(cp->cpu_id);
 482 
 483         if (error == ETIMEDOUT) {
 484                 /*
 485                  * The cpu was started, but never *seemed* to run any
 486                  * code in the kernel; it's probably off spinning in its
 487                  * own private world, though with potential references to
 488                  * our kmem-allocated IDTs and GDTs (for example).
 489                  *
 490                  * Worse still, it may actually wake up some time later,
 491                  * so rather than guess what it might or might not do, we
 492                  * leave the fundamental data structures intact.
 493                  */
 494                 cp->cpu_flags = 0;
 495                 return;
 496         }
 497 
 498         /*
 499          * At this point, the only threads bound to this CPU should
 500          * special per-cpu threads: it's idle thread, it's pause threads,
 501          * and it's interrupt threads.  Clean these up.
 502          */
 503         cpu_destroy_bound_threads(cp);
 504         cp->cpu_idle_thread = NULL;
 505 
 506         /*
 507          * Free the interrupt stack.
 508          */
 509         segkp_release(segkp,
 510             cp->cpu_intr_stack - (INTR_STACK_SIZE - SA(MINFRAME)));
 511         cp->cpu_intr_stack = NULL;
 512 
 513 #ifdef TRAPTRACE
 514         /*
 515          * Discard the trap trace buffer
 516          */
 517         {
 518                 trap_trace_ctl_t *ttc = &trap_trace_ctl[cp->cpu_id];
 519 
 520                 kmem_free((void *)ttc->ttc_first, trap_trace_bufsize);
 521                 ttc->ttc_first = NULL;
 522         }
 523 #endif
 524 
 525         hat_cpu_offline(cp);
 526 
 527         ucode_free_space(cp);
 528 
 529         /* Free CPU ID string and brand string. */
 530         if (cp->cpu_idstr) {
 531                 kmem_free(cp->cpu_idstr, CPU_IDSTRLEN);
 532                 cp->cpu_idstr = NULL;
 533         }
 534         if (cp->cpu_brandstr) {
 535                 kmem_free(cp->cpu_brandstr, CPU_IDSTRLEN);
 536                 cp->cpu_brandstr = NULL;
 537         }
 538 
 539 #if !defined(__xpv)
 540         if (cp->cpu_m.mcpu_mwait != NULL) {
 541                 cpuid_mwait_free(cp);
 542                 cp->cpu_m.mcpu_mwait = NULL;
 543         }
 544 #endif
 545         cpuid_free_space(cp);
 546 
 547         if (cp->cpu_idt != CPU->cpu_idt)
 548                 kmem_free(cp->cpu_idt, PAGESIZE);
 549         cp->cpu_idt = NULL;
 550 
 551         kmem_free(cp->cpu_gdt, PAGESIZE);
 552         cp->cpu_gdt = NULL;
 553 
 554         if (cp->cpu_supp_freqs != NULL) {
 555                 size_t len = strlen(cp->cpu_supp_freqs) + 1;
 556                 kmem_free(cp->cpu_supp_freqs, len);
 557                 cp->cpu_supp_freqs = NULL;
 558         }
 559 
 560         teardown_vaddr_for_ppcopy(cp);
 561 
 562         kcpc_hw_fini(cp);
 563 
 564         cp->cpu_dispthread = NULL;
 565         cp->cpu_thread = NULL;       /* discarded by cpu_destroy_bound_threads() */
 566 
 567         cpu_vm_data_destroy(cp);
 568 
 569         xc_fini_cpu(cp);
 570         disp_cpu_fini(cp);
 571 
 572         ASSERT(cp != CPU0);
 573         bzero(cp, sizeof (*cp));
 574         cp->cpu_next_free = cpu_free_list;
 575         cpu_free_list = cp;
 576 }
 577 
 578 /*
 579  * Apply workarounds for known errata, and warn about those that are absent.
 580  *
 581  * System vendors occasionally create configurations which contain different
 582  * revisions of the CPUs that are almost but not exactly the same.  At the
 583  * time of writing, this meant that their clock rates were the same, their
 584  * feature sets were the same, but the required workaround were -not-
 585  * necessarily the same.  So, this routine is invoked on -every- CPU soon
 586  * after starting to make sure that the resulting system contains the most
 587  * pessimal set of workarounds needed to cope with *any* of the CPUs in the
 588  * system.
 589  *
 590  * workaround_errata is invoked early in mlsetup() for CPU 0, and in
 591  * mp_startup_common() for all slave CPUs. Slaves process workaround_errata
 592  * prior to acknowledging their readiness to the master, so this routine will
 593  * never be executed by multiple CPUs in parallel, thus making updates to
 594  * global data safe.
 595  *
 596  * These workarounds are based on Rev 3.57 of the Revision Guide for
 597  * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005.
 598  */
 599 
 600 #if defined(OPTERON_ERRATUM_88)
 601 int opteron_erratum_88;         /* if non-zero -> at least one cpu has it */
 602 #endif
 603 
 604 #if defined(OPTERON_ERRATUM_91)
 605 int opteron_erratum_91;         /* if non-zero -> at least one cpu has it */
 606 #endif
 607 
 608 #if defined(OPTERON_ERRATUM_93)
 609 int opteron_erratum_93;         /* if non-zero -> at least one cpu has it */
 610 #endif
 611 
 612 #if defined(OPTERON_ERRATUM_95)
 613 int opteron_erratum_95;         /* if non-zero -> at least one cpu has it */
 614 #endif
 615 
 616 #if defined(OPTERON_ERRATUM_100)
 617 int opteron_erratum_100;        /* if non-zero -> at least one cpu has it */
 618 #endif
 619 
 620 #if defined(OPTERON_ERRATUM_108)
 621 int opteron_erratum_108;        /* if non-zero -> at least one cpu has it */
 622 #endif
 623 
 624 #if defined(OPTERON_ERRATUM_109)
 625 int opteron_erratum_109;        /* if non-zero -> at least one cpu has it */
 626 #endif
 627 
 628 #if defined(OPTERON_ERRATUM_121)
 629 int opteron_erratum_121;        /* if non-zero -> at least one cpu has it */
 630 #endif
 631 
 632 #if defined(OPTERON_ERRATUM_122)
 633 int opteron_erratum_122;        /* if non-zero -> at least one cpu has it */
 634 #endif
 635 
 636 #if defined(OPTERON_ERRATUM_123)
 637 int opteron_erratum_123;        /* if non-zero -> at least one cpu has it */
 638 #endif
 639 
 640 #if defined(OPTERON_ERRATUM_131)
 641 int opteron_erratum_131;        /* if non-zero -> at least one cpu has it */
 642 #endif
 643 
 644 #if defined(OPTERON_WORKAROUND_6336786)
 645 int opteron_workaround_6336786; /* non-zero -> WA relevant and applied */
 646 int opteron_workaround_6336786_UP = 0;  /* Not needed for UP */
 647 #endif
 648 
 649 #if defined(OPTERON_WORKAROUND_6323525)
 650 int opteron_workaround_6323525; /* if non-zero -> at least one cpu has it */
 651 #endif
 652 
 653 #if defined(OPTERON_ERRATUM_298)
 654 int opteron_erratum_298;
 655 #endif
 656 
 657 #if defined(OPTERON_ERRATUM_721)
 658 int opteron_erratum_721;
 659 #endif
 660 
 661 static void
 662 workaround_warning(cpu_t *cp, uint_t erratum)
 663 {
 664         cmn_err(CE_WARN, "cpu%d: no workaround for erratum %u",
 665             cp->cpu_id, erratum);
 666 }
 667 
 668 static void
 669 workaround_applied(uint_t erratum)
 670 {
 671         if (erratum > 1000000)
 672                 cmn_err(CE_CONT, "?workaround applied for cpu issue #%d\n",
 673                     erratum);
 674         else
 675                 cmn_err(CE_CONT, "?workaround applied for cpu erratum #%d\n",
 676                     erratum);
 677 }
 678 
 679 static void
 680 msr_warning(cpu_t *cp, const char *rw, uint_t msr, int error)
 681 {
 682         cmn_err(CE_WARN, "cpu%d: couldn't %smsr 0x%x, error %d",
 683             cp->cpu_id, rw, msr, error);
 684 }
 685 
 686 /*
 687  * Determine the number of nodes in a Hammer / Greyhound / Griffin family
 688  * system.
 689  */
 690 static uint_t
 691 opteron_get_nnodes(void)
 692 {
 693         static uint_t nnodes = 0;
 694 
 695         if (nnodes == 0) {
 696 #ifdef  DEBUG
 697                 uint_t family;
 698 
 699                 /*
 700                  * This routine uses a PCI config space based mechanism
 701                  * for retrieving the number of nodes in the system.
 702                  * Device 24, function 0, offset 0x60 as used here is not
 703                  * AMD processor architectural, and may not work on processor
 704                  * families other than those listed below.
 705                  *
 706                  * Callers of this routine must ensure that we're running on
 707                  * a processor which supports this mechanism.
 708                  * The assertion below is meant to catch calls on unsupported
 709                  * processors.
 710                  */
 711                 family = cpuid_getfamily(CPU);
 712                 ASSERT(family == 0xf || family == 0x10 || family == 0x11);
 713 #endif  /* DEBUG */
 714 
 715                 /*
 716                  * Obtain the number of nodes in the system from
 717                  * bits [6:4] of the Node ID register on node 0.
 718                  *
 719                  * The actual node count is NodeID[6:4] + 1
 720                  *
 721                  * The Node ID register is accessed via function 0,
 722                  * offset 0x60. Node 0 is device 24.
 723                  */
 724                 nnodes = ((pci_getl_func(0, 24, 0, 0x60) & 0x70) >> 4) + 1;
 725         }
 726         return (nnodes);
 727 }
 728 
 729 uint_t
 730 do_erratum_298(struct cpu *cpu)
 731 {
 732         static int      osvwrc = -3;
 733         extern int      osvw_opteron_erratum(cpu_t *, uint_t);
 734 
 735         /*
 736          * L2 Eviction May Occur During Processor Operation To Set
 737          * Accessed or Dirty Bit.
 738          */
 739         if (osvwrc == -3) {
 740                 osvwrc = osvw_opteron_erratum(cpu, 298);
 741         } else {
 742                 /* osvw return codes should be consistent for all cpus */
 743                 ASSERT(osvwrc == osvw_opteron_erratum(cpu, 298));
 744         }
 745 
 746         switch (osvwrc) {
 747         case 0:         /* erratum is not present: do nothing */
 748                 break;
 749         case 1:         /* erratum is present: BIOS workaround applied */
 750                 /*
 751                  * check if workaround is actually in place and issue warning
 752                  * if not.
 753                  */
 754                 if (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
 755                     ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0)) {
 756 #if defined(OPTERON_ERRATUM_298)
 757                         opteron_erratum_298++;
 758 #else
 759                         workaround_warning(cpu, 298);
 760                         return (1);
 761 #endif
 762                 }
 763                 break;
 764         case -1:        /* cannot determine via osvw: check cpuid */
 765                 if ((cpuid_opteron_erratum(cpu, 298) > 0) &&
 766                     (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
 767                     ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0))) {
 768 #if defined(OPTERON_ERRATUM_298)
 769                         opteron_erratum_298++;
 770 #else
 771                         workaround_warning(cpu, 298);
 772                         return (1);
 773 #endif
 774                 }
 775                 break;
 776         }
 777         return (0);
 778 }
 779 
 780 uint_t
 781 workaround_errata(struct cpu *cpu)
 782 {
 783         uint_t missing = 0;
 784 
 785         ASSERT(cpu == CPU);
 786 
 787         /*LINTED*/
 788         if (cpuid_opteron_erratum(cpu, 88) > 0) {
 789                 /*
 790                  * SWAPGS May Fail To Read Correct GS Base
 791                  */
 792 #if defined(OPTERON_ERRATUM_88)
 793                 /*
 794                  * The workaround is an mfence in the relevant assembler code
 795                  */
 796                 opteron_erratum_88++;
 797 #else
 798                 workaround_warning(cpu, 88);
 799                 missing++;
 800 #endif
 801         }
 802 
 803         if (cpuid_opteron_erratum(cpu, 91) > 0) {
 804                 /*
 805                  * Software Prefetches May Report A Page Fault
 806                  */
 807 #if defined(OPTERON_ERRATUM_91)
 808                 /*
 809                  * fix is in trap.c
 810                  */
 811                 opteron_erratum_91++;
 812 #else
 813                 workaround_warning(cpu, 91);
 814                 missing++;
 815 #endif
 816         }
 817 
 818         if (cpuid_opteron_erratum(cpu, 93) > 0) {
 819                 /*
 820                  * RSM Auto-Halt Restart Returns to Incorrect RIP
 821                  */
 822 #if defined(OPTERON_ERRATUM_93)
 823                 /*
 824                  * fix is in trap.c
 825                  */
 826                 opteron_erratum_93++;
 827 #else
 828                 workaround_warning(cpu, 93);
 829                 missing++;
 830 #endif
 831         }
 832 
 833         /*LINTED*/
 834         if (cpuid_opteron_erratum(cpu, 95) > 0) {
 835                 /*
 836                  * RET Instruction May Return to Incorrect EIP
 837                  */
 838 #if defined(OPTERON_ERRATUM_95)
 839 #if defined(_LP64)
 840                 /*
 841                  * Workaround this by ensuring that 32-bit user code and
 842                  * 64-bit kernel code never occupy the same address
 843                  * range mod 4G.
 844                  */
 845                 if (_userlimit32 > 0xc0000000ul)
 846                         *(uintptr_t *)&_userlimit32 = 0xc0000000ul;
 847 
 848                 /*LINTED*/
 849                 ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u);
 850                 opteron_erratum_95++;
 851 #endif  /* _LP64 */
 852 #else
 853                 workaround_warning(cpu, 95);
 854                 missing++;
 855 #endif
 856         }
 857 
 858         if (cpuid_opteron_erratum(cpu, 100) > 0) {
 859                 /*
 860                  * Compatibility Mode Branches Transfer to Illegal Address
 861                  */
 862 #if defined(OPTERON_ERRATUM_100)
 863                 /*
 864                  * fix is in trap.c
 865                  */
 866                 opteron_erratum_100++;
 867 #else
 868                 workaround_warning(cpu, 100);
 869                 missing++;
 870 #endif
 871         }
 872 
 873         /*LINTED*/
 874         if (cpuid_opteron_erratum(cpu, 108) > 0) {
 875                 /*
 876                  * CPUID Instruction May Return Incorrect Model Number In
 877                  * Some Processors
 878                  */
 879 #if defined(OPTERON_ERRATUM_108)
 880                 /*
 881                  * (Our cpuid-handling code corrects the model number on
 882                  * those processors)
 883                  */
 884 #else
 885                 workaround_warning(cpu, 108);
 886                 missing++;
 887 #endif
 888         }
 889 
 890         /*LINTED*/
 891         if (cpuid_opteron_erratum(cpu, 109) > 0) do {
 892                 /*
 893                  * Certain Reverse REP MOVS May Produce Unpredictable Behavior
 894                  */
 895 #if defined(OPTERON_ERRATUM_109)
 896                 /*
 897                  * The "workaround" is to print a warning to upgrade the BIOS
 898                  */
 899                 uint64_t value;
 900                 const uint_t msr = MSR_AMD_PATCHLEVEL;
 901                 int err;
 902 
 903                 if ((err = checked_rdmsr(msr, &value)) != 0) {
 904                         msr_warning(cpu, "rd", msr, err);
 905                         workaround_warning(cpu, 109);
 906                         missing++;
 907                 }
 908                 if (value == 0)
 909                         opteron_erratum_109++;
 910 #else
 911                 workaround_warning(cpu, 109);
 912                 missing++;
 913 #endif
 914         /*CONSTANTCONDITION*/
 915         } while (0);
 916 
 917         /*LINTED*/
 918         if (cpuid_opteron_erratum(cpu, 121) > 0) {
 919                 /*
 920                  * Sequential Execution Across Non_Canonical Boundary Caused
 921                  * Processor Hang
 922                  */
 923 #if defined(OPTERON_ERRATUM_121)
 924 #if defined(_LP64)
 925                 /*
 926                  * Erratum 121 is only present in long (64 bit) mode.
 927                  * Workaround is to include the page immediately before the
 928                  * va hole to eliminate the possibility of system hangs due to
 929                  * sequential execution across the va hole boundary.
 930                  */
 931                 if (opteron_erratum_121)
 932                         opteron_erratum_121++;
 933                 else {
 934                         if (hole_start) {
 935                                 hole_start -= PAGESIZE;
 936                         } else {
 937                                 /*
 938                                  * hole_start not yet initialized by
 939                                  * mmu_init. Initialize hole_start
 940                                  * with value to be subtracted.
 941                                  */
 942                                 hole_start = PAGESIZE;
 943                         }
 944                         opteron_erratum_121++;
 945                 }
 946 #endif  /* _LP64 */
 947 #else
 948                 workaround_warning(cpu, 121);
 949                 missing++;
 950 #endif
 951         }
 952 
 953         /*LINTED*/
 954         if (cpuid_opteron_erratum(cpu, 122) > 0) do {
 955                 /*
 956                  * TLB Flush Filter May Cause Coherency Problem in
 957                  * Multiprocessor Systems
 958                  */
 959 #if defined(OPTERON_ERRATUM_122)
 960                 uint64_t value;
 961                 const uint_t msr = MSR_AMD_HWCR;
 962                 int error;
 963 
 964                 /*
 965                  * Erratum 122 is only present in MP configurations (multi-core
 966                  * or multi-processor).
 967                  */
 968 #if defined(__xpv)
 969                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
 970                         break;
 971                 if (!opteron_erratum_122 && xpv_nr_phys_cpus() == 1)
 972                         break;
 973 #else
 974                 if (!opteron_erratum_122 && opteron_get_nnodes() == 1 &&
 975                     cpuid_get_ncpu_per_chip(cpu) == 1)
 976                         break;
 977 #endif
 978                 /* disable TLB Flush Filter */
 979 
 980                 if ((error = checked_rdmsr(msr, &value)) != 0) {
 981                         msr_warning(cpu, "rd", msr, error);
 982                         workaround_warning(cpu, 122);
 983                         missing++;
 984                 } else {
 985                         value |= (uint64_t)AMD_HWCR_FFDIS;
 986                         if ((error = checked_wrmsr(msr, value)) != 0) {
 987                                 msr_warning(cpu, "wr", msr, error);
 988                                 workaround_warning(cpu, 122);
 989                                 missing++;
 990                         }
 991                 }
 992                 opteron_erratum_122++;
 993 #else
 994                 workaround_warning(cpu, 122);
 995                 missing++;
 996 #endif
 997         /*CONSTANTCONDITION*/
 998         } while (0);
 999 
1000         /*LINTED*/
1001         if (cpuid_opteron_erratum(cpu, 123) > 0) do {
1002                 /*
1003                  * Bypassed Reads May Cause Data Corruption of System Hang in
1004                  * Dual Core Processors
1005                  */
1006 #if defined(OPTERON_ERRATUM_123)
1007                 uint64_t value;
1008                 const uint_t msr = MSR_AMD_PATCHLEVEL;
1009                 int err;
1010 
1011                 /*
1012                  * Erratum 123 applies only to multi-core cpus.
1013                  */
1014                 if (cpuid_get_ncpu_per_chip(cpu) < 2)
1015                         break;
1016 #if defined(__xpv)
1017                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1018                         break;
1019 #endif
1020                 /*
1021                  * The "workaround" is to print a warning to upgrade the BIOS
1022                  */
1023                 if ((err = checked_rdmsr(msr, &value)) != 0) {
1024                         msr_warning(cpu, "rd", msr, err);
1025                         workaround_warning(cpu, 123);
1026                         missing++;
1027                 }
1028                 if (value == 0)
1029                         opteron_erratum_123++;
1030 #else
1031                 workaround_warning(cpu, 123);
1032                 missing++;
1033 
1034 #endif
1035         /*CONSTANTCONDITION*/
1036         } while (0);
1037 
1038         /*LINTED*/
1039         if (cpuid_opteron_erratum(cpu, 131) > 0) do {
1040                 /*
1041                  * Multiprocessor Systems with Four or More Cores May Deadlock
1042                  * Waiting for a Probe Response
1043                  */
1044 #if defined(OPTERON_ERRATUM_131)
1045                 uint64_t nbcfg;
1046                 const uint_t msr = MSR_AMD_NB_CFG;
1047                 const uint64_t wabits =
1048                     AMD_NB_CFG_SRQ_HEARTBEAT | AMD_NB_CFG_SRQ_SPR;
1049                 int error;
1050 
1051                 /*
1052                  * Erratum 131 applies to any system with four or more cores.
1053                  */
1054                 if (opteron_erratum_131)
1055                         break;
1056 #if defined(__xpv)
1057                 if (!DOMAIN_IS_INITDOMAIN(xen_info))
1058                         break;
1059                 if (xpv_nr_phys_cpus() < 4)
1060                         break;
1061 #else
1062                 if (opteron_get_nnodes() * cpuid_get_ncpu_per_chip(cpu) < 4)
1063                         break;
1064 #endif
1065                 /*
1066                  * Print a warning if neither of the workarounds for
1067                  * erratum 131 is present.
1068                  */
1069                 if ((error = checked_rdmsr(msr, &nbcfg)) != 0) {
1070                         msr_warning(cpu, "rd", msr, error);
1071                         workaround_warning(cpu, 131);
1072                         missing++;
1073                 } else if ((nbcfg & wabits) == 0) {
1074                         opteron_erratum_131++;
1075                 } else {
1076                         /* cannot have both workarounds set */
1077                         ASSERT((nbcfg & wabits) != wabits);
1078                 }
1079 #else
1080                 workaround_warning(cpu, 131);
1081                 missing++;
1082 #endif
1083         /*CONSTANTCONDITION*/
1084         } while (0);
1085 
1086         /*
1087          * This isn't really an erratum, but for convenience the
1088          * detection/workaround code lives here and in cpuid_opteron_erratum.
1089          */
1090         if (cpuid_opteron_erratum(cpu, 6336786) > 0) {
1091 #if defined(OPTERON_WORKAROUND_6336786)
1092                 /*
1093                  * Disable C1-Clock ramping on multi-core/multi-processor
1094                  * K8 platforms to guard against TSC drift.
1095                  */
1096                 if (opteron_workaround_6336786) {
1097                         opteron_workaround_6336786++;
1098 #if defined(__xpv)
1099                 } else if ((DOMAIN_IS_INITDOMAIN(xen_info) &&
1100                     xpv_nr_phys_cpus() > 1) ||
1101                     opteron_workaround_6336786_UP) {
1102                         /*
1103                          * XXPV Hmm.  We can't walk the Northbridges on
1104                          *      the hypervisor; so just complain and drive
1105                          *      on.  This probably needs to be fixed in
1106                          *      the hypervisor itself.
1107                          */
1108                         opteron_workaround_6336786++;
1109                         workaround_warning(cpu, 6336786);
1110 #else   /* __xpv */
1111                 } else if ((opteron_get_nnodes() *
1112                     cpuid_get_ncpu_per_chip(cpu) > 1) ||
1113                     opteron_workaround_6336786_UP) {
1114 
1115                         uint_t  node, nnodes;
1116                         uint8_t data;
1117 
1118                         nnodes = opteron_get_nnodes();
1119                         for (node = 0; node < nnodes; node++) {
1120                                 /*
1121                                  * Clear PMM7[1:0] (function 3, offset 0x87)
1122                                  * Northbridge device is the node id + 24.
1123                                  */
1124                                 data = pci_getb_func(0, node + 24, 3, 0x87);
1125                                 data &= 0xFC;
1126                                 pci_putb_func(0, node + 24, 3, 0x87, data);
1127                         }
1128                         opteron_workaround_6336786++;
1129 #endif  /* __xpv */
1130                 }
1131 #else
1132                 workaround_warning(cpu, 6336786);
1133                 missing++;
1134 #endif
1135         }
1136 
1137         /*LINTED*/
1138         /*
1139          * Mutex primitives don't work as expected.
1140          */
1141         if (cpuid_opteron_erratum(cpu, 6323525) > 0) {
1142 #if defined(OPTERON_WORKAROUND_6323525)
1143                 /*
1144                  * This problem only occurs with 2 or more cores. If bit in
1145                  * MSR_AMD_BU_CFG set, then not applicable. The workaround
1146                  * is to patch the semaphone routines with the lfence
1147                  * instruction to provide necessary load memory barrier with
1148                  * possible subsequent read-modify-write ops.
1149                  *
1150                  * It is too early in boot to call the patch routine so
1151                  * set erratum variable to be done in startup_end().
1152                  */
1153                 if (opteron_workaround_6323525) {
1154                         opteron_workaround_6323525++;
1155 #if defined(__xpv)
1156                 } else if (is_x86_feature(x86_featureset, X86FSET_SSE2)) {
1157                         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1158                                 /*
1159                                  * XXPV Use dom0_msr here when extended
1160                                  *      operations are supported?
1161                                  */
1162                                 if (xpv_nr_phys_cpus() > 1)
1163                                         opteron_workaround_6323525++;
1164                         } else {
1165                                 /*
1166                                  * We have no way to tell how many physical
1167                                  * cpus there are, or even if this processor
1168                                  * has the problem, so enable the workaround
1169                                  * unconditionally (at some performance cost).
1170                                  */
1171                                 opteron_workaround_6323525++;
1172                         }
1173 #else   /* __xpv */
1174                 } else if (is_x86_feature(x86_featureset, X86FSET_SSE2) &&
1175                     ((opteron_get_nnodes() *
1176                     cpuid_get_ncpu_per_chip(cpu)) > 1)) {
1177                         if ((xrdmsr(MSR_AMD_BU_CFG) & (UINT64_C(1) << 33)) == 0)
1178                                 opteron_workaround_6323525++;
1179 #endif  /* __xpv */
1180                 }
1181 #else
1182                 workaround_warning(cpu, 6323525);
1183                 missing++;
1184 #endif
1185         }
1186 
1187         missing += do_erratum_298(cpu);
1188 
1189         if (cpuid_opteron_erratum(cpu, 721) > 0) {
1190 #if defined(OPTERON_ERRATUM_721)
1191                 on_trap_data_t otd;
1192 
1193                 if (!on_trap(&otd, OT_DATA_ACCESS))
1194                         wrmsr(MSR_AMD_DE_CFG,
1195                             rdmsr(MSR_AMD_DE_CFG) | AMD_DE_CFG_E721);
1196                 no_trap();
1197 
1198                 opteron_erratum_721++;
1199 #else
1200                 workaround_warning(cpu, 721);
1201                 missing++;
1202 #endif
1203         }
1204 
1205 #ifdef __xpv
1206         return (0);
1207 #else
1208         return (missing);
1209 #endif
1210 }
1211 
1212 void
1213 workaround_errata_end()
1214 {
1215 #if defined(OPTERON_ERRATUM_88)
1216         if (opteron_erratum_88)
1217                 workaround_applied(88);
1218 #endif
1219 #if defined(OPTERON_ERRATUM_91)
1220         if (opteron_erratum_91)
1221                 workaround_applied(91);
1222 #endif
1223 #if defined(OPTERON_ERRATUM_93)
1224         if (opteron_erratum_93)
1225                 workaround_applied(93);
1226 #endif
1227 #if defined(OPTERON_ERRATUM_95)
1228         if (opteron_erratum_95)
1229                 workaround_applied(95);
1230 #endif
1231 #if defined(OPTERON_ERRATUM_100)
1232         if (opteron_erratum_100)
1233                 workaround_applied(100);
1234 #endif
1235 #if defined(OPTERON_ERRATUM_108)
1236         if (opteron_erratum_108)
1237                 workaround_applied(108);
1238 #endif
1239 #if defined(OPTERON_ERRATUM_109)
1240         if (opteron_erratum_109) {
1241                 cmn_err(CE_WARN,
1242                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1243                     " processor\nerratum 109 was not detected; updating your"
1244                     " system's BIOS to a version\ncontaining this"
1245                     " microcode patch is HIGHLY recommended or erroneous"
1246                     " system\noperation may occur.\n");
1247         }
1248 #endif
1249 #if defined(OPTERON_ERRATUM_121)
1250         if (opteron_erratum_121)
1251                 workaround_applied(121);
1252 #endif
1253 #if defined(OPTERON_ERRATUM_122)
1254         if (opteron_erratum_122)
1255                 workaround_applied(122);
1256 #endif
1257 #if defined(OPTERON_ERRATUM_123)
1258         if (opteron_erratum_123) {
1259                 cmn_err(CE_WARN,
1260                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1261                     " processor\nerratum 123 was not detected; updating your"
1262                     " system's BIOS to a version\ncontaining this"
1263                     " microcode patch is HIGHLY recommended or erroneous"
1264                     " system\noperation may occur.\n");
1265         }
1266 #endif
1267 #if defined(OPTERON_ERRATUM_131)
1268         if (opteron_erratum_131) {
1269                 cmn_err(CE_WARN,
1270                     "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1271                     " processor\nerratum 131 was not detected; updating your"
1272                     " system's BIOS to a version\ncontaining this"
1273                     " microcode patch is HIGHLY recommended or erroneous"
1274                     " system\noperation may occur.\n");
1275         }
1276 #endif
1277 #if defined(OPTERON_WORKAROUND_6336786)
1278         if (opteron_workaround_6336786)
1279                 workaround_applied(6336786);
1280 #endif
1281 #if defined(OPTERON_WORKAROUND_6323525)
1282         if (opteron_workaround_6323525)
1283                 workaround_applied(6323525);
1284 #endif
1285 #if defined(OPTERON_ERRATUM_298)
1286         if (opteron_erratum_298) {
1287                 cmn_err(CE_WARN,
1288                     "BIOS microcode patch for AMD 64/Opteron(tm)"
1289                     " processor\nerratum 298 was not detected; updating your"
1290                     " system's BIOS to a version\ncontaining this"
1291                     " microcode patch is HIGHLY recommended or erroneous"
1292                     " system\noperation may occur.\n");
1293         }
1294 #endif
1295 #if defined(OPTERON_ERRATUM_721)
1296         if (opteron_erratum_721)
1297                 workaround_applied(721);
1298 #endif
1299 }
1300 
1301 /*
1302  * The procset_slave and procset_master are used to synchronize
1303  * between the control CPU and the target CPU when starting CPUs.
1304  */
1305 static cpuset_t procset_slave, procset_master;
1306 
1307 static void
1308 mp_startup_wait(cpuset_t *sp, processorid_t cpuid)
1309 {
1310         cpuset_t tempset;
1311 
1312         for (tempset = *sp; !CPU_IN_SET(tempset, cpuid);
1313             tempset = *(volatile cpuset_t *)sp) {
1314                 SMT_PAUSE();
1315         }
1316         CPUSET_ATOMIC_DEL(*(cpuset_t *)sp, cpuid);
1317 }
1318 
1319 static void
1320 mp_startup_signal(cpuset_t *sp, processorid_t cpuid)
1321 {
1322         cpuset_t tempset;
1323 
1324         CPUSET_ATOMIC_ADD(*(cpuset_t *)sp, cpuid);
1325         for (tempset = *sp; CPU_IN_SET(tempset, cpuid);
1326             tempset = *(volatile cpuset_t *)sp) {
1327                 SMT_PAUSE();
1328         }
1329 }
1330 
1331 int
1332 mp_start_cpu_common(cpu_t *cp, boolean_t boot)
1333 {
1334         _NOTE(ARGUNUSED(boot));
1335 
1336         void *ctx;
1337         int delays;
1338         int error = 0;
1339         cpuset_t tempset;
1340         processorid_t cpuid;
1341 #ifndef __xpv
1342         extern void cpupm_init(cpu_t *);
1343 #endif
1344 
1345         ASSERT(cp != NULL);
1346         cpuid = cp->cpu_id;
1347         ctx = mach_cpucontext_alloc(cp);
1348         if (ctx == NULL) {
1349                 cmn_err(CE_WARN,
1350                     "cpu%d: failed to allocate context", cp->cpu_id);
1351                 return (EAGAIN);
1352         }
1353         error = mach_cpu_start(cp, ctx);
1354         if (error != 0) {
1355                 cmn_err(CE_WARN,
1356                     "cpu%d: failed to start, error %d", cp->cpu_id, error);
1357                 mach_cpucontext_free(cp, ctx, error);
1358                 return (error);
1359         }
1360 
1361         for (delays = 0, tempset = procset_slave; !CPU_IN_SET(tempset, cpuid);
1362             delays++) {
1363                 if (delays == 500) {
1364                         /*
1365                          * After five seconds, things are probably looking
1366                          * a bit bleak - explain the hang.
1367                          */
1368                         cmn_err(CE_NOTE, "cpu%d: started, "
1369                             "but not running in the kernel yet", cpuid);
1370                 } else if (delays > 2000) {
1371                         /*
1372                          * We waited at least 20 seconds, bail ..
1373                          */
1374                         error = ETIMEDOUT;
1375                         cmn_err(CE_WARN, "cpu%d: timed out", cpuid);
1376                         mach_cpucontext_free(cp, ctx, error);
1377                         return (error);
1378                 }
1379 
1380                 /*
1381                  * wait at least 10ms, then check again..
1382                  */
1383                 delay(USEC_TO_TICK_ROUNDUP(10000));
1384                 tempset = *((volatile cpuset_t *)&procset_slave);
1385         }
1386         CPUSET_ATOMIC_DEL(procset_slave, cpuid);
1387 
1388         mach_cpucontext_free(cp, ctx, 0);
1389 
1390 #ifndef __xpv
1391         if (tsc_gethrtime_enable)
1392                 tsc_sync_master(cpuid);
1393 #endif
1394 
1395         if (dtrace_cpu_init != NULL) {
1396                 (*dtrace_cpu_init)(cpuid);
1397         }
1398 
1399         /*
1400          * During CPU DR operations, the cpu_lock is held by current
1401          * (the control) thread. We can't release the cpu_lock here
1402          * because that will break the CPU DR logic.
1403          * On the other hand, CPUPM and processor group initialization
1404          * routines need to access the cpu_lock. So we invoke those
1405          * routines here on behalf of mp_startup_common().
1406          *
1407          * CPUPM and processor group initialization routines depend
1408          * on the cpuid probing results. Wait for mp_startup_common()
1409          * to signal that cpuid probing is done.
1410          */
1411         mp_startup_wait(&procset_slave, cpuid);
1412 #ifndef __xpv
1413         cpupm_init(cp);
1414 #endif
1415         (void) pg_cpu_init(cp, B_FALSE);
1416         cpu_set_state(cp);
1417         mp_startup_signal(&procset_master, cpuid);
1418 
1419         return (0);
1420 }
1421 
1422 /*
1423  * Start a single cpu, assuming that the kernel context is available
1424  * to successfully start another cpu.
1425  *
1426  * (For example, real mode code is mapped into the right place
1427  * in memory and is ready to be run.)
1428  */
1429 int
1430 start_cpu(processorid_t who)
1431 {
1432         cpu_t *cp;
1433         int error = 0;
1434         cpuset_t tempset;
1435 
1436         ASSERT(who != 0);
1437 
1438         /*
1439          * Check if there's at least a Mbyte of kmem available
1440          * before attempting to start the cpu.
1441          */
1442         if (kmem_avail() < 1024 * 1024) {
1443                 /*
1444                  * Kick off a reap in case that helps us with
1445                  * later attempts ..
1446                  */
1447                 kmem_reap();
1448                 return (ENOMEM);
1449         }
1450 
1451         /*
1452          * First configure cpu.
1453          */
1454         cp = mp_cpu_configure_common(who, B_TRUE);
1455         ASSERT(cp != NULL);
1456 
1457         /*
1458          * Then start cpu.
1459          */
1460         error = mp_start_cpu_common(cp, B_TRUE);
1461         if (error != 0) {
1462                 mp_cpu_unconfigure_common(cp, error);
1463                 return (error);
1464         }
1465 
1466         mutex_exit(&cpu_lock);
1467         tempset = cpu_ready_set;
1468         while (!CPU_IN_SET(tempset, who)) {
1469                 drv_usecwait(1);
1470                 tempset = *((volatile cpuset_t *)&cpu_ready_set);
1471         }
1472         mutex_enter(&cpu_lock);
1473 
1474         return (0);
1475 }
1476 
1477 void
1478 start_other_cpus(int cprboot)
1479 {
1480         _NOTE(ARGUNUSED(cprboot));
1481 
1482         uint_t who;
1483         uint_t bootcpuid = 0;
1484 
1485         /*
1486          * Initialize our own cpu_info.
1487          */
1488         init_cpu_info(CPU);
1489 
1490         cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_idstr);
1491         cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_brandstr);
1492 
1493         /*
1494          * Initialize our syscall handlers
1495          */
1496         init_cpu_syscall(CPU);
1497 
1498         /*
1499          * Take the boot cpu out of the mp_cpus set because we know
1500          * it's already running.  Add it to the cpu_ready_set for
1501          * precisely the same reason.
1502          */
1503         CPUSET_DEL(mp_cpus, bootcpuid);
1504         CPUSET_ADD(cpu_ready_set, bootcpuid);
1505 
1506         /*
1507          * skip the rest of this if
1508          * . only 1 cpu dectected and system isn't hotplug-capable
1509          * . not using MP
1510          */
1511         if ((CPUSET_ISNULL(mp_cpus) && plat_dr_support_cpu() == 0) ||
1512             use_mp == 0) {
1513                 if (use_mp == 0)
1514                         cmn_err(CE_CONT, "?***** Not in MP mode\n");
1515                 goto done;
1516         }
1517 
1518         /*
1519          * perform such initialization as is needed
1520          * to be able to take CPUs on- and off-line.
1521          */
1522         cpu_pause_init();
1523 
1524         xc_init_cpu(CPU);               /* initialize processor crosscalls */
1525 
1526         if (mach_cpucontext_init() != 0)
1527                 goto done;
1528 
1529         flushes_require_xcalls = 1;
1530 
1531         /*
1532          * We lock our affinity to the master CPU to ensure that all slave CPUs
1533          * do their TSC syncs with the same CPU.
1534          */
1535         affinity_set(CPU_CURRENT);
1536 
1537         for (who = 0; who < NCPU; who++) {
1538                 if (!CPU_IN_SET(mp_cpus, who))
1539                         continue;
1540                 ASSERT(who != bootcpuid);
1541 
1542                 mutex_enter(&cpu_lock);
1543                 if (start_cpu(who) != 0)
1544                         CPUSET_DEL(mp_cpus, who);
1545                 cpu_state_change_notify(who, CPU_SETUP);
1546                 mutex_exit(&cpu_lock);
1547         }
1548 
1549         /* Free the space allocated to hold the microcode file */
1550         ucode_cleanup();
1551 
1552         affinity_clear();
1553 
1554         mach_cpucontext_fini();
1555 
1556 done:
1557         if (get_hwenv() == HW_NATIVE)
1558                 workaround_errata_end();
1559         cmi_post_mpstartup();
1560 
1561         if (use_mp && ncpus != boot_max_ncpus) {
1562                 cmn_err(CE_NOTE,
1563                     "System detected %d cpus, but "
1564                     "only %d cpu(s) were enabled during boot.",
1565                     boot_max_ncpus, ncpus);
1566                 cmn_err(CE_NOTE,
1567                     "Use \"boot-ncpus\" parameter to enable more CPU(s). "
1568                     "See eeprom(1M).");
1569         }
1570 }
1571 
1572 int
1573 mp_cpu_configure(int cpuid)
1574 {
1575         cpu_t *cp;
1576 
1577         if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1578                 return (ENOTSUP);
1579         }
1580 
1581         cp = cpu_get(cpuid);
1582         if (cp != NULL) {
1583                 return (EALREADY);
1584         }
1585 
1586         /*
1587          * Check if there's at least a Mbyte of kmem available
1588          * before attempting to start the cpu.
1589          */
1590         if (kmem_avail() < 1024 * 1024) {
1591                 /*
1592                  * Kick off a reap in case that helps us with
1593                  * later attempts ..
1594                  */
1595                 kmem_reap();
1596                 return (ENOMEM);
1597         }
1598 
1599         cp = mp_cpu_configure_common(cpuid, B_FALSE);
1600         ASSERT(cp != NULL && cpu_get(cpuid) == cp);
1601 
1602         return (cp != NULL ? 0 : EAGAIN);
1603 }
1604 
1605 int
1606 mp_cpu_unconfigure(int cpuid)
1607 {
1608         cpu_t *cp;
1609 
1610         if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1611                 return (ENOTSUP);
1612         } else if (cpuid < 0 || cpuid >= max_ncpus) {
1613                 return (EINVAL);
1614         }
1615 
1616         cp = cpu_get(cpuid);
1617         if (cp == NULL) {
1618                 return (ENODEV);
1619         }
1620         mp_cpu_unconfigure_common(cp, 0);
1621 
1622         return (0);
1623 }
1624 
1625 /*
1626  * Startup function for 'other' CPUs (besides boot cpu).
1627  * Called from real_mode_start.
1628  *
1629  * WARNING: until CPU_READY is set, mp_startup_common and routines called by
1630  * mp_startup_common should not call routines (e.g. kmem_free) that could call
1631  * hat_unload which requires CPU_READY to be set.
1632  */
1633 static void
1634 mp_startup_common(boolean_t boot)
1635 {
1636         cpu_t *cp = CPU;
1637         uchar_t new_x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
1638         extern void cpu_event_init_cpu(cpu_t *);
1639 
1640         /*
1641          * We need to get TSC on this proc synced (i.e., any delta
1642          * from cpu0 accounted for) as soon as we can, because many
1643          * many things use gethrtime/pc_gethrestime, including
1644          * interrupts, cmn_err, etc.
1645          */
1646 
1647         /* Let the control CPU continue into tsc_sync_master() */
1648         mp_startup_signal(&procset_slave, cp->cpu_id);
1649 
1650 #ifndef __xpv
1651         if (tsc_gethrtime_enable)
1652                 tsc_sync_slave();
1653 #endif
1654 
1655         /*
1656          * Once this was done from assembly, but it's safer here; if
1657          * it blocks, we need to be able to swtch() to and from, and
1658          * since we get here by calling t_pc, we need to do that call
1659          * before swtch() overwrites it.
1660          */
1661         (void) (*ap_mlsetup)();
1662 
1663         bzero(new_x86_featureset, BT_SIZEOFMAP(NUM_X86_FEATURES));
1664         cpuid_pass1(cp, new_x86_featureset);
1665 
1666 #ifndef __xpv
1667         /*
1668          * Program this cpu's PAT
1669          */
1670         if (is_x86_feature(x86_featureset, X86FSET_PAT))
1671                 pat_sync();
1672 #endif
1673 
1674         /*
1675          * Set up TSC_AUX to contain the cpuid for this processor
1676          * for the rdtscp instruction.
1677          */
1678         if (is_x86_feature(x86_featureset, X86FSET_TSCP))
1679                 (void) wrmsr(MSR_AMD_TSCAUX, cp->cpu_id);
1680 
1681         /*
1682          * Initialize this CPU's syscall handlers
1683          */
1684         init_cpu_syscall(cp);
1685 
1686         /*
1687          * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the
1688          * highest level at which a routine is permitted to block on
1689          * an adaptive mutex (allows for cpu poke interrupt in case
1690          * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks
1691          * device interrupts that may end up in the hat layer issuing cross
1692          * calls before CPU_READY is set.
1693          */
1694         splx(ipltospl(LOCK_LEVEL));
1695         sti();
1696 
1697         /*
1698          * Do a sanity check to make sure this new CPU is a sane thing
1699          * to add to the collection of processors running this system.
1700          *
1701          * XXX  Clearly this needs to get more sophisticated, if x86
1702          * systems start to get built out of heterogenous CPUs; as is
1703          * likely to happen once the number of processors in a configuration
1704          * gets large enough.
1705          */
1706         if (compare_x86_featureset(x86_featureset, new_x86_featureset) ==
1707             B_FALSE) {
1708                 cmn_err(CE_CONT, "cpu%d: featureset\n", cp->cpu_id);
1709                 print_x86_featureset(new_x86_featureset);
1710                 cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id);
1711         }
1712 
1713         /*
1714          * We do not support cpus with mixed monitor/mwait support if the
1715          * boot cpu supports monitor/mwait.
1716          */
1717         if (is_x86_feature(x86_featureset, X86FSET_MWAIT) !=
1718             is_x86_feature(new_x86_featureset, X86FSET_MWAIT))
1719                 panic("unsupported mixed cpu monitor/mwait support detected");
1720 
1721         /*
1722          * We could be more sophisticated here, and just mark the CPU
1723          * as "faulted" but at this point we'll opt for the easier
1724          * answer of dying horribly.  Provided the boot cpu is ok,
1725          * the system can be recovered by booting with use_mp set to zero.
1726          */
1727         if (workaround_errata(cp) != 0)
1728                 panic("critical workaround(s) missing for cpu%d", cp->cpu_id);
1729 
1730         /*
1731          * We can touch cpu_flags here without acquiring the cpu_lock here
1732          * because the cpu_lock is held by the control CPU which is running
1733          * mp_start_cpu_common().
1734          * Need to clear CPU_QUIESCED flag before calling any function which
1735          * may cause thread context switching, such as kmem_alloc() etc.
1736          * The idle thread checks for CPU_QUIESCED flag and loops for ever if
1737          * it's set. So the startup thread may have no chance to switch back
1738          * again if it's switched away with CPU_QUIESCED set.
1739          */
1740         cp->cpu_flags &= ~(CPU_POWEROFF | CPU_QUIESCED);
1741 
1742         /*
1743          * Setup this processor for XSAVE.
1744          */
1745         if (fp_save_mech == FP_XSAVE) {
1746                 xsave_setup_msr(cp);
1747         }
1748 
1749         cpuid_pass2(cp);
1750         cpuid_pass3(cp);
1751         cpuid_pass4(cp, NULL);
1752 
1753         /*
1754          * Correct cpu_idstr and cpu_brandstr on target CPU after
1755          * cpuid_pass1() is done.
1756          */
1757         (void) cpuid_getidstr(cp, cp->cpu_idstr, CPU_IDSTRLEN);
1758         (void) cpuid_getbrandstr(cp, cp->cpu_brandstr, CPU_IDSTRLEN);
1759 
1760         cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS;
1761 
1762         post_startup_cpu_fixups();
1763 
1764         cpu_event_init_cpu(cp);
1765 
1766         /*
1767          * Enable preemption here so that contention for any locks acquired
1768          * later in mp_startup_common may be preempted if the thread owning
1769          * those locks is continuously executing on other CPUs (for example,
1770          * this CPU must be preemptible to allow other CPUs to pause it during
1771          * their startup phases).  It's safe to enable preemption here because
1772          * the CPU state is pretty-much fully constructed.
1773          */
1774         curthread->t_preempt = 0;
1775 
1776         /* The base spl should still be at LOCK LEVEL here */
1777         ASSERT(cp->cpu_base_spl == ipltospl(LOCK_LEVEL));
1778         set_base_spl();         /* Restore the spl to its proper value */
1779 
1780         pghw_physid_create(cp);
1781         /*
1782          * Delegate initialization tasks, which need to access the cpu_lock,
1783          * to mp_start_cpu_common() because we can't acquire the cpu_lock here
1784          * during CPU DR operations.
1785          */
1786         mp_startup_signal(&procset_slave, cp->cpu_id);
1787         mp_startup_wait(&procset_master, cp->cpu_id);
1788         pg_cmt_cpu_startup(cp);
1789 
1790         if (boot) {
1791                 mutex_enter(&cpu_lock);
1792                 cp->cpu_flags &= ~CPU_OFFLINE;
1793                 cpu_enable_intr(cp);
1794                 cpu_add_active(cp);
1795                 mutex_exit(&cpu_lock);
1796         }
1797 
1798         /* Enable interrupts */
1799         (void) spl0();
1800 
1801         /*
1802          * Fill out cpu_ucode_info.  Update microcode if necessary.
1803          */
1804         ucode_check(cp);
1805 
1806 #ifndef __xpv
1807         {
1808                 /*
1809                  * Set up the CPU module for this CPU.  This can't be done
1810                  * before this CPU is made CPU_READY, because we may (in
1811                  * heterogeneous systems) need to go load another CPU module.
1812                  * The act of attempting to load a module may trigger a
1813                  * cross-call, which will ASSERT unless this cpu is CPU_READY.
1814                  */
1815                 cmi_hdl_t hdl;
1816 
1817                 if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
1818                     cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) {
1819                         if (is_x86_feature(x86_featureset, X86FSET_MCA))
1820                                 cmi_mca_init(hdl);
1821                         cp->cpu_m.mcpu_cmi_hdl = hdl;
1822                 }
1823         }
1824 #endif /* __xpv */
1825 
1826         if (boothowto & RB_DEBUG)
1827                 kdi_cpu_init();
1828 
1829         /*
1830          * Setting the bit in cpu_ready_set must be the last operation in
1831          * processor initialization; the boot CPU will continue to boot once
1832          * it sees this bit set for all active CPUs.
1833          */
1834         CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id);
1835 
1836         (void) mach_cpu_create_device_node(cp, NULL);
1837 
1838         cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr);
1839         cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr);
1840         cmn_err(CE_CONT, "?cpu%d initialization complete - online\n",
1841             cp->cpu_id);
1842 
1843         /*
1844          * Now we are done with the startup thread, so free it up.
1845          */
1846         thread_exit();
1847         panic("mp_startup: cannot return");
1848         /*NOTREACHED*/
1849 }
1850 
1851 /*
1852  * Startup function for 'other' CPUs at boot time (besides boot cpu).
1853  */
1854 static void
1855 mp_startup_boot(void)
1856 {
1857         mp_startup_common(B_TRUE);
1858 }
1859 
1860 /*
1861  * Startup function for hotplug CPUs at runtime.
1862  */
1863 void
1864 mp_startup_hotplug(void)
1865 {
1866         mp_startup_common(B_FALSE);
1867 }
1868 
1869 /*
1870  * Start CPU on user request.
1871  */
1872 /* ARGSUSED */
1873 int
1874 mp_cpu_start(struct cpu *cp)
1875 {
1876         ASSERT(MUTEX_HELD(&cpu_lock));
1877         return (0);
1878 }
1879 
1880 /*
1881  * Stop CPU on user request.
1882  */
1883 int
1884 mp_cpu_stop(struct cpu *cp)
1885 {
1886         extern int cbe_psm_timer_mode;
1887         ASSERT(MUTEX_HELD(&cpu_lock));
1888 
1889 #ifdef __xpv
1890         /*
1891          * We can't offline vcpu0.
1892          */
1893         if (cp->cpu_id == 0)
1894                 return (EBUSY);
1895 #endif
1896 
1897         /*
1898          * If TIMER_PERIODIC mode is used, CPU0 is the one running it;
1899          * can't stop it.  (This is true only for machines with no TSC.)
1900          */
1901 
1902         if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0))
1903                 return (EBUSY);
1904 
1905         return (0);
1906 }
1907 
1908 /*
1909  * Take the specified CPU out of participation in interrupts.
1910  */
1911 int
1912 cpu_disable_intr(struct cpu *cp)
1913 {
1914         if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS)
1915                 return (EBUSY);
1916 
1917         cp->cpu_flags &= ~CPU_ENABLE;
1918         return (0);
1919 }
1920 
1921 /*
1922  * Allow the specified CPU to participate in interrupts.
1923  */
1924 void
1925 cpu_enable_intr(struct cpu *cp)
1926 {
1927         ASSERT(MUTEX_HELD(&cpu_lock));
1928         cp->cpu_flags |= CPU_ENABLE;
1929         psm_enable_intr(cp->cpu_id);
1930 }
1931 
1932 void
1933 mp_cpu_faulted_enter(struct cpu *cp)
1934 {
1935 #ifdef __xpv
1936         _NOTE(ARGUNUSED(cp));
1937 #else
1938         cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
1939 
1940         if (hdl != NULL) {
1941                 cmi_hdl_hold(hdl);
1942         } else {
1943                 hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
1944                     cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
1945         }
1946         if (hdl != NULL) {
1947                 cmi_faulted_enter(hdl);
1948                 cmi_hdl_rele(hdl);
1949         }
1950 #endif
1951 }
1952 
1953 void
1954 mp_cpu_faulted_exit(struct cpu *cp)
1955 {
1956 #ifdef __xpv
1957         _NOTE(ARGUNUSED(cp));
1958 #else
1959         cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
1960 
1961         if (hdl != NULL) {
1962                 cmi_hdl_hold(hdl);
1963         } else {
1964                 hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
1965                     cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
1966         }
1967         if (hdl != NULL) {
1968                 cmi_faulted_exit(hdl);
1969                 cmi_hdl_rele(hdl);
1970         }
1971 #endif
1972 }
1973 
1974 /*
1975  * The following two routines are used as context operators on threads belonging
1976  * to processes with a private LDT (see sysi86).  Due to the rarity of such
1977  * processes, these routines are currently written for best code readability and
1978  * organization rather than speed.  We could avoid checking x86_featureset at
1979  * every context switch by installing different context ops, depending on
1980  * x86_featureset, at LDT creation time -- one for each combination of fast
1981  * syscall features.
1982  */
1983 
1984 /*ARGSUSED*/
1985 void
1986 cpu_fast_syscall_disable(void *arg)
1987 {
1988         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
1989             is_x86_feature(x86_featureset, X86FSET_SEP))
1990                 cpu_sep_disable();
1991         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
1992             is_x86_feature(x86_featureset, X86FSET_ASYSC))
1993                 cpu_asysc_disable();
1994 }
1995 
1996 /*ARGSUSED*/
1997 void
1998 cpu_fast_syscall_enable(void *arg)
1999 {
2000         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2001             is_x86_feature(x86_featureset, X86FSET_SEP))
2002                 cpu_sep_enable();
2003         if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2004             is_x86_feature(x86_featureset, X86FSET_ASYSC))
2005                 cpu_asysc_enable();
2006 }
2007 
2008 static void
2009 cpu_sep_enable(void)
2010 {
2011         ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2012         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2013 
2014         wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL);
2015 }
2016 
2017 static void
2018 cpu_sep_disable(void)
2019 {
2020         ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2021         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2022 
2023         /*
2024          * Setting the SYSENTER_CS_MSR register to 0 causes software executing
2025          * the sysenter or sysexit instruction to trigger a #gp fault.
2026          */
2027         wrmsr(MSR_INTC_SEP_CS, 0);
2028 }
2029 
2030 static void
2031 cpu_asysc_enable(void)
2032 {
2033         ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2034         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2035 
2036         wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) |
2037             (uint64_t)(uintptr_t)AMD_EFER_SCE);
2038 }
2039 
2040 static void
2041 cpu_asysc_disable(void)
2042 {
2043         ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2044         ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2045 
2046         /*
2047          * Turn off the SCE (syscall enable) bit in the EFER register. Software
2048          * executing syscall or sysret with this bit off will incur a #ud trap.
2049          */
2050         wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) &
2051             ~((uint64_t)(uintptr_t)AMD_EFER_SCE));
2052 }