1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2017, Joyent, Inc.
  25  */
  26 /*
  27  * Copyright (c) 2010, Intel Corporation.
  28  * All rights reserved.
  29  */
  30 
  31 #include <sys/types.h>
  32 #include <sys/t_lock.h>
  33 #include <sys/param.h>
  34 #include <sys/segments.h>
  35 #include <sys/sysmacros.h>
  36 #include <sys/signal.h>
  37 #include <sys/systm.h>
  38 #include <sys/user.h>
  39 #include <sys/mman.h>
  40 #include <sys/vm.h>
  41 
  42 #include <sys/disp.h>
  43 #include <sys/class.h>
  44 
  45 #include <sys/proc.h>
  46 #include <sys/buf.h>
  47 #include <sys/kmem.h>
  48 
  49 #include <sys/reboot.h>
  50 #include <sys/uadmin.h>
  51 #include <sys/callb.h>
  52 
  53 #include <sys/cred.h>
  54 #include <sys/vnode.h>
  55 #include <sys/file.h>
  56 
  57 #include <sys/procfs.h>
  58 #include <sys/acct.h>
  59 
  60 #include <sys/vfs.h>
  61 #include <sys/dnlc.h>
  62 #include <sys/var.h>
  63 #include <sys/cmn_err.h>
  64 #include <sys/utsname.h>
  65 #include <sys/debug.h>
  66 
  67 #include <sys/dumphdr.h>
  68 #include <sys/bootconf.h>
  69 #include <sys/varargs.h>
  70 #include <sys/promif.h>
  71 #include <sys/modctl.h>
  72 
  73 #include <sys/consdev.h>
  74 #include <sys/frame.h>
  75 
  76 #include <sys/sunddi.h>
  77 #include <sys/ddidmareq.h>
  78 #include <sys/psw.h>
  79 #include <sys/regset.h>
  80 #include <sys/privregs.h>
  81 #include <sys/clock.h>
  82 #include <sys/tss.h>
  83 #include <sys/cpu.h>
  84 #include <sys/stack.h>
  85 #include <sys/trap.h>
  86 #include <sys/pic.h>
  87 #include <vm/hat.h>
  88 #include <vm/anon.h>
  89 #include <vm/as.h>
  90 #include <vm/page.h>
  91 #include <vm/seg.h>
  92 #include <vm/seg_kmem.h>
  93 #include <vm/seg_map.h>
  94 #include <vm/seg_vn.h>
  95 #include <vm/seg_kp.h>
  96 #include <vm/hat_i86.h>
  97 #include <sys/swap.h>
  98 #include <sys/thread.h>
  99 #include <sys/sysconf.h>
 100 #include <sys/vm_machparam.h>
 101 #include <sys/archsystm.h>
 102 #include <sys/machsystm.h>
 103 #include <sys/machlock.h>
 104 #include <sys/x_call.h>
 105 #include <sys/instance.h>
 106 
 107 #include <sys/time.h>
 108 #include <sys/smp_impldefs.h>
 109 #include <sys/psm_types.h>
 110 #include <sys/atomic.h>
 111 #include <sys/panic.h>
 112 #include <sys/cpuvar.h>
 113 #include <sys/dtrace.h>
 114 #include <sys/bl.h>
 115 #include <sys/nvpair.h>
 116 #include <sys/x86_archext.h>
 117 #include <sys/pool_pset.h>
 118 #include <sys/autoconf.h>
 119 #include <sys/mem.h>
 120 #include <sys/dumphdr.h>
 121 #include <sys/compress.h>
 122 #include <sys/cpu_module.h>
 123 #if defined(__xpv)
 124 #include <sys/hypervisor.h>
 125 #include <sys/xpv_panic.h>
 126 #endif
 127 
 128 #include <sys/fastboot.h>
 129 #include <sys/machelf.h>
 130 #include <sys/kobj.h>
 131 #include <sys/multiboot.h>
 132 
 133 #ifdef  TRAPTRACE
 134 #include <sys/traptrace.h>
 135 #endif  /* TRAPTRACE */
 136 
 137 #include <c2/audit.h>
 138 #include <sys/clock_impl.h>
 139 
 140 extern void audit_enterprom(int);
 141 extern void audit_exitprom(int);
 142 
 143 /*
 144  * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used.
 145  */
 146 int     apix_enable = 1;
 147 
 148 int     apic_nvidia_io_max = 0; /* no. of NVIDIA i/o apics */
 149 
 150 /*
 151  * Occassionally the kernel knows better whether to power-off or reboot.
 152  */
 153 int force_shutdown_method = AD_UNKNOWN;
 154 
 155 /*
 156  * The panicbuf array is used to record messages and state:
 157  */
 158 char panicbuf[PANICBUFSIZE];
 159 
 160 /*
 161  * Flags to control Dynamic Reconfiguration features.
 162  */
 163 uint64_t plat_dr_options;
 164 
 165 /*
 166  * Maximum physical address for memory DR operations.
 167  */
 168 uint64_t plat_dr_physmax;
 169 
 170 /*
 171  * maxphys - used during physio
 172  * klustsize - used for klustering by swapfs and specfs
 173  */
 174 int maxphys = 56 * 1024;    /* XXX See vm_subr.c - max b_count in physio */
 175 int klustsize = 56 * 1024;
 176 
 177 caddr_t p0_va;          /* Virtual address for accessing physical page 0 */
 178 
 179 /*
 180  * defined here, though unused on x86,
 181  * to make kstat_fr.c happy.
 182  */
 183 int vac;
 184 
 185 void debug_enter(char *);
 186 
 187 extern void pm_cfb_check_and_powerup(void);
 188 extern void pm_cfb_rele(void);
 189 
 190 extern fastboot_info_t newkernel;
 191 
 192 /*
 193  * Instructions to enable or disable SMAP, respectively.
 194  */
 195 static const uint8_t clac_instr[3] = { 0x0f, 0x01, 0xca };
 196 static const uint8_t stac_instr[3] = { 0x0f, 0x01, 0xcb };
 197 
 198 /*
 199  * Machine dependent code to reboot.
 200  * "mdep" is interpreted as a character pointer; if non-null, it is a pointer
 201  * to a string to be used as the argument string when rebooting.
 202  *
 203  * "invoke_cb" is a boolean. It is set to true when mdboot() can safely
 204  * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when
 205  * we are in a normal shutdown sequence (interrupts are not blocked, the
 206  * system is not panic'ing or being suspended).
 207  */
 208 /*ARGSUSED*/
 209 void
 210 mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb)
 211 {
 212         processorid_t bootcpuid = 0;
 213         static int is_first_quiesce = 1;
 214         static int is_first_reset = 1;
 215         int reset_status = 0;
 216         static char fallback_str[] = "Falling back to regular reboot.\n";
 217 
 218         if (fcn == AD_FASTREBOOT && !newkernel.fi_valid)
 219                 fcn = AD_BOOT;
 220 
 221         if (!panicstr) {
 222                 kpreempt_disable();
 223                 if (fcn == AD_FASTREBOOT) {
 224                         mutex_enter(&cpu_lock);
 225                         if (CPU_ACTIVE(cpu_get(bootcpuid))) {
 226                                 affinity_set(bootcpuid);
 227                         }
 228                         mutex_exit(&cpu_lock);
 229                 } else {
 230                         affinity_set(CPU_CURRENT);
 231                 }
 232         }
 233 
 234         if (force_shutdown_method != AD_UNKNOWN)
 235                 fcn = force_shutdown_method;
 236 
 237         /*
 238          * XXX - rconsvp is set to NULL to ensure that output messages
 239          * are sent to the underlying "hardware" device using the
 240          * monitor's printf routine since we are in the process of
 241          * either rebooting or halting the machine.
 242          */
 243         rconsvp = NULL;
 244 
 245         /*
 246          * Print the reboot message now, before pausing other cpus.
 247          * There is a race condition in the printing support that
 248          * can deadlock multiprocessor machines.
 249          */
 250         if (!(fcn == AD_HALT || fcn == AD_POWEROFF))
 251                 prom_printf("rebooting...\n");
 252 
 253         if (IN_XPV_PANIC())
 254                 reset();
 255 
 256         /*
 257          * We can't bring up the console from above lock level, so do it now
 258          */
 259         pm_cfb_check_and_powerup();
 260 
 261         /* make sure there are no more changes to the device tree */
 262         devtree_freeze();
 263 
 264         if (invoke_cb)
 265                 (void) callb_execute_class(CB_CL_MDBOOT, 0);
 266 
 267         /*
 268          * Clear any unresolved UEs from memory.
 269          */
 270         page_retire_mdboot();
 271 
 272 #if defined(__xpv)
 273         /*
 274          * XXPV Should probably think some more about how we deal
 275          *      with panicing before it's really safe to panic.
 276          *      On hypervisors, we reboot very quickly..  Perhaps panic
 277          *      should only attempt to recover by rebooting if,
 278          *      say, we were able to mount the root filesystem,
 279          *      or if we successfully launched init(1m).
 280          */
 281         if (panicstr && proc_init == NULL)
 282                 (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff);
 283 #endif
 284         /*
 285          * stop other cpus and raise our priority.  since there is only
 286          * one active cpu after this, and our priority will be too high
 287          * for us to be preempted, we're essentially single threaded
 288          * from here on out.
 289          */
 290         (void) spl6();
 291         if (!panicstr) {
 292                 mutex_enter(&cpu_lock);
 293                 pause_cpus(NULL, NULL);
 294                 mutex_exit(&cpu_lock);
 295         }
 296 
 297         /*
 298          * If the system is panicking, the preloaded kernel is valid, and
 299          * fastreboot_onpanic has been set, and the system has been up for
 300          * longer than fastreboot_onpanic_uptime (default to 10 minutes),
 301          * choose Fast Reboot.
 302          */
 303         if (fcn == AD_BOOT && panicstr && newkernel.fi_valid &&
 304             fastreboot_onpanic &&
 305             (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) {
 306                 fcn = AD_FASTREBOOT;
 307         }
 308 
 309         /*
 310          * Try to quiesce devices.
 311          */
 312         if (is_first_quiesce) {
 313                 /*
 314                  * Clear is_first_quiesce before calling quiesce_devices()
 315                  * so that if quiesce_devices() causes panics, it will not
 316                  * be invoked again.
 317                  */
 318                 is_first_quiesce = 0;
 319 
 320                 quiesce_active = 1;
 321                 quiesce_devices(ddi_root_node(), &reset_status);
 322                 if (reset_status == -1) {
 323                         if (fcn == AD_FASTREBOOT && !force_fastreboot) {
 324                                 prom_printf("Driver(s) not capable of fast "
 325                                     "reboot.\n");
 326                                 prom_printf(fallback_str);
 327                                 fastreboot_capable = 0;
 328                                 fcn = AD_BOOT;
 329                         } else if (fcn != AD_FASTREBOOT)
 330                                 fastreboot_capable = 0;
 331                 }
 332                 quiesce_active = 0;
 333         }
 334 
 335         /*
 336          * Try to reset devices. reset_leaves() should only be called
 337          * a) when there are no other threads that could be accessing devices,
 338          *    and
 339          * b) on a system that's not capable of fast reboot (fastreboot_capable
 340          *    being 0), or on a system where quiesce_devices() failed to
 341          *    complete (quiesce_active being 1).
 342          */
 343         if (is_first_reset && (!fastreboot_capable || quiesce_active)) {
 344                 /*
 345                  * Clear is_first_reset before calling reset_devices()
 346                  * so that if reset_devices() causes panics, it will not
 347                  * be invoked again.
 348                  */
 349                 is_first_reset = 0;
 350                 reset_leaves();
 351         }
 352 
 353         /* Verify newkernel checksum */
 354         if (fastreboot_capable && fcn == AD_FASTREBOOT &&
 355             fastboot_cksum_verify(&newkernel) != 0) {
 356                 fastreboot_capable = 0;
 357                 prom_printf("Fast reboot: checksum failed for the new "
 358                     "kernel.\n");
 359                 prom_printf(fallback_str);
 360         }
 361 
 362         (void) spl8();
 363 
 364         if (fastreboot_capable && fcn == AD_FASTREBOOT) {
 365                 /*
 366                  * psm_shutdown is called within fast_reboot()
 367                  */
 368                 fast_reboot();
 369         } else {
 370                 (*psm_shutdownf)(cmd, fcn);
 371 
 372                 if (fcn == AD_HALT || fcn == AD_POWEROFF)
 373                         halt((char *)NULL);
 374                 else
 375                         prom_reboot("");
 376         }
 377         /*NOTREACHED*/
 378 }
 379 
 380 /* mdpreboot - may be called prior to mdboot while root fs still mounted */
 381 /*ARGSUSED*/
 382 void
 383 mdpreboot(int cmd, int fcn, char *mdep)
 384 {
 385         if (fcn == AD_FASTREBOOT && !fastreboot_capable) {
 386                 fcn = AD_BOOT;
 387 #ifdef  __xpv
 388                 cmn_err(CE_WARN, "Fast reboot is not supported on xVM");
 389 #else
 390                 cmn_err(CE_WARN,
 391                     "Fast reboot is not supported on this platform%s",
 392                     fastreboot_nosup_message());
 393 #endif
 394         }
 395 
 396         if (fcn == AD_FASTREBOOT) {
 397                 fastboot_load_kernel(mdep);
 398                 if (!newkernel.fi_valid)
 399                         fcn = AD_BOOT;
 400         }
 401 
 402         (*psm_preshutdownf)(cmd, fcn);
 403 }
 404 
 405 static void
 406 stop_other_cpus(void)
 407 {
 408         ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */
 409         cpuset_t xcset;
 410 
 411         CPUSET_ALL_BUT(xcset, CPU->cpu_id);
 412         xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)mach_cpu_halt);
 413         restore_int_flag(s);
 414 }
 415 
 416 /*
 417  *      Machine dependent abort sequence handling
 418  */
 419 void
 420 abort_sequence_enter(char *msg)
 421 {
 422         if (abort_enable == 0) {
 423                 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
 424                         audit_enterprom(0);
 425                 return;
 426         }
 427         if (AU_ZONE_AUDITING(GET_KCTX_GZ))
 428                 audit_enterprom(1);
 429         debug_enter(msg);
 430         if (AU_ZONE_AUDITING(GET_KCTX_GZ))
 431                 audit_exitprom(1);
 432 }
 433 
 434 /*
 435  * Enter debugger.  Called when the user types ctrl-alt-d or whenever
 436  * code wants to enter the debugger and possibly resume later.
 437  *
 438  * msg: message to print, possibly NULL.
 439  */
 440 void
 441 debug_enter(char *msg)
 442 {
 443         if (dtrace_debugger_init != NULL)
 444                 (*dtrace_debugger_init)();
 445 
 446         if (msg != NULL || (boothowto & RB_DEBUG))
 447                 prom_printf("\n");
 448 
 449         if (msg != NULL)
 450                 prom_printf("%s\n", msg);
 451 
 452         if (boothowto & RB_DEBUG)
 453                 kmdb_enter();
 454 
 455         if (dtrace_debugger_fini != NULL)
 456                 (*dtrace_debugger_fini)();
 457 }
 458 
 459 void
 460 reset(void)
 461 {
 462         extern  void acpi_reset_system();
 463 #if !defined(__xpv)
 464         ushort_t *bios_memchk;
 465 
 466         /*
 467          * Can't use psm_map_phys or acpi_reset_system before the hat is
 468          * initialized.
 469          */
 470         if (khat_running) {
 471                 bios_memchk = (ushort_t *)psm_map_phys(0x472,
 472                     sizeof (ushort_t), PROT_READ | PROT_WRITE);
 473                 if (bios_memchk)
 474                         *bios_memchk = 0x1234;  /* bios memory check disable */
 475 
 476                 if (options_dip != NULL &&
 477                     ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0,
 478                     "efi-systab")) {
 479                         if (bootops == NULL)
 480                                 acpi_reset_system();
 481                         efi_reset();
 482                 }
 483 
 484                 /*
 485                  * The problem with using stubs is that we can call
 486                  * acpi_reset_system only after the kernel is up and running.
 487                  *
 488                  * We should create a global state to keep track of how far
 489                  * up the kernel is but for the time being we will depend on
 490                  * bootops. bootops cleared in startup_end().
 491                  */
 492                 if (bootops == NULL)
 493                         acpi_reset_system();
 494         }
 495 
 496         pc_reset();
 497 #else
 498         if (IN_XPV_PANIC()) {
 499                 if (khat_running && bootops == NULL) {
 500                         acpi_reset_system();
 501                 }
 502 
 503                 pc_reset();
 504         }
 505 
 506         (void) HYPERVISOR_shutdown(SHUTDOWN_reboot);
 507         panic("HYPERVISOR_shutdown() failed");
 508 #endif
 509         /*NOTREACHED*/
 510 }
 511 
 512 /*
 513  * Halt the machine and return to the monitor
 514  */
 515 void
 516 halt(char *s)
 517 {
 518         stop_other_cpus();      /* send stop signal to other CPUs */
 519         if (s)
 520                 prom_printf("(%s) \n", s);
 521         prom_exit_to_mon();
 522         /*NOTREACHED*/
 523 }
 524 
 525 /*
 526  * Initiate interrupt redistribution.
 527  */
 528 void
 529 i_ddi_intr_redist_all_cpus()
 530 {
 531 }
 532 
 533 /*
 534  * XXX These probably ought to live somewhere else
 535  * XXX They are called from mem.c
 536  */
 537 
 538 /*
 539  * Convert page frame number to an OBMEM page frame number
 540  * (i.e. put in the type bits -- zero for this implementation)
 541  */
 542 pfn_t
 543 impl_obmem_pfnum(pfn_t pf)
 544 {
 545         return (pf);
 546 }
 547 
 548 #ifdef  NM_DEBUG
 549 int nmi_test = 0;       /* checked in intentry.s during clock int */
 550 int nmtest = -1;
 551 nmfunc1(int arg, struct regs *rp)
 552 {
 553         printf("nmi called with arg = %x, regs = %x\n", arg, rp);
 554         nmtest += 50;
 555         if (arg == nmtest) {
 556                 printf("ip = %x\n", rp->r_pc);
 557                 return (1);
 558         }
 559         return (0);
 560 }
 561 
 562 #endif
 563 
 564 #include <sys/bootsvcs.h>
 565 
 566 /* Hacked up initialization for initial kernel check out is HERE. */
 567 /* The basic steps are: */
 568 /*      kernel bootfuncs definition/initialization for KADB */
 569 /*      kadb bootfuncs pointer initialization */
 570 /*      putchar/getchar (interrupts disabled) */
 571 
 572 /* kadb bootfuncs pointer initialization */
 573 
 574 int
 575 sysp_getchar()
 576 {
 577         int i;
 578         ulong_t s;
 579 
 580         if (cons_polledio == NULL) {
 581                 /* Uh oh */
 582                 prom_printf("getchar called with no console\n");
 583                 for (;;)
 584                         /* LOOP FOREVER */;
 585         }
 586 
 587         s = clear_int_flag();
 588         i = cons_polledio->cons_polledio_getchar(
 589             cons_polledio->cons_polledio_argument);
 590         restore_int_flag(s);
 591         return (i);
 592 }
 593 
 594 void
 595 sysp_putchar(int c)
 596 {
 597         ulong_t s;
 598 
 599         /*
 600          * We have no alternative but to drop the output on the floor.
 601          */
 602         if (cons_polledio == NULL ||
 603             cons_polledio->cons_polledio_putchar == NULL)
 604                 return;
 605 
 606         s = clear_int_flag();
 607         cons_polledio->cons_polledio_putchar(
 608             cons_polledio->cons_polledio_argument, c);
 609         restore_int_flag(s);
 610 }
 611 
 612 int
 613 sysp_ischar()
 614 {
 615         int i;
 616         ulong_t s;
 617 
 618         if (cons_polledio == NULL ||
 619             cons_polledio->cons_polledio_ischar == NULL)
 620                 return (0);
 621 
 622         s = clear_int_flag();
 623         i = cons_polledio->cons_polledio_ischar(
 624             cons_polledio->cons_polledio_argument);
 625         restore_int_flag(s);
 626         return (i);
 627 }
 628 
 629 int
 630 goany(void)
 631 {
 632         prom_printf("Type any key to continue ");
 633         (void) prom_getchar();
 634         prom_printf("\n");
 635         return (1);
 636 }
 637 
 638 static struct boot_syscalls kern_sysp = {
 639         sysp_getchar,   /*      unchar  (*getchar)();   7  */
 640         sysp_putchar,   /*      int     (*putchar)();   8  */
 641         sysp_ischar,    /*      int     (*ischar)();    9  */
 642 };
 643 
 644 #if defined(__xpv)
 645 int using_kern_polledio;
 646 #endif
 647 
 648 void
 649 kadb_uses_kernel()
 650 {
 651         /*
 652          * This routine is now totally misnamed, since it does not in fact
 653          * control kadb's I/O; it only controls the kernel's prom_* I/O.
 654          */
 655         sysp = &kern_sysp;
 656 #if defined(__xpv)
 657         using_kern_polledio = 1;
 658 #endif
 659 }
 660 
 661 /*
 662  *      the interface to the outside world
 663  */
 664 
 665 /*
 666  * poll_port -- wait for a register to achieve a
 667  *              specific state.  Arguments are a mask of bits we care about,
 668  *              and two sub-masks.  To return normally, all the bits in the
 669  *              first sub-mask must be ON, all the bits in the second sub-
 670  *              mask must be OFF.  If about seconds pass without the register
 671  *              achieving the desired bit configuration, we return 1, else
 672  *              0.
 673  */
 674 int
 675 poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits)
 676 {
 677         int i;
 678         ushort_t maskval;
 679 
 680         for (i = 500000; i; i--) {
 681                 maskval = inb(port) & mask;
 682                 if (((maskval & onbits) == onbits) &&
 683                     ((maskval & offbits) == 0))
 684                         return (0);
 685                 drv_usecwait(10);
 686         }
 687         return (1);
 688 }
 689 
 690 /*
 691  * set_idle_cpu is called from idle() when a CPU becomes idle.
 692  */
 693 /*LINTED: static unused */
 694 static uint_t last_idle_cpu;
 695 
 696 /*ARGSUSED*/
 697 void
 698 set_idle_cpu(int cpun)
 699 {
 700         last_idle_cpu = cpun;
 701         (*psm_set_idle_cpuf)(cpun);
 702 }
 703 
 704 /*
 705  * unset_idle_cpu is called from idle() when a CPU is no longer idle.
 706  */
 707 /*ARGSUSED*/
 708 void
 709 unset_idle_cpu(int cpun)
 710 {
 711         (*psm_unset_idle_cpuf)(cpun);
 712 }
 713 
 714 /*
 715  * This routine is almost correct now, but not quite.  It still needs the
 716  * equivalent concept of "hres_last_tick", just like on the sparc side.
 717  * The idea is to take a snapshot of the hi-res timer while doing the
 718  * hrestime_adj updates under hres_lock in locore, so that the small
 719  * interval between interrupt assertion and interrupt processing is
 720  * accounted for correctly.  Once we have this, the code below should
 721  * be modified to subtract off hres_last_tick rather than hrtime_base.
 722  *
 723  * I'd have done this myself, but I don't have source to all of the
 724  * vendor-specific hi-res timer routines (grrr...).  The generic hook I
 725  * need is something like "gethrtime_unlocked()", which would be just like
 726  * gethrtime() but would assume that you're already holding CLOCK_LOCK().
 727  * This is what the GET_HRTIME() macro is for on sparc (although it also
 728  * serves the function of making time available without a function call
 729  * so you don't take a register window overflow while traps are disabled).
 730  */
 731 void
 732 pc_gethrestime(timestruc_t *tp)
 733 {
 734         int lock_prev;
 735         timestruc_t now;
 736         int nslt;               /* nsec since last tick */
 737         int adj;                /* amount of adjustment to apply */
 738 
 739 loop:
 740         lock_prev = hres_lock;
 741         now = hrestime;
 742         nslt = (int)(gethrtime() - hres_last_tick);
 743         if (nslt < 0) {
 744                 /*
 745                  * nslt < 0 means a tick came between sampling
 746                  * gethrtime() and hres_last_tick; restart the loop
 747                  */
 748 
 749                 goto loop;
 750         }
 751         now.tv_nsec += nslt;
 752         if (hrestime_adj != 0) {
 753                 if (hrestime_adj > 0) {
 754                         adj = (nslt >> ADJ_SHIFT);
 755                         if (adj > hrestime_adj)
 756                                 adj = (int)hrestime_adj;
 757                 } else {
 758                         adj = -(nslt >> ADJ_SHIFT);
 759                         if (adj < hrestime_adj)
 760                                 adj = (int)hrestime_adj;
 761                 }
 762                 now.tv_nsec += adj;
 763         }
 764         while ((unsigned long)now.tv_nsec >= NANOSEC) {
 765 
 766                 /*
 767                  * We might have a large adjustment or have been in the
 768                  * debugger for a long time; take care of (at most) four
 769                  * of those missed seconds (tv_nsec is 32 bits, so
 770                  * anything >4s will be wrapping around).  However,
 771                  * anything more than 2 seconds out of sync will trigger
 772                  * timedelta from clock() to go correct the time anyway,
 773                  * so do what we can, and let the big crowbar do the
 774                  * rest.  A similar correction while loop exists inside
 775                  * hres_tick(); in all cases we'd like tv_nsec to
 776                  * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing
 777                  * user processes, but if tv_sec's a little behind for a
 778                  * little while, that's OK; time still monotonically
 779                  * increases.
 780                  */
 781 
 782                 now.tv_nsec -= NANOSEC;
 783                 now.tv_sec++;
 784         }
 785         if ((hres_lock & ~1) != lock_prev)
 786                 goto loop;
 787 
 788         *tp = now;
 789 }
 790 
 791 void
 792 gethrestime_lasttick(timespec_t *tp)
 793 {
 794         int s;
 795 
 796         s = hr_clock_lock();
 797         *tp = hrestime;
 798         hr_clock_unlock(s);
 799 }
 800 
 801 time_t
 802 gethrestime_sec(void)
 803 {
 804         timestruc_t now;
 805 
 806         gethrestime(&now);
 807         return (now.tv_sec);
 808 }
 809 
 810 /*
 811  * Initialize a kernel thread's stack
 812  */
 813 
 814 caddr_t
 815 thread_stk_init(caddr_t stk)
 816 {
 817         ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0);
 818         return (stk - SA(MINFRAME));
 819 }
 820 
 821 /*
 822  * Initialize lwp's kernel stack.
 823  */
 824 
 825 #ifdef TRAPTRACE
 826 /*
 827  * There's a tricky interdependency here between use of sysenter and
 828  * TRAPTRACE which needs recording to avoid future confusion (this is
 829  * about the third time I've re-figured this out ..)
 830  *
 831  * Here's how debugging lcall works with TRAPTRACE.
 832  *
 833  * 1 We're in userland with a breakpoint on the lcall instruction.
 834  * 2 We execute the instruction - the instruction pushes the userland
 835  *   %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel
 836  *   via the call gate.
 837  * 3 The hardware raises a debug trap in kernel mode, the hardware
 838  *   pushes %efl, %cs, %eip and gets to dbgtrap via the idt.
 839  * 4 dbgtrap pushes the error code and trapno and calls cmntrap
 840  * 5 cmntrap finishes building a trap frame
 841  * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk
 842  *   off the stack into the traptrace buffer.
 843  *
 844  * This means that the traptrace buffer contains the wrong values in
 845  * %esp and %ss, but everything else in there is correct.
 846  *
 847  * Here's how debugging sysenter works with TRAPTRACE.
 848  *
 849  * a We're in userland with a breakpoint on the sysenter instruction.
 850  * b We execute the instruction - the instruction pushes -nothing-
 851  *   on the stack, but sets %cs, %eip, %ss, %esp to prearranged
 852  *   values to take us to sys_sysenter, at the top of the lwp's
 853  *   stack.
 854  * c goto 3
 855  *
 856  * At this point, because we got into the kernel without the requisite
 857  * five pushes on the stack, if we didn't make extra room, we'd
 858  * end up with the TRACE_REGS macro fetching the saved %ss and %esp
 859  * values from negative (unmapped) stack addresses -- which really bites.
 860  * That's why we do the '-= 8' below.
 861  *
 862  * XXX  Note that reading "up" lwp0's stack works because t0 is declared
 863  *      right next to t0stack in locore.s
 864  */
 865 #endif
 866 
 867 caddr_t
 868 lwp_stk_init(klwp_t *lwp, caddr_t stk)
 869 {
 870         caddr_t oldstk;
 871         struct pcb *pcb = &lwp->lwp_pcb;
 872 
 873         oldstk = stk;
 874         stk -= SA(sizeof (struct regs) + SA(MINFRAME));
 875 #ifdef TRAPTRACE
 876         stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */
 877 #endif
 878         stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul));
 879         bzero(stk, oldstk - stk);
 880         lwp->lwp_regs = (void *)(stk + SA(MINFRAME));
 881 
 882         /*
 883          * Arrange that the virtualized %fs and %gs GDT descriptors
 884          * have a well-defined initial state (present, ring 3
 885          * and of type data).
 886          */
 887 #if defined(__amd64)
 888         if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE)
 889                 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
 890         else
 891                 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;
 892 #elif defined(__i386)
 893         pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
 894 #endif  /* __i386 */
 895         lwp_installctx(lwp);
 896         return (stk);
 897 }
 898 
 899 /*
 900  * Use this opportunity to free any dynamically allocated fp storage.
 901  */
 902 void
 903 lwp_stk_fini(klwp_t *lwp)
 904 {
 905         fp_lwp_cleanup(lwp);
 906 }
 907 
 908 void
 909 lwp_fp_init(klwp_t *lwp)
 910 {
 911         fp_lwp_init(lwp);
 912 }
 913 
 914 /*
 915  * If we're not the panic CPU, we wait in panic_idle for reboot.
 916  */
 917 void
 918 panic_idle(void)
 919 {
 920         splx(ipltospl(CLOCK_LEVEL));
 921         (void) setjmp(&curthread->t_pcb);
 922 
 923         dumpsys_helper();
 924 
 925 #ifndef __xpv
 926         for (;;)
 927                 i86_halt();
 928 #else
 929         for (;;)
 930                 ;
 931 #endif
 932 }
 933 
 934 /*
 935  * Stop the other CPUs by cross-calling them and forcing them to enter
 936  * the panic_idle() loop above.
 937  */
 938 /*ARGSUSED*/
 939 void
 940 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl)
 941 {
 942         processorid_t i;
 943         cpuset_t xcset;
 944 
 945         /*
 946          * In the case of a Xen panic, the hypervisor has already stopped
 947          * all of the CPUs.
 948          */
 949         if (!IN_XPV_PANIC()) {
 950                 (void) splzs();
 951 
 952                 CPUSET_ALL_BUT(xcset, cp->cpu_id);
 953                 xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle);
 954         }
 955 
 956         for (i = 0; i < NCPU; i++) {
 957                 if (i != cp->cpu_id && cpu[i] != NULL &&
 958                     (cpu[i]->cpu_flags & CPU_EXISTS))
 959                         cpu[i]->cpu_flags |= CPU_QUIESCED;
 960         }
 961 }
 962 
 963 /*
 964  * Platform callback following each entry to panicsys().
 965  */
 966 /*ARGSUSED*/
 967 void
 968 panic_enter_hw(int spl)
 969 {
 970         /* Nothing to do here */
 971 }
 972 
 973 /*
 974  * Platform-specific code to execute after panicstr is set: we invoke
 975  * the PSM entry point to indicate that a panic has occurred.
 976  */
 977 /*ARGSUSED*/
 978 void
 979 panic_quiesce_hw(panic_data_t *pdp)
 980 {
 981         psm_notifyf(PSM_PANIC_ENTER);
 982 
 983         cmi_panic_callback();
 984 
 985 #ifdef  TRAPTRACE
 986         /*
 987          * Turn off TRAPTRACE
 988          */
 989         TRAPTRACE_FREEZE;
 990 #endif  /* TRAPTRACE */
 991 }
 992 
 993 /*
 994  * Platform callback prior to writing crash dump.
 995  */
 996 /*ARGSUSED*/
 997 void
 998 panic_dump_hw(int spl)
 999 {
1000         /* Nothing to do here */
1001 }
1002 
1003 void *
1004 plat_traceback(void *fpreg)
1005 {
1006 #ifdef __xpv
1007         if (IN_XPV_PANIC())
1008                 return (xpv_traceback(fpreg));
1009 #endif
1010         return (fpreg);
1011 }
1012 
1013 /*ARGSUSED*/
1014 void
1015 plat_tod_fault(enum tod_fault_type tod_bad)
1016 {}
1017 
1018 /*ARGSUSED*/
1019 int
1020 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class)
1021 {
1022         return (ENOTSUP);
1023 }
1024 
1025 /*
1026  * The underlying console output routines are protected by raising IPL in case
1027  * we are still calling into the early boot services.  Once we start calling
1028  * the kernel console emulator, it will disable interrupts completely during
1029  * character rendering (see sysp_putchar, for example).  Refer to the comments
1030  * and code in common/os/console.c for more information on these callbacks.
1031  */
1032 /*ARGSUSED*/
1033 int
1034 console_enter(int busy)
1035 {
1036         return (splzs());
1037 }
1038 
1039 /*ARGSUSED*/
1040 void
1041 console_exit(int busy, int spl)
1042 {
1043         splx(spl);
1044 }
1045 
1046 /*
1047  * Allocate a region of virtual address space, unmapped.
1048  * Stubbed out except on sparc, at least for now.
1049  */
1050 /*ARGSUSED*/
1051 void *
1052 boot_virt_alloc(void *addr, size_t size)
1053 {
1054         return (addr);
1055 }
1056 
1057 volatile unsigned long  tenmicrodata;
1058 
1059 void
1060 tenmicrosec(void)
1061 {
1062         extern int gethrtime_hires;
1063 
1064         if (gethrtime_hires) {
1065                 hrtime_t start, end;
1066                 start = end =  gethrtime();
1067                 while ((end - start) < (10 * (NANOSEC / MICROSEC))) {
1068                         SMT_PAUSE();
1069                         end = gethrtime();
1070                 }
1071         } else {
1072 #if defined(__xpv)
1073                 hrtime_t newtime;
1074 
1075                 newtime = xpv_gethrtime() + 10000; /* now + 10 us */
1076                 while (xpv_gethrtime() < newtime)
1077                         SMT_PAUSE();
1078 #else   /* __xpv */
1079                 int i;
1080 
1081                 /*
1082                  * Artificial loop to induce delay.
1083                  */
1084                 for (i = 0; i < microdata; i++)
1085                         tenmicrodata = microdata;
1086 #endif  /* __xpv */
1087         }
1088 }
1089 
1090 /*
1091  * get_cpu_mstate() is passed an array of timestamps, NCMSTATES
1092  * long, and it fills in the array with the time spent on cpu in
1093  * each of the mstates, where time is returned in nsec.
1094  *
1095  * No guarantee is made that the returned values in times[] will
1096  * monotonically increase on sequential calls, although this will
1097  * be true in the long run. Any such guarantee must be handled by
1098  * the caller, if needed. This can happen if we fail to account
1099  * for elapsed time due to a generation counter conflict, yet we
1100  * did account for it on a prior call (see below).
1101  *
1102  * The complication is that the cpu in question may be updating
1103  * its microstate at the same time that we are reading it.
1104  * Because the microstate is only updated when the CPU's state
1105  * changes, the values in cpu_intracct[] can be indefinitely out
1106  * of date. To determine true current values, it is necessary to
1107  * compare the current time with cpu_mstate_start, and add the
1108  * difference to times[cpu_mstate].
1109  *
1110  * This can be a problem if those values are changing out from
1111  * under us. Because the code path in new_cpu_mstate() is
1112  * performance critical, we have not added a lock to it. Instead,
1113  * we have added a generation counter. Before beginning
1114  * modifications, the counter is set to 0. After modifications,
1115  * it is set to the old value plus one.
1116  *
1117  * get_cpu_mstate() will not consider the values of cpu_mstate
1118  * and cpu_mstate_start to be usable unless the value of
1119  * cpu_mstate_gen is both non-zero and unchanged, both before and
1120  * after reading the mstate information. Note that we must
1121  * protect against out-of-order loads around accesses to the
1122  * generation counter. Also, this is a best effort approach in
1123  * that we do not retry should the counter be found to have
1124  * changed.
1125  *
1126  * cpu_intracct[] is used to identify time spent in each CPU
1127  * mstate while handling interrupts. Such time should be reported
1128  * against system time, and so is subtracted out from its
1129  * corresponding cpu_acct[] time and added to
1130  * cpu_acct[CMS_SYSTEM].
1131  */
1132 
1133 void
1134 get_cpu_mstate(cpu_t *cpu, hrtime_t *times)
1135 {
1136         int i;
1137         hrtime_t now, start;
1138         uint16_t gen;
1139         uint16_t state;
1140         hrtime_t intracct[NCMSTATES];
1141 
1142         /*
1143          * Load all volatile state under the protection of membar.
1144          * cpu_acct[cpu_mstate] must be loaded to avoid double counting
1145          * of (now - cpu_mstate_start) by a change in CPU mstate that
1146          * arrives after we make our last check of cpu_mstate_gen.
1147          */
1148 
1149         now = gethrtime_unscaled();
1150         gen = cpu->cpu_mstate_gen;
1151 
1152         membar_consumer();      /* guarantee load ordering */
1153         start = cpu->cpu_mstate_start;
1154         state = cpu->cpu_mstate;
1155         for (i = 0; i < NCMSTATES; i++) {
1156                 intracct[i] = cpu->cpu_intracct[i];
1157                 times[i] = cpu->cpu_acct[i];
1158         }
1159         membar_consumer();      /* guarantee load ordering */
1160 
1161         if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start)
1162                 times[state] += now - start;
1163 
1164         for (i = 0; i < NCMSTATES; i++) {
1165                 if (i == CMS_SYSTEM)
1166                         continue;
1167                 times[i] -= intracct[i];
1168                 if (times[i] < 0) {
1169                         intracct[i] += times[i];
1170                         times[i] = 0;
1171                 }
1172                 times[CMS_SYSTEM] += intracct[i];
1173                 scalehrtime(&times[i]);
1174         }
1175         scalehrtime(&times[CMS_SYSTEM]);
1176 }
1177 
1178 /*
1179  * This is a version of the rdmsr instruction that allows
1180  * an error code to be returned in the case of failure.
1181  */
1182 int
1183 checked_rdmsr(uint_t msr, uint64_t *value)
1184 {
1185         if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1186                 return (ENOTSUP);
1187         *value = rdmsr(msr);
1188         return (0);
1189 }
1190 
1191 /*
1192  * This is a version of the wrmsr instruction that allows
1193  * an error code to be returned in the case of failure.
1194  */
1195 int
1196 checked_wrmsr(uint_t msr, uint64_t value)
1197 {
1198         if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1199                 return (ENOTSUP);
1200         wrmsr(msr, value);
1201         return (0);
1202 }
1203 
1204 /*
1205  * The mem driver's usual method of using hat_devload() to establish a
1206  * temporary mapping will not work for foreign pages mapped into this
1207  * domain or for the special hypervisor-provided pages.  For the foreign
1208  * pages, we often don't know which domain owns them, so we can't ask the
1209  * hypervisor to set up a new mapping.  For the other pages, we don't have
1210  * a pfn, so we can't create a new PTE.  For these special cases, we do a
1211  * direct uiomove() from the existing kernel virtual address.
1212  */
1213 /*ARGSUSED*/
1214 int
1215 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw)
1216 {
1217 #if defined(__xpv)
1218         void *va = (void *)(uintptr_t)uio->uio_loffset;
1219         off_t pageoff = uio->uio_loffset & PAGEOFFSET;
1220         size_t nbytes = MIN((size_t)(PAGESIZE - pageoff),
1221             (size_t)uio->uio_iov->iov_len);
1222 
1223         if ((rw == UIO_READ &&
1224             (va == HYPERVISOR_shared_info || va == xen_info)) ||
1225             (pfn_is_foreign(hat_getpfnum(kas.a_hat, va))))
1226                 return (uiomove(va, nbytes, rw, uio));
1227 #endif
1228         return (ENOTSUP);
1229 }
1230 
1231 pgcnt_t
1232 num_phys_pages()
1233 {
1234         pgcnt_t npages = 0;
1235         struct memlist *mp;
1236 
1237 #if defined(__xpv)
1238         if (DOMAIN_IS_INITDOMAIN(xen_info))
1239                 return (xpv_nr_phys_pages());
1240 #endif /* __xpv */
1241 
1242         for (mp = phys_install; mp != NULL; mp = mp->ml_next)
1243                 npages += mp->ml_size >> PAGESHIFT;
1244 
1245         return (npages);
1246 }
1247 
1248 /* cpu threshold for compressed dumps */
1249 #ifdef _LP64
1250 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_64_MINCPU;
1251 #else
1252 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_32_MINCPU;
1253 #endif
1254 
1255 int
1256 dump_plat_addr()
1257 {
1258 #ifdef __xpv
1259         pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1260         mem_vtop_t mem_vtop;
1261         int cnt;
1262 
1263         /*
1264          * On the hypervisor, we want to dump the page with shared_info on it.
1265          */
1266         if (!IN_XPV_PANIC()) {
1267                 mem_vtop.m_as = &kas;
1268                 mem_vtop.m_va = HYPERVISOR_shared_info;
1269                 mem_vtop.m_pfn = pfn;
1270                 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t));
1271                 cnt = 1;
1272         } else {
1273                 cnt = dump_xpv_addr();
1274         }
1275         return (cnt);
1276 #else
1277         return (0);
1278 #endif
1279 }
1280 
1281 void
1282 dump_plat_pfn()
1283 {
1284 #ifdef __xpv
1285         pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1286 
1287         if (!IN_XPV_PANIC())
1288                 dumpvp_write(&pfn, sizeof (pfn));
1289         else
1290                 dump_xpv_pfn();
1291 #endif
1292 }
1293 
1294 /*ARGSUSED*/
1295 int
1296 dump_plat_data(void *dump_cbuf)
1297 {
1298 #ifdef __xpv
1299         uint32_t csize;
1300         int cnt;
1301 
1302         if (!IN_XPV_PANIC()) {
1303                 csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf,
1304                     PAGESIZE);
1305                 dumpvp_write(&csize, sizeof (uint32_t));
1306                 dumpvp_write(dump_cbuf, csize);
1307                 cnt = 1;
1308         } else {
1309                 cnt = dump_xpv_data(dump_cbuf);
1310         }
1311         return (cnt);
1312 #else
1313         return (0);
1314 #endif
1315 }
1316 
1317 /*
1318  * Calculates a linear address, given the CS selector and PC values
1319  * by looking up the %cs selector process's LDT or the CPU's GDT.
1320  * proc->p_ldtlock must be held across this call.
1321  */
1322 int
1323 linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1324 {
1325         user_desc_t     *descrp;
1326         caddr_t         baseaddr;
1327         uint16_t        idx = SELTOIDX(rp->r_cs);
1328 
1329         ASSERT(rp->r_cs <= 0xFFFF);
1330         ASSERT(MUTEX_HELD(&p->p_ldtlock));
1331 
1332         if (SELISLDT(rp->r_cs)) {
1333                 /*
1334                  * Currently 64 bit processes cannot have private LDTs.
1335                  */
1336                 ASSERT(p->p_model != DATAMODEL_LP64);
1337 
1338                 if (p->p_ldt == NULL)
1339                         return (-1);
1340 
1341                 descrp = &p->p_ldt[idx];
1342                 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1343 
1344                 /*
1345                  * Calculate the linear address (wraparound is not only ok,
1346                  * it's expected behavior).  The cast to uint32_t is because
1347                  * LDT selectors are only allowed in 32-bit processes.
1348                  */
1349                 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1350                     rp->r_pc);
1351         } else {
1352 #ifdef DEBUG
1353                 descrp = &CPU->cpu_gdt[idx];
1354                 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1355                 /* GDT-based descriptors' base addresses should always be 0 */
1356                 ASSERT(baseaddr == 0);
1357 #endif
1358                 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1359         }
1360 
1361         return (0);
1362 }
1363 
1364 /*
1365  * The implementation of dtrace_linear_pc is similar to the that of
1366  * linear_pc, above, but here we acquire p_ldtlock before accessing
1367  * p_ldt.  This implementation is used by the pid provider; we prefix
1368  * it with "dtrace_" to avoid inducing spurious tracing events.
1369  */
1370 int
1371 dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1372 {
1373         user_desc_t     *descrp;
1374         caddr_t         baseaddr;
1375         uint16_t        idx = SELTOIDX(rp->r_cs);
1376 
1377         ASSERT(rp->r_cs <= 0xFFFF);
1378 
1379         if (SELISLDT(rp->r_cs)) {
1380                 /*
1381                  * Currently 64 bit processes cannot have private LDTs.
1382                  */
1383                 ASSERT(p->p_model != DATAMODEL_LP64);
1384 
1385                 mutex_enter(&p->p_ldtlock);
1386                 if (p->p_ldt == NULL) {
1387                         mutex_exit(&p->p_ldtlock);
1388                         return (-1);
1389                 }
1390                 descrp = &p->p_ldt[idx];
1391                 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1392                 mutex_exit(&p->p_ldtlock);
1393 
1394                 /*
1395                  * Calculate the linear address (wraparound is not only ok,
1396                  * it's expected behavior).  The cast to uint32_t is because
1397                  * LDT selectors are only allowed in 32-bit processes.
1398                  */
1399                 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1400                     rp->r_pc);
1401         } else {
1402 #ifdef DEBUG
1403                 descrp = &CPU->cpu_gdt[idx];
1404                 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1405                 /* GDT-based descriptors' base addresses should always be 0 */
1406                 ASSERT(baseaddr == 0);
1407 #endif
1408                 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1409         }
1410 
1411         return (0);
1412 }
1413 
1414 /*
1415  * We need to post a soft interrupt to reprogram the lbolt cyclic when
1416  * switching from event to cyclic driven lbolt. The following code adds
1417  * and posts the softint for x86.
1418  */
1419 static ddi_softint_hdl_impl_t lbolt_softint_hdl =
1420         {0, 0, NULL, NULL, 0, NULL, NULL, NULL};
1421 
1422 void
1423 lbolt_softint_add(void)
1424 {
1425         (void) add_avsoftintr((void *)&lbolt_softint_hdl, LOCK_LEVEL,
1426             (avfunc)lbolt_ev_to_cyclic, "lbolt_ev_to_cyclic", NULL, NULL);
1427 }
1428 
1429 void
1430 lbolt_softint_post(void)
1431 {
1432         (*setsoftint)(CBE_LOCK_PIL, lbolt_softint_hdl.ih_pending);
1433 }
1434 
1435 boolean_t
1436 plat_dr_check_capability(uint64_t features)
1437 {
1438         return ((plat_dr_options & features) == features);
1439 }
1440 
1441 boolean_t
1442 plat_dr_support_cpu(void)
1443 {
1444         return (plat_dr_options & PLAT_DR_FEATURE_CPU);
1445 }
1446 
1447 boolean_t
1448 plat_dr_support_memory(void)
1449 {
1450         return (plat_dr_options & PLAT_DR_FEATURE_MEMORY);
1451 }
1452 
1453 void
1454 plat_dr_enable_capability(uint64_t features)
1455 {
1456         atomic_or_64(&plat_dr_options, features);
1457 }
1458 
1459 void
1460 plat_dr_disable_capability(uint64_t features)
1461 {
1462         atomic_and_64(&plat_dr_options, ~features);
1463 }
1464 
1465 /*
1466  * If SMAP is supported, look through hi_calls and inline
1467  * calls to smap_enable() to clac and smap_disable() to stac.
1468  */
1469 void
1470 hotinline_smap(hotinline_desc_t *hid)
1471 {
1472         if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE)
1473                 return;
1474 
1475         if (strcmp(hid->hid_symname, "smap_enable") == 0) {
1476                 bcopy(clac_instr, (void *)hid->hid_instr_offset,
1477                     sizeof (clac_instr));
1478         } else if (strcmp(hid->hid_symname, "smap_disable") == 0) {
1479                 bcopy(stac_instr, (void *)hid->hid_instr_offset,
1480                     sizeof (stac_instr));
1481         }
1482 }
1483 
1484 /*
1485  * Loop through hi_calls and hand off the inlining to
1486  * the appropriate calls.
1487  */
1488 void
1489 do_hotinlines(struct module *mp)
1490 {
1491         for (hotinline_desc_t *hid = mp->hi_calls; hid != NULL;
1492             hid = hid->hid_next) {
1493 #if !defined(__xpv)
1494                 hotinline_smap(hid);
1495 #endif  /* __xpv */
1496         }
1497 }