1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2020 Joyent, Inc.
25 */
26 /*
27 * Copyright (c) 2010, Intel Corporation.
28 * All rights reserved.
29 */
30
31 #include <sys/types.h>
32 #include <sys/t_lock.h>
33 #include <sys/param.h>
34 #include <sys/segments.h>
35 #include <sys/sysmacros.h>
36 #include <sys/signal.h>
37 #include <sys/systm.h>
38 #include <sys/user.h>
39 #include <sys/mman.h>
40 #include <sys/vm.h>
41
42 #include <sys/disp.h>
43 #include <sys/class.h>
44
45 #include <sys/proc.h>
46 #include <sys/buf.h>
47 #include <sys/kmem.h>
48
49 #include <sys/reboot.h>
50 #include <sys/uadmin.h>
51 #include <sys/callb.h>
52
53 #include <sys/cred.h>
54 #include <sys/vnode.h>
55 #include <sys/file.h>
56
57 #include <sys/procfs.h>
58 #include <sys/acct.h>
59
60 #include <sys/vfs.h>
61 #include <sys/dnlc.h>
62 #include <sys/var.h>
63 #include <sys/cmn_err.h>
64 #include <sys/utsname.h>
65 #include <sys/debug.h>
66
67 #include <sys/dumphdr.h>
68 #include <sys/bootconf.h>
69 #include <sys/varargs.h>
70 #include <sys/promif.h>
71 #include <sys/modctl.h>
72
73 #include <sys/consdev.h>
74 #include <sys/frame.h>
75
76 #include <sys/sunddi.h>
77 #include <sys/ddidmareq.h>
78 #include <sys/psw.h>
79 #include <sys/regset.h>
80 #include <sys/privregs.h>
81 #include <sys/clock.h>
82 #include <sys/tss.h>
83 #include <sys/cpu.h>
84 #include <sys/stack.h>
85 #include <sys/trap.h>
86 #include <sys/pic.h>
87 #include <vm/hat.h>
88 #include <vm/anon.h>
89 #include <vm/as.h>
90 #include <vm/page.h>
91 #include <vm/seg.h>
92 #include <vm/seg_kmem.h>
93 #include <vm/seg_map.h>
94 #include <vm/seg_vn.h>
95 #include <vm/seg_kp.h>
96 #include <vm/hat_i86.h>
97 #include <sys/swap.h>
98 #include <sys/thread.h>
99 #include <sys/sysconf.h>
100 #include <sys/vm_machparam.h>
101 #include <sys/archsystm.h>
102 #include <sys/machsystm.h>
103 #include <sys/machlock.h>
104 #include <sys/x_call.h>
105 #include <sys/instance.h>
106
107 #include <sys/time.h>
108 #include <sys/smp_impldefs.h>
109 #include <sys/psm_types.h>
110 #include <sys/atomic.h>
111 #include <sys/panic.h>
112 #include <sys/cpuvar.h>
113 #include <sys/dtrace.h>
114 #include <sys/bl.h>
115 #include <sys/nvpair.h>
116 #include <sys/x86_archext.h>
117 #include <sys/pool_pset.h>
118 #include <sys/autoconf.h>
119 #include <sys/mem.h>
120 #include <sys/dumphdr.h>
121 #include <sys/compress.h>
122 #include <sys/cpu_module.h>
123 #if defined(__xpv)
124 #include <sys/hypervisor.h>
125 #include <sys/xpv_panic.h>
126 #endif
127
128 #include <sys/fastboot.h>
129 #include <sys/machelf.h>
130 #include <sys/kobj.h>
131 #include <sys/multiboot.h>
132
133 #ifdef TRAPTRACE
134 #include <sys/traptrace.h>
135 #endif /* TRAPTRACE */
136
137 #include <c2/audit.h>
138 #include <sys/clock_impl.h>
139
140 extern void audit_enterprom(int);
141 extern void audit_exitprom(int);
142
143 /*
144 * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used.
145 */
146 int apix_enable = 1;
147
148 int apic_nvidia_io_max = 0; /* no. of NVIDIA i/o apics */
149
150 /*
151 * Occassionally the kernel knows better whether to power-off or reboot.
152 */
153 int force_shutdown_method = AD_UNKNOWN;
154
155 /*
156 * The panicbuf array is used to record messages and state:
157 */
158 char panicbuf[PANICBUFSIZE];
159
160 /*
161 * Flags to control Dynamic Reconfiguration features.
162 */
163 uint64_t plat_dr_options;
164
165 /*
166 * Maximum physical address for memory DR operations.
167 */
168 uint64_t plat_dr_physmax;
169
170 /*
171 * maxphys - used during physio
172 * klustsize - used for klustering by swapfs and specfs
173 */
174 int maxphys = 56 * 1024; /* XXX See vm_subr.c - max b_count in physio */
175 int klustsize = 56 * 1024;
176
177 caddr_t p0_va; /* Virtual address for accessing physical page 0 */
178
179 /*
180 * defined here, though unused on x86,
181 * to make kstat_fr.c happy.
182 */
183 int vac;
184
185 void debug_enter(char *);
186
187 extern void pm_cfb_check_and_powerup(void);
188 extern void pm_cfb_rele(void);
189
190 extern fastboot_info_t newkernel;
191
192 /*
193 * Instructions to enable or disable SMAP, respectively.
194 */
195 static const uint8_t clac_instr[3] = { 0x0f, 0x01, 0xca };
196 static const uint8_t stac_instr[3] = { 0x0f, 0x01, 0xcb };
197
198 /*
199 * Machine dependent code to reboot.
200 * "mdep" is interpreted as a character pointer; if non-null, it is a pointer
201 * to a string to be used as the argument string when rebooting.
202 *
203 * "invoke_cb" is a boolean. It is set to true when mdboot() can safely
204 * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when
205 * we are in a normal shutdown sequence (interrupts are not blocked, the
206 * system is not panic'ing or being suspended).
207 */
208 /*ARGSUSED*/
209 void
210 mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb)
211 {
212 processorid_t bootcpuid = 0;
213 static int is_first_quiesce = 1;
214 static int is_first_reset = 1;
215 int reset_status = 0;
216 static char fallback_str[] = "Falling back to regular reboot.\n";
217
218 if (fcn == AD_FASTREBOOT && !newkernel.fi_valid)
219 fcn = AD_BOOT;
220
221 if (!panicstr) {
222 kpreempt_disable();
223 if (fcn == AD_FASTREBOOT) {
224 mutex_enter(&cpu_lock);
225 if (CPU_ACTIVE(cpu_get(bootcpuid))) {
226 affinity_set(bootcpuid);
227 }
228 mutex_exit(&cpu_lock);
229 } else {
230 affinity_set(CPU_CURRENT);
231 }
232 }
233
234 if (force_shutdown_method != AD_UNKNOWN)
235 fcn = force_shutdown_method;
236
237 /*
238 * XXX - rconsvp is set to NULL to ensure that output messages
239 * are sent to the underlying "hardware" device using the
240 * monitor's printf routine since we are in the process of
241 * either rebooting or halting the machine.
242 */
243 rconsvp = NULL;
244
245 /*
246 * Print the reboot message now, before pausing other cpus.
247 * There is a race condition in the printing support that
248 * can deadlock multiprocessor machines.
249 */
250 if (!(fcn == AD_HALT || fcn == AD_POWEROFF))
251 prom_printf("rebooting...\n");
252
253 if (IN_XPV_PANIC())
254 reset();
255
256 /*
257 * We can't bring up the console from above lock level, so do it now
258 */
259 pm_cfb_check_and_powerup();
260
261 /* make sure there are no more changes to the device tree */
262 devtree_freeze();
263
264 if (invoke_cb)
265 (void) callb_execute_class(CB_CL_MDBOOT, 0);
266
267 /*
268 * Clear any unresolved UEs from memory.
269 */
270 page_retire_mdboot();
271
272 #if defined(__xpv)
273 /*
274 * XXPV Should probably think some more about how we deal
275 * with panicing before it's really safe to panic.
276 * On hypervisors, we reboot very quickly.. Perhaps panic
277 * should only attempt to recover by rebooting if,
278 * say, we were able to mount the root filesystem,
279 * or if we successfully launched init(1m).
280 */
281 if (panicstr && proc_init == NULL)
282 (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff);
283 #endif
284 /*
285 * stop other cpus and raise our priority. since there is only
286 * one active cpu after this, and our priority will be too high
287 * for us to be preempted, we're essentially single threaded
288 * from here on out.
289 */
290 (void) spl6();
291 if (!panicstr) {
292 mutex_enter(&cpu_lock);
293 pause_cpus(NULL, NULL);
294 mutex_exit(&cpu_lock);
295 }
296
297 /*
298 * If the system is panicking, the preloaded kernel is valid, and
299 * fastreboot_onpanic has been set, and the system has been up for
300 * longer than fastreboot_onpanic_uptime (default to 10 minutes),
301 * choose Fast Reboot.
302 */
303 if (fcn == AD_BOOT && panicstr && newkernel.fi_valid &&
304 fastreboot_onpanic &&
305 (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) {
306 fcn = AD_FASTREBOOT;
307 }
308
309 /*
310 * Try to quiesce devices.
311 */
312 if (is_first_quiesce) {
313 /*
314 * Clear is_first_quiesce before calling quiesce_devices()
315 * so that if quiesce_devices() causes panics, it will not
316 * be invoked again.
317 */
318 is_first_quiesce = 0;
319
320 quiesce_active = 1;
321 quiesce_devices(ddi_root_node(), &reset_status);
322 if (reset_status == -1) {
323 if (fcn == AD_FASTREBOOT && !force_fastreboot) {
324 prom_printf("Driver(s) not capable of fast "
325 "reboot.\n");
326 prom_printf(fallback_str);
327 fastreboot_capable = 0;
328 fcn = AD_BOOT;
329 } else if (fcn != AD_FASTREBOOT)
330 fastreboot_capable = 0;
331 }
332 quiesce_active = 0;
333 }
334
335 /*
336 * Try to reset devices. reset_leaves() should only be called
337 * a) when there are no other threads that could be accessing devices,
338 * and
339 * b) on a system that's not capable of fast reboot (fastreboot_capable
340 * being 0), or on a system where quiesce_devices() failed to
341 * complete (quiesce_active being 1).
342 */
343 if (is_first_reset && (!fastreboot_capable || quiesce_active)) {
344 /*
345 * Clear is_first_reset before calling reset_devices()
346 * so that if reset_devices() causes panics, it will not
347 * be invoked again.
348 */
349 is_first_reset = 0;
350 reset_leaves();
351 }
352
353 /* Verify newkernel checksum */
354 if (fastreboot_capable && fcn == AD_FASTREBOOT &&
355 fastboot_cksum_verify(&newkernel) != 0) {
356 fastreboot_capable = 0;
357 prom_printf("Fast reboot: checksum failed for the new "
358 "kernel.\n");
359 prom_printf(fallback_str);
360 }
361
362 (void) spl8();
363
364 if (fastreboot_capable && fcn == AD_FASTREBOOT) {
365 /*
366 * psm_shutdown is called within fast_reboot()
367 */
368 fast_reboot();
369 } else {
370 (*psm_shutdownf)(cmd, fcn);
371
372 if (fcn == AD_HALT || fcn == AD_POWEROFF)
373 halt((char *)NULL);
374 else
375 prom_reboot("");
376 }
377 /*NOTREACHED*/
378 }
379
380 /* mdpreboot - may be called prior to mdboot while root fs still mounted */
381 /*ARGSUSED*/
382 void
383 mdpreboot(int cmd, int fcn, char *mdep)
384 {
385 if (fcn == AD_FASTREBOOT && !fastreboot_capable) {
386 fcn = AD_BOOT;
387 #ifdef __xpv
388 cmn_err(CE_WARN, "Fast reboot is not supported on xVM");
389 #else
390 cmn_err(CE_WARN,
391 "Fast reboot is not supported on this platform%s",
392 fastreboot_nosup_message());
393 #endif
394 }
395
396 if (fcn == AD_FASTREBOOT) {
397 fastboot_load_kernel(mdep);
398 if (!newkernel.fi_valid)
399 fcn = AD_BOOT;
400 }
401
402 (*psm_preshutdownf)(cmd, fcn);
403 }
404
405 static void
406 stop_other_cpus(void)
407 {
408 ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */
409 cpuset_t xcset;
410
411 CPUSET_ALL_BUT(xcset, CPU->cpu_id);
412 xc_priority(0, 0, 0, CPUSET2BV(xcset), mach_cpu_halt);
413 restore_int_flag(s);
414 }
415
416 /*
417 * Machine dependent abort sequence handling
418 */
419 void
420 abort_sequence_enter(char *msg)
421 {
422 if (abort_enable == 0) {
423 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
424 audit_enterprom(0);
425 return;
426 }
427 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
428 audit_enterprom(1);
429 debug_enter(msg);
430 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
431 audit_exitprom(1);
432 }
433
434 /*
435 * Enter debugger. Called when the user types ctrl-alt-d or whenever
436 * code wants to enter the debugger and possibly resume later.
437 *
438 * msg: message to print, possibly NULL.
439 */
440 void
441 debug_enter(char *msg)
442 {
443 if (dtrace_debugger_init != NULL)
444 (*dtrace_debugger_init)();
445
446 if (msg != NULL || (boothowto & RB_DEBUG))
447 prom_printf("\n");
448
449 if (msg != NULL)
450 prom_printf("%s\n", msg);
451
452 if (boothowto & RB_DEBUG)
453 kmdb_enter();
454
455 if (dtrace_debugger_fini != NULL)
456 (*dtrace_debugger_fini)();
457 }
458
459 void
460 reset(void)
461 {
462 extern void acpi_reset_system();
463 #if !defined(__xpv)
464 ushort_t *bios_memchk;
465
466 /*
467 * Can't use psm_map_phys or acpi_reset_system before the hat is
468 * initialized.
469 */
470 if (khat_running) {
471 bios_memchk = (ushort_t *)psm_map_phys(0x472,
472 sizeof (ushort_t), PROT_READ | PROT_WRITE);
473 if (bios_memchk)
474 *bios_memchk = 0x1234; /* bios memory check disable */
475
476 if (options_dip != NULL &&
477 ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0,
478 "efi-systab")) {
479 if (bootops == NULL)
480 acpi_reset_system();
481 efi_reset();
482 }
483
484 /*
485 * The problem with using stubs is that we can call
486 * acpi_reset_system only after the kernel is up and running.
487 *
488 * We should create a global state to keep track of how far
489 * up the kernel is but for the time being we will depend on
490 * bootops. bootops cleared in startup_end().
491 */
492 if (bootops == NULL)
493 acpi_reset_system();
494 }
495
496 pc_reset();
497 #else
498 if (IN_XPV_PANIC()) {
499 if (khat_running && bootops == NULL) {
500 acpi_reset_system();
501 }
502
503 pc_reset();
504 }
505
506 (void) HYPERVISOR_shutdown(SHUTDOWN_reboot);
507 panic("HYPERVISOR_shutdown() failed");
508 #endif
509 /*NOTREACHED*/
510 }
511
512 /*
513 * Halt the machine and return to the monitor
514 */
515 void
516 halt(char *s)
517 {
518 stop_other_cpus(); /* send stop signal to other CPUs */
519 if (s)
520 prom_printf("(%s) \n", s);
521 prom_exit_to_mon();
522 /*NOTREACHED*/
523 }
524
525 /*
526 * Initiate interrupt redistribution.
527 */
528 void
529 i_ddi_intr_redist_all_cpus()
530 {
531 }
532
533 /*
534 * XXX These probably ought to live somewhere else
535 * XXX They are called from mem.c
536 */
537
538 /*
539 * Convert page frame number to an OBMEM page frame number
540 * (i.e. put in the type bits -- zero for this implementation)
541 */
542 pfn_t
543 impl_obmem_pfnum(pfn_t pf)
544 {
545 return (pf);
546 }
547
548 #ifdef NM_DEBUG
549 int nmi_test = 0; /* checked in intentry.s during clock int */
550 int nmtest = -1;
551 nmfunc1(int arg, struct regs *rp)
552 {
553 printf("nmi called with arg = %x, regs = %x\n", arg, rp);
554 nmtest += 50;
555 if (arg == nmtest) {
556 printf("ip = %x\n", rp->r_pc);
557 return (1);
558 }
559 return (0);
560 }
561
562 #endif
563
564 #include <sys/bootsvcs.h>
565
566 /* Hacked up initialization for initial kernel check out is HERE. */
567 /* The basic steps are: */
568 /* kernel bootfuncs definition/initialization for KADB */
569 /* kadb bootfuncs pointer initialization */
570 /* putchar/getchar (interrupts disabled) */
571
572 /* kadb bootfuncs pointer initialization */
573
574 int
575 sysp_getchar()
576 {
577 int i;
578 ulong_t s;
579
580 if (cons_polledio == NULL) {
581 /* Uh oh */
582 prom_printf("getchar called with no console\n");
583 for (;;)
584 /* LOOP FOREVER */;
585 }
586
587 s = clear_int_flag();
588 i = cons_polledio->cons_polledio_getchar(
589 cons_polledio->cons_polledio_argument);
590 restore_int_flag(s);
591 return (i);
592 }
593
594 void
595 sysp_putchar(int c)
596 {
597 ulong_t s;
598
599 /*
600 * We have no alternative but to drop the output on the floor.
601 */
602 if (cons_polledio == NULL ||
603 cons_polledio->cons_polledio_putchar == NULL)
604 return;
605
606 s = clear_int_flag();
607 cons_polledio->cons_polledio_putchar(
608 cons_polledio->cons_polledio_argument, c);
609 restore_int_flag(s);
610 }
611
612 int
613 sysp_ischar()
614 {
615 int i;
616 ulong_t s;
617
618 if (cons_polledio == NULL ||
619 cons_polledio->cons_polledio_ischar == NULL)
620 return (0);
621
622 s = clear_int_flag();
623 i = cons_polledio->cons_polledio_ischar(
624 cons_polledio->cons_polledio_argument);
625 restore_int_flag(s);
626 return (i);
627 }
628
629 int
630 goany(void)
631 {
632 prom_printf("Type any key to continue ");
633 (void) prom_getchar();
634 prom_printf("\n");
635 return (1);
636 }
637
638 static struct boot_syscalls kern_sysp = {
639 sysp_getchar, /* unchar (*getchar)(); 7 */
640 sysp_putchar, /* int (*putchar)(); 8 */
641 sysp_ischar, /* int (*ischar)(); 9 */
642 };
643
644 #if defined(__xpv)
645 int using_kern_polledio;
646 #endif
647
648 void
649 kadb_uses_kernel()
650 {
651 /*
652 * This routine is now totally misnamed, since it does not in fact
653 * control kadb's I/O; it only controls the kernel's prom_* I/O.
654 */
655 sysp = &kern_sysp;
656 #if defined(__xpv)
657 using_kern_polledio = 1;
658 #endif
659 }
660
661 /*
662 * the interface to the outside world
663 */
664
665 /*
666 * poll_port -- wait for a register to achieve a
667 * specific state. Arguments are a mask of bits we care about,
668 * and two sub-masks. To return normally, all the bits in the
669 * first sub-mask must be ON, all the bits in the second sub-
670 * mask must be OFF. If about seconds pass without the register
671 * achieving the desired bit configuration, we return 1, else
672 * 0.
673 */
674 int
675 poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits)
676 {
677 int i;
678 ushort_t maskval;
679
680 for (i = 500000; i; i--) {
681 maskval = inb(port) & mask;
682 if (((maskval & onbits) == onbits) &&
683 ((maskval & offbits) == 0))
684 return (0);
685 drv_usecwait(10);
686 }
687 return (1);
688 }
689
690 /*
691 * set_idle_cpu is called from idle() when a CPU becomes idle.
692 */
693 /*LINTED: static unused */
694 static uint_t last_idle_cpu;
695
696 /*ARGSUSED*/
697 void
698 set_idle_cpu(int cpun)
699 {
700 last_idle_cpu = cpun;
701 (*psm_set_idle_cpuf)(cpun);
702 }
703
704 /*
705 * unset_idle_cpu is called from idle() when a CPU is no longer idle.
706 */
707 /*ARGSUSED*/
708 void
709 unset_idle_cpu(int cpun)
710 {
711 (*psm_unset_idle_cpuf)(cpun);
712 }
713
714 /*
715 * This routine is almost correct now, but not quite. It still needs the
716 * equivalent concept of "hres_last_tick", just like on the sparc side.
717 * The idea is to take a snapshot of the hi-res timer while doing the
718 * hrestime_adj updates under hres_lock in locore, so that the small
719 * interval between interrupt assertion and interrupt processing is
720 * accounted for correctly. Once we have this, the code below should
721 * be modified to subtract off hres_last_tick rather than hrtime_base.
722 *
723 * I'd have done this myself, but I don't have source to all of the
724 * vendor-specific hi-res timer routines (grrr...). The generic hook I
725 * need is something like "gethrtime_unlocked()", which would be just like
726 * gethrtime() but would assume that you're already holding CLOCK_LOCK().
727 * This is what the GET_HRTIME() macro is for on sparc (although it also
728 * serves the function of making time available without a function call
729 * so you don't take a register window overflow while traps are disabled).
730 */
731 void
732 pc_gethrestime(timestruc_t *tp)
733 {
734 int lock_prev;
735 timestruc_t now;
736 int nslt; /* nsec since last tick */
737 int adj; /* amount of adjustment to apply */
738
739 loop:
740 lock_prev = hres_lock;
741 now = hrestime;
742 nslt = (int)(gethrtime() - hres_last_tick);
743 if (nslt < 0) {
744 /*
745 * nslt < 0 means a tick came between sampling
746 * gethrtime() and hres_last_tick; restart the loop
747 */
748
749 goto loop;
750 }
751 now.tv_nsec += nslt;
752 if (hrestime_adj != 0) {
753 if (hrestime_adj > 0) {
754 adj = (nslt >> ADJ_SHIFT);
755 if (adj > hrestime_adj)
756 adj = (int)hrestime_adj;
757 } else {
758 adj = -(nslt >> ADJ_SHIFT);
759 if (adj < hrestime_adj)
760 adj = (int)hrestime_adj;
761 }
762 now.tv_nsec += adj;
763 }
764 while ((unsigned long)now.tv_nsec >= NANOSEC) {
765
766 /*
767 * We might have a large adjustment or have been in the
768 * debugger for a long time; take care of (at most) four
769 * of those missed seconds (tv_nsec is 32 bits, so
770 * anything >4s will be wrapping around). However,
771 * anything more than 2 seconds out of sync will trigger
772 * timedelta from clock() to go correct the time anyway,
773 * so do what we can, and let the big crowbar do the
774 * rest. A similar correction while loop exists inside
775 * hres_tick(); in all cases we'd like tv_nsec to
776 * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing
777 * user processes, but if tv_sec's a little behind for a
778 * little while, that's OK; time still monotonically
779 * increases.
780 */
781
782 now.tv_nsec -= NANOSEC;
783 now.tv_sec++;
784 }
785 if ((hres_lock & ~1) != lock_prev)
786 goto loop;
787
788 *tp = now;
789 }
790
791 void
792 gethrestime_lasttick(timespec_t *tp)
793 {
794 int s;
795
796 s = hr_clock_lock();
797 *tp = hrestime;
798 hr_clock_unlock(s);
799 }
800
801 time_t
802 gethrestime_sec(void)
803 {
804 timestruc_t now;
805
806 gethrestime(&now);
807 return (now.tv_sec);
808 }
809
810 /*
811 * Initialize a kernel thread's stack
812 */
813
814 caddr_t
815 thread_stk_init(caddr_t stk)
816 {
817 ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0);
818 return (stk - SA(MINFRAME));
819 }
820
821 /*
822 * Initialize lwp's kernel stack.
823 */
824
825 #ifdef TRAPTRACE
826 /*
827 * There's a tricky interdependency here between use of sysenter and
828 * TRAPTRACE which needs recording to avoid future confusion (this is
829 * about the third time I've re-figured this out ..)
830 *
831 * Here's how debugging lcall works with TRAPTRACE.
832 *
833 * 1 We're in userland with a breakpoint on the lcall instruction.
834 * 2 We execute the instruction - the instruction pushes the userland
835 * %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel
836 * via the call gate.
837 * 3 The hardware raises a debug trap in kernel mode, the hardware
838 * pushes %efl, %cs, %eip and gets to dbgtrap via the idt.
839 * 4 dbgtrap pushes the error code and trapno and calls cmntrap
840 * 5 cmntrap finishes building a trap frame
841 * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk
842 * off the stack into the traptrace buffer.
843 *
844 * This means that the traptrace buffer contains the wrong values in
845 * %esp and %ss, but everything else in there is correct.
846 *
847 * Here's how debugging sysenter works with TRAPTRACE.
848 *
849 * a We're in userland with a breakpoint on the sysenter instruction.
850 * b We execute the instruction - the instruction pushes -nothing-
851 * on the stack, but sets %cs, %eip, %ss, %esp to prearranged
852 * values to take us to sys_sysenter, at the top of the lwp's
853 * stack.
854 * c goto 3
855 *
856 * At this point, because we got into the kernel without the requisite
857 * five pushes on the stack, if we didn't make extra room, we'd
858 * end up with the TRACE_REGS macro fetching the saved %ss and %esp
859 * values from negative (unmapped) stack addresses -- which really bites.
860 * That's why we do the '-= 8' below.
861 *
862 * XXX Note that reading "up" lwp0's stack works because t0 is declared
863 * right next to t0stack in locore.s
864 */
865 #endif
866
867 caddr_t
868 lwp_stk_init(klwp_t *lwp, caddr_t stk)
869 {
870 caddr_t oldstk;
871 struct pcb *pcb = &lwp->lwp_pcb;
872
873 oldstk = stk;
874 stk -= SA(sizeof (struct regs) + SA(MINFRAME));
875 #ifdef TRAPTRACE
876 stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */
877 #endif
878 stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul));
879 bzero(stk, oldstk - stk);
880 lwp->lwp_regs = (void *)(stk + SA(MINFRAME));
881
882 /*
883 * Arrange that the virtualized %fs and %gs GDT descriptors
884 * have a well-defined initial state (present, ring 3
885 * and of type data).
886 */
887 #if defined(__amd64)
888 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE)
889 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
890 else
891 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;
892 #elif defined(__i386)
893 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
894 #endif /* __i386 */
895 lwp_installctx(lwp);
896 return (stk);
897 }
898
899 /*
900 * Use this opportunity to free any dynamically allocated fp storage.
901 */
902 void
903 lwp_stk_fini(klwp_t *lwp)
904 {
905 fp_lwp_cleanup(lwp);
906 }
907
908 void
909 lwp_fp_init(klwp_t *lwp)
910 {
911 fp_lwp_init(lwp);
912 }
913
914 /*
915 * If we're not the panic CPU, we wait in panic_idle for reboot.
916 */
917 void
918 panic_idle(void)
919 {
920 splx(ipltospl(CLOCK_LEVEL));
921 (void) setjmp(&curthread->t_pcb);
922
923 dumpsys_helper();
924
925 #ifndef __xpv
926 for (;;)
927 i86_halt();
928 #else
929 for (;;)
930 ;
931 #endif
932 }
933
934 /*
935 * Stop the other CPUs by cross-calling them and forcing them to enter
936 * the panic_idle() loop above.
937 */
938 /*ARGSUSED*/
939 void
940 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl)
941 {
942 processorid_t i;
943 cpuset_t xcset;
944
945 /*
946 * In the case of a Xen panic, the hypervisor has already stopped
947 * all of the CPUs.
948 */
949 if (!IN_XPV_PANIC()) {
950 (void) splzs();
951
952 CPUSET_ALL_BUT(xcset, cp->cpu_id);
953 xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle);
954 }
955
956 for (i = 0; i < NCPU; i++) {
957 if (i != cp->cpu_id && cpu[i] != NULL &&
958 (cpu[i]->cpu_flags & CPU_EXISTS))
959 cpu[i]->cpu_flags |= CPU_QUIESCED;
960 }
961 }
962
963 /*
964 * Platform callback following each entry to panicsys().
965 */
966 /*ARGSUSED*/
967 void
968 panic_enter_hw(int spl)
969 {
970 /* Nothing to do here */
971 }
972
973 /*
974 * Platform-specific code to execute after panicstr is set: we invoke
975 * the PSM entry point to indicate that a panic has occurred.
976 */
977 /*ARGSUSED*/
978 void
979 panic_quiesce_hw(panic_data_t *pdp)
980 {
981 psm_notifyf(PSM_PANIC_ENTER);
982
983 cmi_panic_callback();
984
985 #ifdef TRAPTRACE
986 /*
987 * Turn off TRAPTRACE
988 */
989 TRAPTRACE_FREEZE;
990 #endif /* TRAPTRACE */
991 }
992
993 /*
994 * Platform callback prior to writing crash dump.
995 */
996 /*ARGSUSED*/
997 void
998 panic_dump_hw(int spl)
999 {
1000 /* Nothing to do here */
1001 }
1002
1003 void *
1004 plat_traceback(void *fpreg)
1005 {
1006 #ifdef __xpv
1007 if (IN_XPV_PANIC())
1008 return (xpv_traceback(fpreg));
1009 #endif
1010 return (fpreg);
1011 }
1012
1013 /*ARGSUSED*/
1014 void
1015 plat_tod_fault(enum tod_fault_type tod_bad)
1016 {}
1017
1018 /*ARGSUSED*/
1019 int
1020 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class)
1021 {
1022 return (ENOTSUP);
1023 }
1024
1025 /*
1026 * The underlying console output routines are protected by raising IPL in case
1027 * we are still calling into the early boot services. Once we start calling
1028 * the kernel console emulator, it will disable interrupts completely during
1029 * character rendering (see sysp_putchar, for example). Refer to the comments
1030 * and code in common/os/console.c for more information on these callbacks.
1031 */
1032 /*ARGSUSED*/
1033 int
1034 console_enter(int busy)
1035 {
1036 return (splzs());
1037 }
1038
1039 /*ARGSUSED*/
1040 void
1041 console_exit(int busy, int spl)
1042 {
1043 splx(spl);
1044 }
1045
1046 /*
1047 * Allocate a region of virtual address space, unmapped.
1048 * Stubbed out except on sparc, at least for now.
1049 */
1050 /*ARGSUSED*/
1051 void *
1052 boot_virt_alloc(void *addr, size_t size)
1053 {
1054 return (addr);
1055 }
1056
1057 volatile unsigned long tenmicrodata;
1058
1059 void
1060 tenmicrosec(void)
1061 {
1062 extern int gethrtime_hires;
1063
1064 if (gethrtime_hires) {
1065 hrtime_t start, end;
1066 start = end = gethrtime();
1067 while ((end - start) < (10 * (NANOSEC / MICROSEC))) {
1068 SMT_PAUSE();
1069 end = gethrtime();
1070 }
1071 } else {
1072 #if defined(__xpv)
1073 hrtime_t newtime;
1074
1075 newtime = xpv_gethrtime() + 10000; /* now + 10 us */
1076 while (xpv_gethrtime() < newtime)
1077 SMT_PAUSE();
1078 #else /* __xpv */
1079 int i;
1080
1081 /*
1082 * Artificial loop to induce delay.
1083 */
1084 for (i = 0; i < microdata; i++)
1085 tenmicrodata = microdata;
1086 #endif /* __xpv */
1087 }
1088 }
1089
1090 /*
1091 * get_cpu_mstate() is passed an array of timestamps, NCMSTATES
1092 * long, and it fills in the array with the time spent on cpu in
1093 * each of the mstates, where time is returned in nsec.
1094 *
1095 * No guarantee is made that the returned values in times[] will
1096 * monotonically increase on sequential calls, although this will
1097 * be true in the long run. Any such guarantee must be handled by
1098 * the caller, if needed. This can happen if we fail to account
1099 * for elapsed time due to a generation counter conflict, yet we
1100 * did account for it on a prior call (see below).
1101 *
1102 * The complication is that the cpu in question may be updating
1103 * its microstate at the same time that we are reading it.
1104 * Because the microstate is only updated when the CPU's state
1105 * changes, the values in cpu_intracct[] can be indefinitely out
1106 * of date. To determine true current values, it is necessary to
1107 * compare the current time with cpu_mstate_start, and add the
1108 * difference to times[cpu_mstate].
1109 *
1110 * This can be a problem if those values are changing out from
1111 * under us. Because the code path in new_cpu_mstate() is
1112 * performance critical, we have not added a lock to it. Instead,
1113 * we have added a generation counter. Before beginning
1114 * modifications, the counter is set to 0. After modifications,
1115 * it is set to the old value plus one.
1116 *
1117 * get_cpu_mstate() will not consider the values of cpu_mstate
1118 * and cpu_mstate_start to be usable unless the value of
1119 * cpu_mstate_gen is both non-zero and unchanged, both before and
1120 * after reading the mstate information. Note that we must
1121 * protect against out-of-order loads around accesses to the
1122 * generation counter. Also, this is a best effort approach in
1123 * that we do not retry should the counter be found to have
1124 * changed.
1125 *
1126 * cpu_intracct[] is used to identify time spent in each CPU
1127 * mstate while handling interrupts. Such time should be reported
1128 * against system time, and so is subtracted out from its
1129 * corresponding cpu_acct[] time and added to
1130 * cpu_acct[CMS_SYSTEM].
1131 */
1132
1133 void
1134 get_cpu_mstate(cpu_t *cpu, hrtime_t *times)
1135 {
1136 int i;
1137 hrtime_t now, start;
1138 uint16_t gen;
1139 uint16_t state;
1140 hrtime_t intracct[NCMSTATES];
1141
1142 /*
1143 * Load all volatile state under the protection of membar.
1144 * cpu_acct[cpu_mstate] must be loaded to avoid double counting
1145 * of (now - cpu_mstate_start) by a change in CPU mstate that
1146 * arrives after we make our last check of cpu_mstate_gen.
1147 */
1148
1149 now = gethrtime_unscaled();
1150 gen = cpu->cpu_mstate_gen;
1151
1152 membar_consumer(); /* guarantee load ordering */
1153 start = cpu->cpu_mstate_start;
1154 state = cpu->cpu_mstate;
1155 for (i = 0; i < NCMSTATES; i++) {
1156 intracct[i] = cpu->cpu_intracct[i];
1157 times[i] = cpu->cpu_acct[i];
1158 }
1159 membar_consumer(); /* guarantee load ordering */
1160
1161 if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start)
1162 times[state] += now - start;
1163
1164 for (i = 0; i < NCMSTATES; i++) {
1165 if (i == CMS_SYSTEM)
1166 continue;
1167 times[i] -= intracct[i];
1168 if (times[i] < 0) {
1169 intracct[i] += times[i];
1170 times[i] = 0;
1171 }
1172 times[CMS_SYSTEM] += intracct[i];
1173 scalehrtime(×[i]);
1174 }
1175 scalehrtime(×[CMS_SYSTEM]);
1176 }
1177
1178 /*
1179 * This is a version of the rdmsr instruction that allows
1180 * an error code to be returned in the case of failure.
1181 */
1182 int
1183 checked_rdmsr(uint_t msr, uint64_t *value)
1184 {
1185 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1186 return (ENOTSUP);
1187 *value = rdmsr(msr);
1188 return (0);
1189 }
1190
1191 /*
1192 * This is a version of the wrmsr instruction that allows
1193 * an error code to be returned in the case of failure.
1194 */
1195 int
1196 checked_wrmsr(uint_t msr, uint64_t value)
1197 {
1198 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1199 return (ENOTSUP);
1200 wrmsr(msr, value);
1201 return (0);
1202 }
1203
1204 /*
1205 * The mem driver's usual method of using hat_devload() to establish a
1206 * temporary mapping will not work for foreign pages mapped into this
1207 * domain or for the special hypervisor-provided pages. For the foreign
1208 * pages, we often don't know which domain owns them, so we can't ask the
1209 * hypervisor to set up a new mapping. For the other pages, we don't have
1210 * a pfn, so we can't create a new PTE. For these special cases, we do a
1211 * direct uiomove() from the existing kernel virtual address.
1212 */
1213 /*ARGSUSED*/
1214 int
1215 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw)
1216 {
1217 #if defined(__xpv)
1218 void *va = (void *)(uintptr_t)uio->uio_loffset;
1219 off_t pageoff = uio->uio_loffset & PAGEOFFSET;
1220 size_t nbytes = MIN((size_t)(PAGESIZE - pageoff),
1221 (size_t)uio->uio_iov->iov_len);
1222
1223 if ((rw == UIO_READ &&
1224 (va == HYPERVISOR_shared_info || va == xen_info)) ||
1225 (pfn_is_foreign(hat_getpfnum(kas.a_hat, va))))
1226 return (uiomove(va, nbytes, rw, uio));
1227 #endif
1228 return (ENOTSUP);
1229 }
1230
1231 pgcnt_t
1232 num_phys_pages()
1233 {
1234 pgcnt_t npages = 0;
1235 struct memlist *mp;
1236
1237 #if defined(__xpv)
1238 if (DOMAIN_IS_INITDOMAIN(xen_info))
1239 return (xpv_nr_phys_pages());
1240 #endif /* __xpv */
1241
1242 for (mp = phys_install; mp != NULL; mp = mp->ml_next)
1243 npages += mp->ml_size >> PAGESHIFT;
1244
1245 return (npages);
1246 }
1247
1248 /* cpu threshold for compressed dumps */
1249 #ifdef _LP64
1250 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_64_MINCPU;
1251 #else
1252 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_32_MINCPU;
1253 #endif
1254
1255 int
1256 dump_plat_addr()
1257 {
1258 #ifdef __xpv
1259 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1260 mem_vtop_t mem_vtop;
1261 int cnt;
1262
1263 /*
1264 * On the hypervisor, we want to dump the page with shared_info on it.
1265 */
1266 if (!IN_XPV_PANIC()) {
1267 mem_vtop.m_as = &kas;
1268 mem_vtop.m_va = HYPERVISOR_shared_info;
1269 mem_vtop.m_pfn = pfn;
1270 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t));
1271 cnt = 1;
1272 } else {
1273 cnt = dump_xpv_addr();
1274 }
1275 return (cnt);
1276 #else
1277 return (0);
1278 #endif
1279 }
1280
1281 void
1282 dump_plat_pfn()
1283 {
1284 #ifdef __xpv
1285 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1286
1287 if (!IN_XPV_PANIC())
1288 dumpvp_write(&pfn, sizeof (pfn));
1289 else
1290 dump_xpv_pfn();
1291 #endif
1292 }
1293
1294 /*ARGSUSED*/
1295 int
1296 dump_plat_data(void *dump_cbuf)
1297 {
1298 #ifdef __xpv
1299 uint32_t csize;
1300 int cnt;
1301
1302 if (!IN_XPV_PANIC()) {
1303 csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf,
1304 PAGESIZE);
1305 dumpvp_write(&csize, sizeof (uint32_t));
1306 dumpvp_write(dump_cbuf, csize);
1307 cnt = 1;
1308 } else {
1309 cnt = dump_xpv_data(dump_cbuf);
1310 }
1311 return (cnt);
1312 #else
1313 return (0);
1314 #endif
1315 }
1316
1317 /*
1318 * Calculates a linear address, given the CS selector and PC values
1319 * by looking up the %cs selector process's LDT or the CPU's GDT.
1320 * proc->p_ldtlock must be held across this call.
1321 */
1322 int
1323 linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1324 {
1325 user_desc_t *descrp;
1326 caddr_t baseaddr;
1327 uint16_t idx = SELTOIDX(rp->r_cs);
1328
1329 ASSERT(rp->r_cs <= 0xFFFF);
1330 ASSERT(MUTEX_HELD(&p->p_ldtlock));
1331
1332 if (SELISLDT(rp->r_cs)) {
1333 /*
1334 * Currently 64 bit processes cannot have private LDTs.
1335 */
1336 ASSERT(p->p_model != DATAMODEL_LP64);
1337
1338 if (p->p_ldt == NULL)
1339 return (-1);
1340
1341 descrp = &p->p_ldt[idx];
1342 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1343
1344 /*
1345 * Calculate the linear address (wraparound is not only ok,
1346 * it's expected behavior). The cast to uint32_t is because
1347 * LDT selectors are only allowed in 32-bit processes.
1348 */
1349 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1350 rp->r_pc);
1351 } else {
1352 #ifdef DEBUG
1353 descrp = &CPU->cpu_gdt[idx];
1354 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1355 /* GDT-based descriptors' base addresses should always be 0 */
1356 ASSERT(baseaddr == 0);
1357 #endif
1358 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1359 }
1360
1361 return (0);
1362 }
1363
1364 /*
1365 * The implementation of dtrace_linear_pc is similar to the that of
1366 * linear_pc, above, but here we acquire p_ldtlock before accessing
1367 * p_ldt. This implementation is used by the pid provider; we prefix
1368 * it with "dtrace_" to avoid inducing spurious tracing events.
1369 */
1370 int
1371 dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1372 {
1373 user_desc_t *descrp;
1374 caddr_t baseaddr;
1375 uint16_t idx = SELTOIDX(rp->r_cs);
1376
1377 ASSERT(rp->r_cs <= 0xFFFF);
1378
1379 if (SELISLDT(rp->r_cs)) {
1380 /*
1381 * Currently 64 bit processes cannot have private LDTs.
1382 */
1383 ASSERT(p->p_model != DATAMODEL_LP64);
1384
1385 mutex_enter(&p->p_ldtlock);
1386 if (p->p_ldt == NULL) {
1387 mutex_exit(&p->p_ldtlock);
1388 return (-1);
1389 }
1390 descrp = &p->p_ldt[idx];
1391 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1392 mutex_exit(&p->p_ldtlock);
1393
1394 /*
1395 * Calculate the linear address (wraparound is not only ok,
1396 * it's expected behavior). The cast to uint32_t is because
1397 * LDT selectors are only allowed in 32-bit processes.
1398 */
1399 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1400 rp->r_pc);
1401 } else {
1402 #ifdef DEBUG
1403 descrp = &CPU->cpu_gdt[idx];
1404 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1405 /* GDT-based descriptors' base addresses should always be 0 */
1406 ASSERT(baseaddr == 0);
1407 #endif
1408 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1409 }
1410
1411 return (0);
1412 }
1413
1414 /*
1415 * We need to post a soft interrupt to reprogram the lbolt cyclic when
1416 * switching from event to cyclic driven lbolt. The following code adds
1417 * and posts the softint for x86.
1418 */
1419 static ddi_softint_hdl_impl_t lbolt_softint_hdl =
1420 {0, 0, NULL, NULL, 0, NULL, NULL, NULL};
1421
1422 void
1423 lbolt_softint_add(void)
1424 {
1425 (void) add_avsoftintr((void *)&lbolt_softint_hdl, LOCK_LEVEL,
1426 (avfunc)lbolt_ev_to_cyclic, "lbolt_ev_to_cyclic", NULL, NULL);
1427 }
1428
1429 void
1430 lbolt_softint_post(void)
1431 {
1432 (*setsoftint)(CBE_LOCK_PIL, lbolt_softint_hdl.ih_pending);
1433 }
1434
1435 boolean_t
1436 plat_dr_check_capability(uint64_t features)
1437 {
1438 return ((plat_dr_options & features) == features);
1439 }
1440
1441 boolean_t
1442 plat_dr_support_cpu(void)
1443 {
1444 return (plat_dr_options & PLAT_DR_FEATURE_CPU);
1445 }
1446
1447 boolean_t
1448 plat_dr_support_memory(void)
1449 {
1450 return (plat_dr_options & PLAT_DR_FEATURE_MEMORY);
1451 }
1452
1453 void
1454 plat_dr_enable_capability(uint64_t features)
1455 {
1456 atomic_or_64(&plat_dr_options, features);
1457 }
1458
1459 void
1460 plat_dr_disable_capability(uint64_t features)
1461 {
1462 atomic_and_64(&plat_dr_options, ~features);
1463 }
1464
1465 /*
1466 * If SMAP is supported, look through hi_calls and inline
1467 * calls to smap_enable() to clac and smap_disable() to stac.
1468 */
1469 void
1470 hotinline_smap(hotinline_desc_t *hid)
1471 {
1472 if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE)
1473 return;
1474
1475 if (strcmp(hid->hid_symname, "smap_enable") == 0) {
1476 bcopy(clac_instr, (void *)hid->hid_instr_offset,
1477 sizeof (clac_instr));
1478 } else if (strcmp(hid->hid_symname, "smap_disable") == 0) {
1479 bcopy(stac_instr, (void *)hid->hid_instr_offset,
1480 sizeof (stac_instr));
1481 }
1482 }
1483
1484 /*
1485 * Loop through hi_calls and hand off the inlining to
1486 * the appropriate calls.
1487 */
1488 void
1489 do_hotinlines(struct module *mp)
1490 {
1491 for (hotinline_desc_t *hid = mp->hi_calls; hid != NULL;
1492 hid = hid->hid_next) {
1493 #if !defined(__xpv)
1494 hotinline_smap(hid);
1495 #endif /* __xpv */
1496 }
1497 }