1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2018 Nexenta Systems, Inc. All rights reserved.
25 * Copyright 2017, Joyent, Inc.
26 */
27 /*
28 * Copyright (c) 2010, Intel Corporation.
29 * All rights reserved.
30 */
31
32 #include <sys/types.h>
33 #include <sys/t_lock.h>
34 #include <sys/param.h>
35 #include <sys/segments.h>
36 #include <sys/sysmacros.h>
37 #include <sys/signal.h>
38 #include <sys/systm.h>
39 #include <sys/user.h>
40 #include <sys/mman.h>
41 #include <sys/vm.h>
42
43 #include <sys/disp.h>
44 #include <sys/class.h>
45
46 #include <sys/proc.h>
47 #include <sys/buf.h>
48 #include <sys/kmem.h>
49
50 #include <sys/reboot.h>
51 #include <sys/uadmin.h>
52 #include <sys/callb.h>
53
54 #include <sys/cred.h>
55 #include <sys/vnode.h>
56 #include <sys/file.h>
57
58 #include <sys/procfs.h>
59 #include <sys/acct.h>
60
61 #include <sys/vfs.h>
62 #include <sys/dnlc.h>
63 #include <sys/var.h>
64 #include <sys/cmn_err.h>
65 #include <sys/utsname.h>
66 #include <sys/debug.h>
67
68 #include <sys/dumphdr.h>
69 #include <sys/bootconf.h>
70 #include <sys/varargs.h>
71 #include <sys/promif.h>
72 #include <sys/modctl.h>
73
74 #include <sys/consdev.h>
75 #include <sys/frame.h>
76
77 #include <sys/sunddi.h>
78 #include <sys/ddidmareq.h>
79 #include <sys/psw.h>
80 #include <sys/regset.h>
81 #include <sys/privregs.h>
82 #include <sys/clock.h>
83 #include <sys/tss.h>
84 #include <sys/cpu.h>
85 #include <sys/stack.h>
86 #include <sys/trap.h>
87 #include <sys/pic.h>
88 #include <vm/hat.h>
89 #include <vm/anon.h>
90 #include <vm/as.h>
91 #include <vm/page.h>
92 #include <vm/seg.h>
93 #include <vm/seg_kmem.h>
94 #include <vm/seg_map.h>
95 #include <vm/seg_vn.h>
96 #include <vm/seg_kp.h>
97 #include <vm/hat_i86.h>
98 #include <sys/swap.h>
99 #include <sys/thread.h>
100 #include <sys/sysconf.h>
101 #include <sys/vm_machparam.h>
102 #include <sys/archsystm.h>
103 #include <sys/machsystm.h>
104 #include <sys/machlock.h>
105 #include <sys/x_call.h>
106 #include <sys/instance.h>
107
108 #include <sys/time.h>
109 #include <sys/smp_impldefs.h>
110 #include <sys/psm_types.h>
111 #include <sys/atomic.h>
112 #include <sys/panic.h>
113 #include <sys/cpuvar.h>
114 #include <sys/dtrace.h>
115 #include <sys/bl.h>
116 #include <sys/nvpair.h>
117 #include <sys/x86_archext.h>
118 #include <sys/pool_pset.h>
119 #include <sys/autoconf.h>
120 #include <sys/mem.h>
121 #include <sys/dumphdr.h>
122 #include <sys/compress.h>
123 #include <sys/cpu_module.h>
124 #if defined(__xpv)
125 #include <sys/hypervisor.h>
126 #include <sys/xpv_panic.h>
127 #endif
128
129 #include <sys/fastboot.h>
130 #include <sys/machelf.h>
131 #include <sys/kobj.h>
132 #include <sys/multiboot.h>
133
134 #ifdef TRAPTRACE
135 #include <sys/traptrace.h>
136 #endif /* TRAPTRACE */
137
138 #include <c2/audit.h>
139 #include <sys/clock_impl.h>
140
141 extern void audit_enterprom(int);
142 extern void audit_exitprom(int);
143
144 /*
145 * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used.
146 */
147 int apix_enable = 1;
148
149 int apic_nvidia_io_max = 0; /* no. of NVIDIA i/o apics */
150
151 /*
152 * Occassionally the kernel knows better whether to power-off or reboot.
153 */
154 int force_shutdown_method = AD_UNKNOWN;
155
156 /*
157 * The panicbuf array is used to record messages and state:
158 */
159 char panicbuf[PANICBUFSIZE];
160
161 /*
162 * Flags to control Dynamic Reconfiguration features.
163 */
164 uint64_t plat_dr_options;
165
166 /*
167 * Maximum physical address for memory DR operations.
168 */
169 uint64_t plat_dr_physmax;
170
171 /*
172 * maxphys - used during physio
173 * klustsize - used for klustering by swapfs and specfs
174 */
175 int maxphys = 56 * 1024; /* XXX See vm_subr.c - max b_count in physio */
176 int klustsize = 56 * 1024;
177
178 caddr_t p0_va; /* Virtual address for accessing physical page 0 */
179
180 /*
181 * defined here, though unused on x86,
182 * to make kstat_fr.c happy.
183 */
184 int vac;
185
186 void debug_enter(char *);
187
188 extern void pm_cfb_check_and_powerup(void);
189 extern void pm_cfb_rele(void);
190
191 extern fastboot_info_t newkernel;
192
193 /*
194 * Machine dependent code to reboot.
195 * "mdep" is interpreted as a character pointer; if non-null, it is a pointer
196 * to a string to be used as the argument string when rebooting.
197 *
198 * "invoke_cb" is a boolean. It is set to true when mdboot() can safely
199 * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when
200 * we are in a normal shutdown sequence (interrupts are not blocked, the
201 * system is not panic'ing or being suspended).
202 */
203 /*ARGSUSED*/
204 void
205 mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb)
206 {
207 processorid_t bootcpuid = 0;
208 static int is_first_quiesce = 1;
209 static int is_first_reset = 1;
210 int reset_status = 0;
211 static char fallback_str[] = "Falling back to regular reboot.\n";
212
213 if (fcn == AD_FASTREBOOT && !newkernel.fi_valid)
214 fcn = AD_BOOT;
215
216 if (!panicstr) {
217 kpreempt_disable();
218 if (fcn == AD_FASTREBOOT) {
219 mutex_enter(&cpu_lock);
220 if (CPU_ACTIVE(cpu_get(bootcpuid))) {
221 affinity_set(bootcpuid);
222 }
223 mutex_exit(&cpu_lock);
224 } else {
225 affinity_set(CPU_CURRENT);
226 }
227 }
228
229 if (force_shutdown_method != AD_UNKNOWN)
230 fcn = force_shutdown_method;
231
232 /*
233 * XXX - rconsvp is set to NULL to ensure that output messages
234 * are sent to the underlying "hardware" device using the
235 * monitor's printf routine since we are in the process of
236 * either rebooting or halting the machine.
237 */
238 rconsvp = NULL;
239
240 /*
241 * Print the reboot message now, before pausing other cpus.
242 * There is a race condition in the printing support that
243 * can deadlock multiprocessor machines.
244 */
245 if (!(fcn == AD_HALT || fcn == AD_POWEROFF))
246 prom_printf("rebooting...\n");
247
248 if (IN_XPV_PANIC())
249 reset();
250
251 /*
252 * We can't bring up the console from above lock level, so do it now
253 */
254 pm_cfb_check_and_powerup();
255
256 /* make sure there are no more changes to the device tree */
257 devtree_freeze();
258
259 if (invoke_cb)
260 (void) callb_execute_class(CB_CL_MDBOOT, NULL);
261
262 /*
263 * Clear any unresolved UEs from memory.
264 */
265 page_retire_mdboot();
266
267 #if defined(__xpv)
268 /*
269 * XXPV Should probably think some more about how we deal
270 * with panicing before it's really safe to panic.
271 * On hypervisors, we reboot very quickly.. Perhaps panic
272 * should only attempt to recover by rebooting if,
273 * say, we were able to mount the root filesystem,
274 * or if we successfully launched init(1m).
275 */
276 if (panicstr && proc_init == NULL)
277 (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff);
278 #endif
279 /*
280 * stop other cpus and raise our priority. since there is only
281 * one active cpu after this, and our priority will be too high
282 * for us to be preempted, we're essentially single threaded
283 * from here on out.
284 */
285 (void) spl6();
286 if (!panicstr) {
287 mutex_enter(&cpu_lock);
288 pause_cpus(NULL, NULL);
289 mutex_exit(&cpu_lock);
290 }
291
292 /*
293 * If the system is panicking, the preloaded kernel is valid, and
294 * fastreboot_onpanic has been set, and the system has been up for
295 * longer than fastreboot_onpanic_uptime (default to 10 minutes),
296 * choose Fast Reboot.
297 */
298 if (fcn == AD_BOOT && panicstr && newkernel.fi_valid &&
299 fastreboot_onpanic &&
300 (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) {
301 fcn = AD_FASTREBOOT;
302 }
303
304 /*
305 * Try to quiesce devices.
306 */
307 if (is_first_quiesce) {
308 /*
309 * Clear is_first_quiesce before calling quiesce_devices()
310 * so that if quiesce_devices() causes panics, it will not
311 * be invoked again.
312 */
313 is_first_quiesce = 0;
314
315 quiesce_active = 1;
316 quiesce_devices(ddi_root_node(), &reset_status);
317 if (reset_status == -1) {
318 if (fcn == AD_FASTREBOOT && !force_fastreboot) {
319 prom_printf("Driver(s) not capable of fast "
320 "reboot.\n");
321 prom_printf(fallback_str);
322 fastreboot_capable = 0;
323 fcn = AD_BOOT;
324 } else if (fcn != AD_FASTREBOOT)
325 fastreboot_capable = 0;
326 }
327 quiesce_active = 0;
328 }
329
330 /*
331 * Try to reset devices. reset_leaves() should only be called
332 * a) when there are no other threads that could be accessing devices,
333 * and
334 * b) on a system that's not capable of fast reboot (fastreboot_capable
335 * being 0), or on a system where quiesce_devices() failed to
336 * complete (quiesce_active being 1).
337 */
338 if (is_first_reset && (!fastreboot_capable || quiesce_active)) {
339 /*
340 * Clear is_first_reset before calling reset_devices()
341 * so that if reset_devices() causes panics, it will not
342 * be invoked again.
343 */
344 is_first_reset = 0;
345 reset_leaves();
346 }
347
348 /* Verify newkernel checksum */
349 if (fastreboot_capable && fcn == AD_FASTREBOOT &&
350 fastboot_cksum_verify(&newkernel) != 0) {
351 fastreboot_capable = 0;
352 prom_printf("Fast reboot: checksum failed for the new "
353 "kernel.\n");
354 prom_printf(fallback_str);
355 }
356
357 (void) spl8();
358
359 if (fastreboot_capable && fcn == AD_FASTREBOOT) {
360 /*
361 * psm_shutdown is called within fast_reboot()
362 */
363 fast_reboot();
364 } else {
365 (*psm_shutdownf)(cmd, fcn);
366
367 if (fcn == AD_HALT || fcn == AD_POWEROFF)
368 halt((char *)NULL);
369 else
370 prom_reboot("");
371 }
372 /*NOTREACHED*/
373 }
374
375 /* mdpreboot - may be called prior to mdboot while root fs still mounted */
376 /*ARGSUSED*/
377 void
378 mdpreboot(int cmd, int fcn, char *mdep)
379 {
380 if (fcn == AD_FASTREBOOT && !fastreboot_capable) {
381 fcn = AD_BOOT;
382 #ifdef __xpv
383 cmn_err(CE_WARN, "Fast reboot is not supported on xVM");
384 #else
385 cmn_err(CE_WARN,
386 "Fast reboot is not supported on this platform%s",
387 fastreboot_nosup_message());
388 #endif
389 }
390
391 if (fcn == AD_FASTREBOOT) {
392 fastboot_load_kernel(mdep);
393 if (!newkernel.fi_valid)
394 fcn = AD_BOOT;
395 }
396
397 (*psm_preshutdownf)(cmd, fcn);
398 }
399
400 static void
401 stop_other_cpus(void)
402 {
403 ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */
404 cpuset_t xcset;
405
406 CPUSET_ALL_BUT(xcset, CPU->cpu_id);
407 xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)mach_cpu_halt);
408 restore_int_flag(s);
409 }
410
411 /*
412 * Machine dependent abort sequence handling
413 */
414 void
415 abort_sequence_enter(char *msg)
416 {
417 if (abort_enable == 0) {
418 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
419 audit_enterprom(0);
420 return;
421 }
422 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
423 audit_enterprom(1);
424 debug_enter(msg);
425 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
426 audit_exitprom(1);
427 }
428
429 /*
430 * Enter debugger. Called when the user types ctrl-alt-d or whenever
431 * code wants to enter the debugger and possibly resume later.
432 *
433 * msg: message to print, possibly NULL.
434 */
435 void
436 debug_enter(char *msg)
437 {
438 if (dtrace_debugger_init != NULL)
439 (*dtrace_debugger_init)();
440
441 if (msg != NULL || (boothowto & RB_DEBUG))
442 prom_printf("\n");
443
444 if (msg != NULL)
445 prom_printf("%s\n", msg);
446
447 if (boothowto & RB_DEBUG)
448 kmdb_enter();
449
450 if (dtrace_debugger_fini != NULL)
451 (*dtrace_debugger_fini)();
452 }
453
454 void
455 reset(void)
456 {
457 extern void acpi_reset_system();
458 #if !defined(__xpv)
459 ushort_t *bios_memchk;
460
461 /*
462 * Can't use psm_map_phys or acpi_reset_system before the hat is
463 * initialized.
464 */
465 if (khat_running) {
466 bios_memchk = (ushort_t *)psm_map_phys(0x472,
467 sizeof (ushort_t), PROT_READ | PROT_WRITE);
468 if (bios_memchk)
469 *bios_memchk = 0x1234; /* bios memory check disable */
470
471 if (options_dip != NULL &&
472 ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0,
473 "efi-systab")) {
474 if (bootops == NULL)
475 acpi_reset_system();
476 efi_reset();
477 }
478
479 /*
480 * The problem with using stubs is that we can call
481 * acpi_reset_system only after the kernel is up and running.
482 *
483 * We should create a global state to keep track of how far
484 * up the kernel is but for the time being we will depend on
485 * bootops. bootops cleared in startup_end().
486 */
487 if (bootops == NULL)
488 acpi_reset_system();
489 }
490
491 pc_reset();
492 #else
493 if (IN_XPV_PANIC()) {
494 if (khat_running && bootops == NULL) {
495 acpi_reset_system();
496 }
497
498 pc_reset();
499 }
500
501 (void) HYPERVISOR_shutdown(SHUTDOWN_reboot);
502 panic("HYPERVISOR_shutdown() failed");
503 #endif
504 /*NOTREACHED*/
505 }
506
507 /*
508 * Halt the machine and return to the monitor
509 */
510 void
511 halt(char *s)
512 {
513 stop_other_cpus(); /* send stop signal to other CPUs */
514 if (s)
515 prom_printf("(%s) \n", s);
516 prom_exit_to_mon();
517 /*NOTREACHED*/
518 }
519
520 /*
521 * Initiate interrupt redistribution.
522 */
523 void
524 i_ddi_intr_redist_all_cpus()
525 {
526 }
527
528 /*
529 * XXX These probably ought to live somewhere else
530 * XXX They are called from mem.c
531 */
532
533 /*
534 * Convert page frame number to an OBMEM page frame number
535 * (i.e. put in the type bits -- zero for this implementation)
536 */
537 pfn_t
538 impl_obmem_pfnum(pfn_t pf)
539 {
540 return (pf);
541 }
542
543 #ifdef NM_DEBUG
544 int nmi_test = 0; /* checked in intentry.s during clock int */
545 int nmtest = -1;
546 nmfunc1(int arg, struct regs *rp)
547 {
548 printf("nmi called with arg = %x, regs = %x\n", arg, rp);
549 nmtest += 50;
550 if (arg == nmtest) {
551 printf("ip = %x\n", rp->r_pc);
552 return (1);
553 }
554 return (0);
555 }
556
557 #endif
558
559 #include <sys/bootsvcs.h>
560
561 /* Hacked up initialization for initial kernel check out is HERE. */
562 /* The basic steps are: */
563 /* kernel bootfuncs definition/initialization for KADB */
564 /* kadb bootfuncs pointer initialization */
565 /* putchar/getchar (interrupts disabled) */
566
567 /* kadb bootfuncs pointer initialization */
568
569 int
570 sysp_getchar()
571 {
572 int i;
573 ulong_t s;
574
575 if (cons_polledio == NULL) {
576 /* Uh oh */
577 prom_printf("getchar called with no console\n");
578 for (;;)
579 /* LOOP FOREVER */;
580 }
581
582 s = clear_int_flag();
583 i = cons_polledio->cons_polledio_getchar(
584 cons_polledio->cons_polledio_argument);
585 restore_int_flag(s);
586 return (i);
587 }
588
589 void
590 sysp_putchar(int c)
591 {
592 ulong_t s;
593
594 /*
595 * We have no alternative but to drop the output on the floor.
596 */
597 if (cons_polledio == NULL ||
598 cons_polledio->cons_polledio_putchar == NULL)
599 return;
600
601 s = clear_int_flag();
602 cons_polledio->cons_polledio_putchar(
603 cons_polledio->cons_polledio_argument, c);
604 restore_int_flag(s);
605 }
606
607 int
608 sysp_ischar()
609 {
610 int i;
611 ulong_t s;
612
613 if (cons_polledio == NULL ||
614 cons_polledio->cons_polledio_ischar == NULL)
615 return (0);
616
617 s = clear_int_flag();
618 i = cons_polledio->cons_polledio_ischar(
619 cons_polledio->cons_polledio_argument);
620 restore_int_flag(s);
621 return (i);
622 }
623
624 int
625 goany(void)
626 {
627 prom_printf("Type any key to continue ");
628 (void) prom_getchar();
629 prom_printf("\n");
630 return (1);
631 }
632
633 static struct boot_syscalls kern_sysp = {
634 sysp_getchar, /* unchar (*getchar)(); 7 */
635 sysp_putchar, /* int (*putchar)(); 8 */
636 sysp_ischar, /* int (*ischar)(); 9 */
637 };
638
639 #if defined(__xpv)
640 int using_kern_polledio;
641 #endif
642
643 void
644 kadb_uses_kernel()
645 {
646 /*
647 * This routine is now totally misnamed, since it does not in fact
648 * control kadb's I/O; it only controls the kernel's prom_* I/O.
649 */
650 sysp = &kern_sysp;
651 #if defined(__xpv)
652 using_kern_polledio = 1;
653 #endif
654 }
655
656 /*
657 * the interface to the outside world
658 */
659
660 /*
661 * poll_port -- wait for a register to achieve a
662 * specific state. Arguments are a mask of bits we care about,
663 * and two sub-masks. To return normally, all the bits in the
664 * first sub-mask must be ON, all the bits in the second sub-
665 * mask must be OFF. If about seconds pass without the register
666 * achieving the desired bit configuration, we return 1, else
667 * 0.
668 */
669 int
670 poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits)
671 {
672 int i;
673 ushort_t maskval;
674
675 for (i = 500000; i; i--) {
676 maskval = inb(port) & mask;
677 if (((maskval & onbits) == onbits) &&
678 ((maskval & offbits) == 0))
679 return (0);
680 drv_usecwait(10);
681 }
682 return (1);
683 }
684
685 /*
686 * set_idle_cpu is called from idle() when a CPU becomes idle.
687 */
688 /*LINTED: static unused */
689 static uint_t last_idle_cpu;
690
691 /*ARGSUSED*/
692 void
693 set_idle_cpu(int cpun)
694 {
695 last_idle_cpu = cpun;
696 (*psm_set_idle_cpuf)(cpun);
697 }
698
699 /*
700 * unset_idle_cpu is called from idle() when a CPU is no longer idle.
701 */
702 /*ARGSUSED*/
703 void
704 unset_idle_cpu(int cpun)
705 {
706 (*psm_unset_idle_cpuf)(cpun);
707 }
708
709 /*
710 * This routine is almost correct now, but not quite. It still needs the
711 * equivalent concept of "hres_last_tick", just like on the sparc side.
712 * The idea is to take a snapshot of the hi-res timer while doing the
713 * hrestime_adj updates under hres_lock in locore, so that the small
714 * interval between interrupt assertion and interrupt processing is
715 * accounted for correctly. Once we have this, the code below should
716 * be modified to subtract off hres_last_tick rather than hrtime_base.
717 *
718 * I'd have done this myself, but I don't have source to all of the
719 * vendor-specific hi-res timer routines (grrr...). The generic hook I
720 * need is something like "gethrtime_unlocked()", which would be just like
721 * gethrtime() but would assume that you're already holding CLOCK_LOCK().
722 * This is what the GET_HRTIME() macro is for on sparc (although it also
723 * serves the function of making time available without a function call
724 * so you don't take a register window overflow while traps are disabled).
725 */
726 void
727 pc_gethrestime(timestruc_t *tp)
728 {
729 int lock_prev;
730 timestruc_t now;
731 int nslt; /* nsec since last tick */
732 int adj; /* amount of adjustment to apply */
733
734 loop:
735 lock_prev = hres_lock;
736 now = hrestime;
737 nslt = (int)(gethrtime() - hres_last_tick);
738 if (nslt < 0) {
739 /*
740 * nslt < 0 means a tick came between sampling
741 * gethrtime() and hres_last_tick; restart the loop
742 */
743
744 goto loop;
745 }
746 now.tv_nsec += nslt;
747 if (hrestime_adj != 0) {
748 if (hrestime_adj > 0) {
749 adj = (nslt >> ADJ_SHIFT);
750 if (adj > hrestime_adj)
751 adj = (int)hrestime_adj;
752 } else {
753 adj = -(nslt >> ADJ_SHIFT);
754 if (adj < hrestime_adj)
755 adj = (int)hrestime_adj;
756 }
757 now.tv_nsec += adj;
758 }
759 while ((unsigned long)now.tv_nsec >= NANOSEC) {
760
761 /*
762 * We might have a large adjustment or have been in the
763 * debugger for a long time; take care of (at most) four
764 * of those missed seconds (tv_nsec is 32 bits, so
765 * anything >4s will be wrapping around). However,
766 * anything more than 2 seconds out of sync will trigger
767 * timedelta from clock() to go correct the time anyway,
768 * so do what we can, and let the big crowbar do the
769 * rest. A similar correction while loop exists inside
770 * hres_tick(); in all cases we'd like tv_nsec to
771 * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing
772 * user processes, but if tv_sec's a little behind for a
773 * little while, that's OK; time still monotonically
774 * increases.
775 */
776
777 now.tv_nsec -= NANOSEC;
778 now.tv_sec++;
779 }
780 if ((hres_lock & ~1) != lock_prev)
781 goto loop;
782
783 *tp = now;
784 }
785
786 void
787 gethrestime_lasttick(timespec_t *tp)
788 {
789 int s;
790
791 s = hr_clock_lock();
792 *tp = hrestime;
793 hr_clock_unlock(s);
794 }
795
796 time_t
797 gethrestime_sec(void)
798 {
799 timestruc_t now;
800
801 gethrestime(&now);
802 return (now.tv_sec);
803 }
804
805 /*
806 * Initialize a kernel thread's stack
807 */
808
809 caddr_t
810 thread_stk_init(caddr_t stk)
811 {
812 ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0);
813 return (stk - SA(MINFRAME));
814 }
815
816 /*
817 * Initialize lwp's kernel stack.
818 */
819
820 #ifdef TRAPTRACE
821 /*
822 * There's a tricky interdependency here between use of sysenter and
823 * TRAPTRACE which needs recording to avoid future confusion (this is
824 * about the third time I've re-figured this out ..)
825 *
826 * Here's how debugging lcall works with TRAPTRACE.
827 *
828 * 1 We're in userland with a breakpoint on the lcall instruction.
829 * 2 We execute the instruction - the instruction pushes the userland
830 * %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel
831 * via the call gate.
832 * 3 The hardware raises a debug trap in kernel mode, the hardware
833 * pushes %efl, %cs, %eip and gets to dbgtrap via the idt.
834 * 4 dbgtrap pushes the error code and trapno and calls cmntrap
835 * 5 cmntrap finishes building a trap frame
836 * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk
837 * off the stack into the traptrace buffer.
838 *
839 * This means that the traptrace buffer contains the wrong values in
840 * %esp and %ss, but everything else in there is correct.
841 *
842 * Here's how debugging sysenter works with TRAPTRACE.
843 *
844 * a We're in userland with a breakpoint on the sysenter instruction.
845 * b We execute the instruction - the instruction pushes -nothing-
846 * on the stack, but sets %cs, %eip, %ss, %esp to prearranged
847 * values to take us to sys_sysenter, at the top of the lwp's
848 * stack.
849 * c goto 3
850 *
851 * At this point, because we got into the kernel without the requisite
852 * five pushes on the stack, if we didn't make extra room, we'd
853 * end up with the TRACE_REGS macro fetching the saved %ss and %esp
854 * values from negative (unmapped) stack addresses -- which really bites.
855 * That's why we do the '-= 8' below.
856 *
857 * XXX Note that reading "up" lwp0's stack works because t0 is declared
858 * right next to t0stack in locore.s
859 */
860 #endif
861
862 caddr_t
863 lwp_stk_init(klwp_t *lwp, caddr_t stk)
864 {
865 caddr_t oldstk;
866 struct pcb *pcb = &lwp->lwp_pcb;
867
868 oldstk = stk;
869 stk -= SA(sizeof (struct regs) + SA(MINFRAME));
870 #ifdef TRAPTRACE
871 stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */
872 #endif
873 stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul));
874 bzero(stk, oldstk - stk);
875 lwp->lwp_regs = (void *)(stk + SA(MINFRAME));
876
877 /*
878 * Arrange that the virtualized %fs and %gs GDT descriptors
879 * have a well-defined initial state (present, ring 3
880 * and of type data).
881 */
882 #if defined(__amd64)
883 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE)
884 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
885 else
886 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;
887 #elif defined(__i386)
888 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
889 #endif /* __i386 */
890 lwp_installctx(lwp);
891 return (stk);
892 }
893
894 /*
895 * Use this opportunity to free any dynamically allocated fp storage.
896 */
897 void
898 lwp_stk_fini(klwp_t *lwp)
899 {
900 fp_lwp_cleanup(lwp);
901 }
902
903 void
904 lwp_fp_init(klwp_t *lwp)
905 {
906 fp_lwp_init(lwp);
907 }
908
909 /*
910 * If we're not the panic CPU, we wait in panic_idle for reboot.
911 */
912 void
913 panic_idle(void)
914 {
915 splx(ipltospl(CLOCK_LEVEL));
916 (void) setjmp(&curthread->t_pcb);
917
918 dumpsys_helper();
919
920 #ifndef __xpv
921 for (;;)
922 i86_halt();
923 #else
924 for (;;)
925 ;
926 #endif
927 }
928
929 /*
930 * Stop the other CPUs by cross-calling them and forcing them to enter
931 * the panic_idle() loop above.
932 */
933 /*ARGSUSED*/
934 void
935 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl)
936 {
937 processorid_t i;
938 cpuset_t xcset;
939
940 /*
941 * In the case of a Xen panic, the hypervisor has already stopped
942 * all of the CPUs.
943 */
944 if (!IN_XPV_PANIC()) {
945 (void) splzs();
946
947 CPUSET_ALL_BUT(xcset, cp->cpu_id);
948 xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle);
949 }
950
951 for (i = 0; i < NCPU; i++) {
952 if (i != cp->cpu_id && cpu[i] != NULL &&
953 (cpu[i]->cpu_flags & CPU_EXISTS))
954 cpu[i]->cpu_flags |= CPU_QUIESCED;
955 }
956 }
957
958 /*
959 * Platform callback following each entry to panicsys().
960 */
961 /*ARGSUSED*/
962 void
963 panic_enter_hw(int spl)
964 {
965 /* Nothing to do here */
966 }
967
968 /*
969 * Platform-specific code to execute after panicstr is set: we invoke
970 * the PSM entry point to indicate that a panic has occurred.
971 */
972 /*ARGSUSED*/
973 void
974 panic_quiesce_hw(panic_data_t *pdp)
975 {
976 psm_notifyf(PSM_PANIC_ENTER);
977
978 cmi_panic_callback();
979
980 #ifdef TRAPTRACE
981 /*
982 * Turn off TRAPTRACE
983 */
984 TRAPTRACE_FREEZE;
985 #endif /* TRAPTRACE */
986 }
987
988 /*
989 * Platform callback prior to writing crash dump.
990 */
991 /*ARGSUSED*/
992 void
993 panic_dump_hw(int spl)
994 {
995 /* Nothing to do here */
996 }
997
998 void *
999 plat_traceback(void *fpreg)
1000 {
1001 #ifdef __xpv
1002 if (IN_XPV_PANIC())
1003 return (xpv_traceback(fpreg));
1004 #endif
1005 return (fpreg);
1006 }
1007
1008 /*ARGSUSED*/
1009 void
1010 plat_tod_fault(enum tod_fault_type tod_bad)
1011 {}
1012
1013 /*ARGSUSED*/
1014 int
1015 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class)
1016 {
1017 return (ENOTSUP);
1018 }
1019
1020 /*
1021 * The underlying console output routines are protected by raising IPL in case
1022 * we are still calling into the early boot services. Once we start calling
1023 * the kernel console emulator, it will disable interrupts completely during
1024 * character rendering (see sysp_putchar, for example). Refer to the comments
1025 * and code in common/os/console.c for more information on these callbacks.
1026 */
1027 /*ARGSUSED*/
1028 int
1029 console_enter(int busy)
1030 {
1031 return (splzs());
1032 }
1033
1034 /*ARGSUSED*/
1035 void
1036 console_exit(int busy, int spl)
1037 {
1038 splx(spl);
1039 }
1040
1041 /*
1042 * Allocate a region of virtual address space, unmapped.
1043 * Stubbed out except on sparc, at least for now.
1044 */
1045 /*ARGSUSED*/
1046 void *
1047 boot_virt_alloc(void *addr, size_t size)
1048 {
1049 return (addr);
1050 }
1051
1052 volatile unsigned long tenmicrodata;
1053
1054 void
1055 tenmicrosec(void)
1056 {
1057 extern int gethrtime_hires;
1058
1059 if (gethrtime_hires) {
1060 hrtime_t start, end;
1061 start = end = gethrtime();
1062 while ((end - start) < (10 * (NANOSEC / MICROSEC))) {
1063 SMT_PAUSE();
1064 end = gethrtime();
1065 }
1066 } else {
1067 #if defined(__xpv)
1068 hrtime_t newtime;
1069
1070 newtime = xpv_gethrtime() + 10000; /* now + 10 us */
1071 while (xpv_gethrtime() < newtime)
1072 SMT_PAUSE();
1073 #else /* __xpv */
1074 int i;
1075
1076 /*
1077 * Artificial loop to induce delay.
1078 */
1079 for (i = 0; i < microdata; i++)
1080 tenmicrodata = microdata;
1081 #endif /* __xpv */
1082 }
1083 }
1084
1085 /*
1086 * get_cpu_mstate() is passed an array of timestamps, NCMSTATES
1087 * long, and it fills in the array with the time spent on cpu in
1088 * each of the mstates, where time is returned in nsec.
1089 *
1090 * No guarantee is made that the returned values in times[] will
1091 * monotonically increase on sequential calls, although this will
1092 * be true in the long run. Any such guarantee must be handled by
1093 * the caller, if needed. This can happen if we fail to account
1094 * for elapsed time due to a generation counter conflict, yet we
1095 * did account for it on a prior call (see below).
1096 *
1097 * The complication is that the cpu in question may be updating
1098 * its microstate at the same time that we are reading it.
1099 * Because the microstate is only updated when the CPU's state
1100 * changes, the values in cpu_intracct[] can be indefinitely out
1101 * of date. To determine true current values, it is necessary to
1102 * compare the current time with cpu_mstate_start, and add the
1103 * difference to times[cpu_mstate].
1104 *
1105 * This can be a problem if those values are changing out from
1106 * under us. Because the code path in new_cpu_mstate() is
1107 * performance critical, we have not added a lock to it. Instead,
1108 * we have added a generation counter. Before beginning
1109 * modifications, the counter is set to 0. After modifications,
1110 * it is set to the old value plus one.
1111 *
1112 * get_cpu_mstate() will not consider the values of cpu_mstate
1113 * and cpu_mstate_start to be usable unless the value of
1114 * cpu_mstate_gen is both non-zero and unchanged, both before and
1115 * after reading the mstate information. Note that we must
1116 * protect against out-of-order loads around accesses to the
1117 * generation counter. Also, this is a best effort approach in
1118 * that we do not retry should the counter be found to have
1119 * changed.
1120 *
1121 * cpu_intracct[] is used to identify time spent in each CPU
1122 * mstate while handling interrupts. Such time should be reported
1123 * against system time, and so is subtracted out from its
1124 * corresponding cpu_acct[] time and added to
1125 * cpu_acct[CMS_SYSTEM].
1126 */
1127
1128 void
1129 get_cpu_mstate(cpu_t *cpu, hrtime_t *times)
1130 {
1131 int i;
1132 hrtime_t now, start;
1133 uint16_t gen;
1134 uint16_t state;
1135 hrtime_t intracct[NCMSTATES];
1136
1137 /*
1138 * Load all volatile state under the protection of membar.
1139 * cpu_acct[cpu_mstate] must be loaded to avoid double counting
1140 * of (now - cpu_mstate_start) by a change in CPU mstate that
1141 * arrives after we make our last check of cpu_mstate_gen.
1142 */
1143
1144 now = gethrtime_unscaled();
1145 gen = cpu->cpu_mstate_gen;
1146
1147 membar_consumer(); /* guarantee load ordering */
1148 start = cpu->cpu_mstate_start;
1149 state = cpu->cpu_mstate;
1150 for (i = 0; i < NCMSTATES; i++) {
1151 intracct[i] = cpu->cpu_intracct[i];
1152 times[i] = cpu->cpu_acct[i];
1153 }
1154 membar_consumer(); /* guarantee load ordering */
1155
1156 if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start)
1157 times[state] += now - start;
1158
1159 for (i = 0; i < NCMSTATES; i++) {
1160 if (i == CMS_SYSTEM)
1161 continue;
1162 times[i] -= intracct[i];
1163 if (times[i] < 0) {
1164 intracct[i] += times[i];
1165 times[i] = 0;
1166 }
1167 times[CMS_SYSTEM] += intracct[i];
1168 scalehrtime(×[i]);
1169 }
1170 scalehrtime(×[CMS_SYSTEM]);
1171 }
1172
1173 /*
1174 * This is a version of the rdmsr instruction that allows
1175 * an error code to be returned in the case of failure.
1176 */
1177 int
1178 checked_rdmsr(uint_t msr, uint64_t *value)
1179 {
1180 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1181 return (ENOTSUP);
1182 *value = rdmsr(msr);
1183 return (0);
1184 }
1185
1186 /*
1187 * This is a version of the wrmsr instruction that allows
1188 * an error code to be returned in the case of failure.
1189 */
1190 int
1191 checked_wrmsr(uint_t msr, uint64_t value)
1192 {
1193 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1194 return (ENOTSUP);
1195 wrmsr(msr, value);
1196 return (0);
1197 }
1198
1199 /*
1200 * The mem driver's usual method of using hat_devload() to establish a
1201 * temporary mapping will not work for foreign pages mapped into this
1202 * domain or for the special hypervisor-provided pages. For the foreign
1203 * pages, we often don't know which domain owns them, so we can't ask the
1204 * hypervisor to set up a new mapping. For the other pages, we don't have
1205 * a pfn, so we can't create a new PTE. For these special cases, we do a
1206 * direct uiomove() from the existing kernel virtual address.
1207 */
1208 /*ARGSUSED*/
1209 int
1210 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw)
1211 {
1212 #if defined(__xpv)
1213 void *va = (void *)(uintptr_t)uio->uio_loffset;
1214 off_t pageoff = uio->uio_loffset & PAGEOFFSET;
1215 size_t nbytes = MIN((size_t)(PAGESIZE - pageoff),
1216 (size_t)uio->uio_iov->iov_len);
1217
1218 if ((rw == UIO_READ &&
1219 (va == HYPERVISOR_shared_info || va == xen_info)) ||
1220 (pfn_is_foreign(hat_getpfnum(kas.a_hat, va))))
1221 return (uiomove(va, nbytes, rw, uio));
1222 #endif
1223 return (ENOTSUP);
1224 }
1225
1226 pgcnt_t
1227 num_phys_pages()
1228 {
1229 pgcnt_t npages = 0;
1230 struct memlist *mp;
1231
1232 #if defined(__xpv)
1233 if (DOMAIN_IS_INITDOMAIN(xen_info))
1234 return (xpv_nr_phys_pages());
1235 #endif /* __xpv */
1236
1237 for (mp = phys_install; mp != NULL; mp = mp->ml_next)
1238 npages += mp->ml_size >> PAGESHIFT;
1239
1240 return (npages);
1241 }
1242
1243 int
1244 dump_plat_addr()
1245 {
1246 #ifdef __xpv
1247 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1248 mem_vtop_t mem_vtop;
1249 int cnt;
1250
1251 /*
1252 * On the hypervisor, we want to dump the page with shared_info on it.
1253 */
1254 if (!IN_XPV_PANIC()) {
1255 mem_vtop.m_as = &kas;
1256 mem_vtop.m_va = HYPERVISOR_shared_info;
1257 mem_vtop.m_pfn = pfn;
1258 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t));
1259 cnt = 1;
1260 } else {
1261 cnt = dump_xpv_addr();
1262 }
1263 return (cnt);
1264 #else
1265 return (0);
1266 #endif
1267 }
1268
1269 void
1270 dump_plat_pfn()
1271 {
1272 #ifdef __xpv
1273 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1274
1275 if (!IN_XPV_PANIC())
1276 dumpvp_write(&pfn, sizeof (pfn));
1277 else
1278 dump_xpv_pfn();
1279 #endif
1280 }
1281
1282 /*ARGSUSED*/
1283 int
1284 dump_plat_data(void *dump_cbuf)
1285 {
1286 #ifdef __xpv
1287 uint32_t csize;
1288 int cnt;
1289
1290 if (!IN_XPV_PANIC()) {
1291 csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf,
1292 PAGESIZE);
1293 dumpvp_write(&csize, sizeof (uint32_t));
1294 dumpvp_write(dump_cbuf, csize);
1295 cnt = 1;
1296 } else {
1297 cnt = dump_xpv_data(dump_cbuf);
1298 }
1299 return (cnt);
1300 #else
1301 return (0);
1302 #endif
1303 }
1304
1305 /*
1306 * Calculates a linear address, given the CS selector and PC values
1307 * by looking up the %cs selector process's LDT or the CPU's GDT.
1308 * proc->p_ldtlock must be held across this call.
1309 */
1310 int
1311 linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1312 {
1313 user_desc_t *descrp;
1314 caddr_t baseaddr;
1315 uint16_t idx = SELTOIDX(rp->r_cs);
1316
1317 ASSERT(rp->r_cs <= 0xFFFF);
1318 ASSERT(MUTEX_HELD(&p->p_ldtlock));
1319
1320 if (SELISLDT(rp->r_cs)) {
1321 /*
1322 * Currently 64 bit processes cannot have private LDTs.
1323 */
1324 ASSERT(p->p_model != DATAMODEL_LP64);
1325
1326 if (p->p_ldt == NULL)
1327 return (-1);
1328
1329 descrp = &p->p_ldt[idx];
1330 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1331
1332 /*
1333 * Calculate the linear address (wraparound is not only ok,
1334 * it's expected behavior). The cast to uint32_t is because
1335 * LDT selectors are only allowed in 32-bit processes.
1336 */
1337 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1338 rp->r_pc);
1339 } else {
1340 #ifdef DEBUG
1341 descrp = &CPU->cpu_gdt[idx];
1342 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1343 /* GDT-based descriptors' base addresses should always be 0 */
1344 ASSERT(baseaddr == 0);
1345 #endif
1346 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1347 }
1348
1349 return (0);
1350 }
1351
1352 /*
1353 * The implementation of dtrace_linear_pc is similar to the that of
1354 * linear_pc, above, but here we acquire p_ldtlock before accessing
1355 * p_ldt. This implementation is used by the pid provider; we prefix
1356 * it with "dtrace_" to avoid inducing spurious tracing events.
1357 */
1358 int
1359 dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1360 {
1361 user_desc_t *descrp;
1362 caddr_t baseaddr;
1363 uint16_t idx = SELTOIDX(rp->r_cs);
1364
1365 ASSERT(rp->r_cs <= 0xFFFF);
1366
1367 if (SELISLDT(rp->r_cs)) {
1368 /*
1369 * Currently 64 bit processes cannot have private LDTs.
1370 */
1371 ASSERT(p->p_model != DATAMODEL_LP64);
1372
1373 mutex_enter(&p->p_ldtlock);
1374 if (p->p_ldt == NULL) {
1375 mutex_exit(&p->p_ldtlock);
1376 return (-1);
1377 }
1378 descrp = &p->p_ldt[idx];
1379 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1380 mutex_exit(&p->p_ldtlock);
1381
1382 /*
1383 * Calculate the linear address (wraparound is not only ok,
1384 * it's expected behavior). The cast to uint32_t is because
1385 * LDT selectors are only allowed in 32-bit processes.
1386 */
1387 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1388 rp->r_pc);
1389 } else {
1390 #ifdef DEBUG
1391 descrp = &CPU->cpu_gdt[idx];
1392 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1393 /* GDT-based descriptors' base addresses should always be 0 */
1394 ASSERT(baseaddr == 0);
1395 #endif
1396 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1397 }
1398
1399 return (0);
1400 }
1401
1402 /*
1403 * We need to post a soft interrupt to reprogram the lbolt cyclic when
1404 * switching from event to cyclic driven lbolt. The following code adds
1405 * and posts the softint for x86.
1406 */
1407 static ddi_softint_hdl_impl_t lbolt_softint_hdl =
1408 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL};
1409
1410 void
1411 lbolt_softint_add(void)
1412 {
1413 (void) add_avsoftintr((void *)&lbolt_softint_hdl, LOCK_LEVEL,
1414 (avfunc)lbolt_ev_to_cyclic, "lbolt_ev_to_cyclic", NULL, NULL);
1415 }
1416
1417 void
1418 lbolt_softint_post(void)
1419 {
1420 (*setsoftint)(CBE_LOCK_PIL, lbolt_softint_hdl.ih_pending);
1421 }
1422
1423 boolean_t
1424 plat_dr_check_capability(uint64_t features)
1425 {
1426 return ((plat_dr_options & features) == features);
1427 }
1428
1429 boolean_t
1430 plat_dr_support_cpu(void)
1431 {
1432 return (plat_dr_options & PLAT_DR_FEATURE_CPU);
1433 }
1434
1435 boolean_t
1436 plat_dr_support_memory(void)
1437 {
1438 return (plat_dr_options & PLAT_DR_FEATURE_MEMORY);
1439 }
1440
1441 void
1442 plat_dr_enable_capability(uint64_t features)
1443 {
1444 atomic_or_64(&plat_dr_options, features);
1445 }
1446
1447 void
1448 plat_dr_disable_capability(uint64_t features)
1449 {
1450 atomic_and_64(&plat_dr_options, ~features);
1451 }