1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2017, Joyent, Inc.
25 */
26 /*
27 * Copyright (c) 2010, Intel Corporation.
28 * All rights reserved.
29 */
30
31 #include <sys/types.h>
32 #include <sys/t_lock.h>
33 #include <sys/param.h>
34 #include <sys/segments.h>
35 #include <sys/sysmacros.h>
36 #include <sys/signal.h>
37 #include <sys/systm.h>
38 #include <sys/user.h>
39 #include <sys/mman.h>
40 #include <sys/vm.h>
41
42 #include <sys/disp.h>
43 #include <sys/class.h>
44
45 #include <sys/proc.h>
46 #include <sys/buf.h>
47 #include <sys/kmem.h>
48
49 #include <sys/reboot.h>
50 #include <sys/uadmin.h>
51 #include <sys/callb.h>
52
53 #include <sys/cred.h>
54 #include <sys/vnode.h>
55 #include <sys/file.h>
56
57 #include <sys/procfs.h>
58 #include <sys/acct.h>
59
60 #include <sys/vfs.h>
61 #include <sys/dnlc.h>
62 #include <sys/var.h>
63 #include <sys/cmn_err.h>
64 #include <sys/utsname.h>
65 #include <sys/debug.h>
66
67 #include <sys/dumphdr.h>
68 #include <sys/bootconf.h>
69 #include <sys/varargs.h>
70 #include <sys/promif.h>
71 #include <sys/modctl.h>
72
73 #include <sys/consdev.h>
74 #include <sys/frame.h>
75
76 #include <sys/sunddi.h>
77 #include <sys/ddidmareq.h>
78 #include <sys/psw.h>
79 #include <sys/regset.h>
80 #include <sys/privregs.h>
81 #include <sys/clock.h>
82 #include <sys/tss.h>
83 #include <sys/cpu.h>
84 #include <sys/stack.h>
85 #include <sys/trap.h>
86 #include <sys/pic.h>
87 #include <vm/hat.h>
88 #include <vm/anon.h>
89 #include <vm/as.h>
90 #include <vm/page.h>
91 #include <vm/seg.h>
92 #include <vm/seg_kmem.h>
93 #include <vm/seg_map.h>
94 #include <vm/seg_vn.h>
95 #include <vm/seg_kp.h>
96 #include <vm/hat_i86.h>
97 #include <sys/swap.h>
98 #include <sys/thread.h>
99 #include <sys/sysconf.h>
100 #include <sys/vm_machparam.h>
101 #include <sys/archsystm.h>
102 #include <sys/machsystm.h>
103 #include <sys/machlock.h>
104 #include <sys/x_call.h>
105 #include <sys/instance.h>
106
107 #include <sys/time.h>
108 #include <sys/smp_impldefs.h>
109 #include <sys/psm_types.h>
110 #include <sys/atomic.h>
111 #include <sys/panic.h>
112 #include <sys/cpuvar.h>
113 #include <sys/dtrace.h>
114 #include <sys/bl.h>
115 #include <sys/nvpair.h>
116 #include <sys/x86_archext.h>
117 #include <sys/pool_pset.h>
118 #include <sys/autoconf.h>
119 #include <sys/mem.h>
120 #include <sys/dumphdr.h>
121 #include <sys/compress.h>
122 #include <sys/cpu_module.h>
123 #if defined(__xpv)
124 #include <sys/hypervisor.h>
125 #include <sys/xpv_panic.h>
126 #endif
127
128 #include <sys/fastboot.h>
129 #include <sys/machelf.h>
130 #include <sys/kobj.h>
131 #include <sys/multiboot.h>
132
133 #ifdef TRAPTRACE
134 #include <sys/traptrace.h>
135 #endif /* TRAPTRACE */
136
137 #include <c2/audit.h>
138 #include <sys/clock_impl.h>
139
140 extern void audit_enterprom(int);
141 extern void audit_exitprom(int);
142
143 /*
144 * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used.
145 */
146 int apix_enable = 1;
147
148 int apic_nvidia_io_max = 0; /* no. of NVIDIA i/o apics */
149
150 /*
151 * Occassionally the kernel knows better whether to power-off or reboot.
152 */
153 int force_shutdown_method = AD_UNKNOWN;
154
155 /*
156 * The panicbuf array is used to record messages and state:
157 */
158 char panicbuf[PANICBUFSIZE];
159
160 /*
161 * Flags to control Dynamic Reconfiguration features.
162 */
163 uint64_t plat_dr_options;
164
165 /*
166 * Maximum physical address for memory DR operations.
167 */
168 uint64_t plat_dr_physmax;
169
170 /*
171 * maxphys - used during physio
172 * klustsize - used for klustering by swapfs and specfs
173 */
174 int maxphys = 56 * 1024; /* XXX See vm_subr.c - max b_count in physio */
175 int klustsize = 56 * 1024;
176
177 caddr_t p0_va; /* Virtual address for accessing physical page 0 */
178
179 /*
180 * defined here, though unused on x86,
181 * to make kstat_fr.c happy.
182 */
183 int vac;
184
185 void debug_enter(char *);
186
187 extern void pm_cfb_check_and_powerup(void);
188 extern void pm_cfb_rele(void);
189
190 extern fastboot_info_t newkernel;
191
192 /*
193 * Machine dependent code to reboot.
194 * "mdep" is interpreted as a character pointer; if non-null, it is a pointer
195 * to a string to be used as the argument string when rebooting.
196 *
197 * "invoke_cb" is a boolean. It is set to true when mdboot() can safely
198 * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when
199 * we are in a normal shutdown sequence (interrupts are not blocked, the
200 * system is not panic'ing or being suspended).
201 */
202 /*ARGSUSED*/
203 void
204 mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb)
205 {
206 processorid_t bootcpuid = 0;
207 static int is_first_quiesce = 1;
208 static int is_first_reset = 1;
209 int reset_status = 0;
210 static char fallback_str[] = "Falling back to regular reboot.\n";
211
212 if (fcn == AD_FASTREBOOT && !newkernel.fi_valid)
213 fcn = AD_BOOT;
214
215 if (!panicstr) {
216 kpreempt_disable();
217 if (fcn == AD_FASTREBOOT) {
218 mutex_enter(&cpu_lock);
219 if (CPU_ACTIVE(cpu_get(bootcpuid))) {
220 affinity_set(bootcpuid);
221 }
222 mutex_exit(&cpu_lock);
223 } else {
224 affinity_set(CPU_CURRENT);
225 }
226 }
227
228 if (force_shutdown_method != AD_UNKNOWN)
229 fcn = force_shutdown_method;
230
231 /*
232 * XXX - rconsvp is set to NULL to ensure that output messages
233 * are sent to the underlying "hardware" device using the
234 * monitor's printf routine since we are in the process of
235 * either rebooting or halting the machine.
236 */
237 rconsvp = NULL;
238
239 /*
240 * Print the reboot message now, before pausing other cpus.
241 * There is a race condition in the printing support that
242 * can deadlock multiprocessor machines.
243 */
244 if (!(fcn == AD_HALT || fcn == AD_POWEROFF))
245 prom_printf("rebooting...\n");
246
247 if (IN_XPV_PANIC())
248 reset();
249
250 /*
251 * We can't bring up the console from above lock level, so do it now
252 */
253 pm_cfb_check_and_powerup();
254
255 /* make sure there are no more changes to the device tree */
256 devtree_freeze();
257
258 if (invoke_cb)
259 (void) callb_execute_class(CB_CL_MDBOOT, 0);
260
261 /*
262 * Clear any unresolved UEs from memory.
263 */
264 page_retire_mdboot();
265
266 #if defined(__xpv)
267 /*
268 * XXPV Should probably think some more about how we deal
269 * with panicing before it's really safe to panic.
270 * On hypervisors, we reboot very quickly.. Perhaps panic
271 * should only attempt to recover by rebooting if,
272 * say, we were able to mount the root filesystem,
273 * or if we successfully launched init(1m).
274 */
275 if (panicstr && proc_init == NULL)
276 (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff);
277 #endif
278 /*
279 * stop other cpus and raise our priority. since there is only
280 * one active cpu after this, and our priority will be too high
281 * for us to be preempted, we're essentially single threaded
282 * from here on out.
283 */
284 (void) spl6();
285 if (!panicstr) {
286 mutex_enter(&cpu_lock);
287 pause_cpus(NULL, NULL);
288 mutex_exit(&cpu_lock);
289 }
290
291 /*
292 * If the system is panicking, the preloaded kernel is valid, and
293 * fastreboot_onpanic has been set, and the system has been up for
294 * longer than fastreboot_onpanic_uptime (default to 10 minutes),
295 * choose Fast Reboot.
296 */
297 if (fcn == AD_BOOT && panicstr && newkernel.fi_valid &&
298 fastreboot_onpanic &&
299 (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) {
300 fcn = AD_FASTREBOOT;
301 }
302
303 /*
304 * Try to quiesce devices.
305 */
306 if (is_first_quiesce) {
307 /*
308 * Clear is_first_quiesce before calling quiesce_devices()
309 * so that if quiesce_devices() causes panics, it will not
310 * be invoked again.
311 */
312 is_first_quiesce = 0;
313
314 quiesce_active = 1;
315 quiesce_devices(ddi_root_node(), &reset_status);
316 if (reset_status == -1) {
317 if (fcn == AD_FASTREBOOT && !force_fastreboot) {
318 prom_printf("Driver(s) not capable of fast "
319 "reboot.\n");
320 prom_printf(fallback_str);
321 fastreboot_capable = 0;
322 fcn = AD_BOOT;
323 } else if (fcn != AD_FASTREBOOT)
324 fastreboot_capable = 0;
325 }
326 quiesce_active = 0;
327 }
328
329 /*
330 * Try to reset devices. reset_leaves() should only be called
331 * a) when there are no other threads that could be accessing devices,
332 * and
333 * b) on a system that's not capable of fast reboot (fastreboot_capable
334 * being 0), or on a system where quiesce_devices() failed to
335 * complete (quiesce_active being 1).
336 */
337 if (is_first_reset && (!fastreboot_capable || quiesce_active)) {
338 /*
339 * Clear is_first_reset before calling reset_devices()
340 * so that if reset_devices() causes panics, it will not
341 * be invoked again.
342 */
343 is_first_reset = 0;
344 reset_leaves();
345 }
346
347 /* Verify newkernel checksum */
348 if (fastreboot_capable && fcn == AD_FASTREBOOT &&
349 fastboot_cksum_verify(&newkernel) != 0) {
350 fastreboot_capable = 0;
351 prom_printf("Fast reboot: checksum failed for the new "
352 "kernel.\n");
353 prom_printf(fallback_str);
354 }
355
356 (void) spl8();
357
358 if (fastreboot_capable && fcn == AD_FASTREBOOT) {
359 /*
360 * psm_shutdown is called within fast_reboot()
361 */
362 fast_reboot();
363 } else {
364 (*psm_shutdownf)(cmd, fcn);
365
366 if (fcn == AD_HALT || fcn == AD_POWEROFF)
367 halt((char *)NULL);
368 else
369 prom_reboot("");
370 }
371 /*NOTREACHED*/
372 }
373
374 /* mdpreboot - may be called prior to mdboot while root fs still mounted */
375 /*ARGSUSED*/
376 void
377 mdpreboot(int cmd, int fcn, char *mdep)
378 {
379 if (fcn == AD_FASTREBOOT && !fastreboot_capable) {
380 fcn = AD_BOOT;
381 #ifdef __xpv
382 cmn_err(CE_WARN, "Fast reboot is not supported on xVM");
383 #else
384 cmn_err(CE_WARN,
385 "Fast reboot is not supported on this platform%s",
386 fastreboot_nosup_message());
387 #endif
388 }
389
390 if (fcn == AD_FASTREBOOT) {
391 fastboot_load_kernel(mdep);
392 if (!newkernel.fi_valid)
393 fcn = AD_BOOT;
394 }
395
396 (*psm_preshutdownf)(cmd, fcn);
397 }
398
399 static void
400 stop_other_cpus(void)
401 {
402 ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */
403 cpuset_t xcset;
404
405 CPUSET_ALL_BUT(xcset, CPU->cpu_id);
406 xc_priority(0, 0, 0, CPUSET2BV(xcset), mach_cpu_halt);
407 restore_int_flag(s);
408 }
409
410 /*
411 * Machine dependent abort sequence handling
412 */
413 void
414 abort_sequence_enter(char *msg)
415 {
416 if (abort_enable == 0) {
417 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
418 audit_enterprom(0);
419 return;
420 }
421 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
422 audit_enterprom(1);
423 debug_enter(msg);
424 if (AU_ZONE_AUDITING(GET_KCTX_GZ))
425 audit_exitprom(1);
426 }
427
428 /*
429 * Enter debugger. Called when the user types ctrl-alt-d or whenever
430 * code wants to enter the debugger and possibly resume later.
431 *
432 * msg: message to print, possibly NULL.
433 */
434 void
435 debug_enter(char *msg)
436 {
437 if (dtrace_debugger_init != NULL)
438 (*dtrace_debugger_init)();
439
440 if (msg != NULL || (boothowto & RB_DEBUG))
441 prom_printf("\n");
442
443 if (msg != NULL)
444 prom_printf("%s\n", msg);
445
446 if (boothowto & RB_DEBUG)
447 kmdb_enter();
448
449 if (dtrace_debugger_fini != NULL)
450 (*dtrace_debugger_fini)();
451 }
452
453 void
454 reset(void)
455 {
456 extern void acpi_reset_system();
457 #if !defined(__xpv)
458 ushort_t *bios_memchk;
459
460 /*
461 * Can't use psm_map_phys or acpi_reset_system before the hat is
462 * initialized.
463 */
464 if (khat_running) {
465 bios_memchk = (ushort_t *)psm_map_phys(0x472,
466 sizeof (ushort_t), PROT_READ | PROT_WRITE);
467 if (bios_memchk)
468 *bios_memchk = 0x1234; /* bios memory check disable */
469
470 if (options_dip != NULL &&
471 ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0,
472 "efi-systab")) {
473 if (bootops == NULL)
474 acpi_reset_system();
475 efi_reset();
476 }
477
478 /*
479 * The problem with using stubs is that we can call
480 * acpi_reset_system only after the kernel is up and running.
481 *
482 * We should create a global state to keep track of how far
483 * up the kernel is but for the time being we will depend on
484 * bootops. bootops cleared in startup_end().
485 */
486 if (bootops == NULL)
487 acpi_reset_system();
488 }
489
490 pc_reset();
491 #else
492 if (IN_XPV_PANIC()) {
493 if (khat_running && bootops == NULL) {
494 acpi_reset_system();
495 }
496
497 pc_reset();
498 }
499
500 (void) HYPERVISOR_shutdown(SHUTDOWN_reboot);
501 panic("HYPERVISOR_shutdown() failed");
502 #endif
503 /*NOTREACHED*/
504 }
505
506 /*
507 * Halt the machine and return to the monitor
508 */
509 void
510 halt(char *s)
511 {
512 stop_other_cpus(); /* send stop signal to other CPUs */
513 if (s)
514 prom_printf("(%s) \n", s);
515 prom_exit_to_mon();
516 /*NOTREACHED*/
517 }
518
519 /*
520 * Initiate interrupt redistribution.
521 */
522 void
523 i_ddi_intr_redist_all_cpus()
524 {
525 }
526
527 /*
528 * XXX These probably ought to live somewhere else
529 * XXX They are called from mem.c
530 */
531
532 /*
533 * Convert page frame number to an OBMEM page frame number
534 * (i.e. put in the type bits -- zero for this implementation)
535 */
536 pfn_t
537 impl_obmem_pfnum(pfn_t pf)
538 {
539 return (pf);
540 }
541
542 #ifdef NM_DEBUG
543 int nmi_test = 0; /* checked in intentry.s during clock int */
544 int nmtest = -1;
545 nmfunc1(int arg, struct regs *rp)
546 {
547 printf("nmi called with arg = %x, regs = %x\n", arg, rp);
548 nmtest += 50;
549 if (arg == nmtest) {
550 printf("ip = %x\n", rp->r_pc);
551 return (1);
552 }
553 return (0);
554 }
555
556 #endif
557
558 #include <sys/bootsvcs.h>
559
560 /* Hacked up initialization for initial kernel check out is HERE. */
561 /* The basic steps are: */
562 /* kernel bootfuncs definition/initialization for KADB */
563 /* kadb bootfuncs pointer initialization */
564 /* putchar/getchar (interrupts disabled) */
565
566 /* kadb bootfuncs pointer initialization */
567
568 int
569 sysp_getchar()
570 {
571 int i;
572 ulong_t s;
573
574 if (cons_polledio == NULL) {
575 /* Uh oh */
576 prom_printf("getchar called with no console\n");
577 for (;;)
578 /* LOOP FOREVER */;
579 }
580
581 s = clear_int_flag();
582 i = cons_polledio->cons_polledio_getchar(
583 cons_polledio->cons_polledio_argument);
584 restore_int_flag(s);
585 return (i);
586 }
587
588 void
589 sysp_putchar(int c)
590 {
591 ulong_t s;
592
593 /*
594 * We have no alternative but to drop the output on the floor.
595 */
596 if (cons_polledio == NULL ||
597 cons_polledio->cons_polledio_putchar == NULL)
598 return;
599
600 s = clear_int_flag();
601 cons_polledio->cons_polledio_putchar(
602 cons_polledio->cons_polledio_argument, c);
603 restore_int_flag(s);
604 }
605
606 int
607 sysp_ischar()
608 {
609 int i;
610 ulong_t s;
611
612 if (cons_polledio == NULL ||
613 cons_polledio->cons_polledio_ischar == NULL)
614 return (0);
615
616 s = clear_int_flag();
617 i = cons_polledio->cons_polledio_ischar(
618 cons_polledio->cons_polledio_argument);
619 restore_int_flag(s);
620 return (i);
621 }
622
623 int
624 goany(void)
625 {
626 prom_printf("Type any key to continue ");
627 (void) prom_getchar();
628 prom_printf("\n");
629 return (1);
630 }
631
632 static struct boot_syscalls kern_sysp = {
633 sysp_getchar, /* unchar (*getchar)(); 7 */
634 sysp_putchar, /* int (*putchar)(); 8 */
635 sysp_ischar, /* int (*ischar)(); 9 */
636 };
637
638 #if defined(__xpv)
639 int using_kern_polledio;
640 #endif
641
642 void
643 kadb_uses_kernel()
644 {
645 /*
646 * This routine is now totally misnamed, since it does not in fact
647 * control kadb's I/O; it only controls the kernel's prom_* I/O.
648 */
649 sysp = &kern_sysp;
650 #if defined(__xpv)
651 using_kern_polledio = 1;
652 #endif
653 }
654
655 /*
656 * the interface to the outside world
657 */
658
659 /*
660 * poll_port -- wait for a register to achieve a
661 * specific state. Arguments are a mask of bits we care about,
662 * and two sub-masks. To return normally, all the bits in the
663 * first sub-mask must be ON, all the bits in the second sub-
664 * mask must be OFF. If about seconds pass without the register
665 * achieving the desired bit configuration, we return 1, else
666 * 0.
667 */
668 int
669 poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits)
670 {
671 int i;
672 ushort_t maskval;
673
674 for (i = 500000; i; i--) {
675 maskval = inb(port) & mask;
676 if (((maskval & onbits) == onbits) &&
677 ((maskval & offbits) == 0))
678 return (0);
679 drv_usecwait(10);
680 }
681 return (1);
682 }
683
684 /*
685 * set_idle_cpu is called from idle() when a CPU becomes idle.
686 */
687 /*LINTED: static unused */
688 static uint_t last_idle_cpu;
689
690 /*ARGSUSED*/
691 void
692 set_idle_cpu(int cpun)
693 {
694 last_idle_cpu = cpun;
695 (*psm_set_idle_cpuf)(cpun);
696 }
697
698 /*
699 * unset_idle_cpu is called from idle() when a CPU is no longer idle.
700 */
701 /*ARGSUSED*/
702 void
703 unset_idle_cpu(int cpun)
704 {
705 (*psm_unset_idle_cpuf)(cpun);
706 }
707
708 /*
709 * This routine is almost correct now, but not quite. It still needs the
710 * equivalent concept of "hres_last_tick", just like on the sparc side.
711 * The idea is to take a snapshot of the hi-res timer while doing the
712 * hrestime_adj updates under hres_lock in locore, so that the small
713 * interval between interrupt assertion and interrupt processing is
714 * accounted for correctly. Once we have this, the code below should
715 * be modified to subtract off hres_last_tick rather than hrtime_base.
716 *
717 * I'd have done this myself, but I don't have source to all of the
718 * vendor-specific hi-res timer routines (grrr...). The generic hook I
719 * need is something like "gethrtime_unlocked()", which would be just like
720 * gethrtime() but would assume that you're already holding CLOCK_LOCK().
721 * This is what the GET_HRTIME() macro is for on sparc (although it also
722 * serves the function of making time available without a function call
723 * so you don't take a register window overflow while traps are disabled).
724 */
725 void
726 pc_gethrestime(timestruc_t *tp)
727 {
728 int lock_prev;
729 timestruc_t now;
730 int nslt; /* nsec since last tick */
731 int adj; /* amount of adjustment to apply */
732
733 loop:
734 lock_prev = hres_lock;
735 now = hrestime;
736 nslt = (int)(gethrtime() - hres_last_tick);
737 if (nslt < 0) {
738 /*
739 * nslt < 0 means a tick came between sampling
740 * gethrtime() and hres_last_tick; restart the loop
741 */
742
743 goto loop;
744 }
745 now.tv_nsec += nslt;
746 if (hrestime_adj != 0) {
747 if (hrestime_adj > 0) {
748 adj = (nslt >> ADJ_SHIFT);
749 if (adj > hrestime_adj)
750 adj = (int)hrestime_adj;
751 } else {
752 adj = -(nslt >> ADJ_SHIFT);
753 if (adj < hrestime_adj)
754 adj = (int)hrestime_adj;
755 }
756 now.tv_nsec += adj;
757 }
758 while ((unsigned long)now.tv_nsec >= NANOSEC) {
759
760 /*
761 * We might have a large adjustment or have been in the
762 * debugger for a long time; take care of (at most) four
763 * of those missed seconds (tv_nsec is 32 bits, so
764 * anything >4s will be wrapping around). However,
765 * anything more than 2 seconds out of sync will trigger
766 * timedelta from clock() to go correct the time anyway,
767 * so do what we can, and let the big crowbar do the
768 * rest. A similar correction while loop exists inside
769 * hres_tick(); in all cases we'd like tv_nsec to
770 * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing
771 * user processes, but if tv_sec's a little behind for a
772 * little while, that's OK; time still monotonically
773 * increases.
774 */
775
776 now.tv_nsec -= NANOSEC;
777 now.tv_sec++;
778 }
779 if ((hres_lock & ~1) != lock_prev)
780 goto loop;
781
782 *tp = now;
783 }
784
785 void
786 gethrestime_lasttick(timespec_t *tp)
787 {
788 int s;
789
790 s = hr_clock_lock();
791 *tp = hrestime;
792 hr_clock_unlock(s);
793 }
794
795 time_t
796 gethrestime_sec(void)
797 {
798 timestruc_t now;
799
800 gethrestime(&now);
801 return (now.tv_sec);
802 }
803
804 /*
805 * Initialize a kernel thread's stack
806 */
807
808 caddr_t
809 thread_stk_init(caddr_t stk)
810 {
811 ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0);
812 return (stk - SA(MINFRAME));
813 }
814
815 /*
816 * Initialize lwp's kernel stack.
817 */
818
819 #ifdef TRAPTRACE
820 /*
821 * There's a tricky interdependency here between use of sysenter and
822 * TRAPTRACE which needs recording to avoid future confusion (this is
823 * about the third time I've re-figured this out ..)
824 *
825 * Here's how debugging lcall works with TRAPTRACE.
826 *
827 * 1 We're in userland with a breakpoint on the lcall instruction.
828 * 2 We execute the instruction - the instruction pushes the userland
829 * %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel
830 * via the call gate.
831 * 3 The hardware raises a debug trap in kernel mode, the hardware
832 * pushes %efl, %cs, %eip and gets to dbgtrap via the idt.
833 * 4 dbgtrap pushes the error code and trapno and calls cmntrap
834 * 5 cmntrap finishes building a trap frame
835 * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk
836 * off the stack into the traptrace buffer.
837 *
838 * This means that the traptrace buffer contains the wrong values in
839 * %esp and %ss, but everything else in there is correct.
840 *
841 * Here's how debugging sysenter works with TRAPTRACE.
842 *
843 * a We're in userland with a breakpoint on the sysenter instruction.
844 * b We execute the instruction - the instruction pushes -nothing-
845 * on the stack, but sets %cs, %eip, %ss, %esp to prearranged
846 * values to take us to sys_sysenter, at the top of the lwp's
847 * stack.
848 * c goto 3
849 *
850 * At this point, because we got into the kernel without the requisite
851 * five pushes on the stack, if we didn't make extra room, we'd
852 * end up with the TRACE_REGS macro fetching the saved %ss and %esp
853 * values from negative (unmapped) stack addresses -- which really bites.
854 * That's why we do the '-= 8' below.
855 *
856 * XXX Note that reading "up" lwp0's stack works because t0 is declared
857 * right next to t0stack in locore.s
858 */
859 #endif
860
861 caddr_t
862 lwp_stk_init(klwp_t *lwp, caddr_t stk)
863 {
864 caddr_t oldstk;
865 struct pcb *pcb = &lwp->lwp_pcb;
866
867 oldstk = stk;
868 stk -= SA(sizeof (struct regs) + SA(MINFRAME));
869 #ifdef TRAPTRACE
870 stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */
871 #endif
872 stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul));
873 bzero(stk, oldstk - stk);
874 lwp->lwp_regs = (void *)(stk + SA(MINFRAME));
875
876 /*
877 * Arrange that the virtualized %fs and %gs GDT descriptors
878 * have a well-defined initial state (present, ring 3
879 * and of type data).
880 */
881 #if defined(__amd64)
882 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE)
883 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
884 else
885 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;
886 #elif defined(__i386)
887 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
888 #endif /* __i386 */
889 lwp_installctx(lwp);
890 return (stk);
891 }
892
893 /*
894 * Use this opportunity to free any dynamically allocated fp storage.
895 */
896 void
897 lwp_stk_fini(klwp_t *lwp)
898 {
899 fp_lwp_cleanup(lwp);
900 }
901
902 void
903 lwp_fp_init(klwp_t *lwp)
904 {
905 fp_lwp_init(lwp);
906 }
907
908 /*
909 * If we're not the panic CPU, we wait in panic_idle for reboot.
910 */
911 void
912 panic_idle(void)
913 {
914 splx(ipltospl(CLOCK_LEVEL));
915 (void) setjmp(&curthread->t_pcb);
916
917 dumpsys_helper();
918
919 #ifndef __xpv
920 for (;;)
921 i86_halt();
922 #else
923 for (;;)
924 ;
925 #endif
926 }
927
928 /*
929 * Stop the other CPUs by cross-calling them and forcing them to enter
930 * the panic_idle() loop above.
931 */
932 /*ARGSUSED*/
933 void
934 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl)
935 {
936 processorid_t i;
937 cpuset_t xcset;
938
939 /*
940 * In the case of a Xen panic, the hypervisor has already stopped
941 * all of the CPUs.
942 */
943 if (!IN_XPV_PANIC()) {
944 (void) splzs();
945
946 CPUSET_ALL_BUT(xcset, cp->cpu_id);
947 xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle);
948 }
949
950 for (i = 0; i < NCPU; i++) {
951 if (i != cp->cpu_id && cpu[i] != NULL &&
952 (cpu[i]->cpu_flags & CPU_EXISTS))
953 cpu[i]->cpu_flags |= CPU_QUIESCED;
954 }
955 }
956
957 /*
958 * Platform callback following each entry to panicsys().
959 */
960 /*ARGSUSED*/
961 void
962 panic_enter_hw(int spl)
963 {
964 /* Nothing to do here */
965 }
966
967 /*
968 * Platform-specific code to execute after panicstr is set: we invoke
969 * the PSM entry point to indicate that a panic has occurred.
970 */
971 /*ARGSUSED*/
972 void
973 panic_quiesce_hw(panic_data_t *pdp)
974 {
975 psm_notifyf(PSM_PANIC_ENTER);
976
977 cmi_panic_callback();
978
979 #ifdef TRAPTRACE
980 /*
981 * Turn off TRAPTRACE
982 */
983 TRAPTRACE_FREEZE;
984 #endif /* TRAPTRACE */
985 }
986
987 /*
988 * Platform callback prior to writing crash dump.
989 */
990 /*ARGSUSED*/
991 void
992 panic_dump_hw(int spl)
993 {
994 /* Nothing to do here */
995 }
996
997 void *
998 plat_traceback(void *fpreg)
999 {
1000 #ifdef __xpv
1001 if (IN_XPV_PANIC())
1002 return (xpv_traceback(fpreg));
1003 #endif
1004 return (fpreg);
1005 }
1006
1007 /*ARGSUSED*/
1008 void
1009 plat_tod_fault(enum tod_fault_type tod_bad)
1010 {}
1011
1012 /*ARGSUSED*/
1013 int
1014 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class)
1015 {
1016 return (ENOTSUP);
1017 }
1018
1019 /*
1020 * The underlying console output routines are protected by raising IPL in case
1021 * we are still calling into the early boot services. Once we start calling
1022 * the kernel console emulator, it will disable interrupts completely during
1023 * character rendering (see sysp_putchar, for example). Refer to the comments
1024 * and code in common/os/console.c for more information on these callbacks.
1025 */
1026 /*ARGSUSED*/
1027 int
1028 console_enter(int busy)
1029 {
1030 return (splzs());
1031 }
1032
1033 /*ARGSUSED*/
1034 void
1035 console_exit(int busy, int spl)
1036 {
1037 splx(spl);
1038 }
1039
1040 /*
1041 * Allocate a region of virtual address space, unmapped.
1042 * Stubbed out except on sparc, at least for now.
1043 */
1044 /*ARGSUSED*/
1045 void *
1046 boot_virt_alloc(void *addr, size_t size)
1047 {
1048 return (addr);
1049 }
1050
1051 volatile unsigned long tenmicrodata;
1052
1053 void
1054 tenmicrosec(void)
1055 {
1056 extern int gethrtime_hires;
1057
1058 if (gethrtime_hires) {
1059 hrtime_t start, end;
1060 start = end = gethrtime();
1061 while ((end - start) < (10 * (NANOSEC / MICROSEC))) {
1062 SMT_PAUSE();
1063 end = gethrtime();
1064 }
1065 } else {
1066 #if defined(__xpv)
1067 hrtime_t newtime;
1068
1069 newtime = xpv_gethrtime() + 10000; /* now + 10 us */
1070 while (xpv_gethrtime() < newtime)
1071 SMT_PAUSE();
1072 #else /* __xpv */
1073 int i;
1074
1075 /*
1076 * Artificial loop to induce delay.
1077 */
1078 for (i = 0; i < microdata; i++)
1079 tenmicrodata = microdata;
1080 #endif /* __xpv */
1081 }
1082 }
1083
1084 /*
1085 * get_cpu_mstate() is passed an array of timestamps, NCMSTATES
1086 * long, and it fills in the array with the time spent on cpu in
1087 * each of the mstates, where time is returned in nsec.
1088 *
1089 * No guarantee is made that the returned values in times[] will
1090 * monotonically increase on sequential calls, although this will
1091 * be true in the long run. Any such guarantee must be handled by
1092 * the caller, if needed. This can happen if we fail to account
1093 * for elapsed time due to a generation counter conflict, yet we
1094 * did account for it on a prior call (see below).
1095 *
1096 * The complication is that the cpu in question may be updating
1097 * its microstate at the same time that we are reading it.
1098 * Because the microstate is only updated when the CPU's state
1099 * changes, the values in cpu_intracct[] can be indefinitely out
1100 * of date. To determine true current values, it is necessary to
1101 * compare the current time with cpu_mstate_start, and add the
1102 * difference to times[cpu_mstate].
1103 *
1104 * This can be a problem if those values are changing out from
1105 * under us. Because the code path in new_cpu_mstate() is
1106 * performance critical, we have not added a lock to it. Instead,
1107 * we have added a generation counter. Before beginning
1108 * modifications, the counter is set to 0. After modifications,
1109 * it is set to the old value plus one.
1110 *
1111 * get_cpu_mstate() will not consider the values of cpu_mstate
1112 * and cpu_mstate_start to be usable unless the value of
1113 * cpu_mstate_gen is both non-zero and unchanged, both before and
1114 * after reading the mstate information. Note that we must
1115 * protect against out-of-order loads around accesses to the
1116 * generation counter. Also, this is a best effort approach in
1117 * that we do not retry should the counter be found to have
1118 * changed.
1119 *
1120 * cpu_intracct[] is used to identify time spent in each CPU
1121 * mstate while handling interrupts. Such time should be reported
1122 * against system time, and so is subtracted out from its
1123 * corresponding cpu_acct[] time and added to
1124 * cpu_acct[CMS_SYSTEM].
1125 */
1126
1127 void
1128 get_cpu_mstate(cpu_t *cpu, hrtime_t *times)
1129 {
1130 int i;
1131 hrtime_t now, start;
1132 uint16_t gen;
1133 uint16_t state;
1134 hrtime_t intracct[NCMSTATES];
1135
1136 /*
1137 * Load all volatile state under the protection of membar.
1138 * cpu_acct[cpu_mstate] must be loaded to avoid double counting
1139 * of (now - cpu_mstate_start) by a change in CPU mstate that
1140 * arrives after we make our last check of cpu_mstate_gen.
1141 */
1142
1143 now = gethrtime_unscaled();
1144 gen = cpu->cpu_mstate_gen;
1145
1146 membar_consumer(); /* guarantee load ordering */
1147 start = cpu->cpu_mstate_start;
1148 state = cpu->cpu_mstate;
1149 for (i = 0; i < NCMSTATES; i++) {
1150 intracct[i] = cpu->cpu_intracct[i];
1151 times[i] = cpu->cpu_acct[i];
1152 }
1153 membar_consumer(); /* guarantee load ordering */
1154
1155 if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start)
1156 times[state] += now - start;
1157
1158 for (i = 0; i < NCMSTATES; i++) {
1159 if (i == CMS_SYSTEM)
1160 continue;
1161 times[i] -= intracct[i];
1162 if (times[i] < 0) {
1163 intracct[i] += times[i];
1164 times[i] = 0;
1165 }
1166 times[CMS_SYSTEM] += intracct[i];
1167 scalehrtime(×[i]);
1168 }
1169 scalehrtime(×[CMS_SYSTEM]);
1170 }
1171
1172 /*
1173 * This is a version of the rdmsr instruction that allows
1174 * an error code to be returned in the case of failure.
1175 */
1176 int
1177 checked_rdmsr(uint_t msr, uint64_t *value)
1178 {
1179 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1180 return (ENOTSUP);
1181 *value = rdmsr(msr);
1182 return (0);
1183 }
1184
1185 /*
1186 * This is a version of the wrmsr instruction that allows
1187 * an error code to be returned in the case of failure.
1188 */
1189 int
1190 checked_wrmsr(uint_t msr, uint64_t value)
1191 {
1192 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1193 return (ENOTSUP);
1194 wrmsr(msr, value);
1195 return (0);
1196 }
1197
1198 /*
1199 * The mem driver's usual method of using hat_devload() to establish a
1200 * temporary mapping will not work for foreign pages mapped into this
1201 * domain or for the special hypervisor-provided pages. For the foreign
1202 * pages, we often don't know which domain owns them, so we can't ask the
1203 * hypervisor to set up a new mapping. For the other pages, we don't have
1204 * a pfn, so we can't create a new PTE. For these special cases, we do a
1205 * direct uiomove() from the existing kernel virtual address.
1206 */
1207 /*ARGSUSED*/
1208 int
1209 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw)
1210 {
1211 #if defined(__xpv)
1212 void *va = (void *)(uintptr_t)uio->uio_loffset;
1213 off_t pageoff = uio->uio_loffset & PAGEOFFSET;
1214 size_t nbytes = MIN((size_t)(PAGESIZE - pageoff),
1215 (size_t)uio->uio_iov->iov_len);
1216
1217 if ((rw == UIO_READ &&
1218 (va == HYPERVISOR_shared_info || va == xen_info)) ||
1219 (pfn_is_foreign(hat_getpfnum(kas.a_hat, va))))
1220 return (uiomove(va, nbytes, rw, uio));
1221 #endif
1222 return (ENOTSUP);
1223 }
1224
1225 pgcnt_t
1226 num_phys_pages()
1227 {
1228 pgcnt_t npages = 0;
1229 struct memlist *mp;
1230
1231 #if defined(__xpv)
1232 if (DOMAIN_IS_INITDOMAIN(xen_info))
1233 return (xpv_nr_phys_pages());
1234 #endif /* __xpv */
1235
1236 for (mp = phys_install; mp != NULL; mp = mp->ml_next)
1237 npages += mp->ml_size >> PAGESHIFT;
1238
1239 return (npages);
1240 }
1241
1242 /* cpu threshold for compressed dumps */
1243 #ifdef _LP64
1244 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_64_MINCPU;
1245 #else
1246 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_32_MINCPU;
1247 #endif
1248
1249 int
1250 dump_plat_addr()
1251 {
1252 #ifdef __xpv
1253 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1254 mem_vtop_t mem_vtop;
1255 int cnt;
1256
1257 /*
1258 * On the hypervisor, we want to dump the page with shared_info on it.
1259 */
1260 if (!IN_XPV_PANIC()) {
1261 mem_vtop.m_as = &kas;
1262 mem_vtop.m_va = HYPERVISOR_shared_info;
1263 mem_vtop.m_pfn = pfn;
1264 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t));
1265 cnt = 1;
1266 } else {
1267 cnt = dump_xpv_addr();
1268 }
1269 return (cnt);
1270 #else
1271 return (0);
1272 #endif
1273 }
1274
1275 void
1276 dump_plat_pfn()
1277 {
1278 #ifdef __xpv
1279 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1280
1281 if (!IN_XPV_PANIC())
1282 dumpvp_write(&pfn, sizeof (pfn));
1283 else
1284 dump_xpv_pfn();
1285 #endif
1286 }
1287
1288 /*ARGSUSED*/
1289 int
1290 dump_plat_data(void *dump_cbuf)
1291 {
1292 #ifdef __xpv
1293 uint32_t csize;
1294 int cnt;
1295
1296 if (!IN_XPV_PANIC()) {
1297 csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf,
1298 PAGESIZE);
1299 dumpvp_write(&csize, sizeof (uint32_t));
1300 dumpvp_write(dump_cbuf, csize);
1301 cnt = 1;
1302 } else {
1303 cnt = dump_xpv_data(dump_cbuf);
1304 }
1305 return (cnt);
1306 #else
1307 return (0);
1308 #endif
1309 }
1310
1311 /*
1312 * Calculates a linear address, given the CS selector and PC values
1313 * by looking up the %cs selector process's LDT or the CPU's GDT.
1314 * proc->p_ldtlock must be held across this call.
1315 */
1316 int
1317 linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1318 {
1319 user_desc_t *descrp;
1320 caddr_t baseaddr;
1321 uint16_t idx = SELTOIDX(rp->r_cs);
1322
1323 ASSERT(rp->r_cs <= 0xFFFF);
1324 ASSERT(MUTEX_HELD(&p->p_ldtlock));
1325
1326 if (SELISLDT(rp->r_cs)) {
1327 /*
1328 * Currently 64 bit processes cannot have private LDTs.
1329 */
1330 ASSERT(p->p_model != DATAMODEL_LP64);
1331
1332 if (p->p_ldt == NULL)
1333 return (-1);
1334
1335 descrp = &p->p_ldt[idx];
1336 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1337
1338 /*
1339 * Calculate the linear address (wraparound is not only ok,
1340 * it's expected behavior). The cast to uint32_t is because
1341 * LDT selectors are only allowed in 32-bit processes.
1342 */
1343 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1344 rp->r_pc);
1345 } else {
1346 #ifdef DEBUG
1347 descrp = &CPU->cpu_gdt[idx];
1348 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1349 /* GDT-based descriptors' base addresses should always be 0 */
1350 ASSERT(baseaddr == 0);
1351 #endif
1352 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1353 }
1354
1355 return (0);
1356 }
1357
1358 /*
1359 * The implementation of dtrace_linear_pc is similar to the that of
1360 * linear_pc, above, but here we acquire p_ldtlock before accessing
1361 * p_ldt. This implementation is used by the pid provider; we prefix
1362 * it with "dtrace_" to avoid inducing spurious tracing events.
1363 */
1364 int
1365 dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1366 {
1367 user_desc_t *descrp;
1368 caddr_t baseaddr;
1369 uint16_t idx = SELTOIDX(rp->r_cs);
1370
1371 ASSERT(rp->r_cs <= 0xFFFF);
1372
1373 if (SELISLDT(rp->r_cs)) {
1374 /*
1375 * Currently 64 bit processes cannot have private LDTs.
1376 */
1377 ASSERT(p->p_model != DATAMODEL_LP64);
1378
1379 mutex_enter(&p->p_ldtlock);
1380 if (p->p_ldt == NULL) {
1381 mutex_exit(&p->p_ldtlock);
1382 return (-1);
1383 }
1384 descrp = &p->p_ldt[idx];
1385 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1386 mutex_exit(&p->p_ldtlock);
1387
1388 /*
1389 * Calculate the linear address (wraparound is not only ok,
1390 * it's expected behavior). The cast to uint32_t is because
1391 * LDT selectors are only allowed in 32-bit processes.
1392 */
1393 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1394 rp->r_pc);
1395 } else {
1396 #ifdef DEBUG
1397 descrp = &CPU->cpu_gdt[idx];
1398 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1399 /* GDT-based descriptors' base addresses should always be 0 */
1400 ASSERT(baseaddr == 0);
1401 #endif
1402 *linearp = (caddr_t)(uintptr_t)rp->r_pc;
1403 }
1404
1405 return (0);
1406 }
1407
1408 /*
1409 * We need to post a soft interrupt to reprogram the lbolt cyclic when
1410 * switching from event to cyclic driven lbolt. The following code adds
1411 * and posts the softint for x86.
1412 */
1413 static ddi_softint_hdl_impl_t lbolt_softint_hdl =
1414 {0, 0, NULL, NULL, 0, NULL, NULL, NULL};
1415
1416 void
1417 lbolt_softint_add(void)
1418 {
1419 (void) add_avsoftintr((void *)&lbolt_softint_hdl, LOCK_LEVEL,
1420 (avfunc)lbolt_ev_to_cyclic, "lbolt_ev_to_cyclic", NULL, NULL);
1421 }
1422
1423 void
1424 lbolt_softint_post(void)
1425 {
1426 (*setsoftint)(CBE_LOCK_PIL, lbolt_softint_hdl.ih_pending);
1427 }
1428
1429 boolean_t
1430 plat_dr_check_capability(uint64_t features)
1431 {
1432 return ((plat_dr_options & features) == features);
1433 }
1434
1435 boolean_t
1436 plat_dr_support_cpu(void)
1437 {
1438 return (plat_dr_options & PLAT_DR_FEATURE_CPU);
1439 }
1440
1441 boolean_t
1442 plat_dr_support_memory(void)
1443 {
1444 return (plat_dr_options & PLAT_DR_FEATURE_MEMORY);
1445 }
1446
1447 void
1448 plat_dr_enable_capability(uint64_t features)
1449 {
1450 atomic_or_64(&plat_dr_options, features);
1451 }
1452
1453 void
1454 plat_dr_disable_capability(uint64_t features)
1455 {
1456 atomic_and_64(&plat_dr_options, ~features);
1457 }