1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
26 /* All Rights Reserved */
27 /*
28 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 */
31
32 #include <sys/param.h>
33 #include <sys/types.h>
34 #include <sys/vmparam.h>
35 #include <sys/systm.h>
36 #include <sys/signal.h>
37 #include <sys/stack.h>
38 #include <sys/regset.h>
39 #include <sys/privregs.h>
40 #include <sys/frame.h>
41 #include <sys/proc.h>
42 #include <sys/psw.h>
43 #include <sys/siginfo.h>
44 #include <sys/cpuvar.h>
45 #include <sys/asm_linkage.h>
46 #include <sys/kmem.h>
47 #include <sys/errno.h>
48 #include <sys/bootconf.h>
49 #include <sys/archsystm.h>
50 #include <sys/debug.h>
51 #include <sys/elf.h>
52 #include <sys/spl.h>
53 #include <sys/time.h>
54 #include <sys/atomic.h>
55 #include <sys/sysmacros.h>
56 #include <sys/cmn_err.h>
57 #include <sys/modctl.h>
58 #include <sys/kobj.h>
59 #include <sys/panic.h>
60 #include <sys/reboot.h>
61 #include <sys/time.h>
62 #include <sys/fp.h>
63 #include <sys/x86_archext.h>
64 #include <sys/auxv.h>
65 #include <sys/auxv_386.h>
66 #include <sys/dtrace.h>
67 #include <sys/brand.h>
68 #include <sys/machbrand.h>
69 #include <sys/cmn_err.h>
70
71 extern const struct fnsave_state x87_initial;
72 extern const struct fxsave_state sse_initial;
73
74 /*
75 * Map an fnsave-formatted save area into an fxsave-formatted save area.
76 *
77 * Most fields are the same width, content and semantics. However
78 * the tag word is compressed.
79 */
80 static void
81 fnsave_to_fxsave(const struct fnsave_state *fn, struct fxsave_state *fx)
82 {
83 uint_t i, tagbits;
84
85 fx->fx_fcw = fn->f_fcw;
86 fx->fx_fsw = fn->f_fsw;
87
88 /*
89 * copy element by element (because of holes)
90 */
91 for (i = 0; i < 8; i++)
92 bcopy(&fn->f_st[i].fpr_16[0], &fx->fx_st[i].fpr_16[0],
93 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
94
95 /*
96 * synthesize compressed tag bits
97 */
98 fx->fx_fctw = 0;
99 for (tagbits = fn->f_ftw, i = 0; i < 8; i++, tagbits >>= 2)
100 if ((tagbits & 3) != 3)
101 fx->fx_fctw |= (1 << i);
102
103 fx->fx_fop = fn->f_fop;
104
105 #if defined(__amd64)
106 fx->fx_rip = (uint64_t)fn->f_eip;
107 fx->fx_rdp = (uint64_t)fn->f_dp;
108 #else
109 fx->fx_eip = fn->f_eip;
110 fx->fx_cs = fn->f_cs;
111 fx->__fx_ign0 = 0;
112 fx->fx_dp = fn->f_dp;
113 fx->fx_ds = fn->f_ds;
114 fx->__fx_ign1 = 0;
115 #endif
116 }
117
118 /*
119 * Map from an fxsave-format save area to an fnsave-format save area.
120 */
121 static void
122 fxsave_to_fnsave(const struct fxsave_state *fx, struct fnsave_state *fn)
123 {
124 uint_t i, top, tagbits;
125
126 fn->f_fcw = fx->fx_fcw;
127 fn->__f_ign0 = 0;
128 fn->f_fsw = fx->fx_fsw;
129 fn->__f_ign1 = 0;
130
131 top = (fx->fx_fsw & FPS_TOP) >> 11;
132
133 /*
134 * copy element by element (because of holes)
135 */
136 for (i = 0; i < 8; i++)
137 bcopy(&fx->fx_st[i].fpr_16[0], &fn->f_st[i].fpr_16[0],
138 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
139
140 /*
141 * synthesize uncompressed tag bits
142 */
143 fn->f_ftw = 0;
144 for (tagbits = fx->fx_fctw, i = 0; i < 8; i++, tagbits >>= 1) {
145 uint_t ibit, expo;
146 const uint16_t *fpp;
147 static const uint16_t zero[5] = { 0, 0, 0, 0, 0 };
148
149 if ((tagbits & 1) == 0) {
150 fn->f_ftw |= 3 << (i << 1); /* empty */
151 continue;
152 }
153
154 /*
155 * (tags refer to *physical* registers)
156 */
157 fpp = &fx->fx_st[(i - top + 8) & 7].fpr_16[0];
158 ibit = fpp[3] >> 15;
159 expo = fpp[4] & 0x7fff;
160
161 if (ibit && expo != 0 && expo != 0x7fff)
162 continue; /* valid fp number */
163
164 if (bcmp(fpp, &zero, sizeof (zero)))
165 fn->f_ftw |= 2 << (i << 1); /* NaN */
166 else
167 fn->f_ftw |= 1 << (i << 1); /* fp zero */
168 }
169
170 fn->f_fop = fx->fx_fop;
171
172 fn->__f_ign2 = 0;
173 #if defined(__amd64)
174 fn->f_eip = (uint32_t)fx->fx_rip;
175 fn->f_cs = U32CS_SEL;
176 fn->f_dp = (uint32_t)fx->fx_rdp;
177 fn->f_ds = UDS_SEL;
178 #else
179 fn->f_eip = fx->fx_eip;
180 fn->f_cs = fx->fx_cs;
181 fn->f_dp = fx->fx_dp;
182 fn->f_ds = fx->fx_ds;
183 #endif
184 fn->__f_ign3 = 0;
185 }
186
187 /*
188 * Map from an fpregset_t into an fxsave-format save area
189 */
190 static void
191 fpregset_to_fxsave(const fpregset_t *fp, struct fxsave_state *fx)
192 {
193 #if defined(__amd64)
194 bcopy(fp, fx, sizeof (*fx));
195 #else
196 const struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
197
198 fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
199 fx->fx_mxcsr = fc->mxcsr;
200 bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
201 #endif
202 /*
203 * avoid useless #gp exceptions - mask reserved bits
204 */
205 fx->fx_mxcsr &= sse_mxcsr_mask;
206 }
207
208 /*
209 * Map from an fxsave-format save area into a fpregset_t
210 */
211 static void
212 fxsave_to_fpregset(const struct fxsave_state *fx, fpregset_t *fp)
213 {
214 #if defined(__amd64)
215 bcopy(fx, fp, sizeof (*fx));
216 #else
217 struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
218
219 fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
220 fc->mxcsr = fx->fx_mxcsr;
221 bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
222 #endif
223 }
224
225 #if defined(_SYSCALL32_IMPL)
226 static void
227 fpregset32_to_fxsave(const fpregset32_t *fp, struct fxsave_state *fx)
228 {
229 const struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
230
231 fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
232 /*
233 * avoid useless #gp exceptions - mask reserved bits
234 */
235 fx->fx_mxcsr = sse_mxcsr_mask & fc->mxcsr;
236 bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
237 }
238
239 static void
240 fxsave_to_fpregset32(const struct fxsave_state *fx, fpregset32_t *fp)
241 {
242 struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
243
244 fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
245 fc->mxcsr = fx->fx_mxcsr;
246 bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
247 }
248
249 static void
250 fpregset_nto32(const fpregset_t *src, fpregset32_t *dst)
251 {
252 fxsave_to_fpregset32((struct fxsave_state *)src, dst);
253 dst->fp_reg_set.fpchip_state.status =
254 src->fp_reg_set.fpchip_state.status;
255 dst->fp_reg_set.fpchip_state.xstatus =
256 src->fp_reg_set.fpchip_state.xstatus;
257 }
258
259 static void
260 fpregset_32ton(const fpregset32_t *src, fpregset_t *dst)
261 {
262 fpregset32_to_fxsave(src, (struct fxsave_state *)dst);
263 dst->fp_reg_set.fpchip_state.status =
264 src->fp_reg_set.fpchip_state.status;
265 dst->fp_reg_set.fpchip_state.xstatus =
266 src->fp_reg_set.fpchip_state.xstatus;
267 }
268 #endif
269
270 /*
271 * Set floating-point registers from a native fpregset_t.
272 */
273 void
274 setfpregs(klwp_t *lwp, fpregset_t *fp)
275 {
276 struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
277
278 if (fpu->fpu_flags & FPU_EN) {
279 if (!(fpu->fpu_flags & FPU_VALID)) {
280 /*
281 * FPU context is still active, release the
282 * ownership.
283 */
284 fp_free(fpu, 0);
285 }
286 }
287 /*
288 * Else: if we are trying to change the FPU state of a thread which
289 * hasn't yet initialized floating point, store the state in
290 * the pcb and indicate that the state is valid. When the
291 * thread enables floating point, it will use this state instead
292 * of the default state.
293 */
294
295 switch (fp_save_mech) {
296 #if defined(__i386)
297 case FP_FNSAVE:
298 bcopy(fp, &fpu->fpu_regs.kfpu_u.kfpu_fn,
299 sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn));
300 break;
301 #endif
302 case FP_FXSAVE:
303 fpregset_to_fxsave(fp, &fpu->fpu_regs.kfpu_u.kfpu_fx);
304 fpu->fpu_regs.kfpu_xstatus =
305 fp->fp_reg_set.fpchip_state.xstatus;
306 break;
307
308 case FP_XSAVE:
309 fpregset_to_fxsave(fp,
310 &fpu->fpu_regs.kfpu_u.kfpu_xs.xs_fxsave);
311 fpu->fpu_regs.kfpu_xstatus =
312 fp->fp_reg_set.fpchip_state.xstatus;
313 fpu->fpu_regs.kfpu_u.kfpu_xs.xs_xstate_bv |=
314 (XFEATURE_LEGACY_FP | XFEATURE_SSE);
315 break;
316 default:
317 panic("Invalid fp_save_mech");
318 /*NOTREACHED*/
319 }
320
321 fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status;
322 fpu->fpu_flags |= FPU_VALID;
323 }
324
325 /*
326 * Get floating-point registers into a native fpregset_t.
327 */
328 void
329 getfpregs(klwp_t *lwp, fpregset_t *fp)
330 {
331 struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
332
333 kpreempt_disable();
334 if (fpu->fpu_flags & FPU_EN) {
335 /*
336 * If we have FPU hw and the thread's pcb doesn't have
337 * a valid FPU state then get the state from the hw.
338 */
339 if (fpu_exists && ttolwp(curthread) == lwp &&
340 !(fpu->fpu_flags & FPU_VALID))
341 fp_save(fpu); /* get the current FPU state */
342 }
343
344 /*
345 * There are 3 possible cases we have to be aware of here:
346 *
347 * 1. FPU is enabled. FPU state is stored in the current LWP.
348 *
349 * 2. FPU is not enabled, and there have been no intervening /proc
350 * modifications. Return initial FPU state.
351 *
352 * 3. FPU is not enabled, but a /proc consumer has modified FPU state.
353 * FPU state is stored in the current LWP.
354 */
355 if ((fpu->fpu_flags & FPU_EN) || (fpu->fpu_flags & FPU_VALID)) {
356 /*
357 * Cases 1 and 3.
358 */
359 switch (fp_save_mech) {
360 #if defined(__i386)
361 case FP_FNSAVE:
362 bcopy(&fpu->fpu_regs.kfpu_u.kfpu_fn, fp,
363 sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn));
364 break;
365 #endif
366 case FP_FXSAVE:
367 fxsave_to_fpregset(&fpu->fpu_regs.kfpu_u.kfpu_fx, fp);
368 fp->fp_reg_set.fpchip_state.xstatus =
369 fpu->fpu_regs.kfpu_xstatus;
370 break;
371 case FP_XSAVE:
372 fxsave_to_fpregset(
373 &fpu->fpu_regs.kfpu_u.kfpu_xs.xs_fxsave, fp);
374 fp->fp_reg_set.fpchip_state.xstatus =
375 fpu->fpu_regs.kfpu_xstatus;
376 break;
377 default:
378 panic("Invalid fp_save_mech");
379 /*NOTREACHED*/
380 }
381 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
382 } else {
383 /*
384 * Case 2.
385 */
386 switch (fp_save_mech) {
387 #if defined(__i386)
388 case FP_FNSAVE:
389 bcopy(&x87_initial, fp, sizeof (x87_initial));
390 break;
391 #endif
392 case FP_FXSAVE:
393 case FP_XSAVE:
394 /*
395 * For now, we don't have any AVX specific field in ABI.
396 * If we add any in the future, we need to initial them
397 * as well.
398 */
399 fxsave_to_fpregset(&sse_initial, fp);
400 fp->fp_reg_set.fpchip_state.xstatus =
401 fpu->fpu_regs.kfpu_xstatus;
402 break;
403 default:
404 panic("Invalid fp_save_mech");
405 /*NOTREACHED*/
406 }
407 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
408 }
409 kpreempt_enable();
410 }
411
412 #if defined(_SYSCALL32_IMPL)
413
414 /*
415 * Set floating-point registers from an fpregset32_t.
416 */
417 void
418 setfpregs32(klwp_t *lwp, fpregset32_t *fp)
419 {
420 fpregset_t fpregs;
421
422 fpregset_32ton(fp, &fpregs);
423 setfpregs(lwp, &fpregs);
424 }
425
426 /*
427 * Get floating-point registers into an fpregset32_t.
428 */
429 void
430 getfpregs32(klwp_t *lwp, fpregset32_t *fp)
431 {
432 fpregset_t fpregs;
433
434 getfpregs(lwp, &fpregs);
435 fpregset_nto32(&fpregs, fp);
436 }
437
438 #endif /* _SYSCALL32_IMPL */
439
440 /*
441 * Return the general registers
442 */
443 void
444 getgregs(klwp_t *lwp, gregset_t grp)
445 {
446 struct regs *rp = lwptoregs(lwp);
447 #if defined(__amd64)
448 struct pcb *pcb = &lwp->lwp_pcb;
449 int thisthread = lwptot(lwp) == curthread;
450
451 grp[REG_RDI] = rp->r_rdi;
452 grp[REG_RSI] = rp->r_rsi;
453 grp[REG_RDX] = rp->r_rdx;
454 grp[REG_RCX] = rp->r_rcx;
455 grp[REG_R8] = rp->r_r8;
456 grp[REG_R9] = rp->r_r9;
457 grp[REG_RAX] = rp->r_rax;
458 grp[REG_RBX] = rp->r_rbx;
459 grp[REG_RBP] = rp->r_rbp;
460 grp[REG_R10] = rp->r_r10;
461 grp[REG_R11] = rp->r_r11;
462 grp[REG_R12] = rp->r_r12;
463 grp[REG_R13] = rp->r_r13;
464 grp[REG_R14] = rp->r_r14;
465 grp[REG_R15] = rp->r_r15;
466 grp[REG_FSBASE] = pcb->pcb_fsbase;
467 grp[REG_GSBASE] = pcb->pcb_gsbase;
468 if (thisthread)
469 kpreempt_disable();
470 if (pcb->pcb_rupdate == 1) {
471 grp[REG_DS] = pcb->pcb_ds;
472 grp[REG_ES] = pcb->pcb_es;
473 grp[REG_FS] = pcb->pcb_fs;
474 grp[REG_GS] = pcb->pcb_gs;
475 } else {
476 grp[REG_DS] = rp->r_ds;
477 grp[REG_ES] = rp->r_es;
478 grp[REG_FS] = rp->r_fs;
479 grp[REG_GS] = rp->r_gs;
480 }
481 if (thisthread)
482 kpreempt_enable();
483 grp[REG_TRAPNO] = rp->r_trapno;
484 grp[REG_ERR] = rp->r_err;
485 grp[REG_RIP] = rp->r_rip;
486 grp[REG_CS] = rp->r_cs;
487 grp[REG_SS] = rp->r_ss;
488 grp[REG_RFL] = rp->r_rfl;
489 grp[REG_RSP] = rp->r_rsp;
490 #else
491 bcopy(&rp->r_gs, grp, sizeof (gregset_t));
492 #endif
493 }
494
495 #if defined(_SYSCALL32_IMPL)
496
497 void
498 getgregs32(klwp_t *lwp, gregset32_t grp)
499 {
500 struct regs *rp = lwptoregs(lwp);
501 struct pcb *pcb = &lwp->lwp_pcb;
502 int thisthread = lwptot(lwp) == curthread;
503
504 if (thisthread)
505 kpreempt_disable();
506 if (pcb->pcb_rupdate == 1) {
507 grp[GS] = (uint16_t)pcb->pcb_gs;
508 grp[FS] = (uint16_t)pcb->pcb_fs;
509 grp[DS] = (uint16_t)pcb->pcb_ds;
510 grp[ES] = (uint16_t)pcb->pcb_es;
511 } else {
512 grp[GS] = (uint16_t)rp->r_gs;
513 grp[FS] = (uint16_t)rp->r_fs;
514 grp[DS] = (uint16_t)rp->r_ds;
515 grp[ES] = (uint16_t)rp->r_es;
516 }
517 if (thisthread)
518 kpreempt_enable();
519 grp[EDI] = (greg32_t)rp->r_rdi;
520 grp[ESI] = (greg32_t)rp->r_rsi;
521 grp[EBP] = (greg32_t)rp->r_rbp;
522 grp[ESP] = 0;
523 grp[EBX] = (greg32_t)rp->r_rbx;
524 grp[EDX] = (greg32_t)rp->r_rdx;
525 grp[ECX] = (greg32_t)rp->r_rcx;
526 grp[EAX] = (greg32_t)rp->r_rax;
527 grp[TRAPNO] = (greg32_t)rp->r_trapno;
528 grp[ERR] = (greg32_t)rp->r_err;
529 grp[EIP] = (greg32_t)rp->r_rip;
530 grp[CS] = (uint16_t)rp->r_cs;
531 grp[EFL] = (greg32_t)rp->r_rfl;
532 grp[UESP] = (greg32_t)rp->r_rsp;
533 grp[SS] = (uint16_t)rp->r_ss;
534 }
535
536 void
537 ucontext_32ton(const ucontext32_t *src, ucontext_t *dst)
538 {
539 mcontext_t *dmc = &dst->uc_mcontext;
540 const mcontext32_t *smc = &src->uc_mcontext;
541
542 bzero(dst, sizeof (*dst));
543 dst->uc_flags = src->uc_flags;
544 dst->uc_link = (ucontext_t *)(uintptr_t)src->uc_link;
545
546 bcopy(&src->uc_sigmask, &dst->uc_sigmask, sizeof (dst->uc_sigmask));
547
548 dst->uc_stack.ss_sp = (void *)(uintptr_t)src->uc_stack.ss_sp;
549 dst->uc_stack.ss_size = (size_t)src->uc_stack.ss_size;
550 dst->uc_stack.ss_flags = src->uc_stack.ss_flags;
551
552 dmc->gregs[REG_GS] = (greg_t)(uint32_t)smc->gregs[GS];
553 dmc->gregs[REG_FS] = (greg_t)(uint32_t)smc->gregs[FS];
554 dmc->gregs[REG_ES] = (greg_t)(uint32_t)smc->gregs[ES];
555 dmc->gregs[REG_DS] = (greg_t)(uint32_t)smc->gregs[DS];
556 dmc->gregs[REG_RDI] = (greg_t)(uint32_t)smc->gregs[EDI];
557 dmc->gregs[REG_RSI] = (greg_t)(uint32_t)smc->gregs[ESI];
558 dmc->gregs[REG_RBP] = (greg_t)(uint32_t)smc->gregs[EBP];
559 dmc->gregs[REG_RBX] = (greg_t)(uint32_t)smc->gregs[EBX];
560 dmc->gregs[REG_RDX] = (greg_t)(uint32_t)smc->gregs[EDX];
561 dmc->gregs[REG_RCX] = (greg_t)(uint32_t)smc->gregs[ECX];
562 dmc->gregs[REG_RAX] = (greg_t)(uint32_t)smc->gregs[EAX];
563 dmc->gregs[REG_TRAPNO] = (greg_t)(uint32_t)smc->gregs[TRAPNO];
564 dmc->gregs[REG_ERR] = (greg_t)(uint32_t)smc->gregs[ERR];
565 dmc->gregs[REG_RIP] = (greg_t)(uint32_t)smc->gregs[EIP];
566 dmc->gregs[REG_CS] = (greg_t)(uint32_t)smc->gregs[CS];
567 dmc->gregs[REG_RFL] = (greg_t)(uint32_t)smc->gregs[EFL];
568 dmc->gregs[REG_RSP] = (greg_t)(uint32_t)smc->gregs[UESP];
569 dmc->gregs[REG_SS] = (greg_t)(uint32_t)smc->gregs[SS];
570
571 /*
572 * A valid fpregs is only copied in if uc.uc_flags has UC_FPU set
573 * otherwise there is no guarantee that anything in fpregs is valid.
574 */
575 if (src->uc_flags & UC_FPU)
576 fpregset_32ton(&src->uc_mcontext.fpregs,
577 &dst->uc_mcontext.fpregs);
578 }
579
580 #endif /* _SYSCALL32_IMPL */
581
582 /*
583 * Return the user-level PC.
584 * If in a system call, return the address of the syscall trap.
585 */
586 greg_t
587 getuserpc()
588 {
589 greg_t upc = lwptoregs(ttolwp(curthread))->r_pc;
590 uint32_t insn;
591
592 if (curthread->t_sysnum == 0)
593 return (upc);
594
595 /*
596 * We might've gotten here from sysenter (0xf 0x34),
597 * syscall (0xf 0x5) or lcall (0x9a 0 0 0 0 0x27 0).
598 *
599 * Go peek at the binary to figure it out..
600 */
601 if (fuword32((void *)(upc - 2), &insn) != -1 &&
602 (insn & 0xffff) == 0x340f || (insn & 0xffff) == 0x050f)
603 return (upc - 2);
604 return (upc - 7);
605 }
606
607 /*
608 * Protect segment registers from non-user privilege levels and GDT selectors
609 * other than USER_CS, USER_DS and lwp FS and GS values. If the segment
610 * selector is non-null and not USER_CS/USER_DS, we make sure that the
611 * TI bit is set to point into the LDT and that the RPL is set to 3.
612 *
613 * Since struct regs stores each 16-bit segment register as a 32-bit greg_t, we
614 * also explicitly zero the top 16 bits since they may be coming from the
615 * user's address space via setcontext(2) or /proc.
616 *
617 * Note about null selector. When running on the hypervisor if we allow a
618 * process to set its %cs to null selector with RPL of 0 the hypervisor will
619 * crash the domain. If running on bare metal we would get a #gp fault and
620 * be able to kill the process and continue on. Therefore we make sure to
621 * force RPL to SEL_UPL even for null selector when setting %cs.
622 */
623
624 #if defined(IS_CS) || defined(IS_NOT_CS)
625 #error "IS_CS and IS_NOT_CS already defined"
626 #endif
627
628 #define IS_CS 1
629 #define IS_NOT_CS 0
630
631 /*ARGSUSED*/
632 static greg_t
633 fix_segreg(greg_t sr, int iscs, model_t datamodel)
634 {
635 kthread_t *t = curthread;
636
637 switch (sr &= 0xffff) {
638
639 case 0:
640 if (iscs == IS_CS)
641 return (0 | SEL_UPL);
642 else
643 return (0);
644
645 #if defined(__amd64)
646 /*
647 * If lwp attempts to switch data model then force their
648 * code selector to be null selector.
649 */
650 case U32CS_SEL:
651 if (datamodel == DATAMODEL_NATIVE)
652 return (0 | SEL_UPL);
653 else
654 return (sr);
655
656 case UCS_SEL:
657 if (datamodel == DATAMODEL_ILP32)
658 return (0 | SEL_UPL);
659 #elif defined(__i386)
660 case UCS_SEL:
661 #endif
662 /*FALLTHROUGH*/
663 case UDS_SEL:
664 case LWPFS_SEL:
665 case LWPGS_SEL:
666 case SEL_UPL:
667 return (sr);
668 default:
669 break;
670 }
671
672 /*
673 * Allow this process's brand to do any necessary segment register
674 * manipulation.
675 */
676 if (PROC_IS_BRANDED(t->t_procp) && BRMOP(t->t_procp)->b_fixsegreg) {
677 greg_t bsr = BRMOP(t->t_procp)->b_fixsegreg(sr, datamodel);
678
679 if (bsr == 0 && iscs == IS_CS)
680 return (0 | SEL_UPL);
681 else
682 return (bsr);
683 }
684
685 /*
686 * Force it into the LDT in ring 3 for 32-bit processes, which by
687 * default do not have an LDT, so that any attempt to use an invalid
688 * selector will reference the (non-existant) LDT, and cause a #gp
689 * fault for the process.
690 *
691 * 64-bit processes get the null gdt selector since they
692 * are not allowed to have a private LDT.
693 */
694 #if defined(__amd64)
695 if (datamodel == DATAMODEL_ILP32) {
696 return (sr | SEL_TI_LDT | SEL_UPL);
697 } else {
698 if (iscs == IS_CS)
699 return (0 | SEL_UPL);
700 else
701 return (0);
702 }
703
704 #elif defined(__i386)
705 return (sr | SEL_TI_LDT | SEL_UPL);
706 #endif
707 }
708
709 /*
710 * Set general registers.
711 */
712 void
713 setgregs(klwp_t *lwp, gregset_t grp)
714 {
715 struct regs *rp = lwptoregs(lwp);
716 model_t datamodel = lwp_getdatamodel(lwp);
717
718 #if defined(__amd64)
719 struct pcb *pcb = &lwp->lwp_pcb;
720 int thisthread = lwptot(lwp) == curthread;
721
722 if (datamodel == DATAMODEL_NATIVE) {
723
724 if (thisthread)
725 (void) save_syscall_args(); /* copy the args */
726
727 rp->r_rdi = grp[REG_RDI];
728 rp->r_rsi = grp[REG_RSI];
729 rp->r_rdx = grp[REG_RDX];
730 rp->r_rcx = grp[REG_RCX];
731 rp->r_r8 = grp[REG_R8];
732 rp->r_r9 = grp[REG_R9];
733 rp->r_rax = grp[REG_RAX];
734 rp->r_rbx = grp[REG_RBX];
735 rp->r_rbp = grp[REG_RBP];
736 rp->r_r10 = grp[REG_R10];
737 rp->r_r11 = grp[REG_R11];
738 rp->r_r12 = grp[REG_R12];
739 rp->r_r13 = grp[REG_R13];
740 rp->r_r14 = grp[REG_R14];
741 rp->r_r15 = grp[REG_R15];
742 rp->r_trapno = grp[REG_TRAPNO];
743 rp->r_err = grp[REG_ERR];
744 rp->r_rip = grp[REG_RIP];
745 /*
746 * Setting %cs or %ss to anything else is quietly but
747 * quite definitely forbidden!
748 */
749 rp->r_cs = UCS_SEL;
750 rp->r_ss = UDS_SEL;
751 rp->r_rsp = grp[REG_RSP];
752
753 if (thisthread)
754 kpreempt_disable();
755
756 pcb->pcb_ds = UDS_SEL;
757 pcb->pcb_es = UDS_SEL;
758
759 /*
760 * 64-bit processes -are- allowed to set their fsbase/gsbase
761 * values directly, but only if they're using the segment
762 * selectors that allow that semantic.
763 *
764 * (32-bit processes must use lwp_set_private().)
765 */
766 pcb->pcb_fsbase = grp[REG_FSBASE];
767 pcb->pcb_gsbase = grp[REG_GSBASE];
768 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
769 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
770
771 /*
772 * Ensure that we go out via update_sregs
773 */
774 pcb->pcb_rupdate = 1;
775 lwptot(lwp)->t_post_sys = 1;
776 if (thisthread)
777 kpreempt_enable();
778 #if defined(_SYSCALL32_IMPL)
779 } else {
780 rp->r_rdi = (uint32_t)grp[REG_RDI];
781 rp->r_rsi = (uint32_t)grp[REG_RSI];
782 rp->r_rdx = (uint32_t)grp[REG_RDX];
783 rp->r_rcx = (uint32_t)grp[REG_RCX];
784 rp->r_rax = (uint32_t)grp[REG_RAX];
785 rp->r_rbx = (uint32_t)grp[REG_RBX];
786 rp->r_rbp = (uint32_t)grp[REG_RBP];
787 rp->r_trapno = (uint32_t)grp[REG_TRAPNO];
788 rp->r_err = (uint32_t)grp[REG_ERR];
789 rp->r_rip = (uint32_t)grp[REG_RIP];
790
791 rp->r_cs = fix_segreg(grp[REG_CS], IS_CS, datamodel);
792 rp->r_ss = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
793
794 rp->r_rsp = (uint32_t)grp[REG_RSP];
795
796 if (thisthread)
797 kpreempt_disable();
798
799 pcb->pcb_ds = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
800 pcb->pcb_es = fix_segreg(grp[REG_ES], IS_NOT_CS, datamodel);
801
802 /*
803 * (See fsbase/gsbase commentary above)
804 */
805 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
806 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
807
808 /*
809 * Ensure that we go out via update_sregs
810 */
811 pcb->pcb_rupdate = 1;
812 lwptot(lwp)->t_post_sys = 1;
813 if (thisthread)
814 kpreempt_enable();
815 #endif
816 }
817
818 /*
819 * Only certain bits of the flags register can be modified.
820 */
821 rp->r_rfl = (rp->r_rfl & ~PSL_USERMASK) |
822 (grp[REG_RFL] & PSL_USERMASK);
823
824 #elif defined(__i386)
825
826 /*
827 * Only certain bits of the flags register can be modified.
828 */
829 grp[EFL] = (rp->r_efl & ~PSL_USERMASK) | (grp[EFL] & PSL_USERMASK);
830
831 /*
832 * Copy saved registers from user stack.
833 */
834 bcopy(grp, &rp->r_gs, sizeof (gregset_t));
835
836 rp->r_cs = fix_segreg(rp->r_cs, IS_CS, datamodel);
837 rp->r_ss = fix_segreg(rp->r_ss, IS_NOT_CS, datamodel);
838 rp->r_ds = fix_segreg(rp->r_ds, IS_NOT_CS, datamodel);
839 rp->r_es = fix_segreg(rp->r_es, IS_NOT_CS, datamodel);
840 rp->r_fs = fix_segreg(rp->r_fs, IS_NOT_CS, datamodel);
841 rp->r_gs = fix_segreg(rp->r_gs, IS_NOT_CS, datamodel);
842
843 #endif /* __i386 */
844 }
845
846 /*
847 * Determine whether eip is likely to have an interrupt frame
848 * on the stack. We do this by comparing the address to the
849 * range of addresses spanned by several well-known routines.
850 */
851 extern void _interrupt();
852 extern void _allsyscalls();
853 extern void _cmntrap();
854 extern void fakesoftint();
855
856 extern size_t _interrupt_size;
857 extern size_t _allsyscalls_size;
858 extern size_t _cmntrap_size;
859 extern size_t _fakesoftint_size;
860
861 /*
862 * Get a pc-only stacktrace. Used for kmem_alloc() buffer ownership tracking.
863 * Returns MIN(current stack depth, pcstack_limit).
864 */
865 int
866 getpcstack(pc_t *pcstack, int pcstack_limit)
867 {
868 struct frame *fp = (struct frame *)getfp();
869 struct frame *nextfp, *minfp, *stacktop;
870 int depth = 0;
871 int on_intr;
872 uintptr_t pc;
873
874 if ((on_intr = CPU_ON_INTR(CPU)) != 0)
875 stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME));
876 else
877 stacktop = (struct frame *)curthread->t_stk;
878 minfp = fp;
879
880 pc = ((struct regs *)fp)->r_pc;
881
882 while (depth < pcstack_limit) {
883 nextfp = (struct frame *)fp->fr_savfp;
884 pc = fp->fr_savpc;
885 if (nextfp <= minfp || nextfp >= stacktop) {
886 if (on_intr) {
887 /*
888 * Hop from interrupt stack to thread stack.
889 */
890 stacktop = (struct frame *)curthread->t_stk;
891 minfp = (struct frame *)curthread->t_stkbase;
892 on_intr = 0;
893 continue;
894 }
895 break;
896 }
897 pcstack[depth++] = (pc_t)pc;
898 fp = nextfp;
899 minfp = fp;
900 }
901 return (depth);
902 }
903
904 /*
905 * The following ELF header fields are defined as processor-specific
906 * in the V8 ABI:
907 *
908 * e_ident[EI_DATA] encoding of the processor-specific
909 * data in the object file
910 * e_machine processor identification
911 * e_flags processor-specific flags associated
912 * with the file
913 */
914
915 /*
916 * The value of at_flags reflects a platform's cpu module support.
917 * at_flags is used to check for allowing a binary to execute and
918 * is passed as the value of the AT_FLAGS auxiliary vector.
919 */
920 int at_flags = 0;
921
922 /*
923 * Check the processor-specific fields of an ELF header.
924 *
925 * returns 1 if the fields are valid, 0 otherwise
926 */
927 /*ARGSUSED2*/
928 int
929 elfheadcheck(
930 unsigned char e_data,
931 Elf32_Half e_machine,
932 Elf32_Word e_flags)
933 {
934 if (e_data != ELFDATA2LSB)
935 return (0);
936 #if defined(__amd64)
937 if (e_machine == EM_AMD64)
938 return (1);
939 #endif
940 return (e_machine == EM_386);
941 }
942
943 uint_t auxv_hwcap_include = 0; /* patch to enable unrecognized features */
944 uint_t auxv_hwcap_include_2 = 0; /* second word */
945 uint_t auxv_hwcap_exclude = 0; /* patch for broken cpus, debugging */
946 uint_t auxv_hwcap_exclude_2 = 0; /* second word */
947 #if defined(_SYSCALL32_IMPL)
948 uint_t auxv_hwcap32_include = 0; /* ditto for 32-bit apps */
949 uint_t auxv_hwcap32_include_2 = 0; /* ditto for 32-bit apps */
950 uint_t auxv_hwcap32_exclude = 0; /* ditto for 32-bit apps */
951 uint_t auxv_hwcap32_exclude_2 = 0; /* ditto for 32-bit apps */
952 #endif
953
954 /*
955 * Gather information about the processor and place it into auxv_hwcap
956 * so that it can be exported to the linker via the aux vector.
957 *
958 * We use this seemingly complicated mechanism so that we can ensure
959 * that /etc/system can be used to override what the system can or
960 * cannot discover for itself.
961 */
962 void
963 bind_hwcap(void)
964 {
965 uint_t cpu_hwcap_flags[2];
966 cpuid_pass4(NULL, cpu_hwcap_flags);
967
968 auxv_hwcap = (auxv_hwcap_include | cpu_hwcap_flags[0]) &
969 ~auxv_hwcap_exclude;
970 auxv_hwcap_2 = (auxv_hwcap_include_2 | cpu_hwcap_flags[1]) &
971 ~auxv_hwcap_exclude_2;
972
973 #if defined(__amd64)
974 /*
975 * On AMD processors, sysenter just doesn't work at all
976 * when the kernel is in long mode. On IA-32e processors
977 * it does, but there's no real point in all the alternate
978 * mechanism when syscall works on both.
979 *
980 * Besides, the kernel's sysenter handler is expecting a
981 * 32-bit lwp ...
982 */
983 auxv_hwcap &= ~AV_386_SEP;
984 #else
985 /*
986 * 32-bit processes can -always- use the lahf/sahf instructions
987 */
988 auxv_hwcap |= AV_386_AHF;
989 #endif
990
991 if (auxv_hwcap_include || auxv_hwcap_exclude || auxv_hwcap_include_2 ||
992 auxv_hwcap_exclude_2) {
993 /*
994 * The below assignment is regrettably required to get lint
995 * to accept the validity of our format string. The format
996 * string is in fact valid, but whatever intelligence in lint
997 * understands the cmn_err()-specific %b appears to have an
998 * off-by-one error: it (mistakenly) complains about bit
999 * number 32 (even though this is explicitly permitted).
1000 * Normally, one would will away such warnings with a "LINTED"
1001 * directive, but for reasons unclear and unknown, lint
1002 * refuses to be assuaged in this case. Fortunately, lint
1003 * doesn't pretend to have solved the Halting Problem --
1004 * and as soon as the format string is programmatic, it
1005 * knows enough to shut up.
1006 */
1007 char *fmt = "?user ABI extensions: %b\n";
1008 cmn_err(CE_CONT, fmt, auxv_hwcap, FMT_AV_386);
1009 fmt = "?user ABI extensions (word 2): %b\n";
1010 cmn_err(CE_CONT, fmt, auxv_hwcap_2, FMT_AV_386_2);
1011 }
1012
1013 #if defined(_SYSCALL32_IMPL)
1014 auxv_hwcap32 = (auxv_hwcap32_include | cpu_hwcap_flags[0]) &
1015 ~auxv_hwcap32_exclude;
1016 auxv_hwcap32_2 = (auxv_hwcap32_include_2 | cpu_hwcap_flags[1]) &
1017 ~auxv_hwcap32_exclude_2;
1018
1019 #if defined(__amd64)
1020 /*
1021 * If this is an amd64 architecture machine from Intel, then
1022 * syscall -doesn't- work in compatibility mode, only sysenter does.
1023 *
1024 * Sigh.
1025 */
1026 if (!cpuid_syscall32_insn(NULL))
1027 auxv_hwcap32 &= ~AV_386_AMD_SYSC;
1028
1029 /*
1030 * 32-bit processes can -always- use the lahf/sahf instructions
1031 */
1032 auxv_hwcap32 |= AV_386_AHF;
1033 #endif
1034
1035 if (auxv_hwcap32_include || auxv_hwcap32_exclude ||
1036 auxv_hwcap32_include_2 || auxv_hwcap32_exclude_2) {
1037 /*
1038 * See the block comment in the cmn_err() of auxv_hwcap, above.
1039 */
1040 char *fmt = "?32-bit user ABI extensions: %b\n";
1041 cmn_err(CE_CONT, fmt, auxv_hwcap32, FMT_AV_386);
1042 fmt = "?32-bit user ABI extensions (word 2): %b\n";
1043 cmn_err(CE_CONT, fmt, auxv_hwcap32_2, FMT_AV_386_2);
1044 }
1045 #endif
1046 }
1047
1048 /*
1049 * sync_icache() - this is called
1050 * in proc/fs/prusrio.c. x86 has an unified cache and therefore
1051 * this is a nop.
1052 */
1053 /* ARGSUSED */
1054 void
1055 sync_icache(caddr_t addr, uint_t len)
1056 {
1057 /* Do nothing for now */
1058 }
1059
1060 /*ARGSUSED*/
1061 void
1062 sync_data_memory(caddr_t va, size_t len)
1063 {
1064 /* Not implemented for this platform */
1065 }
1066
1067 int
1068 __ipltospl(int ipl)
1069 {
1070 return (ipltospl(ipl));
1071 }
1072
1073 /*
1074 * The panic code invokes panic_saveregs() to record the contents of a
1075 * regs structure into the specified panic_data structure for debuggers.
1076 */
1077 void
1078 panic_saveregs(panic_data_t *pdp, struct regs *rp)
1079 {
1080 panic_nv_t *pnv = PANICNVGET(pdp);
1081
1082 struct cregs creg;
1083
1084 getcregs(&creg);
1085
1086 #if defined(__amd64)
1087 PANICNVADD(pnv, "rdi", rp->r_rdi);
1088 PANICNVADD(pnv, "rsi", rp->r_rsi);
1089 PANICNVADD(pnv, "rdx", rp->r_rdx);
1090 PANICNVADD(pnv, "rcx", rp->r_rcx);
1091 PANICNVADD(pnv, "r8", rp->r_r8);
1092 PANICNVADD(pnv, "r9", rp->r_r9);
1093 PANICNVADD(pnv, "rax", rp->r_rax);
1094 PANICNVADD(pnv, "rbx", rp->r_rbx);
1095 PANICNVADD(pnv, "rbp", rp->r_rbp);
1096 PANICNVADD(pnv, "r10", rp->r_r10);
1097 PANICNVADD(pnv, "r11", rp->r_r11);
1098 PANICNVADD(pnv, "r12", rp->r_r12);
1099 PANICNVADD(pnv, "r13", rp->r_r13);
1100 PANICNVADD(pnv, "r14", rp->r_r14);
1101 PANICNVADD(pnv, "r15", rp->r_r15);
1102 PANICNVADD(pnv, "fsbase", rdmsr(MSR_AMD_FSBASE));
1103 PANICNVADD(pnv, "gsbase", rdmsr(MSR_AMD_GSBASE));
1104 PANICNVADD(pnv, "ds", rp->r_ds);
1105 PANICNVADD(pnv, "es", rp->r_es);
1106 PANICNVADD(pnv, "fs", rp->r_fs);
1107 PANICNVADD(pnv, "gs", rp->r_gs);
1108 PANICNVADD(pnv, "trapno", rp->r_trapno);
1109 PANICNVADD(pnv, "err", rp->r_err);
1110 PANICNVADD(pnv, "rip", rp->r_rip);
1111 PANICNVADD(pnv, "cs", rp->r_cs);
1112 PANICNVADD(pnv, "rflags", rp->r_rfl);
1113 PANICNVADD(pnv, "rsp", rp->r_rsp);
1114 PANICNVADD(pnv, "ss", rp->r_ss);
1115 PANICNVADD(pnv, "gdt_hi", (uint64_t)(creg.cr_gdt._l[3]));
1116 PANICNVADD(pnv, "gdt_lo", (uint64_t)(creg.cr_gdt._l[0]));
1117 PANICNVADD(pnv, "idt_hi", (uint64_t)(creg.cr_idt._l[3]));
1118 PANICNVADD(pnv, "idt_lo", (uint64_t)(creg.cr_idt._l[0]));
1119 #elif defined(__i386)
1120 PANICNVADD(pnv, "gs", (uint32_t)rp->r_gs);
1121 PANICNVADD(pnv, "fs", (uint32_t)rp->r_fs);
1122 PANICNVADD(pnv, "es", (uint32_t)rp->r_es);
1123 PANICNVADD(pnv, "ds", (uint32_t)rp->r_ds);
1124 PANICNVADD(pnv, "edi", (uint32_t)rp->r_edi);
1125 PANICNVADD(pnv, "esi", (uint32_t)rp->r_esi);
1126 PANICNVADD(pnv, "ebp", (uint32_t)rp->r_ebp);
1127 PANICNVADD(pnv, "esp", (uint32_t)rp->r_esp);
1128 PANICNVADD(pnv, "ebx", (uint32_t)rp->r_ebx);
1129 PANICNVADD(pnv, "edx", (uint32_t)rp->r_edx);
1130 PANICNVADD(pnv, "ecx", (uint32_t)rp->r_ecx);
1131 PANICNVADD(pnv, "eax", (uint32_t)rp->r_eax);
1132 PANICNVADD(pnv, "trapno", (uint32_t)rp->r_trapno);
1133 PANICNVADD(pnv, "err", (uint32_t)rp->r_err);
1134 PANICNVADD(pnv, "eip", (uint32_t)rp->r_eip);
1135 PANICNVADD(pnv, "cs", (uint32_t)rp->r_cs);
1136 PANICNVADD(pnv, "eflags", (uint32_t)rp->r_efl);
1137 PANICNVADD(pnv, "uesp", (uint32_t)rp->r_uesp);
1138 PANICNVADD(pnv, "ss", (uint32_t)rp->r_ss);
1139 PANICNVADD(pnv, "gdt", creg.cr_gdt);
1140 PANICNVADD(pnv, "idt", creg.cr_idt);
1141 #endif /* __i386 */
1142
1143 PANICNVADD(pnv, "ldt", creg.cr_ldt);
1144 PANICNVADD(pnv, "task", creg.cr_task);
1145 PANICNVADD(pnv, "cr0", creg.cr_cr0);
1146 PANICNVADD(pnv, "cr2", creg.cr_cr2);
1147 PANICNVADD(pnv, "cr3", creg.cr_cr3);
1148 if (creg.cr_cr4)
1149 PANICNVADD(pnv, "cr4", creg.cr_cr4);
1150
1151 PANICNVSET(pdp, pnv);
1152 }
1153
1154 #define TR_ARG_MAX 6 /* Max args to print, same as SPARC */
1155
1156 #if !defined(__amd64)
1157
1158 /*
1159 * Given a return address (%eip), determine the likely number of arguments
1160 * that were pushed on the stack prior to its execution. We do this by
1161 * expecting that a typical call sequence consists of pushing arguments on
1162 * the stack, executing a call instruction, and then performing an add
1163 * on %esp to restore it to the value prior to pushing the arguments for
1164 * the call. We attempt to detect such an add, and divide the addend
1165 * by the size of a word to determine the number of pushed arguments.
1166 *
1167 * If we do not find such an add, we punt and return TR_ARG_MAX. It is not
1168 * possible to reliably determine if a function took no arguments (i.e. was
1169 * void) because assembler routines do not reliably perform an add on %esp
1170 * immediately upon returning (eg. _sys_call()), so returning TR_ARG_MAX is
1171 * safer than returning 0.
1172 */
1173 static ulong_t
1174 argcount(uintptr_t eip)
1175 {
1176 const uint8_t *ins = (const uint8_t *)eip;
1177 ulong_t n;
1178
1179 enum {
1180 M_MODRM_ESP = 0xc4, /* Mod/RM byte indicates %esp */
1181 M_ADD_IMM32 = 0x81, /* ADD imm32 to r/m32 */
1182 M_ADD_IMM8 = 0x83 /* ADD imm8 to r/m32 */
1183 };
1184
1185 if (eip < KERNELBASE || ins[1] != M_MODRM_ESP)
1186 return (TR_ARG_MAX);
1187
1188 switch (ins[0]) {
1189 case M_ADD_IMM32:
1190 n = ins[2] + (ins[3] << 8) + (ins[4] << 16) + (ins[5] << 24);
1191 break;
1192
1193 case M_ADD_IMM8:
1194 n = ins[2];
1195 break;
1196
1197 default:
1198 return (TR_ARG_MAX);
1199 }
1200
1201 n /= sizeof (long);
1202 return (MIN(n, TR_ARG_MAX));
1203 }
1204
1205 #endif /* !__amd64 */
1206
1207 /*
1208 * Print a stack backtrace using the specified frame pointer. We delay two
1209 * seconds before continuing, unless this is the panic traceback.
1210 * If we are in the process of panicking, we also attempt to write the
1211 * stack backtrace to a staticly assigned buffer, to allow the panic
1212 * code to find it and write it in to uncompressed pages within the
1213 * system crash dump.
1214 * Note that the frame for the starting stack pointer value is omitted because
1215 * the corresponding %eip is not known.
1216 */
1217
1218 extern char *dump_stack_scratch;
1219
1220 #if defined(__amd64)
1221
1222 void
1223 traceback(caddr_t fpreg)
1224 {
1225 struct frame *fp = (struct frame *)fpreg;
1226 struct frame *nextfp;
1227 uintptr_t pc, nextpc;
1228 ulong_t off;
1229 char args[TR_ARG_MAX * 2 + 16], *sym;
1230 uint_t offset = 0;
1231 uint_t next_offset = 0;
1232 char stack_buffer[1024];
1233
1234 if (!panicstr)
1235 printf("traceback: %%fp = %p\n", (void *)fp);
1236
1237 if (panicstr && !dump_stack_scratch) {
1238 printf("Warning - stack not written to the dump buffer\n");
1239 }
1240
1241 fp = (struct frame *)plat_traceback(fpreg);
1242 if ((uintptr_t)fp < KERNELBASE)
1243 goto out;
1244
1245 pc = fp->fr_savpc;
1246 fp = (struct frame *)fp->fr_savfp;
1247
1248 while ((uintptr_t)fp >= KERNELBASE) {
1249 /*
1250 * XX64 Until port is complete tolerate 8-byte aligned
1251 * frame pointers but flag with a warning so they can
1252 * be fixed.
1253 */
1254 if (((uintptr_t)fp & (STACK_ALIGN - 1)) != 0) {
1255 if (((uintptr_t)fp & (8 - 1)) == 0) {
1256 printf(" >> warning! 8-byte"
1257 " aligned %%fp = %p\n", (void *)fp);
1258 } else {
1259 printf(
1260 " >> mis-aligned %%fp = %p\n", (void *)fp);
1261 break;
1262 }
1263 }
1264
1265 args[0] = '\0';
1266 nextpc = (uintptr_t)fp->fr_savpc;
1267 nextfp = (struct frame *)fp->fr_savfp;
1268 if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1269 printf("%016lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1270 mod_containing_pc((caddr_t)pc), sym, off, args);
1271 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1272 "%s:%s+%lx (%s) | ",
1273 mod_containing_pc((caddr_t)pc), sym, off, args);
1274 } else {
1275 printf("%016lx %lx (%s)\n",
1276 (uintptr_t)fp, pc, args);
1277 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1278 "%lx (%s) | ", pc, args);
1279 }
1280
1281 if (panicstr && dump_stack_scratch) {
1282 next_offset = offset + strlen(stack_buffer);
1283 if (next_offset < STACK_BUF_SIZE) {
1284 bcopy(stack_buffer, dump_stack_scratch + offset,
1285 strlen(stack_buffer));
1286 offset = next_offset;
1287 } else {
1288 /*
1289 * In attempting to save the panic stack
1290 * to the dumpbuf we have overflowed that area.
1291 * Print a warning and continue to printf the
1292 * stack to the msgbuf
1293 */
1294 printf("Warning: stack in the dump buffer"
1295 " may be incomplete\n");
1296 offset = next_offset;
1297 }
1298 }
1299
1300 pc = nextpc;
1301 fp = nextfp;
1302 }
1303 out:
1304 if (!panicstr) {
1305 printf("end of traceback\n");
1306 DELAY(2 * MICROSEC);
1307 } else if (dump_stack_scratch) {
1308 dump_stack_scratch[offset] = '\0';
1309 }
1310 }
1311
1312 #elif defined(__i386)
1313
1314 void
1315 traceback(caddr_t fpreg)
1316 {
1317 struct frame *fp = (struct frame *)fpreg;
1318 struct frame *nextfp, *minfp, *stacktop;
1319 uintptr_t pc, nextpc;
1320 uint_t offset = 0;
1321 uint_t next_offset = 0;
1322 char stack_buffer[1024];
1323
1324 cpu_t *cpu;
1325
1326 /*
1327 * args[] holds TR_ARG_MAX hex long args, plus ", " or '\0'.
1328 */
1329 char args[TR_ARG_MAX * 2 + 8], *p;
1330
1331 int on_intr;
1332 ulong_t off;
1333 char *sym;
1334
1335 if (!panicstr)
1336 printf("traceback: %%fp = %p\n", (void *)fp);
1337
1338 if (panicstr && !dump_stack_scratch) {
1339 printf("Warning - stack not written to the dumpbuf\n");
1340 }
1341
1342 /*
1343 * If we are panicking, all high-level interrupt information in
1344 * CPU was overwritten. panic_cpu has the correct values.
1345 */
1346 kpreempt_disable(); /* prevent migration */
1347
1348 cpu = (panicstr && CPU->cpu_id == panic_cpu.cpu_id)? &panic_cpu : CPU;
1349
1350 if ((on_intr = CPU_ON_INTR(cpu)) != 0)
1351 stacktop = (struct frame *)(cpu->cpu_intr_stack + SA(MINFRAME));
1352 else
1353 stacktop = (struct frame *)curthread->t_stk;
1354
1355 kpreempt_enable();
1356
1357 fp = (struct frame *)plat_traceback(fpreg);
1358 if ((uintptr_t)fp < KERNELBASE)
1359 goto out;
1360
1361 minfp = fp; /* Baseline minimum frame pointer */
1362 pc = fp->fr_savpc;
1363 fp = (struct frame *)fp->fr_savfp;
1364
1365 while ((uintptr_t)fp >= KERNELBASE) {
1366 ulong_t argc;
1367 long *argv;
1368
1369 if (fp <= minfp || fp >= stacktop) {
1370 if (on_intr) {
1371 /*
1372 * Hop from interrupt stack to thread stack.
1373 */
1374 stacktop = (struct frame *)curthread->t_stk;
1375 minfp = (struct frame *)curthread->t_stkbase;
1376 on_intr = 0;
1377 continue;
1378 }
1379 break; /* we're outside of the expected range */
1380 }
1381
1382 if ((uintptr_t)fp & (STACK_ALIGN - 1)) {
1383 printf(" >> mis-aligned %%fp = %p\n", (void *)fp);
1384 break;
1385 }
1386
1387 nextpc = fp->fr_savpc;
1388 nextfp = (struct frame *)fp->fr_savfp;
1389 argc = argcount(nextpc);
1390 argv = (long *)((char *)fp + sizeof (struct frame));
1391
1392 args[0] = '\0';
1393 p = args;
1394 while (argc-- > 0 && argv < (long *)stacktop) {
1395 p += snprintf(p, args + sizeof (args) - p,
1396 "%s%lx", (p == args) ? "" : ", ", *argv++);
1397 }
1398
1399 if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1400 printf("%08lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1401 mod_containing_pc((caddr_t)pc), sym, off, args);
1402 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1403 "%s:%s+%lx (%s) | ",
1404 mod_containing_pc((caddr_t)pc), sym, off, args);
1405
1406 } else {
1407 printf("%08lx %lx (%s)\n",
1408 (uintptr_t)fp, pc, args);
1409 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1410 "%lx (%s) | ", pc, args);
1411
1412 }
1413
1414 if (panicstr && dump_stack_scratch) {
1415 next_offset = offset + strlen(stack_buffer);
1416 if (next_offset < STACK_BUF_SIZE) {
1417 bcopy(stack_buffer, dump_stack_scratch + offset,
1418 strlen(stack_buffer));
1419 offset = next_offset;
1420 } else {
1421 /*
1422 * In attempting to save the panic stack
1423 * to the dumpbuf we have overflowed that area.
1424 * Print a warning and continue to printf the
1425 * stack to the msgbuf
1426 */
1427 printf("Warning: stack in the dumpbuf"
1428 " may be incomplete\n");
1429 offset = next_offset;
1430 }
1431 }
1432
1433 minfp = fp;
1434 pc = nextpc;
1435 fp = nextfp;
1436 }
1437 out:
1438 if (!panicstr) {
1439 printf("end of traceback\n");
1440 DELAY(2 * MICROSEC);
1441 } else if (dump_stack_scratch) {
1442 dump_stack_scratch[offset] = '\0';
1443 }
1444
1445 }
1446
1447 #endif /* __i386 */
1448
1449 /*
1450 * Generate a stack backtrace from a saved register set.
1451 */
1452 void
1453 traceregs(struct regs *rp)
1454 {
1455 traceback((caddr_t)rp->r_fp);
1456 }
1457
1458 void
1459 exec_set_sp(size_t stksize)
1460 {
1461 klwp_t *lwp = ttolwp(curthread);
1462
1463 lwptoregs(lwp)->r_sp = (uintptr_t)curproc->p_usrstack - stksize;
1464 }
1465
1466 hrtime_t
1467 gethrtime_waitfree(void)
1468 {
1469 return (dtrace_gethrtime());
1470 }
1471
1472 hrtime_t
1473 gethrtime(void)
1474 {
1475 return (gethrtimef());
1476 }
1477
1478 hrtime_t
1479 gethrtime_unscaled(void)
1480 {
1481 return (gethrtimeunscaledf());
1482 }
1483
1484 void
1485 scalehrtime(hrtime_t *hrt)
1486 {
1487 scalehrtimef(hrt);
1488 }
1489
1490 uint64_t
1491 unscalehrtime(hrtime_t nsecs)
1492 {
1493 return (unscalehrtimef(nsecs));
1494 }
1495
1496 void
1497 gethrestime(timespec_t *tp)
1498 {
1499 gethrestimef(tp);
1500 }
1501
1502 #if defined(__amd64)
1503 /*
1504 * Part of the implementation of hres_tick(); this routine is
1505 * easier in C than assembler .. called with the hres_lock held.
1506 *
1507 * XX64 Many of these timekeeping variables need to be extern'ed in a header
1508 */
1509
1510 #include <sys/time.h>
1511 #include <sys/machlock.h>
1512
1513 extern int one_sec;
1514 extern int max_hres_adj;
1515
1516 void
1517 __adj_hrestime(void)
1518 {
1519 long long adj;
1520
1521 if (hrestime_adj == 0)
1522 adj = 0;
1523 else if (hrestime_adj > 0) {
1524 if (hrestime_adj < max_hres_adj)
1525 adj = hrestime_adj;
1526 else
1527 adj = max_hres_adj;
1528 } else {
1529 if (hrestime_adj < -max_hres_adj)
1530 adj = -max_hres_adj;
1531 else
1532 adj = hrestime_adj;
1533 }
1534
1535 timedelta -= adj;
1536 hrestime_adj = timedelta;
1537 hrestime.tv_nsec += adj;
1538
1539 while (hrestime.tv_nsec >= NANOSEC) {
1540 one_sec++;
1541 hrestime.tv_sec++;
1542 hrestime.tv_nsec -= NANOSEC;
1543 }
1544 }
1545 #endif
1546
1547 /*
1548 * Wrapper functions to maintain backwards compability
1549 */
1550 int
1551 xcopyin(const void *uaddr, void *kaddr, size_t count)
1552 {
1553 return (xcopyin_nta(uaddr, kaddr, count, UIO_COPY_CACHED));
1554 }
1555
1556 int
1557 xcopyout(const void *kaddr, void *uaddr, size_t count)
1558 {
1559 return (xcopyout_nta(kaddr, uaddr, count, UIO_COPY_CACHED));
1560 }