1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2019 Joyent, Inc. 24 */ 25 26 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */ 28 /* All Rights Reserved */ 29 30 #include <sys/types.h> 31 #include <sys/param.h> 32 #include <sys/sysmacros.h> 33 #include <sys/signal.h> 34 #include <sys/systm.h> 35 #include <sys/user.h> 36 #include <sys/mman.h> 37 #include <sys/class.h> 38 #include <sys/proc.h> 39 #include <sys/procfs.h> 40 #include <sys/buf.h> 41 #include <sys/kmem.h> 42 #include <sys/cred.h> 43 #include <sys/archsystm.h> 44 #include <sys/vmparam.h> 45 #include <sys/prsystm.h> 46 #include <sys/reboot.h> 47 #include <sys/uadmin.h> 48 #include <sys/vfs.h> 49 #include <sys/vnode.h> 50 #include <sys/file.h> 51 #include <sys/session.h> 52 #include <sys/ucontext.h> 53 #include <sys/dnlc.h> 54 #include <sys/var.h> 55 #include <sys/cmn_err.h> 56 #include <sys/debugreg.h> 57 #include <sys/thread.h> 58 #include <sys/vtrace.h> 59 #include <sys/consdev.h> 60 #include <sys/psw.h> 61 #include <sys/regset.h> 62 #include <sys/privregs.h> 63 #include <sys/cpu.h> 64 #include <sys/stack.h> 65 #include <sys/swap.h> 66 #include <vm/hat.h> 67 #include <vm/anon.h> 68 #include <vm/as.h> 69 #include <vm/page.h> 70 #include <vm/seg.h> 71 #include <vm/seg_kmem.h> 72 #include <vm/seg_map.h> 73 #include <vm/seg_vn.h> 74 #include <sys/exec.h> 75 #include <sys/acct.h> 76 #include <sys/core.h> 77 #include <sys/corectl.h> 78 #include <sys/modctl.h> 79 #include <sys/tuneable.h> 80 #include <c2/audit.h> 81 #include <sys/bootconf.h> 82 #include <sys/brand.h> 83 #include <sys/dumphdr.h> 84 #include <sys/promif.h> 85 #include <sys/systeminfo.h> 86 #include <sys/kdi.h> 87 #include <sys/contract_impl.h> 88 #include <sys/x86_archext.h> 89 #include <sys/segments.h> 90 #include <sys/ontrap.h> 91 #include <sys/cpu.h> 92 #ifdef __xpv 93 #include <sys/hypervisor.h> 94 #endif 95 96 /* 97 * Compare the version of boot that boot says it is against 98 * the version of boot the kernel expects. 99 */ 100 int 101 check_boot_version(int boots_version) 102 { 103 if (boots_version == BO_VERSION) 104 return (0); 105 106 prom_printf("Wrong boot interface - kernel needs v%d found v%d\n", 107 BO_VERSION, boots_version); 108 prom_panic("halting"); 109 /*NOTREACHED*/ 110 } 111 112 /* 113 * Process the physical installed list for boot. 114 * Finds: 115 * 1) the pfn of the highest installed physical page, 116 * 2) the number of pages installed 117 * 3) the number of distinct contiguous regions these pages fall into. 118 * 4) the number of contiguous memory ranges 119 */ 120 void 121 installed_top_size_ex( 122 struct memlist *list, /* pointer to start of installed list */ 123 pfn_t *high_pfn, /* return ptr for top value */ 124 pgcnt_t *pgcnt, /* return ptr for sum of installed pages */ 125 int *ranges) /* return ptr for the count of contig. ranges */ 126 { 127 pfn_t top = 0; 128 pgcnt_t sumpages = 0; 129 pfn_t highp; /* high page in a chunk */ 130 int cnt = 0; 131 132 for (; list; list = list->ml_next) { 133 ++cnt; 134 highp = (list->ml_address + list->ml_size - 1) >> PAGESHIFT; 135 if (top < highp) 136 top = highp; 137 sumpages += btop(list->ml_size); 138 } 139 140 *high_pfn = top; 141 *pgcnt = sumpages; 142 *ranges = cnt; 143 } 144 145 void 146 installed_top_size( 147 struct memlist *list, /* pointer to start of installed list */ 148 pfn_t *high_pfn, /* return ptr for top value */ 149 pgcnt_t *pgcnt) /* return ptr for sum of installed pages */ 150 { 151 int ranges; 152 153 installed_top_size_ex(list, high_pfn, pgcnt, &ranges); 154 } 155 156 void 157 phys_install_has_changed(void) 158 {} 159 160 /* 161 * Copy in a memory list from boot to kernel, with a filter function 162 * to remove pages. The filter function can increase the address and/or 163 * decrease the size to filter out pages. It will also align addresses and 164 * sizes to PAGESIZE. 165 */ 166 void 167 copy_memlist_filter( 168 struct memlist *src, 169 struct memlist **dstp, 170 void (*filter)(uint64_t *, uint64_t *)) 171 { 172 struct memlist *dst, *prev; 173 uint64_t addr; 174 uint64_t size; 175 uint64_t eaddr; 176 177 dst = *dstp; 178 prev = dst; 179 180 /* 181 * Move through the memlist applying a filter against 182 * each range of memory. Note that we may apply the 183 * filter multiple times against each memlist entry. 184 */ 185 for (; src; src = src->ml_next) { 186 addr = P2ROUNDUP(src->ml_address, PAGESIZE); 187 eaddr = P2ALIGN(src->ml_address + src->ml_size, PAGESIZE); 188 while (addr < eaddr) { 189 size = eaddr - addr; 190 if (filter != NULL) 191 filter(&addr, &size); 192 if (size == 0) 193 break; 194 dst->ml_address = addr; 195 dst->ml_size = size; 196 dst->ml_next = 0; 197 if (prev == dst) { 198 dst->ml_prev = 0; 199 dst++; 200 } else { 201 dst->ml_prev = prev; 202 prev->ml_next = dst; 203 dst++; 204 prev++; 205 } 206 addr += size; 207 } 208 } 209 210 *dstp = dst; 211 } 212 213 /* 214 * Kernel setup code, called from startup(). 215 */ 216 void 217 kern_setup1(void) 218 { 219 proc_t *pp; 220 221 pp = &p0; 222 223 proc_sched = pp; 224 225 /* 226 * Initialize process 0 data structures 227 */ 228 pp->p_stat = SRUN; 229 pp->p_flag = SSYS; 230 231 pp->p_pidp = &pid0; 232 pp->p_pgidp = &pid0; 233 pp->p_sessp = &session0; 234 pp->p_tlist = &t0; 235 pid0.pid_pglink = pp; 236 pid0.pid_pgtail = pp; 237 238 /* 239 * XXX - we asssume that the u-area is zeroed out except for 240 * ttolwp(curthread)->lwp_regs. 241 */ 242 PTOU(curproc)->u_cmask = (mode_t)CMASK; 243 244 thread_init(); /* init thread_free list */ 245 pid_init(); /* initialize pid (proc) table */ 246 contract_init(); /* initialize contracts */ 247 248 init_pages_pp_maximum(); 249 } 250 251 /* 252 * Load a procedure into a thread. 253 */ 254 void 255 thread_load(kthread_t *t, void (*start)(), caddr_t arg, size_t len) 256 { 257 caddr_t sp; 258 size_t framesz; 259 caddr_t argp; 260 long *p; 261 extern void thread_start(); 262 263 /* 264 * Push a "c" call frame onto the stack to represent 265 * the caller of "start". 266 */ 267 sp = t->t_stk; 268 ASSERT(((uintptr_t)t->t_stk & (STACK_ENTRY_ALIGN - 1)) == 0); 269 if (len != 0) { 270 /* 271 * the object that arg points at is copied into the 272 * caller's frame. 273 */ 274 framesz = SA(len); 275 sp -= framesz; 276 ASSERT(sp > t->t_stkbase); 277 argp = sp + SA(MINFRAME); 278 bcopy(arg, argp, len); 279 arg = argp; 280 } 281 /* 282 * Set up arguments (arg and len) on the caller's stack frame. 283 */ 284 p = (long *)sp; 285 286 *--p = 0; /* fake call */ 287 *--p = 0; /* null frame pointer terminates stack trace */ 288 *--p = (long)len; 289 *--p = (intptr_t)arg; 290 *--p = (intptr_t)start; 291 292 /* 293 * initialize thread to resume at thread_start() which will 294 * turn around and invoke (*start)(arg, len). 295 */ 296 t->t_pc = (uintptr_t)thread_start; 297 t->t_sp = (uintptr_t)p; 298 299 ASSERT((t->t_sp & (STACK_ENTRY_ALIGN - 1)) == 0); 300 } 301 302 /* 303 * load user registers into lwp. 304 */ 305 /*ARGSUSED2*/ 306 void 307 lwp_load(klwp_t *lwp, gregset_t grp, uintptr_t thrptr) 308 { 309 struct regs *rp = lwptoregs(lwp); 310 311 setgregs(lwp, grp); 312 rp->r_ps = PSL_USER; 313 314 /* 315 * For 64-bit lwps, we allow one magic %fs selector value, and one 316 * magic %gs selector to point anywhere in the address space using 317 * %fsbase and %gsbase behind the scenes. libc uses %fs to point 318 * at the ulwp_t structure. 319 * 320 * For 32-bit lwps, libc wedges its lwp thread pointer into the 321 * ucontext ESP slot (which is otherwise irrelevant to setting a 322 * ucontext) and LWPGS_SEL value into gregs[REG_GS]. This is so 323 * syslwp_create() can atomically setup %gs. 324 * 325 * See setup_context() in libc. 326 */ 327 #ifdef _SYSCALL32_IMPL 328 if (lwp_getdatamodel(lwp) == DATAMODEL_ILP32) { 329 if (grp[REG_GS] == LWPGS_SEL) 330 (void) lwp_setprivate(lwp, _LWP_GSBASE, thrptr); 331 } else { 332 /* 333 * See lwp_setprivate in kernel and setup_context in libc. 334 * 335 * Currently libc constructs a ucontext from whole cloth for 336 * every new (not main) lwp created. For 64 bit processes 337 * %fsbase is directly set to point to current thread pointer. 338 * In the past (solaris 10) %fs was also set LWPFS_SEL to 339 * indicate %fsbase. Now we use the null GDT selector for 340 * this purpose. LWP[FS|GS]_SEL are only intended for 32 bit 341 * processes. To ease transition we support older libcs in 342 * the newer kernel by forcing %fs or %gs selector to null 343 * by calling lwp_setprivate if LWP[FS|GS]_SEL is passed in 344 * the ucontext. This is should be ripped out at some future 345 * date. Another fix would be for libc to do a getcontext 346 * and inherit the null %fs/%gs from the current context but 347 * that means an extra system call and could hurt performance. 348 */ 349 if (grp[REG_FS] == 0x1bb) /* hard code legacy LWPFS_SEL */ 350 (void) lwp_setprivate(lwp, _LWP_FSBASE, 351 (uintptr_t)grp[REG_FSBASE]); 352 353 if (grp[REG_GS] == 0x1c3) /* hard code legacy LWPGS_SEL */ 354 (void) lwp_setprivate(lwp, _LWP_GSBASE, 355 (uintptr_t)grp[REG_GSBASE]); 356 } 357 #else 358 if (grp[GS] == LWPGS_SEL) 359 (void) lwp_setprivate(lwp, _LWP_GSBASE, thrptr); 360 #endif 361 362 lwp->lwp_eosys = JUSTRETURN; 363 lwptot(lwp)->t_post_sys = 1; 364 } 365 366 /* 367 * set syscall()'s return values for a lwp. 368 */ 369 void 370 lwp_setrval(klwp_t *lwp, int v1, int v2) 371 { 372 lwptoregs(lwp)->r_ps &= ~PS_C; 373 lwptoregs(lwp)->r_r0 = v1; 374 lwptoregs(lwp)->r_r1 = v2; 375 } 376 377 /* 378 * set syscall()'s return values for a lwp. 379 */ 380 void 381 lwp_setsp(klwp_t *lwp, caddr_t sp) 382 { 383 lwptoregs(lwp)->r_sp = (intptr_t)sp; 384 } 385 386 /* 387 * Copy regs from parent to child. 388 */ 389 void 390 lwp_forkregs(klwp_t *lwp, klwp_t *clwp) 391 { 392 #if defined(__amd64) 393 struct pcb *pcb = &clwp->lwp_pcb; 394 struct regs *rp = lwptoregs(lwp); 395 396 if (!PCB_NEED_UPDATE_SEGS(pcb)) { 397 pcb->pcb_ds = rp->r_ds; 398 pcb->pcb_es = rp->r_es; 399 pcb->pcb_fs = rp->r_fs; 400 pcb->pcb_gs = rp->r_gs; 401 PCB_SET_UPDATE_SEGS(pcb); 402 lwptot(clwp)->t_post_sys = 1; 403 } 404 ASSERT(lwptot(clwp)->t_post_sys); 405 #endif 406 407 fp_lwp_dup(clwp); 408 409 bcopy(lwp->lwp_regs, clwp->lwp_regs, sizeof (struct regs)); 410 } 411 412 /* 413 * This function is currently unused on x86. 414 */ 415 /*ARGSUSED*/ 416 void 417 lwp_freeregs(klwp_t *lwp, int isexec) 418 {} 419 420 /* 421 * This function is currently unused on x86. 422 */ 423 void 424 lwp_pcb_exit(void) 425 {} 426 427 /* 428 * Lwp context ops for segment registers. 429 */ 430 431 /* 432 * Every time we come into the kernel (syscall, interrupt or trap 433 * but not fast-traps) we capture the current values of the user's 434 * segment registers into the lwp's reg structure. This includes 435 * lcall for i386 generic system call support since it is handled 436 * as a segment-not-present trap. 437 * 438 * Here we save the current values from the lwp regs into the pcb 439 * and or PCB_UPDATE_SEGS (1) in pcb->pcb_rupdate to tell the rest 440 * of the kernel that the pcb copy of the segment registers is the 441 * current one. This ensures the lwp's next trip to user land via 442 * update_sregs. Finally we set t_post_sys to ensure that no 443 * system call fast-path's its way out of the kernel via sysret. 444 * 445 * (This means that we need to have interrupts disabled when we 446 * test t->t_post_sys in the syscall handlers; if the test fails, 447 * we need to keep interrupts disabled until we return to userland 448 * so we can't be switched away.) 449 * 450 * As a result of all this, we don't really have to do a whole lot 451 * if the thread is just mucking about in the kernel, switching on 452 * and off the cpu for whatever reason it feels like. And yet we 453 * still preserve fast syscalls, cause if we -don't- get 454 * descheduled, we never come here either. 455 */ 456 457 #define VALID_LWP_DESC(udp) ((udp)->usd_type == SDT_MEMRWA && \ 458 (udp)->usd_p == 1 && (udp)->usd_dpl == SEL_UPL) 459 460 /*ARGSUSED*/ 461 void 462 lwp_segregs_save(klwp_t *lwp) 463 { 464 #if defined(__amd64) 465 pcb_t *pcb = &lwp->lwp_pcb; 466 struct regs *rp; 467 468 ASSERT(VALID_LWP_DESC(&pcb->pcb_fsdesc)); 469 ASSERT(VALID_LWP_DESC(&pcb->pcb_gsdesc)); 470 471 if (!PCB_NEED_UPDATE_SEGS(pcb)) { 472 rp = lwptoregs(lwp); 473 474 /* 475 * If there's no update already pending, capture the current 476 * %ds/%es/%fs/%gs values from lwp's regs in case the user 477 * changed them; %fsbase and %gsbase are privileged so the 478 * kernel versions of these registers in pcb_fsbase and 479 * pcb_gsbase are always up-to-date. 480 */ 481 pcb->pcb_ds = rp->r_ds; 482 pcb->pcb_es = rp->r_es; 483 pcb->pcb_fs = rp->r_fs; 484 pcb->pcb_gs = rp->r_gs; 485 PCB_SET_UPDATE_SEGS(pcb); 486 lwp->lwp_thread->t_post_sys = 1; 487 } 488 #endif /* __amd64 */ 489 490 #if !defined(__xpv) /* XXPV not sure if we can re-read gdt? */ 491 ASSERT(bcmp(&CPU->cpu_gdt[GDT_LWPFS], &lwp->lwp_pcb.pcb_fsdesc, 492 sizeof (lwp->lwp_pcb.pcb_fsdesc)) == 0); 493 ASSERT(bcmp(&CPU->cpu_gdt[GDT_LWPGS], &lwp->lwp_pcb.pcb_gsdesc, 494 sizeof (lwp->lwp_pcb.pcb_gsdesc)) == 0); 495 #endif 496 } 497 498 #if defined(__amd64) 499 500 /* 501 * Update the segment registers with new values from the pcb. 502 * 503 * We have to do this carefully, and in the following order, 504 * in case any of the selectors points at a bogus descriptor. 505 * If they do, we'll catch trap with on_trap and return 1. 506 * returns 0 on success. 507 * 508 * This is particularly tricky for %gs. 509 * This routine must be executed under a cli. 510 */ 511 int 512 update_sregs(struct regs *rp, klwp_t *lwp) 513 { 514 pcb_t *pcb = &lwp->lwp_pcb; 515 ulong_t kgsbase; 516 on_trap_data_t otd; 517 int rc = 0; 518 519 if (!on_trap(&otd, OT_SEGMENT_ACCESS)) { 520 521 #if defined(__xpv) 522 /* 523 * On the hyervisor this is easy. The hypercall below will 524 * swapgs and load %gs with the user selector. If the user 525 * selector is bad the hypervisor will catch the fault and 526 * load %gs with the null selector instead. Either way the 527 * kernel's gsbase is not damaged. 528 */ 529 kgsbase = (ulong_t)CPU; 530 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, 531 pcb->pcb_gs) != 0) { 532 no_trap(); 533 return (1); 534 } 535 536 rp->r_gs = pcb->pcb_gs; 537 ASSERT((cpu_t *)kgsbase == CPU); 538 539 #else /* __xpv */ 540 541 /* 542 * A little more complicated running native. 543 */ 544 kgsbase = (ulong_t)CPU; 545 __set_gs(pcb->pcb_gs); 546 547 /* 548 * If __set_gs fails it's because the new %gs is a bad %gs, 549 * we'll be taking a trap but with the original %gs and %gsbase 550 * undamaged (i.e. pointing at curcpu). 551 * 552 * We've just mucked up the kernel's gsbase. Oops. In 553 * particular we can't take any traps at all. Make the newly 554 * computed gsbase be the hidden gs via swapgs, and fix 555 * the kernel's gsbase back again. Later, when we return to 556 * userland we'll swapgs again restoring gsbase just loaded 557 * above. 558 */ 559 __asm__ __volatile__("mfence; swapgs"); 560 561 rp->r_gs = pcb->pcb_gs; 562 563 /* 564 * Restore kernel's gsbase. Note that this also serializes any 565 * attempted speculation from loading the user-controlled 566 * %gsbase. 567 */ 568 wrmsr(MSR_AMD_GSBASE, kgsbase); 569 570 #endif /* __xpv */ 571 572 /* 573 * Only override the descriptor base address if 574 * r_gs == LWPGS_SEL or if r_gs == NULL. A note on 575 * NULL descriptors -- 32-bit programs take faults 576 * if they deference NULL descriptors; however, 577 * when 64-bit programs load them into %fs or %gs, 578 * they DONT fault -- only the base address remains 579 * whatever it was from the last load. Urk. 580 * 581 * XXX - note that lwp_setprivate now sets %fs/%gs to the 582 * null selector for 64 bit processes. Whereas before 583 * %fs/%gs were set to LWP(FS|GS)_SEL regardless of 584 * the process's data model. For now we check for both 585 * values so that the kernel can also support the older 586 * libc. This should be ripped out at some point in the 587 * future. 588 */ 589 if (pcb->pcb_gs == LWPGS_SEL || pcb->pcb_gs == 0) { 590 #if defined(__xpv) 591 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER, 592 pcb->pcb_gsbase)) { 593 no_trap(); 594 return (1); 595 } 596 #else 597 wrmsr(MSR_AMD_KGSBASE, pcb->pcb_gsbase); 598 #endif 599 } 600 601 __set_ds(pcb->pcb_ds); 602 rp->r_ds = pcb->pcb_ds; 603 604 __set_es(pcb->pcb_es); 605 rp->r_es = pcb->pcb_es; 606 607 __set_fs(pcb->pcb_fs); 608 rp->r_fs = pcb->pcb_fs; 609 610 /* 611 * Same as for %gs 612 */ 613 if (pcb->pcb_fs == LWPFS_SEL || pcb->pcb_fs == 0) { 614 #if defined(__xpv) 615 if (HYPERVISOR_set_segment_base(SEGBASE_FS, 616 pcb->pcb_fsbase)) { 617 no_trap(); 618 return (1); 619 } 620 #else 621 wrmsr(MSR_AMD_FSBASE, pcb->pcb_fsbase); 622 #endif 623 } 624 625 } else { 626 cli(); 627 rc = 1; 628 } 629 no_trap(); 630 return (rc); 631 } 632 633 /* 634 * Make sure any stale selectors are cleared from the segment registers 635 * by putting KDS_SEL (the kernel's default %ds gdt selector) into them. 636 * This is necessary because the kernel itself does not use %es, %fs, nor 637 * %ds. (%cs and %ss are necessary, and are set up by the kernel - along with 638 * %gs - to point to the current cpu struct.) If we enter kmdb while in the 639 * kernel and resume with a stale ldt or brandz selector sitting there in a 640 * segment register, kmdb will #gp fault if the stale selector points to, 641 * for example, an ldt in the context of another process. 642 * 643 * WARNING: Intel and AMD chips behave differently when storing 644 * the null selector into %fs and %gs while in long mode. On AMD 645 * chips fsbase and gsbase are not cleared. But on Intel chips, storing 646 * a null selector into %fs or %gs has the side effect of clearing 647 * fsbase or gsbase. For that reason we use KDS_SEL, which has 648 * consistent behavor between AMD and Intel. 649 * 650 * Caller responsible for preventing cpu migration. 651 */ 652 void 653 reset_sregs(void) 654 { 655 ulong_t kgsbase = (ulong_t)CPU; 656 657 ASSERT(curthread->t_preempt != 0 || getpil() >= DISP_LEVEL); 658 659 cli(); 660 __set_gs(KGS_SEL); 661 662 /* 663 * restore kernel gsbase 664 */ 665 #if defined(__xpv) 666 xen_set_segment_base(SEGBASE_GS_KERNEL, kgsbase); 667 #else 668 wrmsr(MSR_AMD_GSBASE, kgsbase); 669 #endif 670 671 sti(); 672 673 __set_ds(KDS_SEL); 674 __set_es(0 | SEL_KPL); /* selector RPL not ring 0 on hypervisor */ 675 __set_fs(KFS_SEL); 676 } 677 678 #endif /* __amd64 */ 679 680 #ifdef _SYSCALL32_IMPL 681 682 /* 683 * Make it impossible for a process to change its data model. 684 * We do this by toggling the present bits for the 32 and 685 * 64-bit user code descriptors. That way if a user lwp attempts 686 * to change its data model (by using the wrong code descriptor in 687 * %cs) it will fault immediately. This also allows us to simplify 688 * assertions and checks in the kernel. 689 */ 690 691 static void 692 gdt_ucode_model(model_t model) 693 { 694 kpreempt_disable(); 695 if (model == DATAMODEL_NATIVE) { 696 gdt_update_usegd(GDT_UCODE, &ucs_on); 697 gdt_update_usegd(GDT_U32CODE, &ucs32_off); 698 } else { 699 gdt_update_usegd(GDT_U32CODE, &ucs32_on); 700 gdt_update_usegd(GDT_UCODE, &ucs_off); 701 } 702 kpreempt_enable(); 703 } 704 705 #endif /* _SYSCALL32_IMPL */ 706 707 /* 708 * Restore lwp private fs and gs segment descriptors 709 * on current cpu's GDT. 710 */ 711 static void 712 lwp_segregs_restore(klwp_t *lwp) 713 { 714 pcb_t *pcb = &lwp->lwp_pcb; 715 716 ASSERT(VALID_LWP_DESC(&pcb->pcb_fsdesc)); 717 ASSERT(VALID_LWP_DESC(&pcb->pcb_gsdesc)); 718 719 #ifdef _SYSCALL32_IMPL 720 gdt_ucode_model(DATAMODEL_NATIVE); 721 #endif 722 723 gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc); 724 gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc); 725 726 } 727 728 #ifdef _SYSCALL32_IMPL 729 730 static void 731 lwp_segregs_restore32(klwp_t *lwp) 732 { 733 /*LINTED*/ 734 cpu_t *cpu = CPU; 735 pcb_t *pcb = &lwp->lwp_pcb; 736 737 ASSERT(VALID_LWP_DESC(&lwp->lwp_pcb.pcb_fsdesc)); 738 ASSERT(VALID_LWP_DESC(&lwp->lwp_pcb.pcb_gsdesc)); 739 740 gdt_ucode_model(DATAMODEL_ILP32); 741 gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc); 742 gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc); 743 } 744 745 #endif /* _SYSCALL32_IMPL */ 746 747 /* 748 * If this is a process in a branded zone, then we want it to use the brand 749 * syscall entry points instead of the standard Solaris entry points. This 750 * routine must be called when a new lwp is created within a branded zone 751 * or when an existing lwp moves into a branded zone via a zone_enter() 752 * operation. 753 */ 754 void 755 lwp_attach_brand_hdlrs(klwp_t *lwp) 756 { 757 kthread_t *t = lwptot(lwp); 758 759 ASSERT(PROC_IS_BRANDED(lwptoproc(lwp))); 760 761 ASSERT(removectx(t, NULL, brand_interpositioning_disable, 762 brand_interpositioning_enable, NULL, NULL, 763 brand_interpositioning_disable, NULL) == 0); 764 installctx(t, NULL, brand_interpositioning_disable, 765 brand_interpositioning_enable, NULL, NULL, 766 brand_interpositioning_disable, NULL); 767 768 if (t == curthread) { 769 kpreempt_disable(); 770 brand_interpositioning_enable(); 771 kpreempt_enable(); 772 } 773 } 774 775 /* 776 * If this is a process in a branded zone, then we want it to disable the 777 * brand syscall entry points. This routine must be called when the last 778 * lwp in a process is exiting in proc_exit(). 779 */ 780 void 781 lwp_detach_brand_hdlrs(klwp_t *lwp) 782 { 783 kthread_t *t = lwptot(lwp); 784 785 ASSERT(PROC_IS_BRANDED(lwptoproc(lwp))); 786 if (t == curthread) 787 kpreempt_disable(); 788 789 /* Remove the original context handlers */ 790 VERIFY(removectx(t, NULL, brand_interpositioning_disable, 791 brand_interpositioning_enable, NULL, NULL, 792 brand_interpositioning_disable, NULL) != 0); 793 794 if (t == curthread) { 795 /* Cleanup our MSR and IDT entries. */ 796 brand_interpositioning_disable(); 797 kpreempt_enable(); 798 } 799 } 800 801 /* 802 * Add any lwp-associated context handlers to the lwp at the beginning 803 * of the lwp's useful life. 804 * 805 * All paths which create lwp's invoke lwp_create(); lwp_create() 806 * invokes lwp_stk_init() which initializes the stack, sets up 807 * lwp_regs, and invokes this routine. 808 * 809 * All paths which destroy lwp's invoke lwp_exit() to rip the lwp 810 * apart and put it on 'lwp_deathrow'; if the lwp is destroyed it 811 * ends up in thread_free() which invokes freectx(t, 0) before 812 * invoking lwp_stk_fini(). When the lwp is recycled from death 813 * row, lwp_stk_fini() is invoked, then thread_free(), and thus 814 * freectx(t, 0) as before. 815 * 816 * In the case of exec, the surviving lwp is thoroughly scrubbed 817 * clean; exec invokes freectx(t, 1) to destroy associated contexts. 818 * On the way back to the new image, it invokes setregs() which 819 * in turn invokes this routine. 820 */ 821 void 822 lwp_installctx(klwp_t *lwp) 823 { 824 kthread_t *t = lwptot(lwp); 825 int thisthread = t == curthread; 826 #ifdef _SYSCALL32_IMPL 827 void (*restop)(klwp_t *) = lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ? 828 lwp_segregs_restore : lwp_segregs_restore32; 829 #else 830 void (*restop)(klwp_t *) = lwp_segregs_restore; 831 #endif 832 833 /* 834 * Install the basic lwp context handlers on each lwp. 835 * 836 * On the amd64 kernel, the context handlers are responsible for 837 * virtualizing %ds, %es, %fs, and %gs to the lwp. The register 838 * values are only ever changed via sys_rtt when the 839 * PCB_UPDATE_SEGS bit (1) is set in pcb->pcb_rupdate. Only 840 * sys_rtt gets to clear the bit. 841 * 842 * On the i386 kernel, the context handlers are responsible for 843 * virtualizing %gs/%fs to the lwp by updating the per-cpu GDTs 844 */ 845 ASSERT(removectx(t, lwp, lwp_segregs_save, restop, 846 NULL, NULL, NULL, NULL) == 0); 847 if (thisthread) 848 kpreempt_disable(); 849 installctx(t, lwp, lwp_segregs_save, restop, 850 NULL, NULL, NULL, NULL); 851 if (thisthread) { 852 /* 853 * Since we're the right thread, set the values in the GDT 854 */ 855 restop(lwp); 856 kpreempt_enable(); 857 } 858 859 /* 860 * If we have sysenter/sysexit instructions enabled, we need 861 * to ensure that the hardware mechanism is kept up-to-date with the 862 * lwp's kernel stack pointer across context switches. 863 * 864 * sep_save zeros the sysenter stack pointer msr; sep_restore sets 865 * it to the lwp's kernel stack pointer (kstktop). 866 */ 867 if (is_x86_feature(x86_featureset, X86FSET_SEP)) { 868 #if defined(__amd64) 869 caddr_t kstktop = (caddr_t)lwp->lwp_regs; 870 #elif defined(__i386) 871 caddr_t kstktop = ((caddr_t)lwp->lwp_regs - MINFRAME) + 872 SA(sizeof (struct regs) + MINFRAME); 873 #endif 874 ASSERT(removectx(t, kstktop, 875 sep_save, sep_restore, NULL, NULL, NULL, NULL) == 0); 876 877 if (thisthread) 878 kpreempt_disable(); 879 installctx(t, kstktop, 880 sep_save, sep_restore, NULL, NULL, NULL, NULL); 881 if (thisthread) { 882 /* 883 * We're the right thread, so set the stack pointer 884 * for the first sysenter instruction to use 885 */ 886 sep_restore(kstktop); 887 kpreempt_enable(); 888 } 889 } 890 891 if (PROC_IS_BRANDED(ttoproc(t))) 892 lwp_attach_brand_hdlrs(lwp); 893 } 894 895 /* 896 * Clear registers on exec(2). 897 */ 898 void 899 setregs(uarg_t *args) 900 { 901 struct regs *rp; 902 kthread_t *t = curthread; 903 klwp_t *lwp = ttolwp(t); 904 pcb_t *pcb = &lwp->lwp_pcb; 905 greg_t sp; 906 907 /* 908 * Initialize user registers 909 */ 910 (void) save_syscall_args(); /* copy args from registers first */ 911 rp = lwptoregs(lwp); 912 sp = rp->r_sp; 913 bzero(rp, sizeof (*rp)); 914 915 rp->r_ss = UDS_SEL; 916 rp->r_sp = sp; 917 rp->r_pc = args->entry; 918 rp->r_ps = PSL_USER; 919 920 #if defined(__amd64) 921 922 pcb->pcb_fs = pcb->pcb_gs = 0; 923 pcb->pcb_fsbase = pcb->pcb_gsbase = 0; 924 925 if (ttoproc(t)->p_model == DATAMODEL_NATIVE) { 926 927 rp->r_cs = UCS_SEL; 928 929 /* 930 * Only allow 64-bit user code descriptor to be present. 931 */ 932 gdt_ucode_model(DATAMODEL_NATIVE); 933 934 /* 935 * Arrange that the virtualized %fs and %gs GDT descriptors 936 * have a well-defined initial state (present, ring 3 937 * and of type data). 938 */ 939 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc; 940 941 /* 942 * thrptr is either NULL or a value used by DTrace. 943 * 64-bit processes use %fs as their "thread" register. 944 */ 945 if (args->thrptr) 946 (void) lwp_setprivate(lwp, _LWP_FSBASE, args->thrptr); 947 948 } else { 949 950 rp->r_cs = U32CS_SEL; 951 rp->r_ds = rp->r_es = UDS_SEL; 952 953 /* 954 * only allow 32-bit user code selector to be present. 955 */ 956 gdt_ucode_model(DATAMODEL_ILP32); 957 958 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc; 959 960 /* 961 * thrptr is either NULL or a value used by DTrace. 962 * 32-bit processes use %gs as their "thread" register. 963 */ 964 if (args->thrptr) 965 (void) lwp_setprivate(lwp, _LWP_GSBASE, args->thrptr); 966 967 } 968 969 pcb->pcb_ds = rp->r_ds; 970 pcb->pcb_es = rp->r_es; 971 PCB_SET_UPDATE_SEGS(pcb); 972 973 #elif defined(__i386) 974 975 rp->r_cs = UCS_SEL; 976 rp->r_ds = rp->r_es = UDS_SEL; 977 978 /* 979 * Arrange that the virtualized %fs and %gs GDT descriptors 980 * have a well-defined initial state (present, ring 3 981 * and of type data). 982 */ 983 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc; 984 985 /* 986 * For %gs we need to reset LWP_GSBASE in pcb and the 987 * per-cpu GDT descriptor. thrptr is either NULL 988 * or a value used by DTrace. 989 */ 990 if (args->thrptr) 991 (void) lwp_setprivate(lwp, _LWP_GSBASE, args->thrptr); 992 #endif 993 994 lwp->lwp_eosys = JUSTRETURN; 995 t->t_post_sys = 1; 996 997 /* 998 * Add the lwp context handlers that virtualize segment registers, 999 * and/or system call stacks etc. 1000 */ 1001 lwp_installctx(lwp); 1002 1003 /* 1004 * Reset the FPU flags and then initialize the FPU for this lwp. 1005 */ 1006 fp_exec(); 1007 } 1008 1009 user_desc_t * 1010 cpu_get_gdt(void) 1011 { 1012 return (CPU->cpu_gdt); 1013 } 1014 1015 1016 #if !defined(lwp_getdatamodel) 1017 1018 /* 1019 * Return the datamodel of the given lwp. 1020 */ 1021 /*ARGSUSED*/ 1022 model_t 1023 lwp_getdatamodel(klwp_t *lwp) 1024 { 1025 return (lwp->lwp_procp->p_model); 1026 } 1027 1028 #endif /* !lwp_getdatamodel */ 1029 1030 #if !defined(get_udatamodel) 1031 1032 model_t 1033 get_udatamodel(void) 1034 { 1035 return (curproc->p_model); 1036 } 1037 1038 #endif /* !get_udatamodel */