1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2013, Joyent, Inc. All rights reserved.
  25  */
  26 
  27 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  28 /*        All Rights Reserved   */
  29 
  30 #include <sys/types.h>
  31 #include <sys/param.h>
  32 #include <sys/sysmacros.h>
  33 #include <sys/signal.h>
  34 #include <sys/cred.h>
  35 #include <sys/policy.h>
  36 #include <sys/user.h>
  37 #include <sys/systm.h>
  38 #include <sys/cpuvar.h>
  39 #include <sys/vfs.h>
  40 #include <sys/vnode.h>
  41 #include <sys/file.h>
  42 #include <sys/errno.h>
  43 #include <sys/time.h>
  44 #include <sys/proc.h>
  45 #include <sys/cmn_err.h>
  46 #include <sys/acct.h>
  47 #include <sys/tuneable.h>
  48 #include <sys/class.h>
  49 #include <sys/kmem.h>
  50 #include <sys/session.h>
  51 #include <sys/ucontext.h>
  52 #include <sys/stack.h>
  53 #include <sys/procfs.h>
  54 #include <sys/prsystm.h>
  55 #include <sys/vmsystm.h>
  56 #include <sys/vtrace.h>
  57 #include <sys/debug.h>
  58 #include <sys/shm_impl.h>
  59 #include <sys/door_data.h>
  60 #include <vm/as.h>
  61 #include <vm/rm.h>
  62 #include <c2/audit.h>
  63 #include <sys/var.h>
  64 #include <sys/schedctl.h>
  65 #include <sys/utrap.h>
  66 #include <sys/task.h>
  67 #include <sys/resource.h>
  68 #include <sys/cyclic.h>
  69 #include <sys/lgrp.h>
  70 #include <sys/rctl.h>
  71 #include <sys/contract_impl.h>
  72 #include <sys/contract/process_impl.h>
  73 #include <sys/list.h>
  74 #include <sys/dtrace.h>
  75 #include <sys/pool.h>
  76 #include <sys/zone.h>
  77 #include <sys/sdt.h>
  78 #include <sys/class.h>
  79 #include <sys/corectl.h>
  80 #include <sys/brand.h>
  81 #include <sys/fork.h>
  82 
  83 static int64_t cfork(int, int, int);
  84 static int getproc(proc_t **, pid_t, uint_t);
  85 #define GETPROC_USER    0x0
  86 #define GETPROC_KERNEL  0x1
  87 
  88 static void fork_fail(proc_t *);
  89 static void forklwp_fail(proc_t *);
  90 
  91 int fork_fail_pending;
  92 
  93 extern struct kmem_cache *process_cache;
  94 
  95 /*
  96  * The vfork() system call trap is no longer invoked by libc.
  97  * It is retained only for the benefit of applications running
  98  * within a solaris10 branded zone.  It should be eliminated
  99  * when we no longer support solaris10 branded zones.
 100  */
 101 int64_t
 102 vfork(void)
 103 {
 104         curthread->t_post_sys = 1;   /* so vfwait() will be called */
 105         return (cfork(1, 1, 0));
 106 }
 107 
 108 /*
 109  * forksys system call - forkx, forkallx, vforkx.  This is the
 110  * interface invoked by libc for fork1(), forkall(), and vfork()
 111  */
 112 int64_t
 113 forksys(int subcode, int flags)
 114 {
 115         switch (subcode) {
 116         case 0:
 117                 return (cfork(0, 1, flags));    /* forkx(flags) */
 118         case 1:
 119                 return (cfork(0, 0, flags));    /* forkallx(flags) */
 120         case 2:
 121                 curthread->t_post_sys = 1;   /* so vfwait() will be called */
 122                 return (cfork(1, 1, flags));    /* vforkx(flags) */
 123         default:
 124                 return ((int64_t)set_errno(EINVAL));
 125         }
 126 }
 127 
 128 /* ARGSUSED */
 129 static int64_t
 130 cfork(int isvfork, int isfork1, int flags)
 131 {
 132         proc_t *p = ttoproc(curthread);
 133         struct as *as;
 134         proc_t *cp, **orphpp;
 135         klwp_t *clone;
 136         kthread_t *t;
 137         task_t *tk;
 138         rval_t  r;
 139         int error;
 140         int i;
 141         rctl_set_t *dup_set;
 142         rctl_alloc_gp_t *dup_gp;
 143         rctl_entity_p_t e;
 144         lwpdir_t *ldp;
 145         lwpent_t *lep;
 146         lwpent_t *clep;
 147 
 148         /*
 149          * Allow only these two flags.
 150          */
 151         if ((flags & ~(FORK_NOSIGCHLD | FORK_WAITPID)) != 0) {
 152                 error = EINVAL;
 153                 atomic_inc_32(&curproc->p_zone->zone_ffmisc);
 154                 goto forkerr;
 155         }
 156 
 157         /*
 158          * fork is not supported for the /proc agent lwp.
 159          */
 160         if (curthread == p->p_agenttp) {
 161                 error = ENOTSUP;
 162                 atomic_inc_32(&curproc->p_zone->zone_ffmisc);
 163                 goto forkerr;
 164         }
 165 
 166         if ((error = secpolicy_basic_fork(CRED())) != 0) {
 167                 atomic_inc_32(&p->p_zone->zone_ffmisc);
 168                 goto forkerr;
 169         }
 170 
 171         /*
 172          * If the calling lwp is doing a fork1() then the
 173          * other lwps in this process are not duplicated and
 174          * don't need to be held where their kernel stacks can be
 175          * cloned.  If doing forkall(), the process is held with
 176          * SHOLDFORK, so that the lwps are at a point where their
 177          * stacks can be copied which is on entry or exit from
 178          * the kernel.
 179          */
 180         if (!holdlwps(isfork1 ? SHOLDFORK1 : SHOLDFORK)) {
 181                 aston(curthread);
 182                 error = EINTR;
 183                 atomic_inc_32(&p->p_zone->zone_ffmisc);
 184                 goto forkerr;
 185         }
 186 
 187 #if defined(__sparc)
 188         /*
 189          * Ensure that the user stack is fully constructed
 190          * before creating the child process structure.
 191          */
 192         (void) flush_user_windows_to_stack(NULL);
 193 #endif
 194 
 195         mutex_enter(&p->p_lock);
 196         /*
 197          * If this is vfork(), cancel any suspend request we might
 198          * have gotten from some other thread via lwp_suspend().
 199          * Otherwise we could end up with a deadlock on return
 200          * from the vfork() in both the parent and the child.
 201          */
 202         if (isvfork)
 203                 curthread->t_proc_flag &= ~TP_HOLDLWP;
 204         /*
 205          * Prevent our resource set associations from being changed during fork.
 206          */
 207         pool_barrier_enter();
 208         mutex_exit(&p->p_lock);
 209 
 210         /*
 211          * Create a child proc struct. Place a VN_HOLD on appropriate vnodes.
 212          */
 213         if (getproc(&cp, 0, GETPROC_USER) < 0) {
 214                 mutex_enter(&p->p_lock);
 215                 pool_barrier_exit();
 216                 continuelwps(p);
 217                 mutex_exit(&p->p_lock);
 218                 error = EAGAIN;
 219                 goto forkerr;
 220         }
 221 
 222         TRACE_2(TR_FAC_PROC, TR_PROC_FORK, "proc_fork:cp %p p %p", cp, p);
 223 
 224         /*
 225          * Assign an address space to child
 226          */
 227         if (isvfork) {
 228                 /*
 229                  * Clear any watched areas and remember the
 230                  * watched pages for restoring in vfwait().
 231                  */
 232                 as = p->p_as;
 233                 if (avl_numnodes(&as->a_wpage) != 0) {
 234                         AS_LOCK_ENTER(as, RW_WRITER);
 235                         as_clearwatch(as);
 236                         p->p_wpage = as->a_wpage;
 237                         avl_create(&as->a_wpage, wp_compare,
 238                             sizeof (struct watched_page),
 239                             offsetof(struct watched_page, wp_link));
 240                         AS_LOCK_EXIT(as);
 241                 }
 242                 cp->p_as = as;
 243                 cp->p_flag |= SVFORK;
 244 
 245                 /*
 246                  * Use the parent's shm segment list information for
 247                  * the child as it uses its address space till it execs.
 248                  */
 249                 cp->p_segacct = p->p_segacct;
 250         } else {
 251                 /*
 252                  * We need to hold P_PR_LOCK until the address space has
 253                  * been duplicated and we've had a chance to remove from the
 254                  * child any DTrace probes that were in the parent. Holding
 255                  * P_PR_LOCK prevents any new probes from being added and any
 256                  * extant probes from being removed.
 257                  */
 258                 mutex_enter(&p->p_lock);
 259                 sprlock_proc(p);
 260                 p->p_flag |= SFORKING;
 261                 mutex_exit(&p->p_lock);
 262 
 263                 error = as_dup(p->p_as, cp);
 264                 if (error != 0) {
 265                         mutex_enter(&p->p_lock);
 266                         sprunlock(p);
 267                         fork_fail(cp);
 268                         mutex_enter(&pidlock);
 269                         orphpp = &p->p_orphan;
 270                         while (*orphpp != cp)
 271                                 orphpp = &(*orphpp)->p_nextorph;
 272                         *orphpp = cp->p_nextorph;
 273                         if (p->p_child == cp)
 274                                 p->p_child = cp->p_sibling;
 275                         if (cp->p_sibling)
 276                                 cp->p_sibling->p_psibling = cp->p_psibling;
 277                         if (cp->p_psibling)
 278                                 cp->p_psibling->p_sibling = cp->p_sibling;
 279                         mutex_enter(&cp->p_lock);
 280                         tk = cp->p_task;
 281                         task_detach(cp);
 282                         ASSERT(cp->p_pool->pool_ref > 0);
 283                         atomic_dec_32(&cp->p_pool->pool_ref);
 284                         mutex_exit(&cp->p_lock);
 285                         pid_exit(cp, tk);
 286                         mutex_exit(&pidlock);
 287                         task_rele(tk);
 288 
 289                         mutex_enter(&p->p_lock);
 290                         p->p_flag &= ~SFORKING;
 291                         pool_barrier_exit();
 292                         continuelwps(p);
 293                         mutex_exit(&p->p_lock);
 294                         /*
 295                          * Preserve ENOMEM error condition but
 296                          * map all others to EAGAIN.
 297                          */
 298                         error = (error == ENOMEM) ? ENOMEM : EAGAIN;
 299                         atomic_inc_32(&p->p_zone->zone_ffnomem);
 300                         goto forkerr;
 301                 }
 302 
 303                 /*
 304                  * Remove all DTrace tracepoints from the child process. We
 305                  * need to do this _before_ duplicating USDT providers since
 306                  * any associated probes may be immediately enabled.
 307                  */
 308                 if (p->p_dtrace_count > 0)
 309                         dtrace_fasttrap_fork(p, cp);
 310 
 311                 mutex_enter(&p->p_lock);
 312                 sprunlock(p);
 313 
 314                 /* Duplicate parent's shared memory */
 315                 if (p->p_segacct)
 316                         shmfork(p, cp);
 317 
 318                 /*
 319                  * Duplicate any helper actions and providers. The SFORKING
 320                  * we set above informs the code to enable USDT probes that
 321                  * sprlock() may fail because the child is being forked.
 322                  */
 323                 if (p->p_dtrace_helpers != NULL) {
 324                         ASSERT(dtrace_helpers_fork != NULL);
 325                         (*dtrace_helpers_fork)(p, cp);
 326                 }
 327 
 328                 mutex_enter(&p->p_lock);
 329                 p->p_flag &= ~SFORKING;
 330                 mutex_exit(&p->p_lock);
 331         }
 332 
 333         /*
 334          * Duplicate parent's resource controls.
 335          */
 336         dup_set = rctl_set_create();
 337         for (;;) {
 338                 dup_gp = rctl_set_dup_prealloc(p->p_rctls);
 339                 mutex_enter(&p->p_rctls->rcs_lock);
 340                 if (rctl_set_dup_ready(p->p_rctls, dup_gp))
 341                         break;
 342                 mutex_exit(&p->p_rctls->rcs_lock);
 343                 rctl_prealloc_destroy(dup_gp);
 344         }
 345         e.rcep_p.proc = cp;
 346         e.rcep_t = RCENTITY_PROCESS;
 347         cp->p_rctls = rctl_set_dup(p->p_rctls, p, cp, &e, dup_set, dup_gp,
 348             RCD_DUP | RCD_CALLBACK);
 349         mutex_exit(&p->p_rctls->rcs_lock);
 350 
 351         rctl_prealloc_destroy(dup_gp);
 352 
 353         /*
 354          * Allocate the child's lwp directory and lwpid hash table.
 355          */
 356         if (isfork1)
 357                 cp->p_lwpdir_sz = 2;
 358         else
 359                 cp->p_lwpdir_sz = p->p_lwpdir_sz;
 360         cp->p_lwpdir = cp->p_lwpfree = ldp =
 361             kmem_zalloc(cp->p_lwpdir_sz * sizeof (lwpdir_t), KM_SLEEP);
 362         for (i = 1; i < cp->p_lwpdir_sz; i++, ldp++)
 363                 ldp->ld_next = ldp + 1;
 364         cp->p_tidhash_sz = (cp->p_lwpdir_sz + 2) / 2;
 365         cp->p_tidhash =
 366             kmem_zalloc(cp->p_tidhash_sz * sizeof (tidhash_t), KM_SLEEP);
 367 
 368         /*
 369          * Duplicate parent's lwps.
 370          * Mutual exclusion is not needed because the process is
 371          * in the hold state and only the current lwp is running.
 372          */
 373         klgrpset_clear(cp->p_lgrpset);
 374         if (isfork1) {
 375                 clone = forklwp(ttolwp(curthread), cp, curthread->t_tid);
 376                 if (clone == NULL)
 377                         goto forklwperr;
 378                 /*
 379                  * Inherit only the lwp_wait()able flag,
 380                  * Daemon threads should not call fork1(), but oh well...
 381                  */
 382                 lwptot(clone)->t_proc_flag |=
 383                     (curthread->t_proc_flag & TP_TWAIT);
 384         } else {
 385                 /* this is forkall(), no one can be in lwp_wait() */
 386                 ASSERT(p->p_lwpwait == 0 && p->p_lwpdwait == 0);
 387                 /* for each entry in the parent's lwp directory... */
 388                 for (i = 0, ldp = p->p_lwpdir; i < p->p_lwpdir_sz; i++, ldp++) {
 389                         klwp_t *clwp;
 390                         kthread_t *ct;
 391 
 392                         if ((lep = ldp->ld_entry) == NULL)
 393                                 continue;
 394 
 395                         if ((t = lep->le_thread) != NULL) {
 396                                 clwp = forklwp(ttolwp(t), cp, t->t_tid);
 397                                 if (clwp == NULL)
 398                                         goto forklwperr;
 399                                 ct = lwptot(clwp);
 400                                 /*
 401                                  * Inherit lwp_wait()able and daemon flags.
 402                                  */
 403                                 ct->t_proc_flag |=
 404                                     (t->t_proc_flag & (TP_TWAIT|TP_DAEMON));
 405                                 /*
 406                                  * Keep track of the clone of curthread to
 407                                  * post return values through lwp_setrval().
 408                                  * Mark other threads for special treatment
 409                                  * by lwp_rtt() / post_syscall().
 410                                  */
 411                                 if (t == curthread)
 412                                         clone = clwp;
 413                                 else
 414                                         ct->t_flag |= T_FORKALL;
 415                         } else {
 416                                 /*
 417                                  * Replicate zombie lwps in the child.
 418                                  */
 419                                 clep = kmem_zalloc(sizeof (*clep), KM_SLEEP);
 420                                 clep->le_lwpid = lep->le_lwpid;
 421                                 clep->le_start = lep->le_start;
 422                                 lwp_hash_in(cp, clep,
 423                                     cp->p_tidhash, cp->p_tidhash_sz, 0);
 424                         }
 425                 }
 426         }
 427 
 428         /*
 429          * Put new process in the parent's process contract, or put it
 430          * in a new one if there is an active process template.  Send a
 431          * fork event (if requested) to whatever contract the child is
 432          * a member of.  Fails if the parent has been SIGKILLed.
 433          */
 434         if (contract_process_fork(NULL, cp, p, B_TRUE) == NULL) {
 435                 atomic_inc_32(&p->p_zone->zone_ffmisc);
 436                 goto forklwperr;
 437         }
 438 
 439         /*
 440          * No fork failures occur beyond this point.
 441          */
 442 
 443         cp->p_lwpid = p->p_lwpid;
 444         if (!isfork1) {
 445                 cp->p_lwpdaemon = p->p_lwpdaemon;
 446                 cp->p_zombcnt = p->p_zombcnt;
 447                 /*
 448                  * If the parent's lwp ids have wrapped around, so have the
 449                  * child's.
 450                  */
 451                 cp->p_flag |= p->p_flag & SLWPWRAP;
 452         }
 453 
 454         mutex_enter(&p->p_lock);
 455         corectl_path_hold(cp->p_corefile = p->p_corefile);
 456         corectl_content_hold(cp->p_content = p->p_content);
 457         mutex_exit(&p->p_lock);
 458 
 459         /*
 460          * Duplicate process context ops, if any.
 461          */
 462         if (p->p_pctx)
 463                 forkpctx(p, cp);
 464 
 465 #ifdef __sparc
 466         utrap_dup(p, cp);
 467 #endif
 468         /*
 469          * If the child process has been marked to stop on exit
 470          * from this fork, arrange for all other lwps to stop in
 471          * sympathy with the active lwp.
 472          */
 473         if (PTOU(cp)->u_systrap &&
 474             prismember(&PTOU(cp)->u_exitmask, curthread->t_sysnum)) {
 475                 mutex_enter(&cp->p_lock);
 476                 t = cp->p_tlist;
 477                 do {
 478                         t->t_proc_flag |= TP_PRSTOP;
 479                         aston(t);       /* so TP_PRSTOP will be seen */
 480                 } while ((t = t->t_forw) != cp->p_tlist);
 481                 mutex_exit(&cp->p_lock);
 482         }
 483         /*
 484          * If the parent process has been marked to stop on exit
 485          * from this fork, and its asynchronous-stop flag has not
 486          * been set, arrange for all other lwps to stop before
 487          * they return back to user level.
 488          */
 489         if (!(p->p_proc_flag & P_PR_ASYNC) && PTOU(p)->u_systrap &&
 490             prismember(&PTOU(p)->u_exitmask, curthread->t_sysnum)) {
 491                 mutex_enter(&p->p_lock);
 492                 t = p->p_tlist;
 493                 do {
 494                         t->t_proc_flag |= TP_PRSTOP;
 495                         aston(t);       /* so TP_PRSTOP will be seen */
 496                 } while ((t = t->t_forw) != p->p_tlist);
 497                 mutex_exit(&p->p_lock);
 498         }
 499 
 500         if (PROC_IS_BRANDED(p))
 501                 BROP(p)->b_lwp_setrval(clone, p->p_pid, 1);
 502         else
 503                 lwp_setrval(clone, p->p_pid, 1);
 504 
 505         /* set return values for parent */
 506         r.r_val1 = (int)cp->p_pid;
 507         r.r_val2 = 0;
 508 
 509         /*
 510          * pool_barrier_exit() can now be called because the child process has:
 511          * - all identifying features cloned or set (p_pid, p_task, p_pool)
 512          * - all resource sets associated (p_tlist->*->t_cpupart, p_as->a_mset)
 513          * - any other fields set which are used in resource set binding.
 514          */
 515         mutex_enter(&p->p_lock);
 516         pool_barrier_exit();
 517         mutex_exit(&p->p_lock);
 518 
 519         mutex_enter(&pidlock);
 520         mutex_enter(&cp->p_lock);
 521 
 522         /*
 523          * Set flags telling the child what (not) to do on exit.
 524          */
 525         if (flags & FORK_NOSIGCHLD)
 526                 cp->p_pidflag |= CLDNOSIGCHLD;
 527         if (flags & FORK_WAITPID)
 528                 cp->p_pidflag |= CLDWAITPID;
 529 
 530         /*
 531          * Now that there are lwps and threads attached, add the new
 532          * process to the process group.
 533          */
 534         pgjoin(cp, p->p_pgidp);
 535         cp->p_stat = SRUN;
 536         /*
 537          * We are now done with all the lwps in the child process.
 538          */
 539         t = cp->p_tlist;
 540         do {
 541                 /*
 542                  * Set the lwp_suspend()ed lwps running.
 543                  * They will suspend properly at syscall exit.
 544                  */
 545                 if (t->t_proc_flag & TP_HOLDLWP)
 546                         lwp_create_done(t);
 547                 else {
 548                         /* set TS_CREATE to allow continuelwps() to work */
 549                         thread_lock(t);
 550                         ASSERT(t->t_state == TS_STOPPED &&
 551                             !(t->t_schedflag & (TS_CREATE|TS_CSTART)));
 552                         t->t_schedflag |= TS_CREATE;
 553                         thread_unlock(t);
 554                 }
 555         } while ((t = t->t_forw) != cp->p_tlist);
 556         mutex_exit(&cp->p_lock);
 557 
 558         if (isvfork) {
 559                 CPU_STATS_ADDQ(CPU, sys, sysvfork, 1);
 560                 mutex_enter(&p->p_lock);
 561                 p->p_flag |= SVFWAIT;
 562                 curthread->t_flag |= T_VFPARENT;
 563                 DTRACE_PROC1(create, proc_t *, cp);
 564                 cv_broadcast(&pr_pid_cv[p->p_slot]);     /* inform /proc */
 565                 mutex_exit(&p->p_lock);
 566                 /*
 567                  * Grab child's p_lock before dropping pidlock to ensure
 568                  * the process will not disappear before we set it running.
 569                  */
 570                 mutex_enter(&cp->p_lock);
 571                 mutex_exit(&pidlock);
 572                 sigdefault(cp);
 573                 continuelwps(cp);
 574                 mutex_exit(&cp->p_lock);
 575         } else {
 576                 CPU_STATS_ADDQ(CPU, sys, sysfork, 1);
 577                 DTRACE_PROC1(create, proc_t *, cp);
 578                 /*
 579                  * It is CL_FORKRET's job to drop pidlock.
 580                  * If we do it here, the process could be set running
 581                  * and disappear before CL_FORKRET() is called.
 582                  */
 583                 CL_FORKRET(curthread, cp->p_tlist);
 584                 schedctl_set_cidpri(curthread);
 585                 ASSERT(MUTEX_NOT_HELD(&pidlock));
 586         }
 587 
 588         return (r.r_vals);
 589 
 590 forklwperr:
 591         if (isvfork) {
 592                 if (avl_numnodes(&p->p_wpage) != 0) {
 593                         /* restore watchpoints to parent */
 594                         as = p->p_as;
 595                         AS_LOCK_ENTER(as, RW_WRITER);
 596                         as->a_wpage = p->p_wpage;
 597                         avl_create(&p->p_wpage, wp_compare,
 598                             sizeof (struct watched_page),
 599                             offsetof(struct watched_page, wp_link));
 600                         as_setwatch(as);
 601                         AS_LOCK_EXIT(as);
 602                 }
 603         } else {
 604                 if (cp->p_segacct)
 605                         shmexit(cp);
 606                 as = cp->p_as;
 607                 cp->p_as = &kas;
 608                 as_free(as);
 609         }
 610 
 611         if (cp->p_lwpdir) {
 612                 for (i = 0, ldp = cp->p_lwpdir; i < cp->p_lwpdir_sz; i++, ldp++)
 613                         if ((lep = ldp->ld_entry) != NULL)
 614                                 kmem_free(lep, sizeof (*lep));
 615                 kmem_free(cp->p_lwpdir,
 616                     cp->p_lwpdir_sz * sizeof (*cp->p_lwpdir));
 617         }
 618         cp->p_lwpdir = NULL;
 619         cp->p_lwpfree = NULL;
 620         cp->p_lwpdir_sz = 0;
 621 
 622         if (cp->p_tidhash)
 623                 kmem_free(cp->p_tidhash,
 624                     cp->p_tidhash_sz * sizeof (*cp->p_tidhash));
 625         cp->p_tidhash = NULL;
 626         cp->p_tidhash_sz = 0;
 627 
 628         forklwp_fail(cp);
 629         fork_fail(cp);
 630         rctl_set_free(cp->p_rctls);
 631         mutex_enter(&pidlock);
 632 
 633         /*
 634          * Detach failed child from task.
 635          */
 636         mutex_enter(&cp->p_lock);
 637         tk = cp->p_task;
 638         task_detach(cp);
 639         ASSERT(cp->p_pool->pool_ref > 0);
 640         atomic_dec_32(&cp->p_pool->pool_ref);
 641         mutex_exit(&cp->p_lock);
 642 
 643         orphpp = &p->p_orphan;
 644         while (*orphpp != cp)
 645                 orphpp = &(*orphpp)->p_nextorph;
 646         *orphpp = cp->p_nextorph;
 647         if (p->p_child == cp)
 648                 p->p_child = cp->p_sibling;
 649         if (cp->p_sibling)
 650                 cp->p_sibling->p_psibling = cp->p_psibling;
 651         if (cp->p_psibling)
 652                 cp->p_psibling->p_sibling = cp->p_sibling;
 653         pid_exit(cp, tk);
 654         mutex_exit(&pidlock);
 655 
 656         task_rele(tk);
 657 
 658         mutex_enter(&p->p_lock);
 659         pool_barrier_exit();
 660         continuelwps(p);
 661         mutex_exit(&p->p_lock);
 662         error = EAGAIN;
 663 forkerr:
 664         return ((int64_t)set_errno(error));
 665 }
 666 
 667 /*
 668  * Free allocated resources from getproc() if a fork failed.
 669  */
 670 static void
 671 fork_fail(proc_t *cp)
 672 {
 673         uf_info_t *fip = P_FINFO(cp);
 674 
 675         fcnt_add(fip, -1);
 676         sigdelq(cp, NULL, 0);
 677 
 678         mutex_enter(&pidlock);
 679         upcount_dec(crgetruid(cp->p_cred), crgetzoneid(cp->p_cred));
 680         mutex_exit(&pidlock);
 681 
 682         /*
 683          * single threaded, so no locking needed here
 684          */
 685         crfree(cp->p_cred);
 686 
 687         kmem_free(fip->fi_list, fip->fi_nfiles * sizeof (uf_entry_t));
 688 
 689         VN_RELE(PTOU(curproc)->u_cdir);
 690         if (PTOU(curproc)->u_rdir)
 691                 VN_RELE(PTOU(curproc)->u_rdir);
 692         if (cp->p_exec)
 693                 VN_RELE(cp->p_exec);
 694         if (cp->p_execdir)
 695                 VN_RELE(cp->p_execdir);
 696         if (PTOU(curproc)->u_cwd)
 697                 refstr_rele(PTOU(curproc)->u_cwd);
 698         if (PROC_IS_BRANDED(cp)) {
 699                 brand_clearbrand(cp, B_TRUE);
 700         }
 701 }
 702 
 703 /*
 704  * Clean up the lwps already created for this child process.
 705  * The fork failed while duplicating all the lwps of the parent
 706  * and those lwps already created must be freed.
 707  * This process is invisible to the rest of the system,
 708  * so we don't need to hold p->p_lock to protect the list.
 709  */
 710 static void
 711 forklwp_fail(proc_t *p)
 712 {
 713         kthread_t *t;
 714         task_t *tk;
 715         int branded = 0;
 716 
 717         if (PROC_IS_BRANDED(p))
 718                 branded = 1;
 719 
 720         while ((t = p->p_tlist) != NULL) {
 721                 /*
 722                  * First remove the lwp from the process's p_tlist.
 723                  */
 724                 if (t != t->t_forw)
 725                         p->p_tlist = t->t_forw;
 726                 else
 727                         p->p_tlist = NULL;
 728                 p->p_lwpcnt--;
 729                 t->t_forw->t_back = t->t_back;
 730                 t->t_back->t_forw = t->t_forw;
 731 
 732                 tk = p->p_task;
 733                 mutex_enter(&p->p_zone->zone_nlwps_lock);
 734                 tk->tk_nlwps--;
 735                 tk->tk_proj->kpj_nlwps--;
 736                 p->p_zone->zone_nlwps--;
 737                 mutex_exit(&p->p_zone->zone_nlwps_lock);
 738 
 739                 ASSERT(t->t_schedctl == NULL);
 740 
 741                 if (branded)
 742                         BROP(p)->b_freelwp(ttolwp(t));
 743 
 744                 if (t->t_door != NULL) {
 745                         kmem_free(t->t_door, sizeof (door_data_t));
 746                         t->t_door = NULL;
 747                 }
 748                 lwp_ctmpl_clear(ttolwp(t));
 749 
 750                 /*
 751                  * Remove the thread from the all threads list.
 752                  * We need to hold pidlock for this.
 753                  */
 754                 mutex_enter(&pidlock);
 755                 t->t_next->t_prev = t->t_prev;
 756                 t->t_prev->t_next = t->t_next;
 757                 CL_EXIT(t);     /* tell the scheduler that we're exiting */
 758                 cv_broadcast(&t->t_joincv);      /* tell anyone in thread_join */
 759                 mutex_exit(&pidlock);
 760 
 761                 /*
 762                  * Let the lgroup load averages know that this thread isn't
 763                  * going to show up (i.e. un-do what was done on behalf of
 764                  * this thread by the earlier lgrp_move_thread()).
 765                  */
 766                 kpreempt_disable();
 767                 lgrp_move_thread(t, NULL, 1);
 768                 kpreempt_enable();
 769 
 770                 /*
 771                  * The thread was created TS_STOPPED.
 772                  * We change it to TS_FREE to avoid an
 773                  * ASSERT() panic in thread_free().
 774                  */
 775                 t->t_state = TS_FREE;
 776                 thread_rele(t);
 777                 thread_free(t);
 778         }
 779 }
 780 
 781 extern struct as kas;
 782 
 783 /*
 784  * fork a kernel process.
 785  */
 786 int
 787 newproc(void (*pc)(), caddr_t arg, id_t cid, int pri, struct contract **ct,
 788     pid_t pid)
 789 {
 790         proc_t *p;
 791         struct user *up;
 792         kthread_t *t;
 793         cont_process_t *ctp = NULL;
 794         rctl_entity_p_t e;
 795 
 796         ASSERT(cid != sysdccid);
 797         ASSERT(cid != syscid || ct == NULL);
 798         if (CLASS_KERNEL(cid)) {
 799                 rctl_alloc_gp_t *init_gp;
 800                 rctl_set_t *init_set;
 801 
 802                 ASSERT(pid != 1);
 803 
 804                 if (getproc(&p, pid, GETPROC_KERNEL) < 0)
 805                         return (EAGAIN);
 806 
 807                 /*
 808                  * Release the hold on the p_exec and p_execdir, these
 809                  * were acquired in getproc()
 810                  */
 811                 if (p->p_execdir != NULL)
 812                         VN_RELE(p->p_execdir);
 813                 if (p->p_exec != NULL)
 814                         VN_RELE(p->p_exec);
 815                 p->p_flag |= SNOWAIT;
 816                 p->p_exec = NULL;
 817                 p->p_execdir = NULL;
 818 
 819                 init_set = rctl_set_create();
 820                 init_gp = rctl_set_init_prealloc(RCENTITY_PROCESS);
 821 
 822                 /*
 823                  * kernel processes do not inherit /proc tracing flags.
 824                  */
 825                 sigemptyset(&p->p_sigmask);
 826                 premptyset(&p->p_fltmask);
 827                 up = PTOU(p);
 828                 up->u_systrap = 0;
 829                 premptyset(&(up->u_entrymask));
 830                 premptyset(&(up->u_exitmask));
 831                 mutex_enter(&p->p_lock);
 832                 e.rcep_p.proc = p;
 833                 e.rcep_t = RCENTITY_PROCESS;
 834                 p->p_rctls = rctl_set_init(RCENTITY_PROCESS, p, &e, init_set,
 835                     init_gp);
 836                 mutex_exit(&p->p_lock);
 837 
 838                 rctl_prealloc_destroy(init_gp);
 839 
 840                 t = lwp_kernel_create(p, pc, arg, TS_STOPPED, pri);
 841         } else {
 842                 rctl_alloc_gp_t *init_gp, *default_gp;
 843                 rctl_set_t *init_set;
 844                 task_t *tk, *tk_old;
 845                 klwp_t *lwp;
 846 
 847                 if (getproc(&p, pid, GETPROC_USER) < 0)
 848                         return (EAGAIN);
 849                 /*
 850                  * init creates a new task, distinct from the task
 851                  * containing kernel "processes".
 852                  */
 853                 tk = task_create(0, p->p_zone);
 854                 mutex_enter(&tk->tk_zone->zone_nlwps_lock);
 855                 tk->tk_proj->kpj_ntasks++;
 856                 tk->tk_nprocs++;
 857                 mutex_exit(&tk->tk_zone->zone_nlwps_lock);
 858 
 859                 default_gp = rctl_rlimit_set_prealloc(RLIM_NLIMITS);
 860                 init_gp = rctl_set_init_prealloc(RCENTITY_PROCESS);
 861                 init_set = rctl_set_create();
 862 
 863                 mutex_enter(&pidlock);
 864                 mutex_enter(&p->p_lock);
 865                 tk_old = p->p_task;  /* switch to new task */
 866 
 867                 task_detach(p);
 868                 task_begin(tk, p);
 869                 mutex_exit(&pidlock);
 870 
 871                 mutex_enter(&tk_old->tk_zone->zone_nlwps_lock);
 872                 tk_old->tk_nprocs--;
 873                 mutex_exit(&tk_old->tk_zone->zone_nlwps_lock);
 874 
 875                 e.rcep_p.proc = p;
 876                 e.rcep_t = RCENTITY_PROCESS;
 877                 p->p_rctls = rctl_set_init(RCENTITY_PROCESS, p, &e, init_set,
 878                     init_gp);
 879                 rctlproc_default_init(p, default_gp);
 880                 mutex_exit(&p->p_lock);
 881 
 882                 task_rele(tk_old);
 883                 rctl_prealloc_destroy(default_gp);
 884                 rctl_prealloc_destroy(init_gp);
 885 
 886                 if ((lwp = lwp_create(pc, arg, 0, p, TS_STOPPED, pri,
 887                     &curthread->t_hold, cid, 1)) == NULL) {
 888                         task_t *tk;
 889                         fork_fail(p);
 890                         mutex_enter(&pidlock);
 891                         mutex_enter(&p->p_lock);
 892                         tk = p->p_task;
 893                         task_detach(p);
 894                         ASSERT(p->p_pool->pool_ref > 0);
 895                         atomic_add_32(&p->p_pool->pool_ref, -1);
 896                         mutex_exit(&p->p_lock);
 897                         pid_exit(p, tk);
 898                         mutex_exit(&pidlock);
 899                         task_rele(tk);
 900 
 901                         return (EAGAIN);
 902                 }
 903                 t = lwptot(lwp);
 904 
 905                 ctp = contract_process_fork(sys_process_tmpl, p, curproc,
 906                     B_FALSE);
 907                 ASSERT(ctp != NULL);
 908                 if (ct != NULL)
 909                         *ct = &ctp->conp_contract;
 910         }
 911 
 912         ASSERT3U(t->t_tid, ==, 1);
 913         p->p_lwpid = 1;
 914         mutex_enter(&pidlock);
 915         pgjoin(p, p->p_parent->p_pgidp);
 916         p->p_stat = SRUN;
 917         mutex_enter(&p->p_lock);
 918         t->t_proc_flag &= ~TP_HOLDLWP;
 919         lwp_create_done(t);
 920         mutex_exit(&p->p_lock);
 921         mutex_exit(&pidlock);
 922         return (0);
 923 }
 924 
 925 /*
 926  * create a child proc struct.
 927  */
 928 static int
 929 getproc(proc_t **cpp, pid_t pid, uint_t flags)
 930 {
 931         proc_t          *pp, *cp;
 932         pid_t           newpid;
 933         struct user     *uarea;
 934         extern uint_t   nproc;
 935         struct cred     *cr;
 936         uid_t           ruid;
 937         zoneid_t        zoneid;
 938         task_t          *task;
 939         kproject_t      *proj;
 940         zone_t          *zone;
 941         int             rctlfail = 0;
 942 
 943         if (zone_status_get(curproc->p_zone) >= ZONE_IS_SHUTTING_DOWN)
 944                 return (-1);    /* no point in starting new processes */
 945 
 946         pp = (flags & GETPROC_KERNEL) ? &p0 : curproc;
 947         task = pp->p_task;
 948         proj = task->tk_proj;
 949         zone = pp->p_zone;
 950 
 951         mutex_enter(&pp->p_lock);
 952         mutex_enter(&zone->zone_nlwps_lock);
 953         if (proj != proj0p) {
 954                 if (task->tk_nprocs >= task->tk_nprocs_ctl)
 955                         if (rctl_test(rc_task_nprocs, task->tk_rctls,
 956                             pp, 1, 0) & RCT_DENY)
 957                                 rctlfail = 1;
 958 
 959                 if (proj->kpj_nprocs >= proj->kpj_nprocs_ctl)
 960                         if (rctl_test(rc_project_nprocs, proj->kpj_rctls,
 961                             pp, 1, 0) & RCT_DENY)
 962                                 rctlfail = 1;
 963 
 964                 if (zone->zone_nprocs >= zone->zone_nprocs_ctl)
 965                         if (rctl_test(rc_zone_nprocs, zone->zone_rctls,
 966                             pp, 1, 0) & RCT_DENY)
 967                                 rctlfail = 1;
 968 
 969                 if (rctlfail) {
 970                         mutex_exit(&zone->zone_nlwps_lock);
 971                         mutex_exit(&pp->p_lock);
 972                         atomic_inc_32(&zone->zone_ffcap);
 973                         goto punish;
 974                 }
 975         }
 976         task->tk_nprocs++;
 977         proj->kpj_nprocs++;
 978         zone->zone_nprocs++;
 979         mutex_exit(&zone->zone_nlwps_lock);
 980         mutex_exit(&pp->p_lock);
 981 
 982         cp = kmem_cache_alloc(process_cache, KM_SLEEP);
 983         bzero(cp, sizeof (proc_t));
 984 
 985         /*
 986          * Make proc entry for child process
 987          */
 988         mutex_init(&cp->p_splock, NULL, MUTEX_DEFAULT, NULL);
 989         mutex_init(&cp->p_crlock, NULL, MUTEX_DEFAULT, NULL);
 990         mutex_init(&cp->p_pflock, NULL, MUTEX_DEFAULT, NULL);
 991 #if defined(__x86)
 992         mutex_init(&cp->p_ldtlock, NULL, MUTEX_DEFAULT, NULL);
 993 #endif
 994         mutex_init(&cp->p_maplock, NULL, MUTEX_DEFAULT, NULL);
 995         cp->p_stat = SIDL;
 996         cp->p_mstart = gethrtime();
 997         cp->p_as = &kas;
 998         /*
 999          * p_zone must be set before we call pid_allocate since the process
1000          * will be visible after that and code such as prfind_zone will
1001          * look at the p_zone field.
1002          */
1003         cp->p_zone = pp->p_zone;
1004         cp->p_t1_lgrpid = LGRP_NONE;
1005         cp->p_tr_lgrpid = LGRP_NONE;
1006 
1007         if ((newpid = pid_allocate(cp, pid, PID_ALLOC_PROC)) == -1) {
1008                 if (nproc == v.v_proc) {
1009                         CPU_STATS_ADDQ(CPU, sys, procovf, 1);
1010                         cmn_err(CE_WARN, "out of processes");
1011                 }
1012                 goto bad;
1013         }
1014 
1015         mutex_enter(&pp->p_lock);
1016         cp->p_exec = pp->p_exec;
1017         cp->p_execdir = pp->p_execdir;
1018         mutex_exit(&pp->p_lock);
1019 
1020         if (cp->p_exec) {
1021                 VN_HOLD(cp->p_exec);
1022                 /*
1023                  * Each VOP_OPEN() must be paired with a corresponding
1024                  * VOP_CLOSE(). In this case, the executable will be
1025                  * closed for the child in either proc_exit() or gexec().
1026                  */
1027                 if (VOP_OPEN(&cp->p_exec, FREAD, CRED(), NULL) != 0) {
1028                         VN_RELE(cp->p_exec);
1029                         cp->p_exec = NULLVP;
1030                         cp->p_execdir = NULLVP;
1031                         goto bad;
1032                 }
1033         }
1034         if (cp->p_execdir)
1035                 VN_HOLD(cp->p_execdir);
1036 
1037         /*
1038          * If not privileged make sure that this user hasn't exceeded
1039          * v.v_maxup processes, and that users collectively haven't
1040          * exceeded v.v_maxupttl processes.
1041          */
1042         mutex_enter(&pidlock);
1043         ASSERT(nproc < v.v_proc);    /* otherwise how'd we get our pid? */
1044         cr = CRED();
1045         ruid = crgetruid(cr);
1046         zoneid = crgetzoneid(cr);
1047         if (nproc >= v.v_maxup &&    /* short-circuit; usually false */
1048             (nproc >= v.v_maxupttl ||
1049             upcount_get(ruid, zoneid) >= v.v_maxup) &&
1050             secpolicy_newproc(cr) != 0) {
1051                 mutex_exit(&pidlock);
1052                 zcmn_err(zoneid, CE_NOTE,
1053                     "out of per-user processes for uid %d", ruid);
1054                 goto bad;
1055         }
1056 
1057         /*
1058          * Everything is cool, put the new proc on the active process list.
1059          * It is already on the pid list and in /proc.
1060          * Increment the per uid process count (upcount).
1061          */
1062         nproc++;
1063         upcount_inc(ruid, zoneid);
1064 
1065         cp->p_next = practive;
1066         practive->p_prev = cp;
1067         practive = cp;
1068 
1069         cp->p_ignore = pp->p_ignore;
1070         cp->p_siginfo = pp->p_siginfo;
1071         cp->p_flag = pp->p_flag & (SJCTL|SNOWAIT|SNOCD);
1072         cp->p_sessp = pp->p_sessp;
1073         sess_hold(pp);
1074         cp->p_brand = pp->p_brand;
1075         if (PROC_IS_BRANDED(pp))
1076                 BROP(pp)->b_copy_procdata(cp, pp);
1077         cp->p_bssbase = pp->p_bssbase;
1078         cp->p_brkbase = pp->p_brkbase;
1079         cp->p_brksize = pp->p_brksize;
1080         cp->p_brkpageszc = pp->p_brkpageszc;
1081         cp->p_stksize = pp->p_stksize;
1082         cp->p_stkpageszc = pp->p_stkpageszc;
1083         cp->p_stkprot = pp->p_stkprot;
1084         cp->p_datprot = pp->p_datprot;
1085         cp->p_usrstack = pp->p_usrstack;
1086         cp->p_model = pp->p_model;
1087         cp->p_ppid = pp->p_pid;
1088         cp->p_ancpid = pp->p_pid;
1089         cp->p_portcnt = pp->p_portcnt;
1090 
1091         /*
1092          * Initialize watchpoint structures
1093          */
1094         avl_create(&cp->p_warea, wa_compare, sizeof (struct watched_area),
1095             offsetof(struct watched_area, wa_link));
1096 
1097         /*
1098          * Initialize immediate resource control values.
1099          */
1100         cp->p_stk_ctl = pp->p_stk_ctl;
1101         cp->p_fsz_ctl = pp->p_fsz_ctl;
1102         cp->p_vmem_ctl = pp->p_vmem_ctl;
1103         cp->p_fno_ctl = pp->p_fno_ctl;
1104 
1105         /*
1106          * Link up to parent-child-sibling chain.  No need to lock
1107          * in general since only a call to freeproc() (done by the
1108          * same parent as newproc()) diddles with the child chain.
1109          */
1110         cp->p_sibling = pp->p_child;
1111         if (pp->p_child)
1112                 pp->p_child->p_psibling = cp;
1113 
1114         cp->p_parent = pp;
1115         pp->p_child = cp;
1116 
1117         cp->p_child_ns = NULL;
1118         cp->p_sibling_ns = NULL;
1119 
1120         cp->p_nextorph = pp->p_orphan;
1121         cp->p_nextofkin = pp;
1122         pp->p_orphan = cp;
1123 
1124         /*
1125          * Inherit profiling state; do not inherit REALPROF profiling state.
1126          */
1127         cp->p_prof = pp->p_prof;
1128         cp->p_rprof_cyclic = CYCLIC_NONE;
1129 
1130         /*
1131          * Inherit pool pointer from the parent.  Kernel processes are
1132          * always bound to the default pool.
1133          */
1134         mutex_enter(&pp->p_lock);
1135         if (flags & GETPROC_KERNEL) {
1136                 cp->p_pool = pool_default;
1137                 cp->p_flag |= SSYS;
1138         } else {
1139                 cp->p_pool = pp->p_pool;
1140         }
1141         atomic_inc_32(&cp->p_pool->pool_ref);
1142         mutex_exit(&pp->p_lock);
1143 
1144         /*
1145          * Add the child process to the current task.  Kernel processes
1146          * are always attached to task0.
1147          */
1148         mutex_enter(&cp->p_lock);
1149         if (flags & GETPROC_KERNEL)
1150                 task_attach(task0p, cp);
1151         else
1152                 task_attach(pp->p_task, cp);
1153         mutex_exit(&cp->p_lock);
1154         mutex_exit(&pidlock);
1155 
1156         avl_create(&cp->p_ct_held, contract_compar, sizeof (contract_t),
1157             offsetof(contract_t, ct_ctlist));
1158 
1159         /*
1160          * Duplicate any audit information kept in the process table
1161          */
1162         if (audit_active)       /* copy audit data to cp */
1163                 audit_newproc(cp);
1164 
1165         crhold(cp->p_cred = cr);
1166 
1167         /*
1168          * Bump up the counts on the file structures pointed at by the
1169          * parent's file table since the child will point at them too.
1170          */
1171         fcnt_add(P_FINFO(pp), 1);
1172 
1173         if (PTOU(pp)->u_cdir) {
1174                 VN_HOLD(PTOU(pp)->u_cdir);
1175         } else {
1176                 ASSERT(pp == &p0);
1177                 /*
1178                  * We must be at or before vfs_mountroot(); it will take care of
1179                  * assigning our current directory.
1180                  */
1181         }
1182         if (PTOU(pp)->u_rdir)
1183                 VN_HOLD(PTOU(pp)->u_rdir);
1184         if (PTOU(pp)->u_cwd)
1185                 refstr_hold(PTOU(pp)->u_cwd);
1186 
1187         /*
1188          * copy the parent's uarea.
1189          */
1190         uarea = PTOU(cp);
1191         bcopy(PTOU(pp), uarea, sizeof (*uarea));
1192         flist_fork(pp, cp);
1193 
1194         gethrestime(&uarea->u_start);
1195         uarea->u_ticks = ddi_get_lbolt();
1196         uarea->u_mem = rm_asrss(pp->p_as);
1197         uarea->u_acflag = AFORK;
1198 
1199         /*
1200          * If inherit-on-fork, copy /proc tracing flags to child.
1201          */
1202         if ((pp->p_proc_flag & P_PR_FORK) != 0) {
1203                 cp->p_proc_flag |= pp->p_proc_flag & (P_PR_TRACE|P_PR_FORK);
1204                 cp->p_sigmask = pp->p_sigmask;
1205                 cp->p_fltmask = pp->p_fltmask;
1206         } else {
1207                 sigemptyset(&cp->p_sigmask);
1208                 premptyset(&cp->p_fltmask);
1209                 uarea->u_systrap = 0;
1210                 premptyset(&uarea->u_entrymask);
1211                 premptyset(&uarea->u_exitmask);
1212         }
1213         /*
1214          * If microstate accounting is being inherited, mark child
1215          */
1216         if ((pp->p_flag & SMSFORK) != 0)
1217                 cp->p_flag |= pp->p_flag & (SMSFORK|SMSACCT);
1218 
1219         /*
1220          * Inherit fixalignment flag from the parent
1221          */
1222         cp->p_fixalignment = pp->p_fixalignment;
1223 
1224         *cpp = cp;
1225         return (0);
1226 
1227 bad:
1228         ASSERT(MUTEX_NOT_HELD(&pidlock));
1229 
1230         mutex_destroy(&cp->p_crlock);
1231         mutex_destroy(&cp->p_pflock);
1232 #if defined(__x86)
1233         mutex_destroy(&cp->p_ldtlock);
1234 #endif
1235         if (newpid != -1) {
1236                 proc_entry_free(cp->p_pidp);
1237                 (void) pid_rele(cp->p_pidp);
1238         }
1239         kmem_cache_free(process_cache, cp);
1240 
1241         mutex_enter(&zone->zone_nlwps_lock);
1242         task->tk_nprocs--;
1243         proj->kpj_nprocs--;
1244         zone->zone_nprocs--;
1245         mutex_exit(&zone->zone_nlwps_lock);
1246         atomic_inc_32(&zone->zone_ffnoproc);
1247 
1248 punish:
1249         /*
1250          * We most likely got into this situation because some process is
1251          * forking out of control.  As punishment, put it to sleep for a
1252          * bit so it can't eat the machine alive.  Sleep interval is chosen
1253          * to allow no more than one fork failure per cpu per clock tick
1254          * on average (yes, I just made this up).  This has two desirable
1255          * properties: (1) it sets a constant limit on the fork failure
1256          * rate, and (2) the busier the system is, the harsher the penalty
1257          * for abusing it becomes.
1258          */
1259         INCR_COUNT(&fork_fail_pending, &pidlock);
1260         delay(fork_fail_pending / ncpus + 1);
1261         DECR_COUNT(&fork_fail_pending, &pidlock);
1262 
1263         return (-1); /* out of memory or proc slots */
1264 }
1265 
1266 /*
1267  * Release virtual memory.
1268  * In the case of vfork(), the child was given exclusive access to its
1269  * parent's address space.  The parent is waiting in vfwait() for the
1270  * child to release its exclusive claim via relvm().
1271  */
1272 void
1273 relvm()
1274 {
1275         proc_t *p = curproc;
1276 
1277         ASSERT((unsigned)p->p_lwpcnt <= 1);
1278 
1279         prrelvm();      /* inform /proc */
1280 
1281         if (p->p_flag & SVFORK) {
1282                 proc_t *pp = p->p_parent;
1283                 /*
1284                  * The child process is either exec'ing or exit'ing.
1285                  * The child is now separated from the parent's address
1286                  * space.  The parent process is made dispatchable.
1287                  *
1288                  * This is a delicate locking maneuver, involving
1289                  * both the parent's p_lock and the child's p_lock.
1290                  * As soon as the SVFORK flag is turned off, the
1291                  * parent is free to run, but it must not run until
1292                  * we wake it up using its p_cv because it might
1293                  * exit and we would be referencing invalid memory.
1294                  * Therefore, we hold the parent with its p_lock
1295                  * while protecting our p_flags with our own p_lock.
1296                  */
1297 try_again:
1298                 mutex_enter(&p->p_lock); /* grab child's lock first */
1299                 prbarrier(p);           /* make sure /proc is blocked out */
1300                 mutex_enter(&pp->p_lock);
1301 
1302                 /*
1303                  * Check if parent is locked by /proc.
1304                  */
1305                 if (pp->p_proc_flag & P_PR_LOCK) {
1306                         /*
1307                          * Delay until /proc is done with the parent.
1308                          * We must drop our (the child's) p->p_lock, wait
1309                          * via prbarrier() on the parent, then start over.
1310                          */
1311                         mutex_exit(&p->p_lock);
1312                         prbarrier(pp);
1313                         mutex_exit(&pp->p_lock);
1314                         goto try_again;
1315                 }
1316                 p->p_flag &= ~SVFORK;
1317                 kpreempt_disable();
1318                 p->p_as = &kas;
1319 
1320                 /*
1321                  * notify hat of change in thread's address space
1322                  */
1323                 hat_thread_exit(curthread);
1324                 kpreempt_enable();
1325 
1326                 /*
1327                  * child sizes are copied back to parent because
1328                  * child may have grown.
1329                  */
1330                 pp->p_brkbase = p->p_brkbase;
1331                 pp->p_brksize = p->p_brksize;
1332                 pp->p_stksize = p->p_stksize;
1333 
1334                 /*
1335                  * Copy back the shm accounting information
1336                  * to the parent process.
1337                  */
1338                 pp->p_segacct = p->p_segacct;
1339                 p->p_segacct = NULL;
1340 
1341                 /*
1342                  * The parent is no longer waiting for the vfork()d child.
1343                  * Restore the parent's watched pages, if any.  This is
1344                  * safe because we know the parent is not locked by /proc
1345                  */
1346                 pp->p_flag &= ~SVFWAIT;
1347                 if (avl_numnodes(&pp->p_wpage) != 0) {
1348                         pp->p_as->a_wpage = pp->p_wpage;
1349                         avl_create(&pp->p_wpage, wp_compare,
1350                             sizeof (struct watched_page),
1351                             offsetof(struct watched_page, wp_link));
1352                 }
1353                 cv_signal(&pp->p_cv);
1354                 mutex_exit(&pp->p_lock);
1355                 mutex_exit(&p->p_lock);
1356         } else {
1357                 if (p->p_as != &kas) {
1358                         struct as *as;
1359 
1360                         if (p->p_segacct)
1361                                 shmexit(p);
1362 
1363                         /*
1364                          * We grab p_lock for the benefit of /proc
1365                          */
1366                         kpreempt_disable();
1367                         mutex_enter(&p->p_lock);
1368                         prbarrier(p);   /* make sure /proc is blocked out */
1369                         as = p->p_as;
1370                         p->p_as = &kas;
1371                         mutex_exit(&p->p_lock);
1372 
1373                         /*
1374                          * notify hat of change in thread's address space
1375                          */
1376                         hat_thread_exit(curthread);
1377                         kpreempt_enable();
1378 
1379                         as_free(as);
1380                         p->p_tr_lgrpid = LGRP_NONE;
1381                 }
1382         }
1383 }
1384 
1385 /*
1386  * Wait for child to exec or exit.
1387  * Called by parent of vfork'ed process.
1388  * See important comments in relvm(), above.
1389  */
1390 void
1391 vfwait(pid_t pid)
1392 {
1393         int signalled = 0;
1394         proc_t *pp = ttoproc(curthread);
1395         proc_t *cp;
1396 
1397         /*
1398          * Wait for child to exec or exit.
1399          */
1400         for (;;) {
1401                 mutex_enter(&pidlock);
1402                 cp = prfind(pid);
1403                 if (cp == NULL || cp->p_parent != pp) {
1404                         /*
1405                          * Child has exit()ed.
1406                          */
1407                         mutex_exit(&pidlock);
1408                         break;
1409                 }
1410                 /*
1411                  * Grab the child's p_lock before releasing pidlock.
1412                  * Otherwise, the child could exit and we would be
1413                  * referencing invalid memory.
1414                  */
1415                 mutex_enter(&cp->p_lock);
1416                 mutex_exit(&pidlock);
1417                 if (!(cp->p_flag & SVFORK)) {
1418                         /*
1419                          * Child has exec()ed or is exit()ing.
1420                          */
1421                         mutex_exit(&cp->p_lock);
1422                         break;
1423                 }
1424                 mutex_enter(&pp->p_lock);
1425                 mutex_exit(&cp->p_lock);
1426                 /*
1427                  * We might be waked up spuriously from the cv_wait().
1428                  * We have to do the whole operation over again to be
1429                  * sure the child's SVFORK flag really is turned off.
1430                  * We cannot make reference to the child because it can
1431                  * exit before we return and we would be referencing
1432                  * invalid memory.
1433                  *
1434                  * Because this is potentially a very long-term wait,
1435                  * we call cv_wait_sig() (for its jobcontrol and /proc
1436                  * side-effects) unless there is a current signal, in
1437                  * which case we use cv_wait() because we cannot return
1438                  * from this function until the child has released the
1439                  * address space.  Calling cv_wait_sig() with a current
1440                  * signal would lead to an indefinite loop here because
1441                  * cv_wait_sig() returns immediately in this case.
1442                  */
1443                 if (signalled)
1444                         cv_wait(&pp->p_cv, &pp->p_lock);
1445                 else
1446                         signalled = !cv_wait_sig(&pp->p_cv, &pp->p_lock);
1447                 mutex_exit(&pp->p_lock);
1448         }
1449 
1450         /* restore watchpoints to parent */
1451         if (pr_watch_active(pp)) {
1452                 struct as *as = pp->p_as;
1453                 AS_LOCK_ENTER(as, RW_WRITER);
1454                 as_setwatch(as);
1455                 AS_LOCK_EXIT(as);
1456         }
1457 
1458         mutex_enter(&pp->p_lock);
1459         prbarrier(pp);  /* barrier against /proc locking */
1460         continuelwps(pp);
1461         mutex_exit(&pp->p_lock);
1462 }