Print this page
11909 THREAD_KPRI_RELEASE does nothing of the sort
Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>


   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2013, Joyent, Inc. All rights reserved.
  25  */
  26 
  27 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  28 /*        All Rights Reserved   */
  29 
  30 #include <sys/types.h>
  31 #include <sys/param.h>
  32 #include <sys/sysmacros.h>
  33 #include <sys/cred.h>
  34 #include <sys/proc.h>
  35 #include <sys/session.h>
  36 #include <sys/strsubr.h>
  37 #include <sys/signal.h>
  38 #include <sys/user.h>
  39 #include <sys/priocntl.h>
  40 #include <sys/class.h>
  41 #include <sys/disp.h>
  42 #include <sys/procset.h>
  43 #include <sys/debug.h>
  44 #include <sys/ts.h>


 212 static pri_t    ts_globpri(kthread_t *);
 213 static void     ts_yield(kthread_t *);
 214 extern tsdpent_t *ts_getdptbl(void);
 215 extern pri_t    *ts_getkmdpris(void);
 216 extern pri_t    td_getmaxumdpri(void);
 217 static int      ts_alloc(void **, int);
 218 static void     ts_free(void *);
 219 
 220 pri_t           ia_init(id_t, int, classfuncs_t **);
 221 static int      ia_getclinfo(void *);
 222 static int      ia_getclpri(pcpri_t *);
 223 static int      ia_parmsin(void *);
 224 static int      ia_vaparmsin(void *, pc_vaparms_t *);
 225 static int      ia_vaparmsout(void *, pc_vaparms_t *);
 226 static int      ia_parmsset(kthread_t *, void *, id_t, cred_t *);
 227 static void     ia_parmsget(kthread_t *, void *);
 228 static void     ia_set_process_group(pid_t, pid_t, pid_t);
 229 
 230 static void     ts_change_priority(kthread_t *, tsproc_t *);
 231 
 232 extern pri_t    ts_maxkmdpri;   /* maximum kernel mode ts priority */
 233 static pri_t    ts_maxglobpri;  /* maximum global priority used by ts class */
 234 static kmutex_t ts_dptblock;    /* protects time sharing dispatch table */
 235 static kmutex_t ts_list_lock[TS_LISTS]; /* protects tsproc lists */
 236 static tsproc_t ts_plisthead[TS_LISTS]; /* dummy tsproc at head of lists */
 237 
 238 static gid_t    IA_gid = 0;
 239 
 240 static struct classfuncs ts_classfuncs = {
 241         /* class functions */
 242         ts_admin,
 243         ts_getclinfo,
 244         ts_parmsin,
 245         ts_parmsout,
 246         ts_vaparmsin,
 247         ts_vaparmsout,
 248         ts_getclpri,
 249         ts_alloc,
 250         ts_free,
 251 
 252         /* thread functions */


 524                         ts_dptbl[i].ts_maxwait = tmpdpp[i].ts_maxwait;
 525                         ts_dptbl[i].ts_lwait = tmpdpp[i].ts_lwait;
 526                 }
 527                 mutex_exit(&ts_dptblock);
 528                 kmem_free(tmpdpp, tsdpsz);
 529                 break;
 530 
 531         default:
 532                 return (EINVAL);
 533         }
 534         return (0);
 535 }
 536 
 537 
 538 /*
 539  * Allocate a time-sharing class specific thread structure and
 540  * initialize it with the parameters supplied. Also move the thread
 541  * to specified time-sharing priority.
 542  */
 543 static int
 544 ts_enterclass(kthread_t *t, id_t cid, void *parmsp,
 545         cred_t *reqpcredp, void *bufp)
 546 {
 547         tsparms_t       *tsparmsp = (tsparms_t *)parmsp;
 548         tsproc_t        *tspp;
 549         pri_t           reqtsuprilim;
 550         pri_t           reqtsupri;
 551         static uint32_t tspexists = 0;  /* set on first occurrence of */
 552                                         /*   a time-sharing process */
 553 
 554         tspp = (tsproc_t *)bufp;
 555         ASSERT(tspp != NULL);
 556 
 557         /*
 558          * Initialize the tsproc structure.
 559          */
 560         tspp->ts_cpupri = tsmedumdpri;
 561         if (cid == ia_cid) {
 562                 /*
 563                  * Check to make sure caller is either privileged or the
 564                  * window system.  When the window system is converted
 565                  * to using privileges, the second check can go away.


 686         tsproc_t        *ptspp;         /* ptr to parent's tsproc structure */
 687         tsproc_t        *ctspp;         /* ptr to child's tsproc structure */
 688 
 689         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
 690 
 691         ctspp = (tsproc_t *)bufp;
 692         ASSERT(ctspp != NULL);
 693         ptspp = (tsproc_t *)t->t_cldata;
 694         /*
 695          * Initialize child's tsproc structure.
 696          */
 697         thread_lock(t);
 698         ctspp->ts_timeleft = ts_dptbl[ptspp->ts_cpupri].ts_quantum;
 699         ctspp->ts_cpupri = ptspp->ts_cpupri;
 700         ctspp->ts_boost = ptspp->ts_boost;
 701         ctspp->ts_uprilim = ptspp->ts_uprilim;
 702         ctspp->ts_upri = ptspp->ts_upri;
 703         TS_NEWUMDPRI(ctspp);
 704         ctspp->ts_nice = ptspp->ts_nice;
 705         ctspp->ts_dispwait = 0;
 706         ctspp->ts_flags = ptspp->ts_flags & ~(TSKPRI | TSBACKQ | TSRESTORE);
 707         ctspp->ts_tp = ct;
 708         cpucaps_sc_init(&ctspp->ts_caps);
 709         thread_unlock(t);
 710 
 711         /*
 712          * Link new structure into tsproc list.
 713          */
 714         ct->t_cldata = (void *)ctspp;
 715         TS_LIST_INSERT(ctspp);
 716         return (0);
 717 }
 718 
 719 
 720 /*
 721  * Child is placed at back of dispatcher queue and parent gives
 722  * up processor so that the child runs first after the fork.
 723  * This allows the child immediately execing to break the multiple
 724  * use of copy on write pages with no disk home. The parent will
 725  * get to steal them back rather than uselessly copying them.
 726  */


 737         /*
 738          * Grab the child's p_lock before dropping pidlock to ensure
 739          * the process does not disappear before we set it running.
 740          */
 741         mutex_enter(&cp->p_lock);
 742         continuelwps(cp);
 743         mutex_exit(&cp->p_lock);
 744 
 745         mutex_enter(&pp->p_lock);
 746         mutex_exit(&pidlock);
 747         continuelwps(pp);
 748 
 749         thread_lock(t);
 750         tspp = (tsproc_t *)(t->t_cldata);
 751         tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_tqexp;
 752         TS_NEWUMDPRI(tspp);
 753         tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
 754         tspp->ts_dispwait = 0;
 755         t->t_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
 756         ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
 757         tspp->ts_flags &= ~TSKPRI;
 758         THREAD_TRANSITION(t);
 759         ts_setrun(t);
 760         thread_unlock(t);
 761         /*
 762          * Safe to drop p_lock now since since it is safe to change
 763          * the scheduling class after this point.
 764          */
 765         mutex_exit(&pp->p_lock);
 766 
 767         swtch();
 768 }
 769 
 770 
 771 /*
 772  * Get information about the time-sharing class into the buffer
 773  * pointed to by tsinfop. The maximum configured user priority
 774  * is the only information we supply.  ts_getclinfo() is called
 775  * for TS threads, and ia_getclinfo() is called for IA threads.
 776  */
 777 static int


1200             reqtsuprilim > tspp->ts_uprilim &&
1201             secpolicy_raisepriority(reqpcredp) != 0)
1202                 return (EPERM);
1203 
1204         /*
1205          * Set ts_nice to the nice value corresponding to the user
1206          * priority we are setting.  Note that setting the nice field
1207          * of the parameter struct won't affect upri or nice.
1208          */
1209         nice = NZERO - (reqtsupri * NZERO) / ts_maxupri;
1210         if (nice >= 2 * NZERO)
1211                 nice = 2 * NZERO - 1;
1212 
1213         thread_lock(tx);
1214 
1215         tspp->ts_uprilim = reqtsuprilim;
1216         tspp->ts_upri = reqtsupri;
1217         TS_NEWUMDPRI(tspp);
1218         tspp->ts_nice = nice;
1219 
1220         if ((tspp->ts_flags & TSKPRI) != 0) {
1221                 thread_unlock(tx);
1222                 return (0);
1223         }
1224 
1225         tspp->ts_dispwait = 0;
1226         ts_change_priority(tx, tspp);
1227         thread_unlock(tx);
1228         return (0);
1229 }
1230 
1231 
1232 static int
1233 ia_parmsset(kthread_t *tx, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1234 {
1235         tsproc_t        *tspp = (tsproc_t *)tx->t_cldata;
1236         iaparms_t       *iaparmsp = (iaparms_t *)parmsp;
1237         proc_t          *p;
1238         pid_t           pid, pgid, sid;
1239         pid_t           on, off;
1240         struct stdata   *stp;
1241         int             sess_held;
1242 
1243         /*
1244          * Handle user priority changes


1356         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1357         tspp = (tsproc_t *)t->t_cldata;
1358         tspri = tsmedumdpri + tspp->ts_upri;
1359         if (tspri > ts_maxumdpri)
1360                 tspri = ts_maxumdpri;
1361         else if (tspri < 0)
1362                 tspri = 0;
1363         return (ts_dptbl[tspri].ts_globpri);
1364 }
1365 
1366 /*
1367  * Arrange for thread to be placed in appropriate location
1368  * on dispatcher queue.
1369  *
1370  * This is called with the current thread in TS_ONPROC and locked.
1371  */
1372 static void
1373 ts_preempt(kthread_t *t)
1374 {
1375         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1376         klwp_t          *lwp = curthread->t_lwp;
1377         pri_t           oldpri = t->t_pri;
1378 
1379         ASSERT(t == curthread);
1380         ASSERT(THREAD_LOCK_HELD(curthread));
1381 
1382         /*
1383          * If preempted in the kernel, make sure the thread has
1384          * a kernel priority if needed.
1385          */
1386         if (!(tspp->ts_flags & TSKPRI) && lwp != NULL && t->t_kpri_req) {
1387                 tspp->ts_flags |= TSKPRI;
1388                 THREAD_CHANGE_PRI(t, ts_kmdpris[0]);
1389                 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1390                 t->t_trapret = 1;            /* so ts_trapret will run */
1391                 aston(t);
1392         }
1393 
1394         /*
1395          * This thread may be placed on wait queue by CPU Caps. In this case we
1396          * do not need to do anything until it is removed from the wait queue.
1397          * Do not enforce CPU caps on threads running at a kernel priority
1398          */
1399         if (CPUCAPS_ON()) {
1400                 (void) cpucaps_charge(t, &tspp->ts_caps,
1401                     CPUCAPS_CHARGE_ENFORCE);
1402                 if (!(tspp->ts_flags & TSKPRI) && CPUCAPS_ENFORCE(t))
1403                         return;
1404         }
1405 
1406         /*
1407          * If thread got preempted in the user-land then we know
1408          * it isn't holding any locks.  Mark it as swappable.
1409          */
1410         ASSERT(t->t_schedflag & TS_DONT_SWAP);
1411         if (lwp != NULL && lwp->lwp_state == LWP_USER)
1412                 t->t_schedflag &= ~TS_DONT_SWAP;
1413 
1414         /*
1415          * Check to see if we're doing "preemption control" here.  If
1416          * we are, and if the user has requested that this thread not
1417          * be preempted, and if preemptions haven't been put off for
1418          * too long, let the preemption happen here but try to make
1419          * sure the thread is rescheduled as soon as possible.  We do
1420          * this by putting it on the front of the highest priority run
1421          * queue in the TS class.  If the preemption has been put off
1422          * for too long, clear the "nopreempt" bit and let the thread
1423          * be preempted.
1424          */
1425         if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1426                 if (tspp->ts_timeleft > -SC_MAX_TICKS) {
1427                         DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
1428                         if (!(tspp->ts_flags & TSKPRI)) {
1429                                 /*
1430                                  * If not already remembered, remember current
1431                                  * priority for restoration in ts_yield().
1432                                  */
1433                                 if (!(tspp->ts_flags & TSRESTORE)) {
1434                                         tspp->ts_scpri = t->t_pri;
1435                                         tspp->ts_flags |= TSRESTORE;
1436                                 }
1437                                 THREAD_CHANGE_PRI(t, ts_maxumdpri);
1438                                 t->t_schedflag |= TS_DONT_SWAP;
1439                         }
1440                         schedctl_set_yield(t, 1);
1441                         setfrontdq(t);
1442                         goto done;
1443                 } else {
1444                         if (tspp->ts_flags & TSRESTORE) {
1445                                 THREAD_CHANGE_PRI(t, tspp->ts_scpri);
1446                                 tspp->ts_flags &= ~TSRESTORE;
1447                         }
1448                         schedctl_set_nopreempt(t, 0);
1449                         DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
1450                         TNF_PROBE_2(schedctl_preempt, "schedctl TS ts_preempt",
1451                             /* CSTYLED */, tnf_pid, pid, ttoproc(t)->p_pid,
1452                             tnf_lwpid, lwpid, t->t_tid);
1453                         /*
1454                          * Fall through and be preempted below.
1455                          */
1456                 }
1457         }
1458 
1459         if ((tspp->ts_flags & (TSBACKQ|TSKPRI)) == TSBACKQ) {
1460                 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1461                 tspp->ts_dispwait = 0;
1462                 tspp->ts_flags &= ~TSBACKQ;
1463                 setbackdq(t);
1464         } else if ((tspp->ts_flags & (TSBACKQ|TSKPRI)) == (TSBACKQ|TSKPRI)) {
1465                 tspp->ts_flags &= ~TSBACKQ;
1466                 setbackdq(t);
1467         } else {
1468                 setfrontdq(t);
1469         }
1470 
1471 done:
1472         TRACE_2(TR_FAC_DISP, TR_PREEMPT,
1473             "preempt:tid %p old pri %d", t, oldpri);
1474 }
1475 
1476 static void
1477 ts_setrun(kthread_t *t)
1478 {
1479         tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1480 
1481         ASSERT(THREAD_LOCK_HELD(t));    /* t should be in transition */
1482 
1483         if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1484                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1485                 TS_NEWUMDPRI(tspp);
1486                 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1487                 tspp->ts_dispwait = 0;
1488                 if ((tspp->ts_flags & TSKPRI) == 0) {
1489                         THREAD_CHANGE_PRI(t,
1490                             ts_dptbl[tspp->ts_umdpri].ts_globpri);
1491                         ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1492                 }
1493         }
1494 
1495         tspp->ts_flags &= ~TSBACKQ;
1496 
1497         if (tspp->ts_flags & TSIA) {
1498                 if (tspp->ts_flags & TSIASET)
1499                         setfrontdq(t);
1500                 else
1501                         setbackdq(t);
1502         } else {
1503                 if (t->t_disp_time != ddi_get_lbolt())
1504                         setbackdq(t);
1505                 else
1506                         setfrontdq(t);
1507         }
1508 }
1509 
1510 
1511 /*
1512  * Prepare thread for sleep. We reset the thread priority so it will
1513  * run at the kernel priority level when it wakes up.
1514  */
1515 static void
1516 ts_sleep(kthread_t *t)
1517 {
1518         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1519         int             flags;
1520         pri_t           old_pri = t->t_pri;
1521 
1522         ASSERT(t == curthread);
1523         ASSERT(THREAD_LOCK_HELD(t));
1524 
1525         /*
1526          * Account for time spent on CPU before going to sleep.
1527          */
1528         (void) CPUCAPS_CHARGE(t, &tspp->ts_caps, CPUCAPS_CHARGE_ENFORCE);
1529 
1530         flags = tspp->ts_flags;
1531         if (t->t_kpri_req) {
1532                 tspp->ts_flags = flags | TSKPRI;
1533                 THREAD_CHANGE_PRI(t, ts_kmdpris[0]);
1534                 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1535                 t->t_trapret = 1;            /* so ts_trapret will run */
1536                 aston(t);
1537         } else if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1538                 /*
1539                  * If thread has blocked in the kernel (as opposed to
1540                  * being merely preempted), recompute the user mode priority.
1541                  */
1542                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1543                 TS_NEWUMDPRI(tspp);
1544                 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1545                 tspp->ts_dispwait = 0;
1546 
1547                 THREAD_CHANGE_PRI(curthread,
1548                     ts_dptbl[tspp->ts_umdpri].ts_globpri);
1549                 ASSERT(curthread->t_pri >= 0 &&
1550                     curthread->t_pri <= ts_maxglobpri);
1551                 tspp->ts_flags = flags & ~TSKPRI;
1552 
1553                 if (DISP_MUST_SURRENDER(curthread))
1554                         cpu_surrender(curthread);
1555         } else if (flags & TSKPRI) {
1556                 THREAD_CHANGE_PRI(curthread,
1557                     ts_dptbl[tspp->ts_umdpri].ts_globpri);
1558                 ASSERT(curthread->t_pri >= 0 &&
1559                     curthread->t_pri <= ts_maxglobpri);
1560                 tspp->ts_flags = flags & ~TSKPRI;
1561 
1562                 if (DISP_MUST_SURRENDER(curthread))
1563                         cpu_surrender(curthread);
1564         }
1565         t->t_stime = ddi_get_lbolt();                /* time stamp for the swapper */
1566         TRACE_2(TR_FAC_DISP, TR_SLEEP,
1567             "sleep:tid %p old pri %d", t, old_pri);
1568 }
1569 
1570 
1571 /*
1572  * Return Values:
1573  *
1574  *      -1 if the thread is loaded or is not eligible to be swapped in.
1575  *
1576  *      effective priority of the specified thread based on swapout time
1577  *              and size of process (epri >= 0 , epri <= SHRT_MAX).
1578  */
1579 /* ARGSUSED */
1580 static pri_t
1581 ts_swapin(kthread_t *t, int flags)
1582 {
1583         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1584         long            epri = -1;
1585         proc_t          *pp = ttoproc(t);
1586 
1587         ASSERT(THREAD_LOCK_HELD(t));
1588 
1589         /*
1590          * We know that pri_t is a short.
1591          * Be sure not to overrun its range.
1592          */
1593         if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
1594                 time_t swapout_time;
1595 
1596                 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
1597                 if (INHERITED(t) || (tspp->ts_flags & (TSKPRI | TSIASET)))
1598                         epri = (long)DISP_PRIO(t) + swapout_time;
1599                 else {
1600                         /*
1601                          * Threads which have been out for a long time,
1602                          * have high user mode priority and are associated
1603                          * with a small address space are more deserving
1604                          */
1605                         epri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
1606                         ASSERT(epri >= 0 && epri <= ts_maxumdpri);
1607                         epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
1608                 }
1609                 /*
1610                  * Scale epri so SHRT_MAX/2 represents zero priority.
1611                  */
1612                 epri += SHRT_MAX/2;
1613                 if (epri < 0)
1614                         epri = 0;
1615                 else if (epri > SHRT_MAX)
1616                         epri = SHRT_MAX;
1617         }
1618         return ((pri_t)epri);
1619 }


1631  *              Hardswap:  Return an effective priority such that threads
1632  *                         which have been in memory for a while and are
1633  *                         associated with a small address space are swapped
1634  *                         in before others.
1635  *
1636  *              (epri >= 0 , epri <= SHRT_MAX).
1637  */
1638 time_t  ts_minrun = 2;          /* XXX - t_pri becomes 59 within 2 secs */
1639 time_t  ts_minslp = 2;          /* min time on sleep queue for hardswap */
1640 
1641 static pri_t
1642 ts_swapout(kthread_t *t, int flags)
1643 {
1644         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1645         long            epri = -1;
1646         proc_t          *pp = ttoproc(t);
1647         time_t          swapin_time;
1648 
1649         ASSERT(THREAD_LOCK_HELD(t));
1650 
1651         if (INHERITED(t) || (tspp->ts_flags & (TSKPRI | TSIASET)) ||
1652             (t->t_proc_flag & TP_LWPEXIT) ||
1653             (t->t_state & (TS_ZOMB | TS_FREE | TS_STOPPED |
1654             TS_ONPROC | TS_WAIT)) ||
1655             !(t->t_schedflag & TS_LOAD) || !SWAP_OK(t))
1656                 return (-1);
1657 
1658         ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
1659 
1660         /*
1661          * We know that pri_t is a short.
1662          * Be sure not to overrun its range.
1663          */
1664         swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
1665         if (flags == SOFTSWAP) {
1666                 if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
1667                         epri = 0;
1668                 } else {
1669                         return ((pri_t)epri);
1670                 }
1671         } else {


1700  * and set runrun to cause preemption.
1701  */
1702 static void
1703 ts_tick(kthread_t *t)
1704 {
1705         tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1706         klwp_t *lwp;
1707         boolean_t call_cpu_surrender = B_FALSE;
1708         pri_t   oldpri = t->t_pri;
1709 
1710         ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1711 
1712         thread_lock(t);
1713 
1714         /*
1715          * Keep track of thread's project CPU usage.  Note that projects
1716          * get charged even when threads are running in the kernel.
1717          */
1718         if (CPUCAPS_ON()) {
1719                 call_cpu_surrender = cpucaps_charge(t, &tspp->ts_caps,
1720                     CPUCAPS_CHARGE_ENFORCE) && !(tspp->ts_flags & TSKPRI);
1721         }
1722 
1723         if ((tspp->ts_flags & TSKPRI) == 0) {
1724                 if (--tspp->ts_timeleft <= 0) {
1725                         pri_t   new_pri;
1726 
1727                         /*
1728                          * If we're doing preemption control and trying to
1729                          * avoid preempting this thread, just note that
1730                          * the thread should yield soon and let it keep
1731                          * running (unless it's been a while).
1732                          */
1733                         if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1734                                 if (tspp->ts_timeleft > -SC_MAX_TICKS) {
1735                                         DTRACE_SCHED1(schedctl__nopreempt,
1736                                             kthread_t *, t);
1737                                         schedctl_set_yield(t, 1);
1738                                         thread_unlock_nopreempt(t);
1739                                         return;
1740                                 }
1741 
1742                                 TNF_PROBE_2(schedctl_failsafe,
1743                                     "schedctl TS ts_tick", /* CSTYLED */,
1744                                     tnf_pid, pid, ttoproc(t)->p_pid,
1745                                     tnf_lwpid, lwpid, t->t_tid);
1746                         }
1747                         tspp->ts_flags &= ~TSRESTORE;
1748                         tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_tqexp;
1749                         TS_NEWUMDPRI(tspp);
1750                         tspp->ts_dispwait = 0;
1751                         new_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
1752                         ASSERT(new_pri >= 0 && new_pri <= ts_maxglobpri);
1753                         /*
1754                          * When the priority of a thread is changed,
1755                          * it may be necessary to adjust its position
1756                          * on a sleep queue or dispatch queue.
1757                          * The function thread_change_pri accomplishes
1758                          * this.
1759                          */
1760                         if (thread_change_pri(t, new_pri, 0)) {
1761                                 if ((t->t_schedflag & TS_LOAD) &&
1762                                     (lwp = t->t_lwp) &&
1763                                     lwp->lwp_state == LWP_USER)
1764                                         t->t_schedflag &= ~TS_DONT_SWAP;
1765                                 tspp->ts_timeleft =
1766                                     ts_dptbl[tspp->ts_cpupri].ts_quantum;
1767                         } else {
1768                                 call_cpu_surrender = B_TRUE;
1769                         }
1770                         TRACE_2(TR_FAC_DISP, TR_TICK,
1771                             "tick:tid %p old pri %d", t, oldpri);
1772                 } else if (t->t_state == TS_ONPROC &&
1773                     t->t_pri < t->t_disp_queue->disp_maxrunpri) {
1774                         call_cpu_surrender = B_TRUE;
1775                 }
1776         }
1777 
1778         if (call_cpu_surrender) {
1779                 tspp->ts_flags |= TSBACKQ;
1780                 cpu_surrender(t);
1781         }
1782 
1783         thread_unlock_nopreempt(t);     /* clock thread can't be preempted */
1784 }
1785 
1786 
1787 /*
1788  * If thread is currently at a kernel mode priority (has slept)
1789  * we assign it the appropriate user mode priority and time quantum
1790  * here.  If we are lowering the thread's priority below that of
1791  * other runnable threads we will normally set runrun via cpu_surrender() to
1792  * cause preemption.
1793  */
1794 static void
1795 ts_trapret(kthread_t *t)
1796 {
1797         tsproc_t        *tspp = (tsproc_t *)t->t_cldata;
1798         cpu_t           *cp = CPU;
1799         pri_t           old_pri = curthread->t_pri;
1800 
1801         ASSERT(THREAD_LOCK_HELD(t));
1802         ASSERT(t == curthread);
1803         ASSERT(cp->cpu_dispthread == t);
1804         ASSERT(t->t_state == TS_ONPROC);
1805 
1806         t->t_kpri_req = 0;
1807         if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1808                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1809                 TS_NEWUMDPRI(tspp);
1810                 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1811                 tspp->ts_dispwait = 0;
1812 
1813                 /*
1814                  * If thread has blocked in the kernel (as opposed to
1815                  * being merely preempted), recompute the user mode priority.
1816                  */
1817                 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);
1818                 cp->cpu_dispatch_pri = DISP_PRIO(t);
1819                 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1820                 tspp->ts_flags &= ~TSKPRI;
1821 
1822                 if (DISP_MUST_SURRENDER(t))
1823                         cpu_surrender(t);
1824         } else if (tspp->ts_flags & TSKPRI) {
1825                 /*
1826                  * If thread has blocked in the kernel (as opposed to
1827                  * being merely preempted), recompute the user mode priority.
1828                  */
1829                 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);
1830                 cp->cpu_dispatch_pri = DISP_PRIO(t);
1831                 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1832                 tspp->ts_flags &= ~TSKPRI;
1833 
1834                 if (DISP_MUST_SURRENDER(t))
1835                         cpu_surrender(t);
1836         }
1837 
1838         /*
1839          * Swapout lwp if the swapper is waiting for this thread to
1840          * reach a safe point.
1841          */
1842         if ((t->t_schedflag & TS_SWAPENQ) && !(tspp->ts_flags & TSIASET)) {
1843                 thread_unlock(t);
1844                 swapout_lwp(ttolwp(t));
1845                 thread_lock(t);
1846         }
1847 
1848         TRACE_2(TR_FAC_DISP, TR_TRAPRET,
1849             "trapret:tid %p old pri %d", t, old_pri);
1850 }
1851 
1852 
1853 /*
1854  * Update the ts_dispwait values of all time sharing threads that
1855  * are currently runnable at a user mode priority and bump the priority
1856  * if ts_dispwait exceeds ts_maxwait.  Called once per second via
1857  * timeout which we reset here.
1858  *
1859  * There are several lists of time sharing threads broken up by a hash on
1860  * the thread pointer.  Each list has its own lock.  This avoids blocking


1914 {
1915         tsproc_t *tspp;
1916         kthread_t *tx;
1917         int updated = 0;
1918 
1919         mutex_enter(&ts_list_lock[i]);
1920         for (tspp = ts_plisthead[i].ts_next; tspp != &ts_plisthead[i];
1921             tspp = tspp->ts_next) {
1922                 tx = tspp->ts_tp;
1923                 /*
1924                  * Lock the thread and verify state.
1925                  */
1926                 thread_lock(tx);
1927                 /*
1928                  * Skip the thread if it is no longer in the TS (or IA) class.
1929                  */
1930                 if (tx->t_clfuncs != &ts_classfuncs.thread &&
1931                     tx->t_clfuncs != &ia_classfuncs.thread)
1932                         goto next;
1933                 tspp->ts_dispwait++;
1934                 if ((tspp->ts_flags & TSKPRI) != 0)
1935                         goto next;
1936                 if (tspp->ts_dispwait <= ts_dptbl[tspp->ts_umdpri].ts_maxwait)
1937                         goto next;
1938                 if (tx->t_schedctl && schedctl_get_nopreempt(tx))
1939                         goto next;
1940                 if (tx->t_state != TS_RUN && tx->t_state != TS_WAIT &&
1941                     (tx->t_state != TS_SLEEP || !ts_sleep_promote)) {
1942                         /* make next syscall/trap do CL_TRAPRET */
1943                         tx->t_trapret = 1;
1944                         aston(tx);
1945                         goto next;
1946                 }
1947                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_lwait;
1948                 TS_NEWUMDPRI(tspp);
1949                 tspp->ts_dispwait = 0;
1950                 updated = 1;
1951 
1952                 /*
1953                  * Only dequeue it if needs to move; otherwise it should
1954                  * just round-robin here.
1955                  */
1956                 if (tx->t_pri != ts_dptbl[tspp->ts_umdpri].ts_globpri) {
1957                         pri_t oldpri = tx->t_pri;
1958                         ts_change_priority(tx, tspp);
1959                         TRACE_2(TR_FAC_DISP, TR_UPDATE,
1960                             "update:tid %p old pri %d", tx, oldpri);
1961                 }
1962 next:
1963                 thread_unlock(tx);
1964         }
1965         mutex_exit(&ts_list_lock[i]);
1966 
1967         return (updated);
1968 }
1969 
1970 /*
1971  * Processes waking up go to the back of their queue.  We don't
1972  * need to assign a time quantum here because thread is still
1973  * at a kernel mode priority and the time slicing is not done
1974  * for threads running in the kernel after sleeping.  The proper
1975  * time quantum will be assigned by ts_trapret before the thread
1976  * returns to user mode.
1977  */
1978 static void
1979 ts_wakeup(kthread_t *t)
1980 {
1981         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1982 
1983         ASSERT(THREAD_LOCK_HELD(t));
1984 
1985         t->t_stime = ddi_get_lbolt();                /* time stamp for the swapper */
1986 
1987         if (tspp->ts_flags & TSKPRI) {
1988                 tspp->ts_flags &= ~TSBACKQ;
1989                 if (tspp->ts_flags & TSIASET)
1990                         setfrontdq(t);
1991                 else
1992                         setbackdq(t);
1993         } else if (t->t_kpri_req) {
1994                 /*
1995                  * Give thread a priority boost if we were asked.
1996                  */
1997                 tspp->ts_flags |= TSKPRI;
1998                 THREAD_CHANGE_PRI(t, ts_kmdpris[0]);
1999                 setbackdq(t);
2000                 t->t_trapret = 1;    /* so that ts_trapret will run */
2001                 aston(t);
2002         } else {
2003                 if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
2004                         tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
2005                         TS_NEWUMDPRI(tspp);
2006                         tspp->ts_timeleft =
2007                             ts_dptbl[tspp->ts_cpupri].ts_quantum;
2008                         tspp->ts_dispwait = 0;
2009                         THREAD_CHANGE_PRI(t,
2010                             ts_dptbl[tspp->ts_umdpri].ts_globpri);
2011                         ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
2012                 }
2013 
2014                 tspp->ts_flags &= ~TSBACKQ;
2015 
2016                 if (tspp->ts_flags & TSIA) {
2017                         if (tspp->ts_flags & TSIASET)
2018                                 setfrontdq(t);
2019                         else
2020                                 setbackdq(t);
2021                 } else {
2022                         if (t->t_disp_time != ddi_get_lbolt())
2023                                 setbackdq(t);
2024                         else
2025                                 setfrontdq(t);
2026                 }
2027         }
2028 }
2029 
2030 
2031 /*
2032  * When a thread yields, put it on the back of the run queue.
2033  */
2034 static void
2035 ts_yield(kthread_t *t)
2036 {
2037         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
2038 
2039         ASSERT(t == curthread);
2040         ASSERT(THREAD_LOCK_HELD(t));
2041 
2042         /*
2043          * Collect CPU usage spent before yielding
2044          */
2045         (void) CPUCAPS_CHARGE(t, &tspp->ts_caps, CPUCAPS_CHARGE_ENFORCE);
2046 
2047         /*


2274                         mutex_enter(&fg->p_lock);
2275                 }
2276 
2277                 if ((tx = proctot(fg)) == NULL) {
2278                         mutex_exit(&fg->p_lock);
2279                         continue;
2280                 }
2281                 do {
2282                         thread_lock(tx);
2283                         /*
2284                          * if this thread is not interactive continue
2285                          */
2286                         if (tx->t_cid != ia_cid) {
2287                                 thread_unlock(tx);
2288                                 continue;
2289                         }
2290                         tspp = tx->t_cldata;
2291                         tspp->ts_flags |= TSIASET;
2292                         tspp->ts_boost = ia_boost;
2293                         TS_NEWUMDPRI(tspp);
2294                         if ((tspp->ts_flags & TSKPRI) != 0) {
2295                                 thread_unlock(tx);
2296                                 continue;
2297                         }
2298                         tspp->ts_dispwait = 0;
2299                         ts_change_priority(tx, tspp);
2300                         thread_unlock(tx);
2301                 } while ((tx = tx->t_forw) != fg->p_tlist);
2302                 mutex_exit(&fg->p_lock);
2303         }
2304 skip:
2305         if (bg_pgid == 0)
2306                 return;
2307         for (bg = (proc_t *)pgfind(bg_pgid); bg != NULL; bg = bg->p_pglink) {
2308                 if (bg->p_stat == SIDL) {
2309                         continue;
2310                 }
2311                 /*
2312                  * sesssion leaders must be turned off explicitly
2313                  * not implicitly as happens to other members of
2314                  * the process group.
2315                  */
2316                 if (bg->p_pid == bg->p_sessp->s_sid) {
2317                         continue;


2327                         mutex_enter(&bg->p_lock);
2328                 }
2329 
2330                 if ((tx = proctot(bg)) == NULL) {
2331                         mutex_exit(&bg->p_lock);
2332                         continue;
2333                 }
2334                 do {
2335                         thread_lock(tx);
2336                         /*
2337                          * if this thread is not interactive continue
2338                          */
2339                         if (tx->t_cid != ia_cid) {
2340                                 thread_unlock(tx);
2341                                 continue;
2342                         }
2343                         tspp = tx->t_cldata;
2344                         tspp->ts_flags &= ~TSIASET;
2345                         tspp->ts_boost = -ia_boost;
2346                         TS_NEWUMDPRI(tspp);
2347                         if ((tspp->ts_flags & TSKPRI) != 0) {
2348                                 thread_unlock(tx);
2349                                 continue;
2350                         }
2351 
2352                         tspp->ts_dispwait = 0;
2353                         ts_change_priority(tx, tspp);
2354                         thread_unlock(tx);
2355                 } while ((tx = tx->t_forw) != bg->p_tlist);
2356                 mutex_exit(&bg->p_lock);
2357         }
2358 }
2359 
2360 
2361 static void
2362 ts_change_priority(kthread_t *t, tsproc_t *tspp)
2363 {
2364         pri_t   new_pri;
2365 
2366         ASSERT(THREAD_LOCK_HELD(t));
2367         new_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
2368         ASSERT(new_pri >= 0 && new_pri <= ts_maxglobpri);
2369         tspp->ts_flags &= ~TSRESTORE;
2370         t->t_cpri = tspp->ts_upri;




   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2019 Joyent, Inc.
  25  */
  26 
  27 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  28 /*        All Rights Reserved   */
  29 
  30 #include <sys/types.h>
  31 #include <sys/param.h>
  32 #include <sys/sysmacros.h>
  33 #include <sys/cred.h>
  34 #include <sys/proc.h>
  35 #include <sys/session.h>
  36 #include <sys/strsubr.h>
  37 #include <sys/signal.h>
  38 #include <sys/user.h>
  39 #include <sys/priocntl.h>
  40 #include <sys/class.h>
  41 #include <sys/disp.h>
  42 #include <sys/procset.h>
  43 #include <sys/debug.h>
  44 #include <sys/ts.h>


 212 static pri_t    ts_globpri(kthread_t *);
 213 static void     ts_yield(kthread_t *);
 214 extern tsdpent_t *ts_getdptbl(void);
 215 extern pri_t    *ts_getkmdpris(void);
 216 extern pri_t    td_getmaxumdpri(void);
 217 static int      ts_alloc(void **, int);
 218 static void     ts_free(void *);
 219 
 220 pri_t           ia_init(id_t, int, classfuncs_t **);
 221 static int      ia_getclinfo(void *);
 222 static int      ia_getclpri(pcpri_t *);
 223 static int      ia_parmsin(void *);
 224 static int      ia_vaparmsin(void *, pc_vaparms_t *);
 225 static int      ia_vaparmsout(void *, pc_vaparms_t *);
 226 static int      ia_parmsset(kthread_t *, void *, id_t, cred_t *);
 227 static void     ia_parmsget(kthread_t *, void *);
 228 static void     ia_set_process_group(pid_t, pid_t, pid_t);
 229 
 230 static void     ts_change_priority(kthread_t *, tsproc_t *);
 231 

 232 static pri_t    ts_maxglobpri;  /* maximum global priority used by ts class */
 233 static kmutex_t ts_dptblock;    /* protects time sharing dispatch table */
 234 static kmutex_t ts_list_lock[TS_LISTS]; /* protects tsproc lists */
 235 static tsproc_t ts_plisthead[TS_LISTS]; /* dummy tsproc at head of lists */
 236 
 237 static gid_t    IA_gid = 0;
 238 
 239 static struct classfuncs ts_classfuncs = {
 240         /* class functions */
 241         ts_admin,
 242         ts_getclinfo,
 243         ts_parmsin,
 244         ts_parmsout,
 245         ts_vaparmsin,
 246         ts_vaparmsout,
 247         ts_getclpri,
 248         ts_alloc,
 249         ts_free,
 250 
 251         /* thread functions */


 523                         ts_dptbl[i].ts_maxwait = tmpdpp[i].ts_maxwait;
 524                         ts_dptbl[i].ts_lwait = tmpdpp[i].ts_lwait;
 525                 }
 526                 mutex_exit(&ts_dptblock);
 527                 kmem_free(tmpdpp, tsdpsz);
 528                 break;
 529 
 530         default:
 531                 return (EINVAL);
 532         }
 533         return (0);
 534 }
 535 
 536 
 537 /*
 538  * Allocate a time-sharing class specific thread structure and
 539  * initialize it with the parameters supplied. Also move the thread
 540  * to specified time-sharing priority.
 541  */
 542 static int
 543 ts_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
 544     void *bufp)
 545 {
 546         tsparms_t       *tsparmsp = (tsparms_t *)parmsp;
 547         tsproc_t        *tspp;
 548         pri_t           reqtsuprilim;
 549         pri_t           reqtsupri;
 550         static uint32_t tspexists = 0;  /* set on first occurrence of */
 551                                         /*   a time-sharing process */
 552 
 553         tspp = (tsproc_t *)bufp;
 554         ASSERT(tspp != NULL);
 555 
 556         /*
 557          * Initialize the tsproc structure.
 558          */
 559         tspp->ts_cpupri = tsmedumdpri;
 560         if (cid == ia_cid) {
 561                 /*
 562                  * Check to make sure caller is either privileged or the
 563                  * window system.  When the window system is converted
 564                  * to using privileges, the second check can go away.


 685         tsproc_t        *ptspp;         /* ptr to parent's tsproc structure */
 686         tsproc_t        *ctspp;         /* ptr to child's tsproc structure */
 687 
 688         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
 689 
 690         ctspp = (tsproc_t *)bufp;
 691         ASSERT(ctspp != NULL);
 692         ptspp = (tsproc_t *)t->t_cldata;
 693         /*
 694          * Initialize child's tsproc structure.
 695          */
 696         thread_lock(t);
 697         ctspp->ts_timeleft = ts_dptbl[ptspp->ts_cpupri].ts_quantum;
 698         ctspp->ts_cpupri = ptspp->ts_cpupri;
 699         ctspp->ts_boost = ptspp->ts_boost;
 700         ctspp->ts_uprilim = ptspp->ts_uprilim;
 701         ctspp->ts_upri = ptspp->ts_upri;
 702         TS_NEWUMDPRI(ctspp);
 703         ctspp->ts_nice = ptspp->ts_nice;
 704         ctspp->ts_dispwait = 0;
 705         ctspp->ts_flags = ptspp->ts_flags & ~(TSBACKQ | TSRESTORE);
 706         ctspp->ts_tp = ct;
 707         cpucaps_sc_init(&ctspp->ts_caps);
 708         thread_unlock(t);
 709 
 710         /*
 711          * Link new structure into tsproc list.
 712          */
 713         ct->t_cldata = (void *)ctspp;
 714         TS_LIST_INSERT(ctspp);
 715         return (0);
 716 }
 717 
 718 
 719 /*
 720  * Child is placed at back of dispatcher queue and parent gives
 721  * up processor so that the child runs first after the fork.
 722  * This allows the child immediately execing to break the multiple
 723  * use of copy on write pages with no disk home. The parent will
 724  * get to steal them back rather than uselessly copying them.
 725  */


 736         /*
 737          * Grab the child's p_lock before dropping pidlock to ensure
 738          * the process does not disappear before we set it running.
 739          */
 740         mutex_enter(&cp->p_lock);
 741         continuelwps(cp);
 742         mutex_exit(&cp->p_lock);
 743 
 744         mutex_enter(&pp->p_lock);
 745         mutex_exit(&pidlock);
 746         continuelwps(pp);
 747 
 748         thread_lock(t);
 749         tspp = (tsproc_t *)(t->t_cldata);
 750         tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_tqexp;
 751         TS_NEWUMDPRI(tspp);
 752         tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
 753         tspp->ts_dispwait = 0;
 754         t->t_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
 755         ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);

 756         THREAD_TRANSITION(t);
 757         ts_setrun(t);
 758         thread_unlock(t);
 759         /*
 760          * Safe to drop p_lock now since since it is safe to change
 761          * the scheduling class after this point.
 762          */
 763         mutex_exit(&pp->p_lock);
 764 
 765         swtch();
 766 }
 767 
 768 
 769 /*
 770  * Get information about the time-sharing class into the buffer
 771  * pointed to by tsinfop. The maximum configured user priority
 772  * is the only information we supply.  ts_getclinfo() is called
 773  * for TS threads, and ia_getclinfo() is called for IA threads.
 774  */
 775 static int


1198             reqtsuprilim > tspp->ts_uprilim &&
1199             secpolicy_raisepriority(reqpcredp) != 0)
1200                 return (EPERM);
1201 
1202         /*
1203          * Set ts_nice to the nice value corresponding to the user
1204          * priority we are setting.  Note that setting the nice field
1205          * of the parameter struct won't affect upri or nice.
1206          */
1207         nice = NZERO - (reqtsupri * NZERO) / ts_maxupri;
1208         if (nice >= 2 * NZERO)
1209                 nice = 2 * NZERO - 1;
1210 
1211         thread_lock(tx);
1212 
1213         tspp->ts_uprilim = reqtsuprilim;
1214         tspp->ts_upri = reqtsupri;
1215         TS_NEWUMDPRI(tspp);
1216         tspp->ts_nice = nice;
1217 





1218         tspp->ts_dispwait = 0;
1219         ts_change_priority(tx, tspp);
1220         thread_unlock(tx);
1221         return (0);
1222 }
1223 
1224 
1225 static int
1226 ia_parmsset(kthread_t *tx, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1227 {
1228         tsproc_t        *tspp = (tsproc_t *)tx->t_cldata;
1229         iaparms_t       *iaparmsp = (iaparms_t *)parmsp;
1230         proc_t          *p;
1231         pid_t           pid, pgid, sid;
1232         pid_t           on, off;
1233         struct stdata   *stp;
1234         int             sess_held;
1235 
1236         /*
1237          * Handle user priority changes


1349         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1350         tspp = (tsproc_t *)t->t_cldata;
1351         tspri = tsmedumdpri + tspp->ts_upri;
1352         if (tspri > ts_maxumdpri)
1353                 tspri = ts_maxumdpri;
1354         else if (tspri < 0)
1355                 tspri = 0;
1356         return (ts_dptbl[tspri].ts_globpri);
1357 }
1358 
1359 /*
1360  * Arrange for thread to be placed in appropriate location
1361  * on dispatcher queue.
1362  *
1363  * This is called with the current thread in TS_ONPROC and locked.
1364  */
1365 static void
1366 ts_preempt(kthread_t *t)
1367 {
1368         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1369         klwp_t          *lwp = ttolwp(t);
1370         pri_t           oldpri = t->t_pri;
1371 
1372         ASSERT(t == curthread);
1373         ASSERT(THREAD_LOCK_HELD(curthread));
1374 
1375         /*












1376          * This thread may be placed on wait queue by CPU Caps. In this case we
1377          * do not need to do anything until it is removed from the wait queue.

1378          */
1379         if (CPUCAPS_ON()) {
1380                 (void) cpucaps_charge(t, &tspp->ts_caps,
1381                     CPUCAPS_CHARGE_ENFORCE);
1382                 if (CPUCAPS_ENFORCE(t))
1383                         return;
1384         }
1385 
1386         /*
1387          * If thread got preempted in the user-land then we know
1388          * it isn't holding any locks.  Mark it as swappable.
1389          */
1390         ASSERT(t->t_schedflag & TS_DONT_SWAP);
1391         if (lwp != NULL && lwp->lwp_state == LWP_USER)
1392                 t->t_schedflag &= ~TS_DONT_SWAP;
1393 
1394         /*
1395          * Check to see if we're doing "preemption control" here.  If
1396          * we are, and if the user has requested that this thread not
1397          * be preempted, and if preemptions haven't been put off for
1398          * too long, let the preemption happen here but try to make
1399          * sure the thread is rescheduled as soon as possible.  We do
1400          * this by putting it on the front of the highest priority run
1401          * queue in the TS class.  If the preemption has been put off
1402          * for too long, clear the "nopreempt" bit and let the thread
1403          * be preempted.
1404          */
1405         if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1406                 if (tspp->ts_timeleft > -SC_MAX_TICKS) {
1407                         DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);

1408                         /*
1409                          * If not already remembered, remember current
1410                          * priority for restoration in ts_yield().
1411                          */
1412                         if (!(tspp->ts_flags & TSRESTORE)) {
1413                                 tspp->ts_scpri = t->t_pri;
1414                                 tspp->ts_flags |= TSRESTORE;
1415                         }
1416                         THREAD_CHANGE_PRI(t, ts_maxumdpri);
1417                         t->t_schedflag |= TS_DONT_SWAP;

1418                         schedctl_set_yield(t, 1);
1419                         setfrontdq(t);
1420                         goto done;
1421                 } else {
1422                         if (tspp->ts_flags & TSRESTORE) {
1423                                 THREAD_CHANGE_PRI(t, tspp->ts_scpri);
1424                                 tspp->ts_flags &= ~TSRESTORE;
1425                         }
1426                         schedctl_set_nopreempt(t, 0);
1427                         DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
1428                         TNF_PROBE_2(schedctl_preempt, "schedctl TS ts_preempt",
1429                             /* CSTYLED */, tnf_pid, pid, ttoproc(t)->p_pid,
1430                             tnf_lwpid, lwpid, t->t_tid);
1431                         /*
1432                          * Fall through and be preempted below.
1433                          */
1434                 }
1435         }
1436 
1437         if ((tspp->ts_flags & TSBACKQ) != 0) {
1438                 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1439                 tspp->ts_dispwait = 0;
1440                 tspp->ts_flags &= ~TSBACKQ;
1441                 setbackdq(t);



1442         } else {
1443                 setfrontdq(t);
1444         }
1445 
1446 done:
1447         TRACE_2(TR_FAC_DISP, TR_PREEMPT,
1448             "preempt:tid %p old pri %d", t, oldpri);
1449 }
1450 
1451 static void
1452 ts_setrun(kthread_t *t)
1453 {
1454         tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1455 
1456         ASSERT(THREAD_LOCK_HELD(t));    /* t should be in transition */
1457 
1458         if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1459                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1460                 TS_NEWUMDPRI(tspp);
1461                 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1462                 tspp->ts_dispwait = 0;
1463                 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);


1464                 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1465         }

1466 
1467         tspp->ts_flags &= ~TSBACKQ;
1468 
1469         if (tspp->ts_flags & TSIA) {
1470                 if (tspp->ts_flags & TSIASET)
1471                         setfrontdq(t);
1472                 else
1473                         setbackdq(t);
1474         } else {
1475                 if (t->t_disp_time != ddi_get_lbolt())
1476                         setbackdq(t);
1477                 else
1478                         setfrontdq(t);
1479         }
1480 }
1481 
1482 
1483 /*
1484  * Prepare thread for sleep.

1485  */
1486 static void
1487 ts_sleep(kthread_t *t)
1488 {
1489         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);

1490         pri_t           old_pri = t->t_pri;
1491 
1492         ASSERT(t == curthread);
1493         ASSERT(THREAD_LOCK_HELD(t));
1494 
1495         /*
1496          * Account for time spent on CPU before going to sleep.
1497          */
1498         (void) CPUCAPS_CHARGE(t, &tspp->ts_caps, CPUCAPS_CHARGE_ENFORCE);
1499 
1500         if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {











1501                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1502                 TS_NEWUMDPRI(tspp);
1503                 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1504                 tspp->ts_dispwait = 0;
1505 
1506                 THREAD_CHANGE_PRI(curthread,
1507                     ts_dptbl[tspp->ts_umdpri].ts_globpri);
1508                 ASSERT(curthread->t_pri >= 0 &&
1509                     curthread->t_pri <= ts_maxglobpri);

1510 
1511                 if (DISP_MUST_SURRENDER(curthread))
1512                         cpu_surrender(curthread);









1513         }
1514         t->t_stime = ddi_get_lbolt();                /* time stamp for the swapper */
1515         TRACE_2(TR_FAC_DISP, TR_SLEEP,
1516             "sleep:tid %p old pri %d", t, old_pri);
1517 }
1518 
1519 
1520 /*
1521  * Return Values:
1522  *
1523  *      -1 if the thread is loaded or is not eligible to be swapped in.
1524  *
1525  *      effective priority of the specified thread based on swapout time
1526  *              and size of process (epri >= 0 , epri <= SHRT_MAX).
1527  */
1528 /* ARGSUSED */
1529 static pri_t
1530 ts_swapin(kthread_t *t, int flags)
1531 {
1532         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1533         long            epri = -1;
1534         proc_t          *pp = ttoproc(t);
1535 
1536         ASSERT(THREAD_LOCK_HELD(t));
1537 
1538         /*
1539          * We know that pri_t is a short.
1540          * Be sure not to overrun its range.
1541          */
1542         if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
1543                 time_t swapout_time;
1544 
1545                 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
1546                 if (INHERITED(t) || (tspp->ts_flags & TSIASET)) {
1547                         epri = (long)DISP_PRIO(t) + swapout_time;
1548                 } else {
1549                         /*
1550                          * Threads which have been out for a long time,
1551                          * have high user mode priority and are associated
1552                          * with a small address space are more deserving
1553                          */
1554                         epri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
1555                         ASSERT(epri >= 0 && epri <= ts_maxumdpri);
1556                         epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
1557                 }
1558                 /*
1559                  * Scale epri so SHRT_MAX/2 represents zero priority.
1560                  */
1561                 epri += SHRT_MAX/2;
1562                 if (epri < 0)
1563                         epri = 0;
1564                 else if (epri > SHRT_MAX)
1565                         epri = SHRT_MAX;
1566         }
1567         return ((pri_t)epri);
1568 }


1580  *              Hardswap:  Return an effective priority such that threads
1581  *                         which have been in memory for a while and are
1582  *                         associated with a small address space are swapped
1583  *                         in before others.
1584  *
1585  *              (epri >= 0 , epri <= SHRT_MAX).
1586  */
1587 time_t  ts_minrun = 2;          /* XXX - t_pri becomes 59 within 2 secs */
1588 time_t  ts_minslp = 2;          /* min time on sleep queue for hardswap */
1589 
1590 static pri_t
1591 ts_swapout(kthread_t *t, int flags)
1592 {
1593         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1594         long            epri = -1;
1595         proc_t          *pp = ttoproc(t);
1596         time_t          swapin_time;
1597 
1598         ASSERT(THREAD_LOCK_HELD(t));
1599 
1600         if (INHERITED(t) || (tspp->ts_flags & TSIASET) ||
1601             (t->t_proc_flag & TP_LWPEXIT) ||
1602             (t->t_state & (TS_ZOMB | TS_FREE | TS_STOPPED |
1603             TS_ONPROC | TS_WAIT)) ||
1604             !(t->t_schedflag & TS_LOAD) || !SWAP_OK(t))
1605                 return (-1);
1606 
1607         ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
1608 
1609         /*
1610          * We know that pri_t is a short.
1611          * Be sure not to overrun its range.
1612          */
1613         swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
1614         if (flags == SOFTSWAP) {
1615                 if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
1616                         epri = 0;
1617                 } else {
1618                         return ((pri_t)epri);
1619                 }
1620         } else {


1649  * and set runrun to cause preemption.
1650  */
1651 static void
1652 ts_tick(kthread_t *t)
1653 {
1654         tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1655         klwp_t *lwp;
1656         boolean_t call_cpu_surrender = B_FALSE;
1657         pri_t   oldpri = t->t_pri;
1658 
1659         ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1660 
1661         thread_lock(t);
1662 
1663         /*
1664          * Keep track of thread's project CPU usage.  Note that projects
1665          * get charged even when threads are running in the kernel.
1666          */
1667         if (CPUCAPS_ON()) {
1668                 call_cpu_surrender = cpucaps_charge(t, &tspp->ts_caps,
1669                     CPUCAPS_CHARGE_ENFORCE);
1670         }
1671 

1672         if (--tspp->ts_timeleft <= 0) {
1673                 pri_t   new_pri;
1674 
1675                 /*
1676                  * If we're doing preemption control and trying to avoid
1677                  * preempting this thread, just note that the thread should
1678                  * yield soon and let it keep running (unless it's been a
1679                  * while).
1680                  */
1681                 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1682                         if (tspp->ts_timeleft > -SC_MAX_TICKS) {
1683                                 DTRACE_SCHED1(schedctl__nopreempt,
1684                                     kthread_t *, t);
1685                                 schedctl_set_yield(t, 1);
1686                                 thread_unlock_nopreempt(t);
1687                                 return;
1688                         }
1689 
1690                         TNF_PROBE_2(schedctl_failsafe,
1691                             "schedctl TS ts_tick", /* CSTYLED */,
1692                             tnf_pid, pid, ttoproc(t)->p_pid,
1693                             tnf_lwpid, lwpid, t->t_tid);
1694                 }
1695                 tspp->ts_flags &= ~TSRESTORE;
1696                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_tqexp;
1697                 TS_NEWUMDPRI(tspp);
1698                 tspp->ts_dispwait = 0;
1699                 new_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
1700                 ASSERT(new_pri >= 0 && new_pri <= ts_maxglobpri);
1701                 /*
1702                  * When the priority of a thread is changed, it may be
1703                  * necessary to adjust its position on a sleep queue or
1704                  * dispatch queue.  The function thread_change_pri accomplishes

1705                  * this.
1706                  */
1707                 if (thread_change_pri(t, new_pri, 0)) {
1708                         if ((t->t_schedflag & TS_LOAD) &&
1709                             (lwp = t->t_lwp) &&
1710                             lwp->lwp_state == LWP_USER)
1711                                 t->t_schedflag &= ~TS_DONT_SWAP;
1712                         tspp->ts_timeleft =
1713                             ts_dptbl[tspp->ts_cpupri].ts_quantum;
1714                 } else {
1715                         call_cpu_surrender = B_TRUE;
1716                 }
1717                 TRACE_2(TR_FAC_DISP, TR_TICK,
1718                     "tick:tid %p old pri %d", t, oldpri);
1719         } else if (t->t_state == TS_ONPROC &&
1720             t->t_pri < t->t_disp_queue->disp_maxrunpri) {
1721                 call_cpu_surrender = B_TRUE;
1722         }

1723 
1724         if (call_cpu_surrender) {
1725                 tspp->ts_flags |= TSBACKQ;
1726                 cpu_surrender(t);
1727         }
1728 
1729         thread_unlock_nopreempt(t);     /* clock thread can't be preempted */
1730 }
1731 
1732 
1733 /*
1734  * If we are lowering the thread's priority below that of other runnable
1735  * threads we will normally set runrun via cpu_surrender() to cause preemption.



1736  */
1737 static void
1738 ts_trapret(kthread_t *t)
1739 {
1740         tsproc_t        *tspp = (tsproc_t *)t->t_cldata;
1741         cpu_t           *cp = CPU;
1742         pri_t           old_pri = curthread->t_pri;
1743 
1744         ASSERT(THREAD_LOCK_HELD(t));
1745         ASSERT(t == curthread);
1746         ASSERT(cp->cpu_dispthread == t);
1747         ASSERT(t->t_state == TS_ONPROC);
1748 

1749         if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1750                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1751                 TS_NEWUMDPRI(tspp);
1752                 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1753                 tspp->ts_dispwait = 0;
1754 
1755                 /*
1756                  * If thread has blocked in the kernel (as opposed to
1757                  * being merely preempted), recompute the user mode priority.
1758                  */
1759                 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);
1760                 cp->cpu_dispatch_pri = DISP_PRIO(t);
1761                 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);

1762 
1763                 if (DISP_MUST_SURRENDER(t))
1764                         cpu_surrender(t);












1765         }
1766 
1767         /*
1768          * Swapout lwp if the swapper is waiting for this thread to reach a
1769          * safe point.
1770          */
1771         if ((t->t_schedflag & TS_SWAPENQ) && !(tspp->ts_flags & TSIASET)) {
1772                 thread_unlock(t);
1773                 swapout_lwp(ttolwp(t));
1774                 thread_lock(t);
1775         }
1776 
1777         TRACE_2(TR_FAC_DISP, TR_TRAPRET,
1778             "trapret:tid %p old pri %d", t, old_pri);
1779 }
1780 
1781 
1782 /*
1783  * Update the ts_dispwait values of all time sharing threads that
1784  * are currently runnable at a user mode priority and bump the priority
1785  * if ts_dispwait exceeds ts_maxwait.  Called once per second via
1786  * timeout which we reset here.
1787  *
1788  * There are several lists of time sharing threads broken up by a hash on
1789  * the thread pointer.  Each list has its own lock.  This avoids blocking


1843 {
1844         tsproc_t *tspp;
1845         kthread_t *tx;
1846         int updated = 0;
1847 
1848         mutex_enter(&ts_list_lock[i]);
1849         for (tspp = ts_plisthead[i].ts_next; tspp != &ts_plisthead[i];
1850             tspp = tspp->ts_next) {
1851                 tx = tspp->ts_tp;
1852                 /*
1853                  * Lock the thread and verify state.
1854                  */
1855                 thread_lock(tx);
1856                 /*
1857                  * Skip the thread if it is no longer in the TS (or IA) class.
1858                  */
1859                 if (tx->t_clfuncs != &ts_classfuncs.thread &&
1860                     tx->t_clfuncs != &ia_classfuncs.thread)
1861                         goto next;
1862                 tspp->ts_dispwait++;


1863                 if (tspp->ts_dispwait <= ts_dptbl[tspp->ts_umdpri].ts_maxwait)
1864                         goto next;
1865                 if (tx->t_schedctl && schedctl_get_nopreempt(tx))
1866                         goto next;
1867                 if (tx->t_state != TS_RUN && tx->t_state != TS_WAIT &&
1868                     (tx->t_state != TS_SLEEP || !ts_sleep_promote)) {
1869                         /* make next syscall/trap do CL_TRAPRET */
1870                         tx->t_trapret = 1;
1871                         aston(tx);
1872                         goto next;
1873                 }
1874                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_lwait;
1875                 TS_NEWUMDPRI(tspp);
1876                 tspp->ts_dispwait = 0;
1877                 updated = 1;
1878 
1879                 /*
1880                  * Only dequeue it if needs to move; otherwise it should
1881                  * just round-robin here.
1882                  */
1883                 if (tx->t_pri != ts_dptbl[tspp->ts_umdpri].ts_globpri) {
1884                         pri_t oldpri = tx->t_pri;
1885                         ts_change_priority(tx, tspp);
1886                         TRACE_2(TR_FAC_DISP, TR_UPDATE,
1887                             "update:tid %p old pri %d", tx, oldpri);
1888                 }
1889 next:
1890                 thread_unlock(tx);
1891         }
1892         mutex_exit(&ts_list_lock[i]);
1893 
1894         return (updated);
1895 }
1896 
1897 /*
1898  * Processes waking up go to the back of their queue.





1899  */
1900 static void
1901 ts_wakeup(kthread_t *t)
1902 {
1903         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1904 
1905         ASSERT(THREAD_LOCK_HELD(t));
1906 
1907         t->t_stime = ddi_get_lbolt();                /* time stamp for the swapper */
1908 
















1909         if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1910                 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1911                 TS_NEWUMDPRI(tspp);
1912                 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;

1913                 tspp->ts_dispwait = 0;
1914                 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);

1915                 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1916         }
1917 
1918         tspp->ts_flags &= ~TSBACKQ;
1919 
1920         if (tspp->ts_flags & TSIA) {
1921                 if (tspp->ts_flags & TSIASET)
1922                         setfrontdq(t);
1923                 else
1924                         setbackdq(t);
1925         } else {
1926                 if (t->t_disp_time != ddi_get_lbolt())
1927                         setbackdq(t);
1928                 else
1929                         setfrontdq(t);
1930         }

1931 }
1932 
1933 
1934 /*
1935  * When a thread yields, put it on the back of the run queue.
1936  */
1937 static void
1938 ts_yield(kthread_t *t)
1939 {
1940         tsproc_t        *tspp = (tsproc_t *)(t->t_cldata);
1941 
1942         ASSERT(t == curthread);
1943         ASSERT(THREAD_LOCK_HELD(t));
1944 
1945         /*
1946          * Collect CPU usage spent before yielding
1947          */
1948         (void) CPUCAPS_CHARGE(t, &tspp->ts_caps, CPUCAPS_CHARGE_ENFORCE);
1949 
1950         /*


2177                         mutex_enter(&fg->p_lock);
2178                 }
2179 
2180                 if ((tx = proctot(fg)) == NULL) {
2181                         mutex_exit(&fg->p_lock);
2182                         continue;
2183                 }
2184                 do {
2185                         thread_lock(tx);
2186                         /*
2187                          * if this thread is not interactive continue
2188                          */
2189                         if (tx->t_cid != ia_cid) {
2190                                 thread_unlock(tx);
2191                                 continue;
2192                         }
2193                         tspp = tx->t_cldata;
2194                         tspp->ts_flags |= TSIASET;
2195                         tspp->ts_boost = ia_boost;
2196                         TS_NEWUMDPRI(tspp);




2197                         tspp->ts_dispwait = 0;
2198                         ts_change_priority(tx, tspp);
2199                         thread_unlock(tx);
2200                 } while ((tx = tx->t_forw) != fg->p_tlist);
2201                 mutex_exit(&fg->p_lock);
2202         }
2203 skip:
2204         if (bg_pgid == 0)
2205                 return;
2206         for (bg = (proc_t *)pgfind(bg_pgid); bg != NULL; bg = bg->p_pglink) {
2207                 if (bg->p_stat == SIDL) {
2208                         continue;
2209                 }
2210                 /*
2211                  * sesssion leaders must be turned off explicitly
2212                  * not implicitly as happens to other members of
2213                  * the process group.
2214                  */
2215                 if (bg->p_pid == bg->p_sessp->s_sid) {
2216                         continue;


2226                         mutex_enter(&bg->p_lock);
2227                 }
2228 
2229                 if ((tx = proctot(bg)) == NULL) {
2230                         mutex_exit(&bg->p_lock);
2231                         continue;
2232                 }
2233                 do {
2234                         thread_lock(tx);
2235                         /*
2236                          * if this thread is not interactive continue
2237                          */
2238                         if (tx->t_cid != ia_cid) {
2239                                 thread_unlock(tx);
2240                                 continue;
2241                         }
2242                         tspp = tx->t_cldata;
2243                         tspp->ts_flags &= ~TSIASET;
2244                         tspp->ts_boost = -ia_boost;
2245                         TS_NEWUMDPRI(tspp);




2246 
2247                         tspp->ts_dispwait = 0;
2248                         ts_change_priority(tx, tspp);
2249                         thread_unlock(tx);
2250                 } while ((tx = tx->t_forw) != bg->p_tlist);
2251                 mutex_exit(&bg->p_lock);
2252         }
2253 }
2254 
2255 
2256 static void
2257 ts_change_priority(kthread_t *t, tsproc_t *tspp)
2258 {
2259         pri_t   new_pri;
2260 
2261         ASSERT(THREAD_LOCK_HELD(t));
2262         new_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
2263         ASSERT(new_pri >= 0 && new_pri <= ts_maxglobpri);
2264         tspp->ts_flags &= ~TSRESTORE;
2265         t->t_cpri = tspp->ts_upri;