4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2013, Joyent, Inc. All rights reserved.
25 */
26
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 /* All Rights Reserved */
29
30 #include <sys/types.h>
31 #include <sys/param.h>
32 #include <sys/sysmacros.h>
33 #include <sys/cred.h>
34 #include <sys/proc.h>
35 #include <sys/session.h>
36 #include <sys/strsubr.h>
37 #include <sys/signal.h>
38 #include <sys/user.h>
39 #include <sys/priocntl.h>
40 #include <sys/class.h>
41 #include <sys/disp.h>
42 #include <sys/procset.h>
43 #include <sys/debug.h>
44 #include <sys/ts.h>
212 static pri_t ts_globpri(kthread_t *);
213 static void ts_yield(kthread_t *);
214 extern tsdpent_t *ts_getdptbl(void);
215 extern pri_t *ts_getkmdpris(void);
216 extern pri_t td_getmaxumdpri(void);
217 static int ts_alloc(void **, int);
218 static void ts_free(void *);
219
220 pri_t ia_init(id_t, int, classfuncs_t **);
221 static int ia_getclinfo(void *);
222 static int ia_getclpri(pcpri_t *);
223 static int ia_parmsin(void *);
224 static int ia_vaparmsin(void *, pc_vaparms_t *);
225 static int ia_vaparmsout(void *, pc_vaparms_t *);
226 static int ia_parmsset(kthread_t *, void *, id_t, cred_t *);
227 static void ia_parmsget(kthread_t *, void *);
228 static void ia_set_process_group(pid_t, pid_t, pid_t);
229
230 static void ts_change_priority(kthread_t *, tsproc_t *);
231
232 extern pri_t ts_maxkmdpri; /* maximum kernel mode ts priority */
233 static pri_t ts_maxglobpri; /* maximum global priority used by ts class */
234 static kmutex_t ts_dptblock; /* protects time sharing dispatch table */
235 static kmutex_t ts_list_lock[TS_LISTS]; /* protects tsproc lists */
236 static tsproc_t ts_plisthead[TS_LISTS]; /* dummy tsproc at head of lists */
237
238 static gid_t IA_gid = 0;
239
240 static struct classfuncs ts_classfuncs = {
241 /* class functions */
242 ts_admin,
243 ts_getclinfo,
244 ts_parmsin,
245 ts_parmsout,
246 ts_vaparmsin,
247 ts_vaparmsout,
248 ts_getclpri,
249 ts_alloc,
250 ts_free,
251
252 /* thread functions */
524 ts_dptbl[i].ts_maxwait = tmpdpp[i].ts_maxwait;
525 ts_dptbl[i].ts_lwait = tmpdpp[i].ts_lwait;
526 }
527 mutex_exit(&ts_dptblock);
528 kmem_free(tmpdpp, tsdpsz);
529 break;
530
531 default:
532 return (EINVAL);
533 }
534 return (0);
535 }
536
537
538 /*
539 * Allocate a time-sharing class specific thread structure and
540 * initialize it with the parameters supplied. Also move the thread
541 * to specified time-sharing priority.
542 */
543 static int
544 ts_enterclass(kthread_t *t, id_t cid, void *parmsp,
545 cred_t *reqpcredp, void *bufp)
546 {
547 tsparms_t *tsparmsp = (tsparms_t *)parmsp;
548 tsproc_t *tspp;
549 pri_t reqtsuprilim;
550 pri_t reqtsupri;
551 static uint32_t tspexists = 0; /* set on first occurrence of */
552 /* a time-sharing process */
553
554 tspp = (tsproc_t *)bufp;
555 ASSERT(tspp != NULL);
556
557 /*
558 * Initialize the tsproc structure.
559 */
560 tspp->ts_cpupri = tsmedumdpri;
561 if (cid == ia_cid) {
562 /*
563 * Check to make sure caller is either privileged or the
564 * window system. When the window system is converted
565 * to using privileges, the second check can go away.
686 tsproc_t *ptspp; /* ptr to parent's tsproc structure */
687 tsproc_t *ctspp; /* ptr to child's tsproc structure */
688
689 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
690
691 ctspp = (tsproc_t *)bufp;
692 ASSERT(ctspp != NULL);
693 ptspp = (tsproc_t *)t->t_cldata;
694 /*
695 * Initialize child's tsproc structure.
696 */
697 thread_lock(t);
698 ctspp->ts_timeleft = ts_dptbl[ptspp->ts_cpupri].ts_quantum;
699 ctspp->ts_cpupri = ptspp->ts_cpupri;
700 ctspp->ts_boost = ptspp->ts_boost;
701 ctspp->ts_uprilim = ptspp->ts_uprilim;
702 ctspp->ts_upri = ptspp->ts_upri;
703 TS_NEWUMDPRI(ctspp);
704 ctspp->ts_nice = ptspp->ts_nice;
705 ctspp->ts_dispwait = 0;
706 ctspp->ts_flags = ptspp->ts_flags & ~(TSKPRI | TSBACKQ | TSRESTORE);
707 ctspp->ts_tp = ct;
708 cpucaps_sc_init(&ctspp->ts_caps);
709 thread_unlock(t);
710
711 /*
712 * Link new structure into tsproc list.
713 */
714 ct->t_cldata = (void *)ctspp;
715 TS_LIST_INSERT(ctspp);
716 return (0);
717 }
718
719
720 /*
721 * Child is placed at back of dispatcher queue and parent gives
722 * up processor so that the child runs first after the fork.
723 * This allows the child immediately execing to break the multiple
724 * use of copy on write pages with no disk home. The parent will
725 * get to steal them back rather than uselessly copying them.
726 */
737 /*
738 * Grab the child's p_lock before dropping pidlock to ensure
739 * the process does not disappear before we set it running.
740 */
741 mutex_enter(&cp->p_lock);
742 continuelwps(cp);
743 mutex_exit(&cp->p_lock);
744
745 mutex_enter(&pp->p_lock);
746 mutex_exit(&pidlock);
747 continuelwps(pp);
748
749 thread_lock(t);
750 tspp = (tsproc_t *)(t->t_cldata);
751 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_tqexp;
752 TS_NEWUMDPRI(tspp);
753 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
754 tspp->ts_dispwait = 0;
755 t->t_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
756 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
757 tspp->ts_flags &= ~TSKPRI;
758 THREAD_TRANSITION(t);
759 ts_setrun(t);
760 thread_unlock(t);
761 /*
762 * Safe to drop p_lock now since since it is safe to change
763 * the scheduling class after this point.
764 */
765 mutex_exit(&pp->p_lock);
766
767 swtch();
768 }
769
770
771 /*
772 * Get information about the time-sharing class into the buffer
773 * pointed to by tsinfop. The maximum configured user priority
774 * is the only information we supply. ts_getclinfo() is called
775 * for TS threads, and ia_getclinfo() is called for IA threads.
776 */
777 static int
1200 reqtsuprilim > tspp->ts_uprilim &&
1201 secpolicy_raisepriority(reqpcredp) != 0)
1202 return (EPERM);
1203
1204 /*
1205 * Set ts_nice to the nice value corresponding to the user
1206 * priority we are setting. Note that setting the nice field
1207 * of the parameter struct won't affect upri or nice.
1208 */
1209 nice = NZERO - (reqtsupri * NZERO) / ts_maxupri;
1210 if (nice >= 2 * NZERO)
1211 nice = 2 * NZERO - 1;
1212
1213 thread_lock(tx);
1214
1215 tspp->ts_uprilim = reqtsuprilim;
1216 tspp->ts_upri = reqtsupri;
1217 TS_NEWUMDPRI(tspp);
1218 tspp->ts_nice = nice;
1219
1220 if ((tspp->ts_flags & TSKPRI) != 0) {
1221 thread_unlock(tx);
1222 return (0);
1223 }
1224
1225 tspp->ts_dispwait = 0;
1226 ts_change_priority(tx, tspp);
1227 thread_unlock(tx);
1228 return (0);
1229 }
1230
1231
1232 static int
1233 ia_parmsset(kthread_t *tx, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1234 {
1235 tsproc_t *tspp = (tsproc_t *)tx->t_cldata;
1236 iaparms_t *iaparmsp = (iaparms_t *)parmsp;
1237 proc_t *p;
1238 pid_t pid, pgid, sid;
1239 pid_t on, off;
1240 struct stdata *stp;
1241 int sess_held;
1242
1243 /*
1244 * Handle user priority changes
1356 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1357 tspp = (tsproc_t *)t->t_cldata;
1358 tspri = tsmedumdpri + tspp->ts_upri;
1359 if (tspri > ts_maxumdpri)
1360 tspri = ts_maxumdpri;
1361 else if (tspri < 0)
1362 tspri = 0;
1363 return (ts_dptbl[tspri].ts_globpri);
1364 }
1365
1366 /*
1367 * Arrange for thread to be placed in appropriate location
1368 * on dispatcher queue.
1369 *
1370 * This is called with the current thread in TS_ONPROC and locked.
1371 */
1372 static void
1373 ts_preempt(kthread_t *t)
1374 {
1375 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1376 klwp_t *lwp = curthread->t_lwp;
1377 pri_t oldpri = t->t_pri;
1378
1379 ASSERT(t == curthread);
1380 ASSERT(THREAD_LOCK_HELD(curthread));
1381
1382 /*
1383 * If preempted in the kernel, make sure the thread has
1384 * a kernel priority if needed.
1385 */
1386 if (!(tspp->ts_flags & TSKPRI) && lwp != NULL && t->t_kpri_req) {
1387 tspp->ts_flags |= TSKPRI;
1388 THREAD_CHANGE_PRI(t, ts_kmdpris[0]);
1389 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1390 t->t_trapret = 1; /* so ts_trapret will run */
1391 aston(t);
1392 }
1393
1394 /*
1395 * This thread may be placed on wait queue by CPU Caps. In this case we
1396 * do not need to do anything until it is removed from the wait queue.
1397 * Do not enforce CPU caps on threads running at a kernel priority
1398 */
1399 if (CPUCAPS_ON()) {
1400 (void) cpucaps_charge(t, &tspp->ts_caps,
1401 CPUCAPS_CHARGE_ENFORCE);
1402 if (!(tspp->ts_flags & TSKPRI) && CPUCAPS_ENFORCE(t))
1403 return;
1404 }
1405
1406 /*
1407 * If thread got preempted in the user-land then we know
1408 * it isn't holding any locks. Mark it as swappable.
1409 */
1410 ASSERT(t->t_schedflag & TS_DONT_SWAP);
1411 if (lwp != NULL && lwp->lwp_state == LWP_USER)
1412 t->t_schedflag &= ~TS_DONT_SWAP;
1413
1414 /*
1415 * Check to see if we're doing "preemption control" here. If
1416 * we are, and if the user has requested that this thread not
1417 * be preempted, and if preemptions haven't been put off for
1418 * too long, let the preemption happen here but try to make
1419 * sure the thread is rescheduled as soon as possible. We do
1420 * this by putting it on the front of the highest priority run
1421 * queue in the TS class. If the preemption has been put off
1422 * for too long, clear the "nopreempt" bit and let the thread
1423 * be preempted.
1424 */
1425 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1426 if (tspp->ts_timeleft > -SC_MAX_TICKS) {
1427 DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
1428 if (!(tspp->ts_flags & TSKPRI)) {
1429 /*
1430 * If not already remembered, remember current
1431 * priority for restoration in ts_yield().
1432 */
1433 if (!(tspp->ts_flags & TSRESTORE)) {
1434 tspp->ts_scpri = t->t_pri;
1435 tspp->ts_flags |= TSRESTORE;
1436 }
1437 THREAD_CHANGE_PRI(t, ts_maxumdpri);
1438 t->t_schedflag |= TS_DONT_SWAP;
1439 }
1440 schedctl_set_yield(t, 1);
1441 setfrontdq(t);
1442 goto done;
1443 } else {
1444 if (tspp->ts_flags & TSRESTORE) {
1445 THREAD_CHANGE_PRI(t, tspp->ts_scpri);
1446 tspp->ts_flags &= ~TSRESTORE;
1447 }
1448 schedctl_set_nopreempt(t, 0);
1449 DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
1450 TNF_PROBE_2(schedctl_preempt, "schedctl TS ts_preempt",
1451 /* CSTYLED */, tnf_pid, pid, ttoproc(t)->p_pid,
1452 tnf_lwpid, lwpid, t->t_tid);
1453 /*
1454 * Fall through and be preempted below.
1455 */
1456 }
1457 }
1458
1459 if ((tspp->ts_flags & (TSBACKQ|TSKPRI)) == TSBACKQ) {
1460 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1461 tspp->ts_dispwait = 0;
1462 tspp->ts_flags &= ~TSBACKQ;
1463 setbackdq(t);
1464 } else if ((tspp->ts_flags & (TSBACKQ|TSKPRI)) == (TSBACKQ|TSKPRI)) {
1465 tspp->ts_flags &= ~TSBACKQ;
1466 setbackdq(t);
1467 } else {
1468 setfrontdq(t);
1469 }
1470
1471 done:
1472 TRACE_2(TR_FAC_DISP, TR_PREEMPT,
1473 "preempt:tid %p old pri %d", t, oldpri);
1474 }
1475
1476 static void
1477 ts_setrun(kthread_t *t)
1478 {
1479 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1480
1481 ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */
1482
1483 if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1484 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1485 TS_NEWUMDPRI(tspp);
1486 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1487 tspp->ts_dispwait = 0;
1488 if ((tspp->ts_flags & TSKPRI) == 0) {
1489 THREAD_CHANGE_PRI(t,
1490 ts_dptbl[tspp->ts_umdpri].ts_globpri);
1491 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1492 }
1493 }
1494
1495 tspp->ts_flags &= ~TSBACKQ;
1496
1497 if (tspp->ts_flags & TSIA) {
1498 if (tspp->ts_flags & TSIASET)
1499 setfrontdq(t);
1500 else
1501 setbackdq(t);
1502 } else {
1503 if (t->t_disp_time != ddi_get_lbolt())
1504 setbackdq(t);
1505 else
1506 setfrontdq(t);
1507 }
1508 }
1509
1510
1511 /*
1512 * Prepare thread for sleep. We reset the thread priority so it will
1513 * run at the kernel priority level when it wakes up.
1514 */
1515 static void
1516 ts_sleep(kthread_t *t)
1517 {
1518 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1519 int flags;
1520 pri_t old_pri = t->t_pri;
1521
1522 ASSERT(t == curthread);
1523 ASSERT(THREAD_LOCK_HELD(t));
1524
1525 /*
1526 * Account for time spent on CPU before going to sleep.
1527 */
1528 (void) CPUCAPS_CHARGE(t, &tspp->ts_caps, CPUCAPS_CHARGE_ENFORCE);
1529
1530 flags = tspp->ts_flags;
1531 if (t->t_kpri_req) {
1532 tspp->ts_flags = flags | TSKPRI;
1533 THREAD_CHANGE_PRI(t, ts_kmdpris[0]);
1534 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1535 t->t_trapret = 1; /* so ts_trapret will run */
1536 aston(t);
1537 } else if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1538 /*
1539 * If thread has blocked in the kernel (as opposed to
1540 * being merely preempted), recompute the user mode priority.
1541 */
1542 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1543 TS_NEWUMDPRI(tspp);
1544 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1545 tspp->ts_dispwait = 0;
1546
1547 THREAD_CHANGE_PRI(curthread,
1548 ts_dptbl[tspp->ts_umdpri].ts_globpri);
1549 ASSERT(curthread->t_pri >= 0 &&
1550 curthread->t_pri <= ts_maxglobpri);
1551 tspp->ts_flags = flags & ~TSKPRI;
1552
1553 if (DISP_MUST_SURRENDER(curthread))
1554 cpu_surrender(curthread);
1555 } else if (flags & TSKPRI) {
1556 THREAD_CHANGE_PRI(curthread,
1557 ts_dptbl[tspp->ts_umdpri].ts_globpri);
1558 ASSERT(curthread->t_pri >= 0 &&
1559 curthread->t_pri <= ts_maxglobpri);
1560 tspp->ts_flags = flags & ~TSKPRI;
1561
1562 if (DISP_MUST_SURRENDER(curthread))
1563 cpu_surrender(curthread);
1564 }
1565 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
1566 TRACE_2(TR_FAC_DISP, TR_SLEEP,
1567 "sleep:tid %p old pri %d", t, old_pri);
1568 }
1569
1570
1571 /*
1572 * Return Values:
1573 *
1574 * -1 if the thread is loaded or is not eligible to be swapped in.
1575 *
1576 * effective priority of the specified thread based on swapout time
1577 * and size of process (epri >= 0 , epri <= SHRT_MAX).
1578 */
1579 /* ARGSUSED */
1580 static pri_t
1581 ts_swapin(kthread_t *t, int flags)
1582 {
1583 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1584 long epri = -1;
1585 proc_t *pp = ttoproc(t);
1586
1587 ASSERT(THREAD_LOCK_HELD(t));
1588
1589 /*
1590 * We know that pri_t is a short.
1591 * Be sure not to overrun its range.
1592 */
1593 if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
1594 time_t swapout_time;
1595
1596 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
1597 if (INHERITED(t) || (tspp->ts_flags & (TSKPRI | TSIASET)))
1598 epri = (long)DISP_PRIO(t) + swapout_time;
1599 else {
1600 /*
1601 * Threads which have been out for a long time,
1602 * have high user mode priority and are associated
1603 * with a small address space are more deserving
1604 */
1605 epri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
1606 ASSERT(epri >= 0 && epri <= ts_maxumdpri);
1607 epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
1608 }
1609 /*
1610 * Scale epri so SHRT_MAX/2 represents zero priority.
1611 */
1612 epri += SHRT_MAX/2;
1613 if (epri < 0)
1614 epri = 0;
1615 else if (epri > SHRT_MAX)
1616 epri = SHRT_MAX;
1617 }
1618 return ((pri_t)epri);
1619 }
1631 * Hardswap: Return an effective priority such that threads
1632 * which have been in memory for a while and are
1633 * associated with a small address space are swapped
1634 * in before others.
1635 *
1636 * (epri >= 0 , epri <= SHRT_MAX).
1637 */
1638 time_t ts_minrun = 2; /* XXX - t_pri becomes 59 within 2 secs */
1639 time_t ts_minslp = 2; /* min time on sleep queue for hardswap */
1640
1641 static pri_t
1642 ts_swapout(kthread_t *t, int flags)
1643 {
1644 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1645 long epri = -1;
1646 proc_t *pp = ttoproc(t);
1647 time_t swapin_time;
1648
1649 ASSERT(THREAD_LOCK_HELD(t));
1650
1651 if (INHERITED(t) || (tspp->ts_flags & (TSKPRI | TSIASET)) ||
1652 (t->t_proc_flag & TP_LWPEXIT) ||
1653 (t->t_state & (TS_ZOMB | TS_FREE | TS_STOPPED |
1654 TS_ONPROC | TS_WAIT)) ||
1655 !(t->t_schedflag & TS_LOAD) || !SWAP_OK(t))
1656 return (-1);
1657
1658 ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
1659
1660 /*
1661 * We know that pri_t is a short.
1662 * Be sure not to overrun its range.
1663 */
1664 swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
1665 if (flags == SOFTSWAP) {
1666 if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
1667 epri = 0;
1668 } else {
1669 return ((pri_t)epri);
1670 }
1671 } else {
1700 * and set runrun to cause preemption.
1701 */
1702 static void
1703 ts_tick(kthread_t *t)
1704 {
1705 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1706 klwp_t *lwp;
1707 boolean_t call_cpu_surrender = B_FALSE;
1708 pri_t oldpri = t->t_pri;
1709
1710 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1711
1712 thread_lock(t);
1713
1714 /*
1715 * Keep track of thread's project CPU usage. Note that projects
1716 * get charged even when threads are running in the kernel.
1717 */
1718 if (CPUCAPS_ON()) {
1719 call_cpu_surrender = cpucaps_charge(t, &tspp->ts_caps,
1720 CPUCAPS_CHARGE_ENFORCE) && !(tspp->ts_flags & TSKPRI);
1721 }
1722
1723 if ((tspp->ts_flags & TSKPRI) == 0) {
1724 if (--tspp->ts_timeleft <= 0) {
1725 pri_t new_pri;
1726
1727 /*
1728 * If we're doing preemption control and trying to
1729 * avoid preempting this thread, just note that
1730 * the thread should yield soon and let it keep
1731 * running (unless it's been a while).
1732 */
1733 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1734 if (tspp->ts_timeleft > -SC_MAX_TICKS) {
1735 DTRACE_SCHED1(schedctl__nopreempt,
1736 kthread_t *, t);
1737 schedctl_set_yield(t, 1);
1738 thread_unlock_nopreempt(t);
1739 return;
1740 }
1741
1742 TNF_PROBE_2(schedctl_failsafe,
1743 "schedctl TS ts_tick", /* CSTYLED */,
1744 tnf_pid, pid, ttoproc(t)->p_pid,
1745 tnf_lwpid, lwpid, t->t_tid);
1746 }
1747 tspp->ts_flags &= ~TSRESTORE;
1748 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_tqexp;
1749 TS_NEWUMDPRI(tspp);
1750 tspp->ts_dispwait = 0;
1751 new_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
1752 ASSERT(new_pri >= 0 && new_pri <= ts_maxglobpri);
1753 /*
1754 * When the priority of a thread is changed,
1755 * it may be necessary to adjust its position
1756 * on a sleep queue or dispatch queue.
1757 * The function thread_change_pri accomplishes
1758 * this.
1759 */
1760 if (thread_change_pri(t, new_pri, 0)) {
1761 if ((t->t_schedflag & TS_LOAD) &&
1762 (lwp = t->t_lwp) &&
1763 lwp->lwp_state == LWP_USER)
1764 t->t_schedflag &= ~TS_DONT_SWAP;
1765 tspp->ts_timeleft =
1766 ts_dptbl[tspp->ts_cpupri].ts_quantum;
1767 } else {
1768 call_cpu_surrender = B_TRUE;
1769 }
1770 TRACE_2(TR_FAC_DISP, TR_TICK,
1771 "tick:tid %p old pri %d", t, oldpri);
1772 } else if (t->t_state == TS_ONPROC &&
1773 t->t_pri < t->t_disp_queue->disp_maxrunpri) {
1774 call_cpu_surrender = B_TRUE;
1775 }
1776 }
1777
1778 if (call_cpu_surrender) {
1779 tspp->ts_flags |= TSBACKQ;
1780 cpu_surrender(t);
1781 }
1782
1783 thread_unlock_nopreempt(t); /* clock thread can't be preempted */
1784 }
1785
1786
1787 /*
1788 * If thread is currently at a kernel mode priority (has slept)
1789 * we assign it the appropriate user mode priority and time quantum
1790 * here. If we are lowering the thread's priority below that of
1791 * other runnable threads we will normally set runrun via cpu_surrender() to
1792 * cause preemption.
1793 */
1794 static void
1795 ts_trapret(kthread_t *t)
1796 {
1797 tsproc_t *tspp = (tsproc_t *)t->t_cldata;
1798 cpu_t *cp = CPU;
1799 pri_t old_pri = curthread->t_pri;
1800
1801 ASSERT(THREAD_LOCK_HELD(t));
1802 ASSERT(t == curthread);
1803 ASSERT(cp->cpu_dispthread == t);
1804 ASSERT(t->t_state == TS_ONPROC);
1805
1806 t->t_kpri_req = 0;
1807 if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1808 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1809 TS_NEWUMDPRI(tspp);
1810 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1811 tspp->ts_dispwait = 0;
1812
1813 /*
1814 * If thread has blocked in the kernel (as opposed to
1815 * being merely preempted), recompute the user mode priority.
1816 */
1817 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);
1818 cp->cpu_dispatch_pri = DISP_PRIO(t);
1819 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1820 tspp->ts_flags &= ~TSKPRI;
1821
1822 if (DISP_MUST_SURRENDER(t))
1823 cpu_surrender(t);
1824 } else if (tspp->ts_flags & TSKPRI) {
1825 /*
1826 * If thread has blocked in the kernel (as opposed to
1827 * being merely preempted), recompute the user mode priority.
1828 */
1829 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);
1830 cp->cpu_dispatch_pri = DISP_PRIO(t);
1831 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1832 tspp->ts_flags &= ~TSKPRI;
1833
1834 if (DISP_MUST_SURRENDER(t))
1835 cpu_surrender(t);
1836 }
1837
1838 /*
1839 * Swapout lwp if the swapper is waiting for this thread to
1840 * reach a safe point.
1841 */
1842 if ((t->t_schedflag & TS_SWAPENQ) && !(tspp->ts_flags & TSIASET)) {
1843 thread_unlock(t);
1844 swapout_lwp(ttolwp(t));
1845 thread_lock(t);
1846 }
1847
1848 TRACE_2(TR_FAC_DISP, TR_TRAPRET,
1849 "trapret:tid %p old pri %d", t, old_pri);
1850 }
1851
1852
1853 /*
1854 * Update the ts_dispwait values of all time sharing threads that
1855 * are currently runnable at a user mode priority and bump the priority
1856 * if ts_dispwait exceeds ts_maxwait. Called once per second via
1857 * timeout which we reset here.
1858 *
1859 * There are several lists of time sharing threads broken up by a hash on
1860 * the thread pointer. Each list has its own lock. This avoids blocking
1914 {
1915 tsproc_t *tspp;
1916 kthread_t *tx;
1917 int updated = 0;
1918
1919 mutex_enter(&ts_list_lock[i]);
1920 for (tspp = ts_plisthead[i].ts_next; tspp != &ts_plisthead[i];
1921 tspp = tspp->ts_next) {
1922 tx = tspp->ts_tp;
1923 /*
1924 * Lock the thread and verify state.
1925 */
1926 thread_lock(tx);
1927 /*
1928 * Skip the thread if it is no longer in the TS (or IA) class.
1929 */
1930 if (tx->t_clfuncs != &ts_classfuncs.thread &&
1931 tx->t_clfuncs != &ia_classfuncs.thread)
1932 goto next;
1933 tspp->ts_dispwait++;
1934 if ((tspp->ts_flags & TSKPRI) != 0)
1935 goto next;
1936 if (tspp->ts_dispwait <= ts_dptbl[tspp->ts_umdpri].ts_maxwait)
1937 goto next;
1938 if (tx->t_schedctl && schedctl_get_nopreempt(tx))
1939 goto next;
1940 if (tx->t_state != TS_RUN && tx->t_state != TS_WAIT &&
1941 (tx->t_state != TS_SLEEP || !ts_sleep_promote)) {
1942 /* make next syscall/trap do CL_TRAPRET */
1943 tx->t_trapret = 1;
1944 aston(tx);
1945 goto next;
1946 }
1947 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_lwait;
1948 TS_NEWUMDPRI(tspp);
1949 tspp->ts_dispwait = 0;
1950 updated = 1;
1951
1952 /*
1953 * Only dequeue it if needs to move; otherwise it should
1954 * just round-robin here.
1955 */
1956 if (tx->t_pri != ts_dptbl[tspp->ts_umdpri].ts_globpri) {
1957 pri_t oldpri = tx->t_pri;
1958 ts_change_priority(tx, tspp);
1959 TRACE_2(TR_FAC_DISP, TR_UPDATE,
1960 "update:tid %p old pri %d", tx, oldpri);
1961 }
1962 next:
1963 thread_unlock(tx);
1964 }
1965 mutex_exit(&ts_list_lock[i]);
1966
1967 return (updated);
1968 }
1969
1970 /*
1971 * Processes waking up go to the back of their queue. We don't
1972 * need to assign a time quantum here because thread is still
1973 * at a kernel mode priority and the time slicing is not done
1974 * for threads running in the kernel after sleeping. The proper
1975 * time quantum will be assigned by ts_trapret before the thread
1976 * returns to user mode.
1977 */
1978 static void
1979 ts_wakeup(kthread_t *t)
1980 {
1981 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1982
1983 ASSERT(THREAD_LOCK_HELD(t));
1984
1985 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
1986
1987 if (tspp->ts_flags & TSKPRI) {
1988 tspp->ts_flags &= ~TSBACKQ;
1989 if (tspp->ts_flags & TSIASET)
1990 setfrontdq(t);
1991 else
1992 setbackdq(t);
1993 } else if (t->t_kpri_req) {
1994 /*
1995 * Give thread a priority boost if we were asked.
1996 */
1997 tspp->ts_flags |= TSKPRI;
1998 THREAD_CHANGE_PRI(t, ts_kmdpris[0]);
1999 setbackdq(t);
2000 t->t_trapret = 1; /* so that ts_trapret will run */
2001 aston(t);
2002 } else {
2003 if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
2004 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
2005 TS_NEWUMDPRI(tspp);
2006 tspp->ts_timeleft =
2007 ts_dptbl[tspp->ts_cpupri].ts_quantum;
2008 tspp->ts_dispwait = 0;
2009 THREAD_CHANGE_PRI(t,
2010 ts_dptbl[tspp->ts_umdpri].ts_globpri);
2011 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
2012 }
2013
2014 tspp->ts_flags &= ~TSBACKQ;
2015
2016 if (tspp->ts_flags & TSIA) {
2017 if (tspp->ts_flags & TSIASET)
2018 setfrontdq(t);
2019 else
2020 setbackdq(t);
2021 } else {
2022 if (t->t_disp_time != ddi_get_lbolt())
2023 setbackdq(t);
2024 else
2025 setfrontdq(t);
2026 }
2027 }
2028 }
2029
2030
2031 /*
2032 * When a thread yields, put it on the back of the run queue.
2033 */
2034 static void
2035 ts_yield(kthread_t *t)
2036 {
2037 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
2038
2039 ASSERT(t == curthread);
2040 ASSERT(THREAD_LOCK_HELD(t));
2041
2042 /*
2043 * Collect CPU usage spent before yielding
2044 */
2045 (void) CPUCAPS_CHARGE(t, &tspp->ts_caps, CPUCAPS_CHARGE_ENFORCE);
2046
2047 /*
2274 mutex_enter(&fg->p_lock);
2275 }
2276
2277 if ((tx = proctot(fg)) == NULL) {
2278 mutex_exit(&fg->p_lock);
2279 continue;
2280 }
2281 do {
2282 thread_lock(tx);
2283 /*
2284 * if this thread is not interactive continue
2285 */
2286 if (tx->t_cid != ia_cid) {
2287 thread_unlock(tx);
2288 continue;
2289 }
2290 tspp = tx->t_cldata;
2291 tspp->ts_flags |= TSIASET;
2292 tspp->ts_boost = ia_boost;
2293 TS_NEWUMDPRI(tspp);
2294 if ((tspp->ts_flags & TSKPRI) != 0) {
2295 thread_unlock(tx);
2296 continue;
2297 }
2298 tspp->ts_dispwait = 0;
2299 ts_change_priority(tx, tspp);
2300 thread_unlock(tx);
2301 } while ((tx = tx->t_forw) != fg->p_tlist);
2302 mutex_exit(&fg->p_lock);
2303 }
2304 skip:
2305 if (bg_pgid == 0)
2306 return;
2307 for (bg = (proc_t *)pgfind(bg_pgid); bg != NULL; bg = bg->p_pglink) {
2308 if (bg->p_stat == SIDL) {
2309 continue;
2310 }
2311 /*
2312 * sesssion leaders must be turned off explicitly
2313 * not implicitly as happens to other members of
2314 * the process group.
2315 */
2316 if (bg->p_pid == bg->p_sessp->s_sid) {
2317 continue;
2327 mutex_enter(&bg->p_lock);
2328 }
2329
2330 if ((tx = proctot(bg)) == NULL) {
2331 mutex_exit(&bg->p_lock);
2332 continue;
2333 }
2334 do {
2335 thread_lock(tx);
2336 /*
2337 * if this thread is not interactive continue
2338 */
2339 if (tx->t_cid != ia_cid) {
2340 thread_unlock(tx);
2341 continue;
2342 }
2343 tspp = tx->t_cldata;
2344 tspp->ts_flags &= ~TSIASET;
2345 tspp->ts_boost = -ia_boost;
2346 TS_NEWUMDPRI(tspp);
2347 if ((tspp->ts_flags & TSKPRI) != 0) {
2348 thread_unlock(tx);
2349 continue;
2350 }
2351
2352 tspp->ts_dispwait = 0;
2353 ts_change_priority(tx, tspp);
2354 thread_unlock(tx);
2355 } while ((tx = tx->t_forw) != bg->p_tlist);
2356 mutex_exit(&bg->p_lock);
2357 }
2358 }
2359
2360
2361 static void
2362 ts_change_priority(kthread_t *t, tsproc_t *tspp)
2363 {
2364 pri_t new_pri;
2365
2366 ASSERT(THREAD_LOCK_HELD(t));
2367 new_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
2368 ASSERT(new_pri >= 0 && new_pri <= ts_maxglobpri);
2369 tspp->ts_flags &= ~TSRESTORE;
2370 t->t_cpri = tspp->ts_upri;
|
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 /* All Rights Reserved */
29
30 #include <sys/types.h>
31 #include <sys/param.h>
32 #include <sys/sysmacros.h>
33 #include <sys/cred.h>
34 #include <sys/proc.h>
35 #include <sys/session.h>
36 #include <sys/strsubr.h>
37 #include <sys/signal.h>
38 #include <sys/user.h>
39 #include <sys/priocntl.h>
40 #include <sys/class.h>
41 #include <sys/disp.h>
42 #include <sys/procset.h>
43 #include <sys/debug.h>
44 #include <sys/ts.h>
212 static pri_t ts_globpri(kthread_t *);
213 static void ts_yield(kthread_t *);
214 extern tsdpent_t *ts_getdptbl(void);
215 extern pri_t *ts_getkmdpris(void);
216 extern pri_t td_getmaxumdpri(void);
217 static int ts_alloc(void **, int);
218 static void ts_free(void *);
219
220 pri_t ia_init(id_t, int, classfuncs_t **);
221 static int ia_getclinfo(void *);
222 static int ia_getclpri(pcpri_t *);
223 static int ia_parmsin(void *);
224 static int ia_vaparmsin(void *, pc_vaparms_t *);
225 static int ia_vaparmsout(void *, pc_vaparms_t *);
226 static int ia_parmsset(kthread_t *, void *, id_t, cred_t *);
227 static void ia_parmsget(kthread_t *, void *);
228 static void ia_set_process_group(pid_t, pid_t, pid_t);
229
230 static void ts_change_priority(kthread_t *, tsproc_t *);
231
232 static pri_t ts_maxglobpri; /* maximum global priority used by ts class */
233 static kmutex_t ts_dptblock; /* protects time sharing dispatch table */
234 static kmutex_t ts_list_lock[TS_LISTS]; /* protects tsproc lists */
235 static tsproc_t ts_plisthead[TS_LISTS]; /* dummy tsproc at head of lists */
236
237 static gid_t IA_gid = 0;
238
239 static struct classfuncs ts_classfuncs = {
240 /* class functions */
241 ts_admin,
242 ts_getclinfo,
243 ts_parmsin,
244 ts_parmsout,
245 ts_vaparmsin,
246 ts_vaparmsout,
247 ts_getclpri,
248 ts_alloc,
249 ts_free,
250
251 /* thread functions */
523 ts_dptbl[i].ts_maxwait = tmpdpp[i].ts_maxwait;
524 ts_dptbl[i].ts_lwait = tmpdpp[i].ts_lwait;
525 }
526 mutex_exit(&ts_dptblock);
527 kmem_free(tmpdpp, tsdpsz);
528 break;
529
530 default:
531 return (EINVAL);
532 }
533 return (0);
534 }
535
536
537 /*
538 * Allocate a time-sharing class specific thread structure and
539 * initialize it with the parameters supplied. Also move the thread
540 * to specified time-sharing priority.
541 */
542 static int
543 ts_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
544 void *bufp)
545 {
546 tsparms_t *tsparmsp = (tsparms_t *)parmsp;
547 tsproc_t *tspp;
548 pri_t reqtsuprilim;
549 pri_t reqtsupri;
550 static uint32_t tspexists = 0; /* set on first occurrence of */
551 /* a time-sharing process */
552
553 tspp = (tsproc_t *)bufp;
554 ASSERT(tspp != NULL);
555
556 /*
557 * Initialize the tsproc structure.
558 */
559 tspp->ts_cpupri = tsmedumdpri;
560 if (cid == ia_cid) {
561 /*
562 * Check to make sure caller is either privileged or the
563 * window system. When the window system is converted
564 * to using privileges, the second check can go away.
685 tsproc_t *ptspp; /* ptr to parent's tsproc structure */
686 tsproc_t *ctspp; /* ptr to child's tsproc structure */
687
688 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
689
690 ctspp = (tsproc_t *)bufp;
691 ASSERT(ctspp != NULL);
692 ptspp = (tsproc_t *)t->t_cldata;
693 /*
694 * Initialize child's tsproc structure.
695 */
696 thread_lock(t);
697 ctspp->ts_timeleft = ts_dptbl[ptspp->ts_cpupri].ts_quantum;
698 ctspp->ts_cpupri = ptspp->ts_cpupri;
699 ctspp->ts_boost = ptspp->ts_boost;
700 ctspp->ts_uprilim = ptspp->ts_uprilim;
701 ctspp->ts_upri = ptspp->ts_upri;
702 TS_NEWUMDPRI(ctspp);
703 ctspp->ts_nice = ptspp->ts_nice;
704 ctspp->ts_dispwait = 0;
705 ctspp->ts_flags = ptspp->ts_flags & ~(TSBACKQ | TSRESTORE);
706 ctspp->ts_tp = ct;
707 cpucaps_sc_init(&ctspp->ts_caps);
708 thread_unlock(t);
709
710 /*
711 * Link new structure into tsproc list.
712 */
713 ct->t_cldata = (void *)ctspp;
714 TS_LIST_INSERT(ctspp);
715 return (0);
716 }
717
718
719 /*
720 * Child is placed at back of dispatcher queue and parent gives
721 * up processor so that the child runs first after the fork.
722 * This allows the child immediately execing to break the multiple
723 * use of copy on write pages with no disk home. The parent will
724 * get to steal them back rather than uselessly copying them.
725 */
736 /*
737 * Grab the child's p_lock before dropping pidlock to ensure
738 * the process does not disappear before we set it running.
739 */
740 mutex_enter(&cp->p_lock);
741 continuelwps(cp);
742 mutex_exit(&cp->p_lock);
743
744 mutex_enter(&pp->p_lock);
745 mutex_exit(&pidlock);
746 continuelwps(pp);
747
748 thread_lock(t);
749 tspp = (tsproc_t *)(t->t_cldata);
750 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_tqexp;
751 TS_NEWUMDPRI(tspp);
752 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
753 tspp->ts_dispwait = 0;
754 t->t_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
755 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
756 THREAD_TRANSITION(t);
757 ts_setrun(t);
758 thread_unlock(t);
759 /*
760 * Safe to drop p_lock now since since it is safe to change
761 * the scheduling class after this point.
762 */
763 mutex_exit(&pp->p_lock);
764
765 swtch();
766 }
767
768
769 /*
770 * Get information about the time-sharing class into the buffer
771 * pointed to by tsinfop. The maximum configured user priority
772 * is the only information we supply. ts_getclinfo() is called
773 * for TS threads, and ia_getclinfo() is called for IA threads.
774 */
775 static int
1198 reqtsuprilim > tspp->ts_uprilim &&
1199 secpolicy_raisepriority(reqpcredp) != 0)
1200 return (EPERM);
1201
1202 /*
1203 * Set ts_nice to the nice value corresponding to the user
1204 * priority we are setting. Note that setting the nice field
1205 * of the parameter struct won't affect upri or nice.
1206 */
1207 nice = NZERO - (reqtsupri * NZERO) / ts_maxupri;
1208 if (nice >= 2 * NZERO)
1209 nice = 2 * NZERO - 1;
1210
1211 thread_lock(tx);
1212
1213 tspp->ts_uprilim = reqtsuprilim;
1214 tspp->ts_upri = reqtsupri;
1215 TS_NEWUMDPRI(tspp);
1216 tspp->ts_nice = nice;
1217
1218 tspp->ts_dispwait = 0;
1219 ts_change_priority(tx, tspp);
1220 thread_unlock(tx);
1221 return (0);
1222 }
1223
1224
1225 static int
1226 ia_parmsset(kthread_t *tx, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1227 {
1228 tsproc_t *tspp = (tsproc_t *)tx->t_cldata;
1229 iaparms_t *iaparmsp = (iaparms_t *)parmsp;
1230 proc_t *p;
1231 pid_t pid, pgid, sid;
1232 pid_t on, off;
1233 struct stdata *stp;
1234 int sess_held;
1235
1236 /*
1237 * Handle user priority changes
1349 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1350 tspp = (tsproc_t *)t->t_cldata;
1351 tspri = tsmedumdpri + tspp->ts_upri;
1352 if (tspri > ts_maxumdpri)
1353 tspri = ts_maxumdpri;
1354 else if (tspri < 0)
1355 tspri = 0;
1356 return (ts_dptbl[tspri].ts_globpri);
1357 }
1358
1359 /*
1360 * Arrange for thread to be placed in appropriate location
1361 * on dispatcher queue.
1362 *
1363 * This is called with the current thread in TS_ONPROC and locked.
1364 */
1365 static void
1366 ts_preempt(kthread_t *t)
1367 {
1368 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1369 klwp_t *lwp = ttolwp(t);
1370 pri_t oldpri = t->t_pri;
1371
1372 ASSERT(t == curthread);
1373 ASSERT(THREAD_LOCK_HELD(curthread));
1374
1375 /*
1376 * This thread may be placed on wait queue by CPU Caps. In this case we
1377 * do not need to do anything until it is removed from the wait queue.
1378 */
1379 if (CPUCAPS_ON()) {
1380 (void) cpucaps_charge(t, &tspp->ts_caps,
1381 CPUCAPS_CHARGE_ENFORCE);
1382 if (CPUCAPS_ENFORCE(t))
1383 return;
1384 }
1385
1386 /*
1387 * If thread got preempted in the user-land then we know
1388 * it isn't holding any locks. Mark it as swappable.
1389 */
1390 ASSERT(t->t_schedflag & TS_DONT_SWAP);
1391 if (lwp != NULL && lwp->lwp_state == LWP_USER)
1392 t->t_schedflag &= ~TS_DONT_SWAP;
1393
1394 /*
1395 * Check to see if we're doing "preemption control" here. If
1396 * we are, and if the user has requested that this thread not
1397 * be preempted, and if preemptions haven't been put off for
1398 * too long, let the preemption happen here but try to make
1399 * sure the thread is rescheduled as soon as possible. We do
1400 * this by putting it on the front of the highest priority run
1401 * queue in the TS class. If the preemption has been put off
1402 * for too long, clear the "nopreempt" bit and let the thread
1403 * be preempted.
1404 */
1405 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1406 if (tspp->ts_timeleft > -SC_MAX_TICKS) {
1407 DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
1408 /*
1409 * If not already remembered, remember current
1410 * priority for restoration in ts_yield().
1411 */
1412 if (!(tspp->ts_flags & TSRESTORE)) {
1413 tspp->ts_scpri = t->t_pri;
1414 tspp->ts_flags |= TSRESTORE;
1415 }
1416 THREAD_CHANGE_PRI(t, ts_maxumdpri);
1417 t->t_schedflag |= TS_DONT_SWAP;
1418 schedctl_set_yield(t, 1);
1419 setfrontdq(t);
1420 goto done;
1421 } else {
1422 if (tspp->ts_flags & TSRESTORE) {
1423 THREAD_CHANGE_PRI(t, tspp->ts_scpri);
1424 tspp->ts_flags &= ~TSRESTORE;
1425 }
1426 schedctl_set_nopreempt(t, 0);
1427 DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
1428 TNF_PROBE_2(schedctl_preempt, "schedctl TS ts_preempt",
1429 /* CSTYLED */, tnf_pid, pid, ttoproc(t)->p_pid,
1430 tnf_lwpid, lwpid, t->t_tid);
1431 /*
1432 * Fall through and be preempted below.
1433 */
1434 }
1435 }
1436
1437 if ((tspp->ts_flags & TSBACKQ) != 0) {
1438 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1439 tspp->ts_dispwait = 0;
1440 tspp->ts_flags &= ~TSBACKQ;
1441 setbackdq(t);
1442 } else {
1443 setfrontdq(t);
1444 }
1445
1446 done:
1447 TRACE_2(TR_FAC_DISP, TR_PREEMPT,
1448 "preempt:tid %p old pri %d", t, oldpri);
1449 }
1450
1451 static void
1452 ts_setrun(kthread_t *t)
1453 {
1454 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1455
1456 ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */
1457
1458 if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1459 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1460 TS_NEWUMDPRI(tspp);
1461 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1462 tspp->ts_dispwait = 0;
1463 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);
1464 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1465 }
1466
1467 tspp->ts_flags &= ~TSBACKQ;
1468
1469 if (tspp->ts_flags & TSIA) {
1470 if (tspp->ts_flags & TSIASET)
1471 setfrontdq(t);
1472 else
1473 setbackdq(t);
1474 } else {
1475 if (t->t_disp_time != ddi_get_lbolt())
1476 setbackdq(t);
1477 else
1478 setfrontdq(t);
1479 }
1480 }
1481
1482
1483 /*
1484 * Prepare thread for sleep.
1485 */
1486 static void
1487 ts_sleep(kthread_t *t)
1488 {
1489 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1490 pri_t old_pri = t->t_pri;
1491
1492 ASSERT(t == curthread);
1493 ASSERT(THREAD_LOCK_HELD(t));
1494
1495 /*
1496 * Account for time spent on CPU before going to sleep.
1497 */
1498 (void) CPUCAPS_CHARGE(t, &tspp->ts_caps, CPUCAPS_CHARGE_ENFORCE);
1499
1500 if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1501 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1502 TS_NEWUMDPRI(tspp);
1503 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1504 tspp->ts_dispwait = 0;
1505
1506 THREAD_CHANGE_PRI(curthread,
1507 ts_dptbl[tspp->ts_umdpri].ts_globpri);
1508 ASSERT(curthread->t_pri >= 0 &&
1509 curthread->t_pri <= ts_maxglobpri);
1510
1511 if (DISP_MUST_SURRENDER(curthread))
1512 cpu_surrender(curthread);
1513 }
1514 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
1515 TRACE_2(TR_FAC_DISP, TR_SLEEP,
1516 "sleep:tid %p old pri %d", t, old_pri);
1517 }
1518
1519
1520 /*
1521 * Return Values:
1522 *
1523 * -1 if the thread is loaded or is not eligible to be swapped in.
1524 *
1525 * effective priority of the specified thread based on swapout time
1526 * and size of process (epri >= 0 , epri <= SHRT_MAX).
1527 */
1528 /* ARGSUSED */
1529 static pri_t
1530 ts_swapin(kthread_t *t, int flags)
1531 {
1532 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1533 long epri = -1;
1534 proc_t *pp = ttoproc(t);
1535
1536 ASSERT(THREAD_LOCK_HELD(t));
1537
1538 /*
1539 * We know that pri_t is a short.
1540 * Be sure not to overrun its range.
1541 */
1542 if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
1543 time_t swapout_time;
1544
1545 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
1546 if (INHERITED(t) || (tspp->ts_flags & TSIASET)) {
1547 epri = (long)DISP_PRIO(t) + swapout_time;
1548 } else {
1549 /*
1550 * Threads which have been out for a long time,
1551 * have high user mode priority and are associated
1552 * with a small address space are more deserving
1553 */
1554 epri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
1555 ASSERT(epri >= 0 && epri <= ts_maxumdpri);
1556 epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
1557 }
1558 /*
1559 * Scale epri so SHRT_MAX/2 represents zero priority.
1560 */
1561 epri += SHRT_MAX/2;
1562 if (epri < 0)
1563 epri = 0;
1564 else if (epri > SHRT_MAX)
1565 epri = SHRT_MAX;
1566 }
1567 return ((pri_t)epri);
1568 }
1580 * Hardswap: Return an effective priority such that threads
1581 * which have been in memory for a while and are
1582 * associated with a small address space are swapped
1583 * in before others.
1584 *
1585 * (epri >= 0 , epri <= SHRT_MAX).
1586 */
1587 time_t ts_minrun = 2; /* XXX - t_pri becomes 59 within 2 secs */
1588 time_t ts_minslp = 2; /* min time on sleep queue for hardswap */
1589
1590 static pri_t
1591 ts_swapout(kthread_t *t, int flags)
1592 {
1593 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1594 long epri = -1;
1595 proc_t *pp = ttoproc(t);
1596 time_t swapin_time;
1597
1598 ASSERT(THREAD_LOCK_HELD(t));
1599
1600 if (INHERITED(t) || (tspp->ts_flags & TSIASET) ||
1601 (t->t_proc_flag & TP_LWPEXIT) ||
1602 (t->t_state & (TS_ZOMB | TS_FREE | TS_STOPPED |
1603 TS_ONPROC | TS_WAIT)) ||
1604 !(t->t_schedflag & TS_LOAD) || !SWAP_OK(t))
1605 return (-1);
1606
1607 ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
1608
1609 /*
1610 * We know that pri_t is a short.
1611 * Be sure not to overrun its range.
1612 */
1613 swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
1614 if (flags == SOFTSWAP) {
1615 if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
1616 epri = 0;
1617 } else {
1618 return ((pri_t)epri);
1619 }
1620 } else {
1649 * and set runrun to cause preemption.
1650 */
1651 static void
1652 ts_tick(kthread_t *t)
1653 {
1654 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1655 klwp_t *lwp;
1656 boolean_t call_cpu_surrender = B_FALSE;
1657 pri_t oldpri = t->t_pri;
1658
1659 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1660
1661 thread_lock(t);
1662
1663 /*
1664 * Keep track of thread's project CPU usage. Note that projects
1665 * get charged even when threads are running in the kernel.
1666 */
1667 if (CPUCAPS_ON()) {
1668 call_cpu_surrender = cpucaps_charge(t, &tspp->ts_caps,
1669 CPUCAPS_CHARGE_ENFORCE);
1670 }
1671
1672 if (--tspp->ts_timeleft <= 0) {
1673 pri_t new_pri;
1674
1675 /*
1676 * If we're doing preemption control and trying to avoid
1677 * preempting this thread, just note that the thread should
1678 * yield soon and let it keep running (unless it's been a
1679 * while).
1680 */
1681 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1682 if (tspp->ts_timeleft > -SC_MAX_TICKS) {
1683 DTRACE_SCHED1(schedctl__nopreempt,
1684 kthread_t *, t);
1685 schedctl_set_yield(t, 1);
1686 thread_unlock_nopreempt(t);
1687 return;
1688 }
1689
1690 TNF_PROBE_2(schedctl_failsafe,
1691 "schedctl TS ts_tick", /* CSTYLED */,
1692 tnf_pid, pid, ttoproc(t)->p_pid,
1693 tnf_lwpid, lwpid, t->t_tid);
1694 }
1695 tspp->ts_flags &= ~TSRESTORE;
1696 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_tqexp;
1697 TS_NEWUMDPRI(tspp);
1698 tspp->ts_dispwait = 0;
1699 new_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
1700 ASSERT(new_pri >= 0 && new_pri <= ts_maxglobpri);
1701 /*
1702 * When the priority of a thread is changed, it may be
1703 * necessary to adjust its position on a sleep queue or
1704 * dispatch queue. The function thread_change_pri accomplishes
1705 * this.
1706 */
1707 if (thread_change_pri(t, new_pri, 0)) {
1708 if ((t->t_schedflag & TS_LOAD) &&
1709 (lwp = t->t_lwp) &&
1710 lwp->lwp_state == LWP_USER)
1711 t->t_schedflag &= ~TS_DONT_SWAP;
1712 tspp->ts_timeleft =
1713 ts_dptbl[tspp->ts_cpupri].ts_quantum;
1714 } else {
1715 call_cpu_surrender = B_TRUE;
1716 }
1717 TRACE_2(TR_FAC_DISP, TR_TICK,
1718 "tick:tid %p old pri %d", t, oldpri);
1719 } else if (t->t_state == TS_ONPROC &&
1720 t->t_pri < t->t_disp_queue->disp_maxrunpri) {
1721 call_cpu_surrender = B_TRUE;
1722 }
1723
1724 if (call_cpu_surrender) {
1725 tspp->ts_flags |= TSBACKQ;
1726 cpu_surrender(t);
1727 }
1728
1729 thread_unlock_nopreempt(t); /* clock thread can't be preempted */
1730 }
1731
1732
1733 /*
1734 * If we are lowering the thread's priority below that of other runnable
1735 * threads we will normally set runrun via cpu_surrender() to cause preemption.
1736 */
1737 static void
1738 ts_trapret(kthread_t *t)
1739 {
1740 tsproc_t *tspp = (tsproc_t *)t->t_cldata;
1741 cpu_t *cp = CPU;
1742 pri_t old_pri = curthread->t_pri;
1743
1744 ASSERT(THREAD_LOCK_HELD(t));
1745 ASSERT(t == curthread);
1746 ASSERT(cp->cpu_dispthread == t);
1747 ASSERT(t->t_state == TS_ONPROC);
1748
1749 if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1750 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1751 TS_NEWUMDPRI(tspp);
1752 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1753 tspp->ts_dispwait = 0;
1754
1755 /*
1756 * If thread has blocked in the kernel (as opposed to
1757 * being merely preempted), recompute the user mode priority.
1758 */
1759 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);
1760 cp->cpu_dispatch_pri = DISP_PRIO(t);
1761 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1762
1763 if (DISP_MUST_SURRENDER(t))
1764 cpu_surrender(t);
1765 }
1766
1767 /*
1768 * Swapout lwp if the swapper is waiting for this thread to reach a
1769 * safe point.
1770 */
1771 if ((t->t_schedflag & TS_SWAPENQ) && !(tspp->ts_flags & TSIASET)) {
1772 thread_unlock(t);
1773 swapout_lwp(ttolwp(t));
1774 thread_lock(t);
1775 }
1776
1777 TRACE_2(TR_FAC_DISP, TR_TRAPRET,
1778 "trapret:tid %p old pri %d", t, old_pri);
1779 }
1780
1781
1782 /*
1783 * Update the ts_dispwait values of all time sharing threads that
1784 * are currently runnable at a user mode priority and bump the priority
1785 * if ts_dispwait exceeds ts_maxwait. Called once per second via
1786 * timeout which we reset here.
1787 *
1788 * There are several lists of time sharing threads broken up by a hash on
1789 * the thread pointer. Each list has its own lock. This avoids blocking
1843 {
1844 tsproc_t *tspp;
1845 kthread_t *tx;
1846 int updated = 0;
1847
1848 mutex_enter(&ts_list_lock[i]);
1849 for (tspp = ts_plisthead[i].ts_next; tspp != &ts_plisthead[i];
1850 tspp = tspp->ts_next) {
1851 tx = tspp->ts_tp;
1852 /*
1853 * Lock the thread and verify state.
1854 */
1855 thread_lock(tx);
1856 /*
1857 * Skip the thread if it is no longer in the TS (or IA) class.
1858 */
1859 if (tx->t_clfuncs != &ts_classfuncs.thread &&
1860 tx->t_clfuncs != &ia_classfuncs.thread)
1861 goto next;
1862 tspp->ts_dispwait++;
1863 if (tspp->ts_dispwait <= ts_dptbl[tspp->ts_umdpri].ts_maxwait)
1864 goto next;
1865 if (tx->t_schedctl && schedctl_get_nopreempt(tx))
1866 goto next;
1867 if (tx->t_state != TS_RUN && tx->t_state != TS_WAIT &&
1868 (tx->t_state != TS_SLEEP || !ts_sleep_promote)) {
1869 /* make next syscall/trap do CL_TRAPRET */
1870 tx->t_trapret = 1;
1871 aston(tx);
1872 goto next;
1873 }
1874 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_lwait;
1875 TS_NEWUMDPRI(tspp);
1876 tspp->ts_dispwait = 0;
1877 updated = 1;
1878
1879 /*
1880 * Only dequeue it if needs to move; otherwise it should
1881 * just round-robin here.
1882 */
1883 if (tx->t_pri != ts_dptbl[tspp->ts_umdpri].ts_globpri) {
1884 pri_t oldpri = tx->t_pri;
1885 ts_change_priority(tx, tspp);
1886 TRACE_2(TR_FAC_DISP, TR_UPDATE,
1887 "update:tid %p old pri %d", tx, oldpri);
1888 }
1889 next:
1890 thread_unlock(tx);
1891 }
1892 mutex_exit(&ts_list_lock[i]);
1893
1894 return (updated);
1895 }
1896
1897 /*
1898 * Processes waking up go to the back of their queue.
1899 */
1900 static void
1901 ts_wakeup(kthread_t *t)
1902 {
1903 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1904
1905 ASSERT(THREAD_LOCK_HELD(t));
1906
1907 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
1908
1909 if (tspp->ts_dispwait > ts_dptbl[tspp->ts_umdpri].ts_maxwait) {
1910 tspp->ts_cpupri = ts_dptbl[tspp->ts_cpupri].ts_slpret;
1911 TS_NEWUMDPRI(tspp);
1912 tspp->ts_timeleft = ts_dptbl[tspp->ts_cpupri].ts_quantum;
1913 tspp->ts_dispwait = 0;
1914 THREAD_CHANGE_PRI(t, ts_dptbl[tspp->ts_umdpri].ts_globpri);
1915 ASSERT(t->t_pri >= 0 && t->t_pri <= ts_maxglobpri);
1916 }
1917
1918 tspp->ts_flags &= ~TSBACKQ;
1919
1920 if (tspp->ts_flags & TSIA) {
1921 if (tspp->ts_flags & TSIASET)
1922 setfrontdq(t);
1923 else
1924 setbackdq(t);
1925 } else {
1926 if (t->t_disp_time != ddi_get_lbolt())
1927 setbackdq(t);
1928 else
1929 setfrontdq(t);
1930 }
1931 }
1932
1933
1934 /*
1935 * When a thread yields, put it on the back of the run queue.
1936 */
1937 static void
1938 ts_yield(kthread_t *t)
1939 {
1940 tsproc_t *tspp = (tsproc_t *)(t->t_cldata);
1941
1942 ASSERT(t == curthread);
1943 ASSERT(THREAD_LOCK_HELD(t));
1944
1945 /*
1946 * Collect CPU usage spent before yielding
1947 */
1948 (void) CPUCAPS_CHARGE(t, &tspp->ts_caps, CPUCAPS_CHARGE_ENFORCE);
1949
1950 /*
2177 mutex_enter(&fg->p_lock);
2178 }
2179
2180 if ((tx = proctot(fg)) == NULL) {
2181 mutex_exit(&fg->p_lock);
2182 continue;
2183 }
2184 do {
2185 thread_lock(tx);
2186 /*
2187 * if this thread is not interactive continue
2188 */
2189 if (tx->t_cid != ia_cid) {
2190 thread_unlock(tx);
2191 continue;
2192 }
2193 tspp = tx->t_cldata;
2194 tspp->ts_flags |= TSIASET;
2195 tspp->ts_boost = ia_boost;
2196 TS_NEWUMDPRI(tspp);
2197 tspp->ts_dispwait = 0;
2198 ts_change_priority(tx, tspp);
2199 thread_unlock(tx);
2200 } while ((tx = tx->t_forw) != fg->p_tlist);
2201 mutex_exit(&fg->p_lock);
2202 }
2203 skip:
2204 if (bg_pgid == 0)
2205 return;
2206 for (bg = (proc_t *)pgfind(bg_pgid); bg != NULL; bg = bg->p_pglink) {
2207 if (bg->p_stat == SIDL) {
2208 continue;
2209 }
2210 /*
2211 * sesssion leaders must be turned off explicitly
2212 * not implicitly as happens to other members of
2213 * the process group.
2214 */
2215 if (bg->p_pid == bg->p_sessp->s_sid) {
2216 continue;
2226 mutex_enter(&bg->p_lock);
2227 }
2228
2229 if ((tx = proctot(bg)) == NULL) {
2230 mutex_exit(&bg->p_lock);
2231 continue;
2232 }
2233 do {
2234 thread_lock(tx);
2235 /*
2236 * if this thread is not interactive continue
2237 */
2238 if (tx->t_cid != ia_cid) {
2239 thread_unlock(tx);
2240 continue;
2241 }
2242 tspp = tx->t_cldata;
2243 tspp->ts_flags &= ~TSIASET;
2244 tspp->ts_boost = -ia_boost;
2245 TS_NEWUMDPRI(tspp);
2246
2247 tspp->ts_dispwait = 0;
2248 ts_change_priority(tx, tspp);
2249 thread_unlock(tx);
2250 } while ((tx = tx->t_forw) != bg->p_tlist);
2251 mutex_exit(&bg->p_lock);
2252 }
2253 }
2254
2255
2256 static void
2257 ts_change_priority(kthread_t *t, tsproc_t *tspp)
2258 {
2259 pri_t new_pri;
2260
2261 ASSERT(THREAD_LOCK_HELD(t));
2262 new_pri = ts_dptbl[tspp->ts_umdpri].ts_globpri;
2263 ASSERT(new_pri >= 0 && new_pri <= ts_maxglobpri);
2264 tspp->ts_flags &= ~TSRESTORE;
2265 t->t_cpri = tspp->ts_upri;
|