1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2013, Joyent, Inc. All rights reserved.
  25  */
  26 
  27 #include <sys/types.h>
  28 #include <sys/param.h>
  29 #include <sys/sysmacros.h>
  30 #include <sys/cred.h>
  31 #include <sys/proc.h>
  32 #include <sys/strsubr.h>
  33 #include <sys/priocntl.h>
  34 #include <sys/class.h>
  35 #include <sys/disp.h>
  36 #include <sys/procset.h>
  37 #include <sys/debug.h>
  38 #include <sys/kmem.h>
  39 #include <sys/errno.h>
  40 #include <sys/systm.h>
  41 #include <sys/schedctl.h>
  42 #include <sys/vmsystm.h>
  43 #include <sys/atomic.h>
  44 #include <sys/project.h>
  45 #include <sys/modctl.h>
  46 #include <sys/fss.h>
  47 #include <sys/fsspriocntl.h>
  48 #include <sys/cpupart.h>
  49 #include <sys/zone.h>
  50 #include <vm/rm.h>
  51 #include <vm/seg_kmem.h>
  52 #include <sys/tnf_probe.h>
  53 #include <sys/policy.h>
  54 #include <sys/sdt.h>
  55 #include <sys/cpucaps.h>
  56 
  57 /*
  58  * The fair share scheduling class ensures that collections of processes
  59  * (zones and projects) each get their configured share of CPU.  This is in
  60  * contrast to the TS class which considers individual processes.
  61  *
  62  * The FSS cpu-share is set on zones using the zone.cpu-shares rctl and on
  63  * projects using the project.cpu-shares rctl.  By default the value is 1
  64  * and it can range from 0 - 64k.  A value of 0 means that processes in the
  65  * collection will only get CPU resources when there are no other processes
  66  * that need CPU. The cpu-share is used as one of the inputs to calculate a
  67  * thread's "user-mode" priority (umdpri) for the scheduler.  The umdpri falls
  68  * in the range 0-59.  FSS calculates other, internal, priorities which are not
  69  * visible outside of the FSS class.
  70  *
  71  * The FSS class should approximate TS behavior when there are excess CPU
  72  * resources.  When there is a backlog of runnable processes, then the share
  73  * is used as input into the runnable process's priority calculation, where
  74  * the final umdpri is used by the scheduler to determine when the process runs.
  75  *
  76  * Projects in a zone compete with each other for CPU time, receiving CPU
  77  * allocation within a zone proportional to the project's share; at a higher
  78  * level zones compete with each other, receiving allocation in a pset
  79  * proportional to the zone's share.
  80  *
  81  * The FSS priority calculation consists of several parts.
  82  *
  83  * 1) Once per second the fss_update function runs. The first thing it does is
  84  *    call fss_decay_usage. This function does three things.
  85  *
  86  * a) fss_decay_usage first decays the maxfsspri value for the pset.  This
  87  *    value is used in the per-process priority calculation described in step
  88  *    (2b).  The maxfsspri is decayed using the following formula:
  89  *
  90  *                      maxfsspri * fss_nice_decay[NZERO])
  91  *        maxfsspri =  ------------------------------------
  92  *                            FSS_DECAY_BASE
  93  *
  94  *
  95  *     - NZERO is the default process priority (i.e. 20)
  96  *
  97  *    The fss_nice_decay array is a fixed set of values used to adjust the
  98  *    decay rate of processes based on their nice value.  Entries in this
  99  *    array are initialized in fss_init using the following formula:
 100  *
 101  *                        (FSS_DECAY_MAX - FSS_DECAY_MIN) * i
 102  *       FSS_DECAY_MIN + -------------------------------------
 103  *                               FSS_NICE_RANGE - 1
 104  *
 105  *     - FSS_DECAY_MIN is 82 = approximates 65% (82/128)
 106  *     - FSS_DECAY_MAX is 108 = approximates 85% (108/128)
 107  *     - FSS_NICE_RANGE is 40 (range is 0 - 39)
 108  *
 109  * b) The second thing fss_decay_usage does is update each project's "usage"
 110  *    for the last second and then recalculates the project's "share usage".
 111  *
 112  *    The usage value is the recent CPU usage for all of the threads in the
 113  *    project. It is decayed and updated this way:
 114  *
 115  *                  (usage * FSS_DECAY_USG)
 116  *        usage =  ------------------------- + ticks;
 117  *                       FSS_DECAY_BASE
 118  *
 119  *     - FSS_DECAY_BASE is 128 - used instead of 100 so we can shift vs divide
 120  *     - FSS_DECAY_USG is 96 - approximates 75% (96/128)
 121  *     - ticks is updated whenever a process in this project is running
 122  *       when the scheduler's tick processing fires. This is not a simple
 123  *       counter, the values are based on the entries in the fss_nice_tick
 124  *       array (see section 3 below). ticks is then reset to 0 so it can track
 125  *       the next seconds worth of nice-adjusted time for the project.
 126  *
 127  * c) The third thing fss_decay_usage does is update each project's "share
 128  *    usage" (shusage). This is the normalized usage value for the project and
 129  *    is calculated this way:
 130  *
 131  *                pset_shares^2    zone_int_shares^2
 132  *        usage * ------------- * ------------------
 133  *                kpj_shares^2     zone_ext_shares^2
 134  *
 135  *    - usage - see (1b) for more details
 136  *    - pset_shares is the total of all *active* zone shares in the pset (by
 137  *      default there is only one pset)
 138  *    - kpj_shares is the individual project's share (project.cpu-shares rctl)
 139  *    - zone_int_shares is the sum of shares of all active projects within the
 140  *      zone (the zone-internal total)
 141  *    - zone_ext_shares is the share value for the zone (zone.cpu-shares rctl)
 142  *
 143  *    The shusage is used in step (2b) to calculate the thread's new internal
 144  *    priority. A larger shusage value leads to a lower priority.
 145  *
 146  * 2) The fss_update function then calls fss_update_list to update the priority
 147  *    of all threads. This does two things.
 148  *
 149  * a) First the thread's internal priority is decayed using the following
 150  *    formula:
 151  *
 152  *                  fsspri * fss_nice_decay[nice_value])
 153  *        fsspri =  ------------------------------------
 154  *                            FSS_DECAY_BASE
 155  *
 156  *     - FSS_DECAY_BASE is 128 as described above
 157  *
 158  * b) Second, if the thread is runnable (TS_RUN or TS_WAIT) calls fss_newpri
 159  *    to update the user-mode priority (umdpri) of the runnable thread.
 160  *    Threads that are running (TS_ONPROC) or waiting for an event (TS_SLEEP)
 161  *    are not updated at this time. The updated user-mode priority can cause
 162  *    threads to change their position in the run queue.
 163  *
 164  *    The process's new internal fsspri is calculated using the following
 165  *    formula. All runnable threads in the project will use the same shusage
 166  *    and nrunnable values in their calculation.
 167  *
 168  *        fsspri += shusage * nrunnable * ticks
 169  *
 170  *     - shusage is the project's share usage, calculated in (1c)
 171  *     - nrunnable is the number of runnable threads in the project
 172  *     - ticks is the number of ticks this thread ran since the last fss_newpri
 173  *       invocation.
 174  *
 175  *    Finally the process's new user-mode priority is calculated using the
 176  *    following formula:
 177  *
 178  *                              (fsspri * umdprirange)
 179  *        umdpri = maxumdpri - ------------------------
 180  *                                    maxfsspri
 181  *
 182  *     - maxumdpri is MINCLSYSPRI - 1 (i.e. 59)
 183  *     - umdprirange is maxumdpri - 1 (i.e. 58)
 184  *     - maxfsspri is the largest fsspri seen so far, as we're iterating all
 185  *       runnable processes
 186  *
 187  *    Thus, a higher internal priority (fsspri) leads to a lower user-mode
 188  *    priority which means the thread runs less. The fsspri is higher when
 189  *    the project's normalized share usage is higher, when the project has
 190  *    more runnable threads, or when the thread has accumulated more run-time.
 191  *
 192  *    This code has various checks to ensure the resulting umdpri is in the
 193  *    range 1-59.  See fss_newpri for more details.
 194  *
 195  * To reiterate, the above processing is performed once per second to recompute
 196  * the runnable thread user-mode priorities.
 197  *
 198  * 3) The final major component in the priority calculation is the tick
 199  *    processing which occurs on a thread that is running when the clock
 200  *    calls fss_tick.
 201  *
 202  *    A thread can run continuously in user-land (compute-bound) for the
 203  *    fss_quantum (see "dispadmin -c FSS -g" for the configurable properties).
 204  *    The fss_quantum defaults to 11 (i.e. 11 ticks).
 205  *
 206  *    Once the quantum has been consumed, the thread will call fss_newpri to
 207  *    recompute its umdpri priority, as described above in (2b). Threads that
 208  *    were T_ONPROC at the one second interval when runnable thread priorities
 209  *    were recalculated will have their umdpri priority recalculated when their
 210  *    quanta expires.
 211  *
 212  *    To ensure that runnable threads within a project see the expected
 213  *    round-robin behavior, there is a special case in fss_newpri for a thread
 214  *    that has run for its quanta within the one second update interval.  See
 215  *    the handling for the quanta_up parameter within fss_newpri.
 216  *
 217  *    Also of interest, the fss_tick code increments the project's tick value
 218  *    using the fss_nice_tick array entry for the thread's nice value. The idea
 219  *    behind the fss_nice_tick array is that the cost of a tick is lower at
 220  *    positive nice values (so that it doesn't increase the project's usage
 221  *    as much as normal) with a 50% drop at the maximum level and a 50%
 222  *    increase at the minimum level. See (1b). The fss_nice_tick array is
 223  *    initialized in fss_init using the following formula:
 224  *
 225  *         FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2) - i)
 226  *        --------------------------------------------------
 227  *                          FSS_NICE_RANGE
 228  *
 229  *     - FSS_TICK_COST is 1000, the tick cost for threads with nice level 0
 230  *
 231  * FSS Data Structures:
 232  *
 233  *                 fsszone
 234  *                  -----           -----
 235  *  -----          |     |         |     |
 236  * |     |-------->|     |<------->|     |<---->...
 237  * |     |          -----           -----
 238  * |     |          ^    ^            ^
 239  * |     |---       |     \            \
 240  *  -----    |      |      \            \
 241  * fsspset   |      |       \            \
 242  *           |      |        \            \
 243  *           |    -----       -----       -----
 244  *            -->|     |<--->|     |<--->|     |
 245  *               |     |     |     |     |     |
 246  *                -----       -----       -----
 247  *               fssproj
 248  *
 249  * That is, fsspsets contain a list of fsszone's that are currently active in
 250  * the pset, and a list of fssproj's, corresponding to projects with runnable
 251  * threads on the pset.  fssproj's in turn point to the fsszone which they
 252  * are a member of.
 253  *
 254  * An fssproj_t is removed when there are no threads in it.
 255  *
 256  * An fsszone_t is removed when there are no projects with threads in it.
 257  */
 258 
 259 static pri_t fss_init(id_t, int, classfuncs_t **);
 260 
 261 static struct sclass fss = {
 262         "FSS",
 263         fss_init,
 264         0
 265 };
 266 
 267 extern struct mod_ops mod_schedops;
 268 
 269 /*
 270  * Module linkage information for the kernel.
 271  */
 272 static struct modlsched modlsched = {
 273         &mod_schedops, "fair share scheduling class", &fss
 274 };
 275 
 276 static struct modlinkage modlinkage = {
 277         MODREV_1, (void *)&modlsched, NULL
 278 };
 279 
 280 #define FSS_MAXUPRI     60
 281 
 282 /*
 283  * The fssproc_t structures are kept in an array of circular doubly linked
 284  * lists.  A hash on the thread pointer is used to determine which list each
 285  * thread should be placed in.  Each list has a dummy "head" which is never
 286  * removed, so the list is never empty.  fss_update traverses these lists to
 287  * update the priorities of threads that have been waiting on the run queue.
 288  */
 289 #define FSS_LISTS               16 /* number of lists, must be power of 2 */
 290 #define FSS_LIST_HASH(t)        (((uintptr_t)(t) >> 9) & (FSS_LISTS - 1))
 291 #define FSS_LIST_NEXT(i)        (((i) + 1) & (FSS_LISTS - 1))
 292 
 293 #define FSS_LIST_INSERT(fssproc)                                \
 294 {                                                               \
 295         int index = FSS_LIST_HASH(fssproc->fss_tp);          \
 296         kmutex_t *lockp = &fss_listlock[index];                     \
 297         fssproc_t *headp = &fss_listhead[index];            \
 298         mutex_enter(lockp);                                     \
 299         fssproc->fss_next = headp->fss_next;                      \
 300         fssproc->fss_prev = headp;                           \
 301         headp->fss_next->fss_prev = fssproc;                      \
 302         headp->fss_next = fssproc;                           \
 303         mutex_exit(lockp);                                      \
 304 }
 305 
 306 #define FSS_LIST_DELETE(fssproc)                                \
 307 {                                                               \
 308         int index = FSS_LIST_HASH(fssproc->fss_tp);          \
 309         kmutex_t *lockp = &fss_listlock[index];                     \
 310         mutex_enter(lockp);                                     \
 311         fssproc->fss_prev->fss_next = fssproc->fss_next;       \
 312         fssproc->fss_next->fss_prev = fssproc->fss_prev;       \
 313         mutex_exit(lockp);                                      \
 314 }
 315 
 316 #define FSS_TICK_COST   1000    /* tick cost for threads with nice level = 0 */
 317 
 318 /*
 319  * Decay rate percentages are based on n/128 rather than n/100 so  that
 320  * calculations can avoid having to do an integer divide by 100 (divide
 321  * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift).
 322  *
 323  * FSS_DECAY_MIN        =  83/128 ~= 65%
 324  * FSS_DECAY_MAX        = 108/128 ~= 85%
 325  * FSS_DECAY_USG        =  96/128 ~= 75%
 326  */
 327 #define FSS_DECAY_MIN   83      /* fsspri decay pct for threads w/ nice -20 */
 328 #define FSS_DECAY_MAX   108     /* fsspri decay pct for threads w/ nice +19 */
 329 #define FSS_DECAY_USG   96      /* fssusage decay pct for projects */
 330 #define FSS_DECAY_BASE  128     /* base for decay percentages above */
 331 
 332 #define FSS_NICE_MIN    0
 333 #define FSS_NICE_MAX    (2 * NZERO - 1)
 334 #define FSS_NICE_RANGE  (FSS_NICE_MAX - FSS_NICE_MIN + 1)
 335 
 336 static int      fss_nice_tick[FSS_NICE_RANGE];
 337 static int      fss_nice_decay[FSS_NICE_RANGE];
 338 
 339 static pri_t    fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
 340 static pri_t    fss_maxumdpri; /* maximum user mode fss priority */
 341 static pri_t    fss_maxglobpri; /* maximum global priority used by fss class */
 342 static pri_t    fss_minglobpri; /* minimum global priority */
 343 
 344 static fssproc_t fss_listhead[FSS_LISTS];
 345 static kmutex_t fss_listlock[FSS_LISTS];
 346 
 347 static fsspset_t *fsspsets;
 348 static kmutex_t fsspsets_lock;  /* protects fsspsets */
 349 
 350 static id_t     fss_cid;
 351 
 352 static time_t   fss_minrun = 2; /* t_pri becomes 59 within 2 secs */
 353 static time_t   fss_minslp = 2; /* min time on sleep queue for hardswap */
 354 static int      fss_quantum = 11;
 355 
 356 static void     fss_newpri(fssproc_t *, boolean_t);
 357 static void     fss_update(void *);
 358 static int      fss_update_list(int);
 359 static void     fss_change_priority(kthread_t *, fssproc_t *);
 360 
 361 static int      fss_admin(caddr_t, cred_t *);
 362 static int      fss_getclinfo(void *);
 363 static int      fss_parmsin(void *);
 364 static int      fss_parmsout(void *, pc_vaparms_t *);
 365 static int      fss_vaparmsin(void *, pc_vaparms_t *);
 366 static int      fss_vaparmsout(void *, pc_vaparms_t *);
 367 static int      fss_getclpri(pcpri_t *);
 368 static int      fss_alloc(void **, int);
 369 static void     fss_free(void *);
 370 
 371 static int      fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
 372 static void     fss_exitclass(void *);
 373 static int      fss_canexit(kthread_t *, cred_t *);
 374 static int      fss_fork(kthread_t *, kthread_t *, void *);
 375 static void     fss_forkret(kthread_t *, kthread_t *);
 376 static void     fss_parmsget(kthread_t *, void *);
 377 static int      fss_parmsset(kthread_t *, void *, id_t, cred_t *);
 378 static void     fss_stop(kthread_t *, int, int);
 379 static void     fss_exit(kthread_t *);
 380 static void     fss_active(kthread_t *);
 381 static void     fss_inactive(kthread_t *);
 382 static pri_t    fss_swapin(kthread_t *, int);
 383 static pri_t    fss_swapout(kthread_t *, int);
 384 static void     fss_trapret(kthread_t *);
 385 static void     fss_preempt(kthread_t *);
 386 static void     fss_setrun(kthread_t *);
 387 static void     fss_sleep(kthread_t *);
 388 static void     fss_tick(kthread_t *);
 389 static void     fss_wakeup(kthread_t *);
 390 static int      fss_donice(kthread_t *, cred_t *, int, int *);
 391 static int      fss_doprio(kthread_t *, cred_t *, int, int *);
 392 static pri_t    fss_globpri(kthread_t *);
 393 static void     fss_yield(kthread_t *);
 394 static void     fss_nullsys();
 395 
 396 static struct classfuncs fss_classfuncs = {
 397         /* class functions */
 398         fss_admin,
 399         fss_getclinfo,
 400         fss_parmsin,
 401         fss_parmsout,
 402         fss_vaparmsin,
 403         fss_vaparmsout,
 404         fss_getclpri,
 405         fss_alloc,
 406         fss_free,
 407 
 408         /* thread functions */
 409         fss_enterclass,
 410         fss_exitclass,
 411         fss_canexit,
 412         fss_fork,
 413         fss_forkret,
 414         fss_parmsget,
 415         fss_parmsset,
 416         fss_stop,
 417         fss_exit,
 418         fss_active,
 419         fss_inactive,
 420         fss_swapin,
 421         fss_swapout,
 422         fss_trapret,
 423         fss_preempt,
 424         fss_setrun,
 425         fss_sleep,
 426         fss_tick,
 427         fss_wakeup,
 428         fss_donice,
 429         fss_globpri,
 430         fss_nullsys,    /* set_process_group */
 431         fss_yield,
 432         fss_doprio,
 433 };
 434 
 435 int
 436 _init()
 437 {
 438         return (mod_install(&modlinkage));
 439 }
 440 
 441 int
 442 _fini()
 443 {
 444         return (EBUSY);
 445 }
 446 
 447 int
 448 _info(struct modinfo *modinfop)
 449 {
 450         return (mod_info(&modlinkage, modinfop));
 451 }
 452 
 453 /*ARGSUSED*/
 454 static int
 455 fss_project_walker(kproject_t *kpj, void *buf)
 456 {
 457         return (0);
 458 }
 459 
 460 void *
 461 fss_allocbuf(int op, int type)
 462 {
 463         fssbuf_t *fssbuf;
 464         void **fsslist;
 465         int cnt;
 466         int i;
 467         size_t size;
 468 
 469         ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF);
 470         ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
 471         ASSERT(MUTEX_HELD(&cpu_lock));
 472 
 473         fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP);
 474         switch (op) {
 475         case FSS_NPSET_BUF:
 476                 cnt = cpupart_list(NULL, 0, CP_NONEMPTY);
 477                 break;
 478         case FSS_NPROJ_BUF:
 479                 cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL);
 480                 break;
 481         case FSS_ONE_BUF:
 482                 cnt = 1;
 483                 break;
 484         }
 485 
 486         switch (type) {
 487         case FSS_ALLOC_PROJ:
 488                 size = sizeof (fssproj_t);
 489                 break;
 490         case FSS_ALLOC_ZONE:
 491                 size = sizeof (fsszone_t);
 492                 break;
 493         }
 494         fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP);
 495         fssbuf->fssb_size = cnt;
 496         fssbuf->fssb_list = fsslist;
 497         for (i = 0; i < cnt; i++)
 498                 fsslist[i] = kmem_zalloc(size, KM_SLEEP);
 499         return (fssbuf);
 500 }
 501 
 502 void
 503 fss_freebuf(fssbuf_t *fssbuf, int type)
 504 {
 505         void **fsslist;
 506         int i;
 507         size_t size;
 508 
 509         ASSERT(fssbuf != NULL);
 510         ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
 511         fsslist = fssbuf->fssb_list;
 512 
 513         switch (type) {
 514         case FSS_ALLOC_PROJ:
 515                 size = sizeof (fssproj_t);
 516                 break;
 517         case FSS_ALLOC_ZONE:
 518                 size = sizeof (fsszone_t);
 519                 break;
 520         }
 521 
 522         for (i = 0; i < fssbuf->fssb_size; i++) {
 523                 if (fsslist[i] != NULL)
 524                         kmem_free(fsslist[i], size);
 525         }
 526         kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size);
 527         kmem_free(fssbuf, sizeof (fssbuf_t));
 528 }
 529 
 530 static fsspset_t *
 531 fss_find_fsspset(cpupart_t *cpupart)
 532 {
 533         int i;
 534         fsspset_t *fsspset = NULL;
 535         int found = 0;
 536 
 537         ASSERT(cpupart != NULL);
 538         ASSERT(MUTEX_HELD(&fsspsets_lock));
 539 
 540         /*
 541          * Search for the cpupart pointer in the array of fsspsets.
 542          */
 543         for (i = 0; i < max_ncpus; i++) {
 544                 fsspset = &fsspsets[i];
 545                 if (fsspset->fssps_cpupart == cpupart) {
 546                         ASSERT(fsspset->fssps_nproj > 0);
 547                         found = 1;
 548                         break;
 549                 }
 550         }
 551         if (found == 0) {
 552                 /*
 553                  * If we didn't find anything, then use the first
 554                  * available slot in the fsspsets array.
 555                  */
 556                 for (i = 0; i < max_ncpus; i++) {
 557                         fsspset = &fsspsets[i];
 558                         if (fsspset->fssps_cpupart == NULL) {
 559                                 ASSERT(fsspset->fssps_nproj == 0);
 560                                 found = 1;
 561                                 break;
 562                         }
 563                 }
 564                 fsspset->fssps_cpupart = cpupart;
 565         }
 566         ASSERT(found == 1);
 567         return (fsspset);
 568 }
 569 
 570 static void
 571 fss_del_fsspset(fsspset_t *fsspset)
 572 {
 573         ASSERT(MUTEX_HELD(&fsspsets_lock));
 574         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 575         ASSERT(fsspset->fssps_nproj == 0);
 576         ASSERT(fsspset->fssps_list == NULL);
 577         ASSERT(fsspset->fssps_zones == NULL);
 578         fsspset->fssps_cpupart = NULL;
 579         fsspset->fssps_maxfsspri = 0;
 580         fsspset->fssps_shares = 0;
 581 }
 582 
 583 /*
 584  * The following routine returns a pointer to the fsszone structure which
 585  * belongs to zone "zone" and cpu partition fsspset, if such structure exists.
 586  */
 587 static fsszone_t *
 588 fss_find_fsszone(fsspset_t *fsspset, zone_t *zone)
 589 {
 590         fsszone_t *fsszone;
 591 
 592         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 593 
 594         if (fsspset->fssps_list != NULL) {
 595                 /*
 596                  * There are projects/zones active on this cpu partition
 597                  * already.  Try to find our zone among them.
 598                  */
 599                 fsszone = fsspset->fssps_zones;
 600                 do {
 601                         if (fsszone->fssz_zone == zone) {
 602                                 return (fsszone);
 603                         }
 604                         fsszone = fsszone->fssz_next;
 605                 } while (fsszone != fsspset->fssps_zones);
 606         }
 607         return (NULL);
 608 }
 609 
 610 /*
 611  * The following routine links new fsszone structure into doubly linked list of
 612  * zones active on the specified cpu partition.
 613  */
 614 static void
 615 fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone)
 616 {
 617         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 618 
 619         fsszone->fssz_zone = zone;
 620         fsszone->fssz_rshares = zone->zone_shares;
 621 
 622         if (fsspset->fssps_zones == NULL) {
 623                 /*
 624                  * This will be the first fsszone for this fsspset
 625                  */
 626                 fsszone->fssz_next = fsszone->fssz_prev = fsszone;
 627                 fsspset->fssps_zones = fsszone;
 628         } else {
 629                 /*
 630                  * Insert this fsszone to the doubly linked list.
 631                  */
 632                 fsszone_t *fssz_head = fsspset->fssps_zones;
 633 
 634                 fsszone->fssz_next = fssz_head;
 635                 fsszone->fssz_prev = fssz_head->fssz_prev;
 636                 fssz_head->fssz_prev->fssz_next = fsszone;
 637                 fssz_head->fssz_prev = fsszone;
 638                 fsspset->fssps_zones = fsszone;
 639         }
 640 }
 641 
 642 /*
 643  * The following routine removes a single fsszone structure from the doubly
 644  * linked list of zones active on the specified cpu partition.  Note that
 645  * global fsspsets_lock must be held in case this fsszone structure is the last
 646  * on the above mentioned list.  Also note that the fsszone structure is not
 647  * freed here, it is the responsibility of the caller to call kmem_free for it.
 648  */
 649 static void
 650 fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone)
 651 {
 652         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 653         ASSERT(fsszone->fssz_nproj == 0);
 654         ASSERT(fsszone->fssz_shares == 0);
 655         ASSERT(fsszone->fssz_runnable == 0);
 656 
 657         if (fsszone->fssz_next != fsszone) {
 658                 /*
 659                  * This is not the last zone in the list.
 660                  */
 661                 fsszone->fssz_prev->fssz_next = fsszone->fssz_next;
 662                 fsszone->fssz_next->fssz_prev = fsszone->fssz_prev;
 663                 if (fsspset->fssps_zones == fsszone)
 664                         fsspset->fssps_zones = fsszone->fssz_next;
 665         } else {
 666                 /*
 667                  * This was the last zone active in this cpu partition.
 668                  */
 669                 fsspset->fssps_zones = NULL;
 670         }
 671 }
 672 
 673 /*
 674  * The following routine returns a pointer to the fssproj structure
 675  * which belongs to project kpj and cpu partition fsspset, if such structure
 676  * exists.
 677  */
 678 static fssproj_t *
 679 fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj)
 680 {
 681         fssproj_t *fssproj;
 682 
 683         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 684 
 685         if (fsspset->fssps_list != NULL) {
 686                 /*
 687                  * There are projects running on this cpu partition already.
 688                  * Try to find our project among them.
 689                  */
 690                 fssproj = fsspset->fssps_list;
 691                 do {
 692                         if (fssproj->fssp_proj == kpj) {
 693                                 ASSERT(fssproj->fssp_pset == fsspset);
 694                                 return (fssproj);
 695                         }
 696                         fssproj = fssproj->fssp_next;
 697                 } while (fssproj != fsspset->fssps_list);
 698         }
 699         return (NULL);
 700 }
 701 
 702 /*
 703  * The following routine links new fssproj structure into doubly linked list
 704  * of projects running on the specified cpu partition.
 705  */
 706 static void
 707 fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone,
 708     fssproj_t *fssproj)
 709 {
 710         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 711 
 712         fssproj->fssp_pset = fsspset;
 713         fssproj->fssp_proj = kpj;
 714         fssproj->fssp_shares = kpj->kpj_shares;
 715 
 716         fsspset->fssps_nproj++;
 717 
 718         if (fsspset->fssps_list == NULL) {
 719                 /*
 720                  * This will be the first fssproj for this fsspset
 721                  */
 722                 fssproj->fssp_next = fssproj->fssp_prev = fssproj;
 723                 fsspset->fssps_list = fssproj;
 724         } else {
 725                 /*
 726                  * Insert this fssproj to the doubly linked list.
 727                  */
 728                 fssproj_t *fssp_head = fsspset->fssps_list;
 729 
 730                 fssproj->fssp_next = fssp_head;
 731                 fssproj->fssp_prev = fssp_head->fssp_prev;
 732                 fssp_head->fssp_prev->fssp_next = fssproj;
 733                 fssp_head->fssp_prev = fssproj;
 734                 fsspset->fssps_list = fssproj;
 735         }
 736         fssproj->fssp_fsszone = fsszone;
 737         fsszone->fssz_nproj++;
 738         ASSERT(fsszone->fssz_nproj != 0);
 739 }
 740 
 741 /*
 742  * The following routine removes a single fssproj structure from the doubly
 743  * linked list of projects running on the specified cpu partition.  Note that
 744  * global fsspsets_lock must be held in case if this fssproj structure is the
 745  * last on the above mentioned list.  Also note that the fssproj structure is
 746  * not freed here, it is the responsibility of the caller to call kmem_free
 747  * for it.
 748  */
 749 static void
 750 fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj)
 751 {
 752         fsszone_t *fsszone;
 753 
 754         ASSERT(MUTEX_HELD(&fsspsets_lock));
 755         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 756         ASSERT(fssproj->fssp_runnable == 0);
 757 
 758         fsspset->fssps_nproj--;
 759 
 760         fsszone = fssproj->fssp_fsszone;
 761         fsszone->fssz_nproj--;
 762 
 763         if (fssproj->fssp_next != fssproj) {
 764                 /*
 765                  * This is not the last part in the list.
 766                  */
 767                 fssproj->fssp_prev->fssp_next = fssproj->fssp_next;
 768                 fssproj->fssp_next->fssp_prev = fssproj->fssp_prev;
 769                 if (fsspset->fssps_list == fssproj)
 770                         fsspset->fssps_list = fssproj->fssp_next;
 771                 if (fsszone->fssz_nproj == 0)
 772                         fss_remove_fsszone(fsspset, fsszone);
 773         } else {
 774                 /*
 775                  * This was the last project part running
 776                  * at this cpu partition.
 777                  */
 778                 fsspset->fssps_list = NULL;
 779                 ASSERT(fsspset->fssps_nproj == 0);
 780                 ASSERT(fsszone->fssz_nproj == 0);
 781                 fss_remove_fsszone(fsspset, fsszone);
 782                 fss_del_fsspset(fsspset);
 783         }
 784 }
 785 
 786 static void
 787 fss_inactive(kthread_t *t)
 788 {
 789         fssproc_t *fssproc;
 790         fssproj_t *fssproj;
 791         fsspset_t *fsspset;
 792         fsszone_t *fsszone;
 793 
 794         ASSERT(THREAD_LOCK_HELD(t));
 795         fssproc = FSSPROC(t);
 796         fssproj = FSSPROC2FSSPROJ(fssproc);
 797         if (fssproj == NULL)    /* if this thread already exited */
 798                 return;
 799         fsspset = FSSPROJ2FSSPSET(fssproj);
 800         fsszone = fssproj->fssp_fsszone;
 801         disp_lock_enter_high(&fsspset->fssps_displock);
 802         ASSERT(fssproj->fssp_runnable > 0);
 803         if (--fssproj->fssp_runnable == 0) {
 804                 fsszone->fssz_shares -= fssproj->fssp_shares;
 805                 if (--fsszone->fssz_runnable == 0)
 806                         fsspset->fssps_shares -= fsszone->fssz_rshares;
 807         }
 808         ASSERT(fssproc->fss_runnable == 1);
 809         fssproc->fss_runnable = 0;
 810         disp_lock_exit_high(&fsspset->fssps_displock);
 811 }
 812 
 813 static void
 814 fss_active(kthread_t *t)
 815 {
 816         fssproc_t *fssproc;
 817         fssproj_t *fssproj;
 818         fsspset_t *fsspset;
 819         fsszone_t *fsszone;
 820 
 821         ASSERT(THREAD_LOCK_HELD(t));
 822         fssproc = FSSPROC(t);
 823         fssproj = FSSPROC2FSSPROJ(fssproc);
 824         if (fssproj == NULL)    /* if this thread already exited */
 825                 return;
 826         fsspset = FSSPROJ2FSSPSET(fssproj);
 827         fsszone = fssproj->fssp_fsszone;
 828         disp_lock_enter_high(&fsspset->fssps_displock);
 829         if (++fssproj->fssp_runnable == 1) {
 830                 fsszone->fssz_shares += fssproj->fssp_shares;
 831                 if (++fsszone->fssz_runnable == 1)
 832                         fsspset->fssps_shares += fsszone->fssz_rshares;
 833         }
 834         ASSERT(fssproc->fss_runnable == 0);
 835         fssproc->fss_runnable = 1;
 836         disp_lock_exit_high(&fsspset->fssps_displock);
 837 }
 838 
 839 /*
 840  * Fair share scheduler initialization. Called by dispinit() at boot time.
 841  * We can ignore clparmsz argument since we know that the smallest possible
 842  * parameter buffer is big enough for us.
 843  */
 844 /*ARGSUSED*/
 845 static pri_t
 846 fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
 847 {
 848         int i;
 849 
 850         ASSERT(MUTEX_HELD(&cpu_lock));
 851 
 852         fss_cid = cid;
 853         fss_maxumdpri = minclsyspri - 1;
 854         fss_maxglobpri = minclsyspri;
 855         fss_minglobpri = 0;
 856         fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP);
 857 
 858         /*
 859          * Initialize the fssproc hash table.
 860          */
 861         for (i = 0; i < FSS_LISTS; i++)
 862                 fss_listhead[i].fss_next = fss_listhead[i].fss_prev =
 863                     &fss_listhead[i];
 864 
 865         *clfuncspp = &fss_classfuncs;
 866 
 867         /*
 868          * Fill in fss_nice_tick and fss_nice_decay arrays:
 869          * The cost of a tick is lower at positive nice values (so that it
 870          * will not increase its project's usage as much as normal) with 50%
 871          * drop at the maximum level and 50% increase at the minimum level.
 872          * The fsspri decay is slower at positive nice values.  fsspri values
 873          * of processes with negative nice levels must decay faster to receive
 874          * time slices more frequently than normal.
 875          */
 876         for (i = 0; i < FSS_NICE_RANGE; i++) {
 877                 fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2)
 878                     - i)) / FSS_NICE_RANGE;
 879                 fss_nice_decay[i] = FSS_DECAY_MIN +
 880                     ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) /
 881                     (FSS_NICE_RANGE - 1);
 882         }
 883 
 884         return (fss_maxglobpri);
 885 }
 886 
 887 /*
 888  * Calculate the new fss_umdpri based on the usage, the normalized share usage
 889  * and the number of active threads.  Reset the tick counter for this thread.
 890  *
 891  * When calculating the new priority using the standard formula we can hit
 892  * a scenario where we don't have good round-robin behavior.  This would be
 893  * most commonly seen when there is a zone with lots of runnable threads.
 894  * In the bad scenario we will see the following behavior when using the
 895  * standard formula and these conditions:
 896  *
 897  *      - there are multiple runnable threads in the zone (project)
 898  *      - the fssps_maxfsspri is a very large value
 899  *      - (we also know all of these threads will use the project's
 900  *          fssp_shusage)
 901  *
 902  * Under these conditions, a thread with a low fss_fsspri value is chosen
 903  * to run and the thread gets a high fss_umdpri.  This thread can run for
 904  * its full quanta (fss_timeleft) at which time fss_newpri is called to
 905  * calculate the thread's new priority.
 906  *
 907  * In this case, because the newly calculated fsspri value is much smaller
 908  * (orders of magnitude) than the fssps_maxfsspri value, if we used the
 909  * standard formula the thread will still get a high fss_umdpri value and
 910  * will run again for another quanta, even though there are other runnable
 911  * threads in the project.
 912  *
 913  * For a thread that is runnable for a long time, the thread can continue
 914  * to run for many quanta (totaling many seconds) before the thread's fsspri
 915  * exceeds the fssps_maxfsspri and the thread's fss_umdpri is reset back
 916  * down to 1.  This behavior also keeps the fssps_maxfsspr at a high value,
 917  * so that the next runnable thread might repeat this cycle.
 918  *
 919  * This leads to the case where we don't have round-robin behavior at quanta
 920  * granularity, but instead, runnable threads within the project only run
 921  * at several second intervals.
 922  *
 923  * To prevent this scenario from occuring, when a thread has consumed its
 924  * quanta and there are multiple runnable threads in the project, we
 925  * immediately cause the thread to hit fssps_maxfsspri so that it gets
 926  * reset back to 1 and another runnable thread in the project can run.
 927  */
 928 static void
 929 fss_newpri(fssproc_t *fssproc, boolean_t quanta_up)
 930 {
 931         kthread_t *tp;
 932         fssproj_t *fssproj;
 933         fsspset_t *fsspset;
 934         fsszone_t *fsszone;
 935         fsspri_t fsspri, maxfsspri;
 936         uint32_t n_runnable;
 937         pri_t invpri;
 938         uint32_t ticks;
 939 
 940         tp = fssproc->fss_tp;
 941         ASSERT(tp != NULL);
 942 
 943         if (tp->t_cid != fss_cid)
 944                 return;
 945 
 946         ASSERT(THREAD_LOCK_HELD(tp));
 947 
 948         fssproj = FSSPROC2FSSPROJ(fssproc);
 949         fsszone = FSSPROJ2FSSZONE(fssproj);
 950         if (fssproj == NULL)
 951                 /*
 952                  * No need to change priority of exited threads.
 953                  */
 954                 return;
 955 
 956         fsspset = FSSPROJ2FSSPSET(fssproj);
 957         disp_lock_enter_high(&fsspset->fssps_displock);
 958 
 959         ticks = fssproc->fss_ticks;
 960         fssproc->fss_ticks = 0;
 961 
 962         if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) {
 963                 /*
 964                  * Special case: threads with no shares.
 965                  */
 966                 fssproc->fss_umdpri = fss_minglobpri;
 967                 disp_lock_exit_high(&fsspset->fssps_displock);
 968                 return;
 969         }
 970 
 971         maxfsspri = fsspset->fssps_maxfsspri;
 972         n_runnable = fssproj->fssp_runnable;
 973 
 974         if (quanta_up && n_runnable > 1) {
 975                 fsspri = maxfsspri;
 976         } else {
 977                 /*
 978                  * fsspri += fssp_shusage * nrunnable * ticks
 979                  * If all three values are non-0, this typically calculates to
 980                  * a large number (sometimes > 1M, sometimes > 100B) due to
 981                  * fssp_shusage which can be > 1T.
 982                  */
 983                 fsspri = fssproc->fss_fsspri;
 984                 fsspri += fssproj->fssp_shusage * n_runnable * ticks;
 985         }
 986 
 987         fssproc->fss_fsspri = fsspri;
 988 
 989         /*
 990          * fss_maxumdpri is normally 59, since FSS priorities are 0-59.
 991          * If the previous calculation resulted in 0 (e.g. was 0 and added 0
 992          * because ticks == 0), then instead of 0, we use the largest priority,
 993          * which is still small in comparison to the large numbers we typically
 994          * see.
 995          */
 996         if (fsspri < fss_maxumdpri)
 997                 fsspri = fss_maxumdpri; /* so that maxfsspri is != 0 */
 998 
 999         /*
1000          * The general priority formula:
1001          *
1002          *                      (fsspri * umdprirange)
1003          *   pri = maxumdpri - ------------------------
1004          *                              maxfsspri
1005          *
1006          * If this thread's fsspri is greater than the previous largest
1007          * fsspri, then record it as the new high and priority for this
1008          * thread will be one (the lowest priority assigned to a thread
1009          * that has non-zero shares). Because of this check, maxfsspri can
1010          * change as this function is called via the
1011          * fss_update -> fss_update_list -> fss_newpri code path to update
1012          * all runnable threads. See the code in fss_update for how we
1013          * mitigate this issue.
1014          *
1015          * Note that this formula cannot produce out of bounds priority
1016          * values (0-59); if it is changed, additional checks may need to be
1017          * added.
1018          */
1019         if (fsspri >= maxfsspri) {
1020                 fsspset->fssps_maxfsspri = fsspri;
1021                 disp_lock_exit_high(&fsspset->fssps_displock);
1022                 fssproc->fss_umdpri = 1;
1023         } else {
1024                 disp_lock_exit_high(&fsspset->fssps_displock);
1025                 invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri;
1026                 fssproc->fss_umdpri = fss_maxumdpri - invpri;
1027         }
1028 }
1029 
1030 /*
1031  * Decays usages of all running projects, resets their tick counters and
1032  * calcluates the projects normalized share usage. Called once per second from
1033  * fss_update().
1034  */
1035 static void
1036 fss_decay_usage()
1037 {
1038         uint32_t zone_ext_shares, zone_int_shares;
1039         uint32_t kpj_shares, pset_shares;
1040         fsspset_t *fsspset;
1041         fssproj_t *fssproj;
1042         fsszone_t *fsszone;
1043         fsspri_t maxfsspri;
1044         int psetid;
1045         struct zone *zp;
1046 
1047         mutex_enter(&fsspsets_lock);
1048         /*
1049          * Go through all active processor sets and decay usages of projects
1050          * running on them.
1051          */
1052         for (psetid = 0; psetid < max_ncpus; psetid++) {
1053                 fsspset = &fsspsets[psetid];
1054                 mutex_enter(&fsspset->fssps_lock);
1055 
1056                 fsspset->fssps_gen++;
1057 
1058                 if (fsspset->fssps_cpupart == NULL ||
1059                     (fssproj = fsspset->fssps_list) == NULL) {
1060                         mutex_exit(&fsspset->fssps_lock);
1061                         continue;
1062                 }
1063 
1064                 /*
1065                  * Decay maxfsspri for this cpu partition with the
1066                  * fastest possible decay rate.
1067                  */
1068                 disp_lock_enter(&fsspset->fssps_displock);
1069 
1070                 pset_shares = fsspset->fssps_shares;
1071 
1072                 maxfsspri = (fsspset->fssps_maxfsspri *
1073                     fss_nice_decay[NZERO]) / FSS_DECAY_BASE;
1074                 if (maxfsspri < fss_maxumdpri)
1075                         maxfsspri = fss_maxumdpri;
1076                 fsspset->fssps_maxfsspri = maxfsspri;
1077 
1078                 do {
1079                         fsszone = fssproj->fssp_fsszone;
1080                         zp = fsszone->fssz_zone;
1081 
1082                         /*
1083                          * Reset zone's FSS stats if they are from a
1084                          * previous cycle.
1085                          */
1086                         if (fsspset->fssps_gen != zp->zone_fss_gen) {
1087                                 zp->zone_fss_gen = fsspset->fssps_gen;
1088                                 zp->zone_run_ticks = 0;
1089                         }
1090 
1091                         /*
1092                          * Decay project usage, then add in this cycle's
1093                          * nice tick value.
1094                          */
1095                         fssproj->fssp_usage =
1096                             (fssproj->fssp_usage * FSS_DECAY_USG) /
1097                             FSS_DECAY_BASE +
1098                             fssproj->fssp_ticks;
1099 
1100                         fssproj->fssp_ticks = 0;
1101                         zp->zone_run_ticks += fssproj->fssp_tick_cnt;
1102                         fssproj->fssp_tick_cnt = 0;
1103 
1104                         /*
1105                          * Readjust the project's number of shares if it has
1106                          * changed since we checked it last time.
1107                          */
1108                         kpj_shares = fssproj->fssp_proj->kpj_shares;
1109                         if (fssproj->fssp_shares != kpj_shares) {
1110                                 if (fssproj->fssp_runnable != 0) {
1111                                         fsszone->fssz_shares -=
1112                                             fssproj->fssp_shares;
1113                                         fsszone->fssz_shares += kpj_shares;
1114                                 }
1115                                 fssproj->fssp_shares = kpj_shares;
1116                         }
1117 
1118                         /*
1119                          * Readjust the zone's number of shares if it
1120                          * has changed since we checked it last time.
1121                          */
1122                         zone_ext_shares = zp->zone_shares;
1123                         if (fsszone->fssz_rshares != zone_ext_shares) {
1124                                 if (fsszone->fssz_runnable != 0) {
1125                                         fsspset->fssps_shares -=
1126                                             fsszone->fssz_rshares;
1127                                         fsspset->fssps_shares +=
1128                                             zone_ext_shares;
1129                                         pset_shares = fsspset->fssps_shares;
1130                                 }
1131                                 fsszone->fssz_rshares = zone_ext_shares;
1132                         }
1133                         zone_int_shares = fsszone->fssz_shares;
1134 
1135                         /*
1136                          * If anything is runnable in the project, track the
1137                          * overall project share percent for monitoring useage.
1138                          */
1139                         if (fssproj->fssp_runnable > 0) {
1140                                 uint32_t zone_shr_pct;
1141                                 uint32_t int_shr_pct;
1142 
1143                                 /*
1144                                  * Times 1000 to get tenths of a percent
1145                                  *
1146                                  *                zone_ext_shares
1147                                  * zone_shr_pct = ---------------
1148                                  *                pset_shares
1149                                  *
1150                                  *                kpj_shares
1151                                  * int_shr_pct =  ---------------
1152                                  *                zone_int_shares
1153                                  */
1154                                 if (pset_shares == 0 || zone_int_shares == 0) {
1155                                         fssproj->fssp_shr_pct = 0;
1156                                 } else {
1157                                         zone_shr_pct =
1158                                             (zone_ext_shares * 1000) /
1159                                             pset_shares;
1160                                         int_shr_pct = (kpj_shares * 1000) /
1161                                             zone_int_shares;
1162                                         fssproj->fssp_shr_pct =
1163                                             (zone_shr_pct * int_shr_pct) /
1164                                             1000;
1165                                 }
1166                         } else {
1167                                 DTRACE_PROBE1(fss__prj__norun, fssproj_t *,
1168                                     fssproj);
1169                         }
1170 
1171                         /*
1172                          * Calculate fssp_shusage value to be used
1173                          * for fsspri increments for the next second.
1174                          */
1175                         if (kpj_shares == 0 || zone_ext_shares == 0) {
1176                                 fssproj->fssp_shusage = 0;
1177                         } else if (FSSPROJ2KPROJ(fssproj) == proj0p) {
1178                                 uint32_t zone_shr_pct;
1179 
1180                                 /*
1181                                  * Project 0 in the global zone has 50%
1182                                  * of its zone. See calculation above for
1183                                  * the zone's share percent.
1184                                  */
1185                                 if (pset_shares == 0)
1186                                         zone_shr_pct = 1000;
1187                                 else
1188                                         zone_shr_pct =
1189                                             (zone_ext_shares * 1000) /
1190                                             pset_shares;
1191 
1192                                 fssproj->fssp_shr_pct = zone_shr_pct / 2;
1193 
1194                                 fssproj->fssp_shusage = (fssproj->fssp_usage *
1195                                     zone_int_shares * zone_int_shares) /
1196                                     (zone_ext_shares * zone_ext_shares);
1197                         } else {
1198                                 /*
1199                                  * Thread's priority is based on its project's
1200                                  * normalized usage (shusage) value which gets
1201                                  * calculated this way:
1202                                  *
1203                                  *         pset_shares^2    zone_int_shares^2
1204                                  * usage * ------------- * ------------------
1205                                  *         kpj_shares^2     zone_ext_shares^2
1206                                  *
1207                                  * Where zone_int_shares is the sum of shares
1208                                  * of all active projects within the zone (and
1209                                  * the pset), and zone_ext_shares is the number
1210                                  * of zone shares (ie, zone.cpu-shares).
1211                                  *
1212                                  * If there is only one zone active on the pset
1213                                  * the above reduces to:
1214                                  *
1215                                  *                      zone_int_shares^2
1216                                  * shusage = usage * ---------------------
1217                                  *                      kpj_shares^2
1218                                  *
1219                                  * If there's only one project active in the
1220                                  * zone this formula reduces to:
1221                                  *
1222                                  *                      pset_shares^2
1223                                  * shusage = usage * ----------------------
1224                                  *                      zone_ext_shares^2
1225                                  *
1226                                  * shusage is one input to calculating fss_pri
1227                                  * in fss_newpri(). Larger values tend toward
1228                                  * lower priorities for processes in the proj.
1229                                  */
1230                                 fssproj->fssp_shusage = fssproj->fssp_usage *
1231                                     pset_shares * zone_int_shares;
1232                                 fssproj->fssp_shusage /=
1233                                     kpj_shares * zone_ext_shares;
1234                                 fssproj->fssp_shusage *=
1235                                     pset_shares * zone_int_shares;
1236                                 fssproj->fssp_shusage /=
1237                                     kpj_shares * zone_ext_shares;
1238                         }
1239                         fssproj = fssproj->fssp_next;
1240                 } while (fssproj != fsspset->fssps_list);
1241 
1242                 disp_lock_exit(&fsspset->fssps_displock);
1243                 mutex_exit(&fsspset->fssps_lock);
1244         }
1245         mutex_exit(&fsspsets_lock);
1246 }
1247 
1248 static void
1249 fss_change_priority(kthread_t *t, fssproc_t *fssproc)
1250 {
1251         pri_t new_pri;
1252 
1253         ASSERT(THREAD_LOCK_HELD(t));
1254         new_pri = fssproc->fss_umdpri;
1255         ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
1256 
1257         t->t_cpri = fssproc->fss_upri;
1258         fssproc->fss_flags &= ~FSSRESTORE;
1259         if (t == curthread || t->t_state == TS_ONPROC) {
1260                 /*
1261                  * curthread is always onproc
1262                  */
1263                 cpu_t *cp = t->t_disp_queue->disp_cpu;
1264                 THREAD_CHANGE_PRI(t, new_pri);
1265                 if (t == cp->cpu_dispthread)
1266                         cp->cpu_dispatch_pri = DISP_PRIO(t);
1267                 if (DISP_MUST_SURRENDER(t)) {
1268                         fssproc->fss_flags |= FSSBACKQ;
1269                         cpu_surrender(t);
1270                 } else {
1271                         fssproc->fss_timeleft = fss_quantum;
1272                 }
1273         } else {
1274                 /*
1275                  * When the priority of a thread is changed, it may be
1276                  * necessary to adjust its position on a sleep queue or
1277                  * dispatch queue.  The function thread_change_pri accomplishes
1278                  * this.
1279                  */
1280                 if (thread_change_pri(t, new_pri, 0)) {
1281                         /*
1282                          * The thread was on a run queue.
1283                          */
1284                         fssproc->fss_timeleft = fss_quantum;
1285                 } else {
1286                         fssproc->fss_flags |= FSSBACKQ;
1287                 }
1288         }
1289 }
1290 
1291 /*
1292  * Update priorities of all fair-sharing threads that are currently runnable
1293  * at a user mode priority based on the number of shares and current usage.
1294  * Called once per second via timeout which we reset here.
1295  *
1296  * There are several lists of fair-sharing threads broken up by a hash on the
1297  * thread pointer.  Each list has its own lock.  This avoids blocking all
1298  * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs.
1299  * fss_update traverses each list in turn.
1300  *
1301  * Each time we're run (once/second) we may start at the next list and iterate
1302  * through all of the lists. By starting with a different list, we mitigate any
1303  * effects we would see updating the fssps_maxfsspri value in fss_newpri.
1304  */
1305 static void
1306 fss_update(void *arg)
1307 {
1308         int i;
1309         int new_marker = -1;
1310         static int fss_update_marker;
1311 
1312         /*
1313          * Decay and update usages for all projects.
1314          */
1315         fss_decay_usage();
1316 
1317         /*
1318          * Start with the fss_update_marker list, then do the rest.
1319          */
1320         i = fss_update_marker;
1321 
1322         /*
1323          * Go around all threads, set new priorities and decay
1324          * per-thread CPU usages.
1325          */
1326         do {
1327                 /*
1328                  * If this is the first list after the current marker to have
1329                  * threads with priority updates, advance the marker to this
1330                  * list for the next time fss_update runs.
1331                  */
1332                 if (fss_update_list(i) &&
1333                     new_marker == -1 && i != fss_update_marker)
1334                         new_marker = i;
1335         } while ((i = FSS_LIST_NEXT(i)) != fss_update_marker);
1336 
1337         /*
1338          * Advance marker for the next fss_update call
1339          */
1340         if (new_marker != -1)
1341                 fss_update_marker = new_marker;
1342 
1343         (void) timeout(fss_update, arg, hz);
1344 }
1345 
1346 /*
1347  * Updates priority for a list of threads.  Returns 1 if the priority of one
1348  * of the threads was actually updated, 0 if none were for various reasons
1349  * (thread is no longer in the FSS class, is not runnable, has the preemption
1350  * control no-preempt bit set, etc.)
1351  */
1352 static int
1353 fss_update_list(int i)
1354 {
1355         fssproc_t *fssproc;
1356         fssproj_t *fssproj;
1357         fsspri_t fsspri;
1358         pri_t fss_umdpri;
1359         kthread_t *t;
1360         int updated = 0;
1361 
1362         mutex_enter(&fss_listlock[i]);
1363         for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i];
1364             fssproc = fssproc->fss_next) {
1365                 t = fssproc->fss_tp;
1366                 /*
1367                  * Lock the thread and verify the state.
1368                  */
1369                 thread_lock(t);
1370                 /*
1371                  * Skip the thread if it is no longer in the FSS class or
1372                  * is running with kernel mode priority.
1373                  */
1374                 if (t->t_cid != fss_cid)
1375                         goto next;
1376                 if ((fssproc->fss_flags & FSSKPRI) != 0)
1377                         goto next;
1378 
1379                 fssproj = FSSPROC2FSSPROJ(fssproc);
1380                 if (fssproj == NULL)
1381                         goto next;
1382 
1383                 if (fssproj->fssp_shares != 0) {
1384                         /*
1385                          * Decay fsspri value.
1386                          */
1387                         fsspri = fssproc->fss_fsspri;
1388                         fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) /
1389                             FSS_DECAY_BASE;
1390                         fssproc->fss_fsspri = fsspri;
1391                 }
1392 
1393                 if (t->t_schedctl && schedctl_get_nopreempt(t))
1394                         goto next;
1395                 if (t->t_state != TS_RUN && t->t_state != TS_WAIT) {
1396                         /*
1397                          * Make next syscall/trap call fss_trapret
1398                          */
1399                         t->t_trapret = 1;
1400                         aston(t);
1401                         if (t->t_state == TS_ONPROC)
1402                                 DTRACE_PROBE1(fss__onproc, fssproc_t *,
1403                                     fssproc);
1404                         goto next;
1405                 }
1406                 fss_newpri(fssproc, B_FALSE);
1407                 updated = 1;
1408 
1409                 fss_umdpri = fssproc->fss_umdpri;
1410 
1411                 /*
1412                  * Only dequeue the thread if it needs to be moved; otherwise
1413                  * it should just round-robin here.
1414                  */
1415                 if (t->t_pri != fss_umdpri)
1416                         fss_change_priority(t, fssproc);
1417 next:
1418                 thread_unlock(t);
1419         }
1420         mutex_exit(&fss_listlock[i]);
1421         return (updated);
1422 }
1423 
1424 /*ARGSUSED*/
1425 static int
1426 fss_admin(caddr_t uaddr, cred_t *reqpcredp)
1427 {
1428         fssadmin_t fssadmin;
1429 
1430         if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t)))
1431                 return (EFAULT);
1432 
1433         switch (fssadmin.fss_cmd) {
1434         case FSS_SETADMIN:
1435                 if (secpolicy_dispadm(reqpcredp) != 0)
1436                         return (EPERM);
1437                 if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz)
1438                         return (EINVAL);
1439                 fss_quantum = fssadmin.fss_quantum;
1440                 break;
1441         case FSS_GETADMIN:
1442                 fssadmin.fss_quantum = fss_quantum;
1443                 if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t)))
1444                         return (EFAULT);
1445                 break;
1446         default:
1447                 return (EINVAL);
1448         }
1449         return (0);
1450 }
1451 
1452 static int
1453 fss_getclinfo(void *infop)
1454 {
1455         fssinfo_t *fssinfo = (fssinfo_t *)infop;
1456         fssinfo->fss_maxupri = fss_maxupri;
1457         return (0);
1458 }
1459 
1460 static int
1461 fss_parmsin(void *parmsp)
1462 {
1463         fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1464 
1465         /*
1466          * Check validity of parameters.
1467          */
1468         if ((fssparmsp->fss_uprilim > fss_maxupri ||
1469             fssparmsp->fss_uprilim < -fss_maxupri) &&
1470             fssparmsp->fss_uprilim != FSS_NOCHANGE)
1471                 return (EINVAL);
1472 
1473         if ((fssparmsp->fss_upri > fss_maxupri ||
1474             fssparmsp->fss_upri < -fss_maxupri) &&
1475             fssparmsp->fss_upri != FSS_NOCHANGE)
1476                 return (EINVAL);
1477 
1478         return (0);
1479 }
1480 
1481 /*ARGSUSED*/
1482 static int
1483 fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1484 {
1485         return (0);
1486 }
1487 
1488 static int
1489 fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp)
1490 {
1491         fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1492         int priflag = 0;
1493         int limflag = 0;
1494         uint_t cnt;
1495         pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1496 
1497         /*
1498          * FSS_NOCHANGE (-32768) is outside of the range of values for
1499          * fss_uprilim and fss_upri.  If the structure fssparms_t is changed,
1500          * FSS_NOCHANGE should be replaced by a flag word.
1501          */
1502         fssparmsp->fss_uprilim = FSS_NOCHANGE;
1503         fssparmsp->fss_upri = FSS_NOCHANGE;
1504 
1505         /*
1506          * Get the varargs parameter and check validity of parameters.
1507          */
1508         if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1509                 return (EINVAL);
1510 
1511         for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1512                 switch (vpp->pc_key) {
1513                 case FSS_KY_UPRILIM:
1514                         if (limflag++)
1515                                 return (EINVAL);
1516                         fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm;
1517                         if (fssparmsp->fss_uprilim > fss_maxupri ||
1518                             fssparmsp->fss_uprilim < -fss_maxupri)
1519                                 return (EINVAL);
1520                         break;
1521                 case FSS_KY_UPRI:
1522                         if (priflag++)
1523                                 return (EINVAL);
1524                         fssparmsp->fss_upri = (pri_t)vpp->pc_parm;
1525                         if (fssparmsp->fss_upri > fss_maxupri ||
1526                             fssparmsp->fss_upri < -fss_maxupri)
1527                                 return (EINVAL);
1528                         break;
1529                 default:
1530                         return (EINVAL);
1531                 }
1532         }
1533 
1534         if (vaparmsp->pc_vaparmscnt == 0) {
1535                 /*
1536                  * Use default parameters.
1537                  */
1538                 fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0;
1539         }
1540 
1541         return (0);
1542 }
1543 
1544 /*
1545  * Copy all selected fair-sharing class parameters to the user.  The parameters
1546  * are specified by a key.
1547  */
1548 static int
1549 fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1550 {
1551         fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1552         int priflag = 0;
1553         int limflag = 0;
1554         uint_t cnt;
1555         pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1556 
1557         ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));
1558 
1559         if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1560                 return (EINVAL);
1561 
1562         for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1563                 switch (vpp->pc_key) {
1564                 case FSS_KY_UPRILIM:
1565                         if (limflag++)
1566                                 return (EINVAL);
1567                         if (copyout(&fssparmsp->fss_uprilim,
1568                             (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1569                                 return (EFAULT);
1570                         break;
1571                 case FSS_KY_UPRI:
1572                         if (priflag++)
1573                                 return (EINVAL);
1574                         if (copyout(&fssparmsp->fss_upri,
1575                             (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1576                                 return (EFAULT);
1577                         break;
1578                 default:
1579                         return (EINVAL);
1580                 }
1581         }
1582 
1583         return (0);
1584 }
1585 
1586 /*
1587  * Return the user mode scheduling priority range.
1588  */
1589 static int
1590 fss_getclpri(pcpri_t *pcprip)
1591 {
1592         pcprip->pc_clpmax = fss_maxupri;
1593         pcprip->pc_clpmin = -fss_maxupri;
1594         return (0);
1595 }
1596 
1597 static int
1598 fss_alloc(void **p, int flag)
1599 {
1600         void *bufp;
1601 
1602         if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) {
1603                 return (ENOMEM);
1604         } else {
1605                 *p = bufp;
1606                 return (0);
1607         }
1608 }
1609 
1610 static void
1611 fss_free(void *bufp)
1612 {
1613         if (bufp)
1614                 kmem_free(bufp, sizeof (fssproc_t));
1615 }
1616 
1617 /*
1618  * Thread functions
1619  */
1620 static int
1621 fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
1622     void *bufp)
1623 {
1624         fssparms_t      *fssparmsp = (fssparms_t *)parmsp;
1625         fssproc_t       *fssproc;
1626         pri_t           reqfssuprilim;
1627         pri_t           reqfssupri;
1628         static uint32_t fssexists = 0;
1629         fsspset_t       *fsspset;
1630         fssproj_t       *fssproj;
1631         fsszone_t       *fsszone;
1632         kproject_t      *kpj;
1633         zone_t          *zone;
1634         int             fsszone_allocated = 0;
1635 
1636         fssproc = (fssproc_t *)bufp;
1637         ASSERT(fssproc != NULL);
1638 
1639         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1640 
1641         /*
1642          * Only root can move threads to FSS class.
1643          */
1644         if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0)
1645                 return (EPERM);
1646         /*
1647          * Initialize the fssproc structure.
1648          */
1649         fssproc->fss_umdpri = fss_maxumdpri / 2;
1650 
1651         if (fssparmsp == NULL) {
1652                 /*
1653                  * Use default values.
1654                  */
1655                 fssproc->fss_nice = NZERO;
1656                 fssproc->fss_uprilim = fssproc->fss_upri = 0;
1657         } else {
1658                 /*
1659                  * Use supplied values.
1660                  */
1661                 if (fssparmsp->fss_uprilim == FSS_NOCHANGE) {
1662                         reqfssuprilim = 0;
1663                 } else {
1664                         if (fssparmsp->fss_uprilim > 0 &&
1665                             secpolicy_setpriority(reqpcredp) != 0)
1666                                 return (EPERM);
1667                         reqfssuprilim = fssparmsp->fss_uprilim;
1668                 }
1669                 if (fssparmsp->fss_upri == FSS_NOCHANGE) {
1670                         reqfssupri = reqfssuprilim;
1671                 } else {
1672                         if (fssparmsp->fss_upri > 0 &&
1673                             secpolicy_setpriority(reqpcredp) != 0)
1674                                 return (EPERM);
1675                         /*
1676                          * Set the user priority to the requested value or
1677                          * the upri limit, whichever is lower.
1678                          */
1679                         reqfssupri = fssparmsp->fss_upri;
1680                         if (reqfssupri > reqfssuprilim)
1681                                 reqfssupri = reqfssuprilim;
1682                 }
1683                 fssproc->fss_uprilim = reqfssuprilim;
1684                 fssproc->fss_upri = reqfssupri;
1685                 fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri;
1686                 if (fssproc->fss_nice > FSS_NICE_MAX)
1687                         fssproc->fss_nice = FSS_NICE_MAX;
1688         }
1689 
1690         fssproc->fss_timeleft = fss_quantum;
1691         fssproc->fss_tp = t;
1692         cpucaps_sc_init(&fssproc->fss_caps);
1693 
1694         /*
1695          * Put a lock on our fsspset structure.
1696          */
1697         mutex_enter(&fsspsets_lock);
1698         fsspset = fss_find_fsspset(t->t_cpupart);
1699         mutex_enter(&fsspset->fssps_lock);
1700         mutex_exit(&fsspsets_lock);
1701 
1702         zone = ttoproc(t)->p_zone;
1703         if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) {
1704                 if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP))
1705                     == NULL) {
1706                         mutex_exit(&fsspset->fssps_lock);
1707                         return (ENOMEM);
1708                 } else {
1709                         fsszone_allocated = 1;
1710                         fss_insert_fsszone(fsspset, zone, fsszone);
1711                 }
1712         }
1713         kpj = ttoproj(t);
1714         if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) {
1715                 if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP))
1716                     == NULL) {
1717                         if (fsszone_allocated) {
1718                                 fss_remove_fsszone(fsspset, fsszone);
1719                                 kmem_free(fsszone, sizeof (fsszone_t));
1720                         }
1721                         mutex_exit(&fsspset->fssps_lock);
1722                         return (ENOMEM);
1723                 } else {
1724                         fss_insert_fssproj(fsspset, kpj, fsszone, fssproj);
1725                 }
1726         }
1727         fssproj->fssp_threads++;
1728         fssproc->fss_proj = fssproj;
1729 
1730         /*
1731          * Reset priority. Process goes to a "user mode" priority here
1732          * regardless of whether or not it has slept since entering the kernel.
1733          */
1734         thread_lock(t);
1735         t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
1736         t->t_cid = cid;
1737         t->t_cldata = (void *)fssproc;
1738         t->t_schedflag |= TS_RUNQMATCH;
1739         fss_change_priority(t, fssproc);
1740         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
1741             t->t_state == TS_WAIT)
1742                 fss_active(t);
1743         thread_unlock(t);
1744 
1745         mutex_exit(&fsspset->fssps_lock);
1746 
1747         /*
1748          * Link new structure into fssproc list.
1749          */
1750         FSS_LIST_INSERT(fssproc);
1751 
1752         /*
1753          * If this is the first fair-sharing thread to occur since boot,
1754          * we set up the initial call to fss_update() here. Use an atomic
1755          * compare-and-swap since that's easier and faster than a mutex
1756          * (but check with an ordinary load first since most of the time
1757          * this will already be done).
1758          */
1759         if (fssexists == 0 && atomic_cas_32(&fssexists, 0, 1) == 0)
1760                 (void) timeout(fss_update, NULL, hz);
1761 
1762         return (0);
1763 }
1764 
1765 /*
1766  * Remove fssproc_t from the list.
1767  */
1768 static void
1769 fss_exitclass(void *procp)
1770 {
1771         fssproc_t *fssproc = (fssproc_t *)procp;
1772         fssproj_t *fssproj;
1773         fsspset_t *fsspset;
1774         fsszone_t *fsszone;
1775         kthread_t *t = fssproc->fss_tp;
1776 
1777         /*
1778          * We should be either getting this thread off the deathrow or
1779          * this thread has already moved to another scheduling class and
1780          * we're being called with its old cldata buffer pointer.  In both
1781          * cases, the content of this buffer can not be changed while we're
1782          * here.
1783          */
1784         mutex_enter(&fsspsets_lock);
1785         thread_lock(t);
1786         if (t->t_cid != fss_cid) {
1787                 /*
1788                  * We're being called as a result of the priocntl() system
1789                  * call -- someone is trying to move our thread to another
1790                  * scheduling class. We can't call fss_inactive() here
1791                  * because our thread's t_cldata pointer already points
1792                  * to another scheduling class specific data.
1793                  */
1794                 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1795 
1796                 fssproj = FSSPROC2FSSPROJ(fssproc);
1797                 fsspset = FSSPROJ2FSSPSET(fssproj);
1798                 fsszone = fssproj->fssp_fsszone;
1799 
1800                 if (fssproc->fss_runnable) {
1801                         disp_lock_enter_high(&fsspset->fssps_displock);
1802                         if (--fssproj->fssp_runnable == 0) {
1803                                 fsszone->fssz_shares -= fssproj->fssp_shares;
1804                                 if (--fsszone->fssz_runnable == 0)
1805                                         fsspset->fssps_shares -=
1806                                             fsszone->fssz_rshares;
1807                         }
1808                         disp_lock_exit_high(&fsspset->fssps_displock);
1809                 }
1810                 thread_unlock(t);
1811 
1812                 mutex_enter(&fsspset->fssps_lock);
1813                 if (--fssproj->fssp_threads == 0) {
1814                         fss_remove_fssproj(fsspset, fssproj);
1815                         if (fsszone->fssz_nproj == 0)
1816                                 kmem_free(fsszone, sizeof (fsszone_t));
1817                         kmem_free(fssproj, sizeof (fssproj_t));
1818                 }
1819                 mutex_exit(&fsspset->fssps_lock);
1820 
1821         } else {
1822                 ASSERT(t->t_state == TS_FREE);
1823                 /*
1824                  * We're being called from thread_free() when our thread
1825                  * is removed from the deathrow. There is nothing we need
1826                  * do here since everything should've been done earlier
1827                  * in fss_exit().
1828                  */
1829                 thread_unlock(t);
1830         }
1831         mutex_exit(&fsspsets_lock);
1832 
1833         FSS_LIST_DELETE(fssproc);
1834         fss_free(fssproc);
1835 }
1836 
1837 /*ARGSUSED*/
1838 static int
1839 fss_canexit(kthread_t *t, cred_t *credp)
1840 {
1841         /*
1842          * A thread is allowed to exit FSS only if we have sufficient
1843          * privileges.
1844          */
1845         if (credp != NULL && secpolicy_setpriority(credp) != 0)
1846                 return (EPERM);
1847         else
1848                 return (0);
1849 }
1850 
1851 /*
1852  * Initialize fair-share class specific proc structure for a child.
1853  */
1854 static int
1855 fss_fork(kthread_t *pt, kthread_t *ct, void *bufp)
1856 {
1857         fssproc_t *pfssproc;    /* ptr to parent's fssproc structure    */
1858         fssproc_t *cfssproc;    /* ptr to child's fssproc structure     */
1859         fssproj_t *fssproj;
1860         fsspset_t *fsspset;
1861 
1862         ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock));
1863         ASSERT(ct->t_state == TS_STOPPED);
1864 
1865         cfssproc = (fssproc_t *)bufp;
1866         ASSERT(cfssproc != NULL);
1867         bzero(cfssproc, sizeof (fssproc_t));
1868 
1869         thread_lock(pt);
1870         pfssproc = FSSPROC(pt);
1871         fssproj = FSSPROC2FSSPROJ(pfssproc);
1872         fsspset = FSSPROJ2FSSPSET(fssproj);
1873         thread_unlock(pt);
1874 
1875         mutex_enter(&fsspset->fssps_lock);
1876         /*
1877          * Initialize child's fssproc structure.
1878          */
1879         thread_lock(pt);
1880         ASSERT(FSSPROJ(pt) == fssproj);
1881         cfssproc->fss_proj = fssproj;
1882         cfssproc->fss_timeleft = fss_quantum;
1883         cfssproc->fss_umdpri = pfssproc->fss_umdpri;
1884         cfssproc->fss_fsspri = 0;
1885         cfssproc->fss_uprilim = pfssproc->fss_uprilim;
1886         cfssproc->fss_upri = pfssproc->fss_upri;
1887         cfssproc->fss_tp = ct;
1888         cfssproc->fss_nice = pfssproc->fss_nice;
1889         cpucaps_sc_init(&cfssproc->fss_caps);
1890 
1891         cfssproc->fss_flags =
1892             pfssproc->fss_flags & ~(FSSKPRI | FSSBACKQ | FSSRESTORE);
1893         ct->t_cldata = (void *)cfssproc;
1894         ct->t_schedflag |= TS_RUNQMATCH;
1895         thread_unlock(pt);
1896 
1897         fssproj->fssp_threads++;
1898         mutex_exit(&fsspset->fssps_lock);
1899 
1900         /*
1901          * Link new structure into fssproc hash table.
1902          */
1903         FSS_LIST_INSERT(cfssproc);
1904         return (0);
1905 }
1906 
1907 /*
1908  * Child is placed at back of dispatcher queue and parent gives up processor
1909  * so that the child runs first after the fork. This allows the child
1910  * immediately execing to break the multiple use of copy on write pages with no
1911  * disk home. The parent will get to steal them back rather than uselessly
1912  * copying them.
1913  */
1914 static void
1915 fss_forkret(kthread_t *t, kthread_t *ct)
1916 {
1917         proc_t *pp = ttoproc(t);
1918         proc_t *cp = ttoproc(ct);
1919         fssproc_t *fssproc;
1920 
1921         ASSERT(t == curthread);
1922         ASSERT(MUTEX_HELD(&pidlock));
1923 
1924         /*
1925          * Grab the child's p_lock before dropping pidlock to ensure the
1926          * process does not disappear before we set it running.
1927          */
1928         mutex_enter(&cp->p_lock);
1929         continuelwps(cp);
1930         mutex_exit(&cp->p_lock);
1931 
1932         mutex_enter(&pp->p_lock);
1933         mutex_exit(&pidlock);
1934         continuelwps(pp);
1935 
1936         thread_lock(t);
1937 
1938         fssproc = FSSPROC(t);
1939         fss_newpri(fssproc, B_FALSE);
1940         fssproc->fss_timeleft = fss_quantum;
1941         t->t_pri = fssproc->fss_umdpri;
1942         ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1943         fssproc->fss_flags &= ~FSSKPRI;
1944         THREAD_TRANSITION(t);
1945 
1946         /*
1947          * We don't want to call fss_setrun(t) here because it may call
1948          * fss_active, which we don't need.
1949          */
1950         fssproc->fss_flags &= ~FSSBACKQ;
1951 
1952         if (t->t_disp_time != ddi_get_lbolt())
1953                 setbackdq(t);
1954         else
1955                 setfrontdq(t);
1956 
1957         thread_unlock(t);
1958         /*
1959          * Safe to drop p_lock now since it is safe to change
1960          * the scheduling class after this point.
1961          */
1962         mutex_exit(&pp->p_lock);
1963 
1964         swtch();
1965 }
1966 
1967 /*
1968  * Get the fair-sharing parameters of the thread pointed to by fssprocp into
1969  * the buffer pointed by fssparmsp.
1970  */
1971 static void
1972 fss_parmsget(kthread_t *t, void *parmsp)
1973 {
1974         fssproc_t *fssproc = FSSPROC(t);
1975         fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1976 
1977         fssparmsp->fss_uprilim = fssproc->fss_uprilim;
1978         fssparmsp->fss_upri = fssproc->fss_upri;
1979 }
1980 
1981 /*ARGSUSED*/
1982 static int
1983 fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1984 {
1985         char            nice;
1986         pri_t           reqfssuprilim;
1987         pri_t           reqfssupri;
1988         fssproc_t       *fssproc = FSSPROC(t);
1989         fssparms_t      *fssparmsp = (fssparms_t *)parmsp;
1990 
1991         ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1992 
1993         if (fssparmsp->fss_uprilim == FSS_NOCHANGE)
1994                 reqfssuprilim = fssproc->fss_uprilim;
1995         else
1996                 reqfssuprilim = fssparmsp->fss_uprilim;
1997 
1998         if (fssparmsp->fss_upri == FSS_NOCHANGE)
1999                 reqfssupri = fssproc->fss_upri;
2000         else
2001                 reqfssupri = fssparmsp->fss_upri;
2002 
2003         /*
2004          * Make sure the user priority doesn't exceed the upri limit.
2005          */
2006         if (reqfssupri > reqfssuprilim)
2007                 reqfssupri = reqfssuprilim;
2008 
2009         /*
2010          * Basic permissions enforced by generic kernel code for all classes
2011          * require that a thread attempting to change the scheduling parameters
2012          * of a target thread be privileged or have a real or effective UID
2013          * matching that of the target thread. We are not called unless these
2014          * basic permission checks have already passed. The fair-sharing class
2015          * requires in addition that the calling thread be privileged if it
2016          * is attempting to raise the upri limit above its current value.
2017          * This may have been checked previously but if our caller passed us
2018          * a non-NULL credential pointer we assume it hasn't and we check it
2019          * here.
2020          */
2021         if ((reqpcredp != NULL) &&
2022             (reqfssuprilim > fssproc->fss_uprilim) &&
2023             secpolicy_raisepriority(reqpcredp) != 0)
2024                 return (EPERM);
2025 
2026         /*
2027          * Set fss_nice to the nice value corresponding to the user priority we
2028          * are setting.  Note that setting the nice field of the parameter
2029          * struct won't affect upri or nice.
2030          */
2031         nice = NZERO - (reqfssupri * NZERO) / fss_maxupri;
2032         if (nice > FSS_NICE_MAX)
2033                 nice = FSS_NICE_MAX;
2034 
2035         thread_lock(t);
2036 
2037         fssproc->fss_uprilim = reqfssuprilim;
2038         fssproc->fss_upri = reqfssupri;
2039         fssproc->fss_nice = nice;
2040         fss_newpri(fssproc, B_FALSE);
2041 
2042         if ((fssproc->fss_flags & FSSKPRI) != 0) {
2043                 thread_unlock(t);
2044                 return (0);
2045         }
2046 
2047         fss_change_priority(t, fssproc);
2048         thread_unlock(t);
2049         return (0);
2050 
2051 }
2052 
2053 /*
2054  * The thread is being stopped.
2055  */
2056 /*ARGSUSED*/
2057 static void
2058 fss_stop(kthread_t *t, int why, int what)
2059 {
2060         ASSERT(THREAD_LOCK_HELD(t));
2061         ASSERT(t == curthread);
2062 
2063         fss_inactive(t);
2064 }
2065 
2066 /*
2067  * The current thread is exiting, do necessary adjustments to its project
2068  */
2069 static void
2070 fss_exit(kthread_t *t)
2071 {
2072         fsspset_t *fsspset;
2073         fssproj_t *fssproj;
2074         fssproc_t *fssproc;
2075         fsszone_t *fsszone;
2076         int free = 0;
2077 
2078         /*
2079          * Thread t here is either a current thread (in which case we hold
2080          * its process' p_lock), or a thread being destroyed by forklwp_fail(),
2081          * in which case we hold pidlock and thread is no longer on the
2082          * thread list.
2083          */
2084         ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock));
2085 
2086         fssproc = FSSPROC(t);
2087         fssproj = FSSPROC2FSSPROJ(fssproc);
2088         fsspset = FSSPROJ2FSSPSET(fssproj);
2089         fsszone = fssproj->fssp_fsszone;
2090 
2091         mutex_enter(&fsspsets_lock);
2092         mutex_enter(&fsspset->fssps_lock);
2093 
2094         thread_lock(t);
2095         disp_lock_enter_high(&fsspset->fssps_displock);
2096         if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
2097                 if (--fssproj->fssp_runnable == 0) {
2098                         fsszone->fssz_shares -= fssproj->fssp_shares;
2099                         if (--fsszone->fssz_runnable == 0)
2100                                 fsspset->fssps_shares -= fsszone->fssz_rshares;
2101                 }
2102                 ASSERT(fssproc->fss_runnable == 1);
2103                 fssproc->fss_runnable = 0;
2104         }
2105         if (--fssproj->fssp_threads == 0) {
2106                 fss_remove_fssproj(fsspset, fssproj);
2107                 free = 1;
2108         }
2109         disp_lock_exit_high(&fsspset->fssps_displock);
2110         fssproc->fss_proj = NULL;    /* mark this thread as already exited */
2111         thread_unlock(t);
2112 
2113         if (free) {
2114                 if (fsszone->fssz_nproj == 0)
2115                         kmem_free(fsszone, sizeof (fsszone_t));
2116                 kmem_free(fssproj, sizeof (fssproj_t));
2117         }
2118         mutex_exit(&fsspset->fssps_lock);
2119         mutex_exit(&fsspsets_lock);
2120 
2121         /*
2122          * A thread could be exiting in between clock ticks, so we need to
2123          * calculate how much CPU time it used since it was charged last time.
2124          *
2125          * CPU caps are not enforced on exiting processes - it is usually
2126          * desirable to exit as soon as possible to free resources.
2127          */
2128         if (CPUCAPS_ON()) {
2129                 thread_lock(t);
2130                 fssproc = FSSPROC(t);
2131                 (void) cpucaps_charge(t, &fssproc->fss_caps,
2132                     CPUCAPS_CHARGE_ONLY);
2133                 thread_unlock(t);
2134         }
2135 }
2136 
2137 static void
2138 fss_nullsys()
2139 {
2140 }
2141 
2142 /*
2143  * fss_swapin() returns -1 if the thread is loaded or is not eligible to be
2144  * swapped in. Otherwise, it returns the thread's effective priority based
2145  * on swapout time and size of process (0 <= epri <= 0 SHRT_MAX).
2146  */
2147 /*ARGSUSED*/
2148 static pri_t
2149 fss_swapin(kthread_t *t, int flags)
2150 {
2151         fssproc_t *fssproc = FSSPROC(t);
2152         long epri = -1;
2153         proc_t *pp = ttoproc(t);
2154 
2155         ASSERT(THREAD_LOCK_HELD(t));
2156 
2157         if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
2158                 time_t swapout_time;
2159 
2160                 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
2161                 if (INHERITED(t) || (fssproc->fss_flags & FSSKPRI)) {
2162                         epri = (long)DISP_PRIO(t) + swapout_time;
2163                 } else {
2164                         /*
2165                          * Threads which have been out for a long time,
2166                          * have high user mode priority and are associated
2167                          * with a small address space are more deserving.
2168                          */
2169                         epri = fssproc->fss_umdpri;
2170                         ASSERT(epri >= 0 && epri <= fss_maxumdpri);
2171                         epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
2172                 }
2173                 /*
2174                  * Scale epri so that SHRT_MAX / 2 represents zero priority.
2175                  */
2176                 epri += SHRT_MAX / 2;
2177                 if (epri < 0)
2178                         epri = 0;
2179                 else if (epri > SHRT_MAX)
2180                         epri = SHRT_MAX;
2181         }
2182         return ((pri_t)epri);
2183 }
2184 
2185 /*
2186  * fss_swapout() returns -1 if the thread isn't loaded or is not eligible to
2187  * be swapped out. Otherwise, it returns the thread's effective priority
2188  * based on if the swapper is in softswap or hardswap mode.
2189  */
2190 static pri_t
2191 fss_swapout(kthread_t *t, int flags)
2192 {
2193         fssproc_t *fssproc = FSSPROC(t);
2194         long epri = -1;
2195         proc_t *pp = ttoproc(t);
2196         time_t swapin_time;
2197 
2198         ASSERT(THREAD_LOCK_HELD(t));
2199 
2200         if (INHERITED(t) ||
2201             (fssproc->fss_flags & FSSKPRI) ||
2202             (t->t_proc_flag & TP_LWPEXIT) ||
2203             (t->t_state & (TS_ZOMB|TS_FREE|TS_STOPPED|TS_ONPROC|TS_WAIT)) ||
2204             !(t->t_schedflag & TS_LOAD) ||
2205             !(SWAP_OK(t)))
2206                 return (-1);
2207 
2208         ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
2209 
2210         swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
2211 
2212         if (flags == SOFTSWAP) {
2213                 if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
2214                         epri = 0;
2215                 } else {
2216                         return ((pri_t)epri);
2217                 }
2218         } else {
2219                 pri_t pri;
2220 
2221                 if ((t->t_state == TS_SLEEP && swapin_time > fss_minslp) ||
2222                     (t->t_state == TS_RUN && swapin_time > fss_minrun)) {
2223                         pri = fss_maxumdpri;
2224                         epri = swapin_time -
2225                             (rm_asrss(pp->p_as) / nz(maxpgio)/2) - (long)pri;
2226                 } else {
2227                         return ((pri_t)epri);
2228                 }
2229         }
2230 
2231         /*
2232          * Scale epri so that SHRT_MAX / 2 represents zero priority.
2233          */
2234         epri += SHRT_MAX / 2;
2235         if (epri < 0)
2236                 epri = 0;
2237         else if (epri > SHRT_MAX)
2238                 epri = SHRT_MAX;
2239 
2240         return ((pri_t)epri);
2241 }
2242 
2243 /*
2244  * If thread is currently at a kernel mode priority (has slept) and is
2245  * returning to the userland we assign it the appropriate user mode priority
2246  * and time quantum here.  If we're lowering the thread's priority below that
2247  * of other runnable threads then we will set runrun via cpu_surrender() to
2248  * cause preemption.
2249  */
2250 static void
2251 fss_trapret(kthread_t *t)
2252 {
2253         fssproc_t *fssproc = FSSPROC(t);
2254         cpu_t *cp = CPU;
2255 
2256         ASSERT(THREAD_LOCK_HELD(t));
2257         ASSERT(t == curthread);
2258         ASSERT(cp->cpu_dispthread == t);
2259         ASSERT(t->t_state == TS_ONPROC);
2260 
2261         t->t_kpri_req = 0;
2262         if (fssproc->fss_flags & FSSKPRI) {
2263                 /*
2264                  * If thread has blocked in the kernel
2265                  */
2266                 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2267                 cp->cpu_dispatch_pri = DISP_PRIO(t);
2268                 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2269                 fssproc->fss_flags &= ~FSSKPRI;
2270 
2271                 if (DISP_MUST_SURRENDER(t))
2272                         cpu_surrender(t);
2273         }
2274 
2275         /*
2276          * Swapout lwp if the swapper is waiting for this thread to reach
2277          * a safe point.
2278          */
2279         if (t->t_schedflag & TS_SWAPENQ) {
2280                 thread_unlock(t);
2281                 swapout_lwp(ttolwp(t));
2282                 thread_lock(t);
2283         }
2284 }
2285 
2286 /*
2287  * Arrange for thread to be placed in appropriate location on dispatcher queue.
2288  * This is called with the current thread in TS_ONPROC and locked.
2289  */
2290 static void
2291 fss_preempt(kthread_t *t)
2292 {
2293         fssproc_t *fssproc = FSSPROC(t);
2294         klwp_t *lwp;
2295         uint_t flags;
2296 
2297         ASSERT(t == curthread);
2298         ASSERT(THREAD_LOCK_HELD(curthread));
2299         ASSERT(t->t_state == TS_ONPROC);
2300 
2301         /*
2302          * If preempted in the kernel, make sure the thread has a kernel
2303          * priority if needed.
2304          */
2305         lwp = curthread->t_lwp;
2306         if (!(fssproc->fss_flags & FSSKPRI) && lwp != NULL && t->t_kpri_req) {
2307                 fssproc->fss_flags |= FSSKPRI;
2308                 THREAD_CHANGE_PRI(t, minclsyspri);
2309                 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2310                 t->t_trapret = 1;    /* so that fss_trapret will run */
2311                 aston(t);
2312         }
2313 
2314         /*
2315          * This thread may be placed on wait queue by CPU Caps. In this case we
2316          * do not need to do anything until it is removed from the wait queue.
2317          * Do not enforce CPU caps on threads running at a kernel priority
2318          */
2319         if (CPUCAPS_ON()) {
2320                 (void) cpucaps_charge(t, &fssproc->fss_caps,
2321                     CPUCAPS_CHARGE_ENFORCE);
2322 
2323                 if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t))
2324                         return;
2325         }
2326 
2327         /*
2328          * If preempted in user-land mark the thread as swappable because it
2329          * cannot be holding any kernel locks.
2330          */
2331         ASSERT(t->t_schedflag & TS_DONT_SWAP);
2332         if (lwp != NULL && lwp->lwp_state == LWP_USER)
2333                 t->t_schedflag &= ~TS_DONT_SWAP;
2334 
2335         /*
2336          * Check to see if we're doing "preemption control" here.  If
2337          * we are, and if the user has requested that this thread not
2338          * be preempted, and if preemptions haven't been put off for
2339          * too long, let the preemption happen here but try to make
2340          * sure the thread is rescheduled as soon as possible.  We do
2341          * this by putting it on the front of the highest priority run
2342          * queue in the FSS class.  If the preemption has been put off
2343          * for too long, clear the "nopreempt" bit and let the thread
2344          * be preempted.
2345          */
2346         if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2347                 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2348                         DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
2349                         if (!(fssproc->fss_flags & FSSKPRI)) {
2350                                 /*
2351                                  * If not already remembered, remember current
2352                                  * priority for restoration in fss_yield().
2353                                  */
2354                                 if (!(fssproc->fss_flags & FSSRESTORE)) {
2355                                         fssproc->fss_scpri = t->t_pri;
2356                                         fssproc->fss_flags |= FSSRESTORE;
2357                                 }
2358                                 THREAD_CHANGE_PRI(t, fss_maxumdpri);
2359                                 t->t_schedflag |= TS_DONT_SWAP;
2360                         }
2361                         schedctl_set_yield(t, 1);
2362                         setfrontdq(t);
2363                         return;
2364                 } else {
2365                         if (fssproc->fss_flags & FSSRESTORE) {
2366                                 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2367                                 fssproc->fss_flags &= ~FSSRESTORE;
2368                         }
2369                         schedctl_set_nopreempt(t, 0);
2370                         DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
2371                         /*
2372                          * Fall through and be preempted below.
2373                          */
2374                 }
2375         }
2376 
2377         flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI);
2378 
2379         if (flags == FSSBACKQ) {
2380                 fssproc->fss_timeleft = fss_quantum;
2381                 fssproc->fss_flags &= ~FSSBACKQ;
2382                 setbackdq(t);
2383         } else if (flags == (FSSBACKQ | FSSKPRI)) {
2384                 fssproc->fss_flags &= ~FSSBACKQ;
2385                 setbackdq(t);
2386         } else {
2387                 setfrontdq(t);
2388         }
2389 }
2390 
2391 /*
2392  * Called when a thread is waking up and is to be placed on the run queue.
2393  */
2394 static void
2395 fss_setrun(kthread_t *t)
2396 {
2397         fssproc_t *fssproc = FSSPROC(t);
2398 
2399         ASSERT(THREAD_LOCK_HELD(t));    /* t should be in transition */
2400 
2401         if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED)
2402                 fss_active(t);
2403 
2404         fssproc->fss_timeleft = fss_quantum;
2405 
2406         fssproc->fss_flags &= ~FSSBACKQ;
2407         /*
2408          * If previously were running at the kernel priority then keep that
2409          * priority and the fss_timeleft doesn't matter.
2410          */
2411         if ((fssproc->fss_flags & FSSKPRI) == 0)
2412                 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2413 
2414         if (t->t_disp_time != ddi_get_lbolt())
2415                 setbackdq(t);
2416         else
2417                 setfrontdq(t);
2418 }
2419 
2420 /*
2421  * Prepare thread for sleep. We reset the thread priority so it will run at the
2422  * kernel priority level when it wakes up.
2423  */
2424 static void
2425 fss_sleep(kthread_t *t)
2426 {
2427         fssproc_t *fssproc = FSSPROC(t);
2428 
2429         ASSERT(t == curthread);
2430         ASSERT(THREAD_LOCK_HELD(t));
2431 
2432         ASSERT(t->t_state == TS_ONPROC);
2433 
2434         /*
2435          * Account for time spent on CPU before going to sleep.
2436          */
2437         (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2438 
2439         fss_inactive(t);
2440 
2441         /*
2442          * Assign a system priority to the thread and arrange for it to be
2443          * retained when the thread is next placed on the run queue (i.e.,
2444          * when it wakes up) instead of being given a new pri.  Also arrange
2445          * for trapret processing as the thread leaves the system call so it
2446          * will drop back to normal priority range.
2447          */
2448         if (t->t_kpri_req) {
2449                 THREAD_CHANGE_PRI(t, minclsyspri);
2450                 fssproc->fss_flags |= FSSKPRI;
2451                 t->t_trapret = 1;    /* so that fss_trapret will run */
2452                 aston(t);
2453         } else if (fssproc->fss_flags & FSSKPRI) {
2454                 /*
2455                  * The thread has done a THREAD_KPRI_REQUEST(), slept, then
2456                  * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again),
2457                  * then slept again all without finishing the current system
2458                  * call so trapret won't have cleared FSSKPRI
2459                  */
2460                 fssproc->fss_flags &= ~FSSKPRI;
2461                 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2462                 if (DISP_MUST_SURRENDER(curthread))
2463                         cpu_surrender(t);
2464         }
2465         t->t_stime = ddi_get_lbolt();        /* time stamp for the swapper */
2466 }
2467 
2468 /*
2469  * A tick interrupt has ocurrend on a running thread. Check to see if our
2470  * time slice has expired.  We must also clear the TS_DONT_SWAP flag in
2471  * t_schedflag if the thread is eligible to be swapped out.
2472  */
2473 static void
2474 fss_tick(kthread_t *t)
2475 {
2476         fssproc_t *fssproc;
2477         fssproj_t *fssproj;
2478         klwp_t *lwp;
2479         boolean_t call_cpu_surrender = B_FALSE;
2480         boolean_t cpucaps_enforce = B_FALSE;
2481 
2482         ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
2483 
2484         /*
2485          * It's safe to access fsspset and fssproj structures because we're
2486          * holding our p_lock here.
2487          */
2488         thread_lock(t);
2489         fssproc = FSSPROC(t);
2490         fssproj = FSSPROC2FSSPROJ(fssproc);
2491         if (fssproj != NULL) {
2492                 fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
2493                 disp_lock_enter_high(&fsspset->fssps_displock);
2494                 fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2495                 fssproj->fssp_tick_cnt++;
2496                 fssproc->fss_ticks++;
2497                 disp_lock_exit_high(&fsspset->fssps_displock);
2498         }
2499 
2500         /*
2501          * Keep track of thread's project CPU usage.  Note that projects
2502          * get charged even when threads are running in the kernel.
2503          * Do not surrender CPU if running in the SYS class.
2504          */
2505         if (CPUCAPS_ON()) {
2506                 cpucaps_enforce = cpucaps_charge(t,
2507                     &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE) &&
2508                     !(fssproc->fss_flags & FSSKPRI);
2509         }
2510 
2511         /*
2512          * A thread's execution time for threads running in the SYS class
2513          * is not tracked.
2514          */
2515         if ((fssproc->fss_flags & FSSKPRI) == 0) {
2516                 /*
2517                  * If thread is not in kernel mode, decrement its fss_timeleft
2518                  */
2519                 if (--fssproc->fss_timeleft <= 0) {
2520                         pri_t new_pri;
2521 
2522                         /*
2523                          * If we're doing preemption control and trying to
2524                          * avoid preempting this thread, just note that the
2525                          * thread should yield soon and let it keep running
2526                          * (unless it's been a while).
2527                          */
2528                         if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2529                                 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2530                                         DTRACE_SCHED1(schedctl__nopreempt,
2531                                             kthread_t *, t);
2532                                         schedctl_set_yield(t, 1);
2533                                         thread_unlock_nopreempt(t);
2534                                         return;
2535                                 }
2536                         }
2537                         fssproc->fss_flags &= ~FSSRESTORE;
2538 
2539                         fss_newpri(fssproc, B_TRUE);
2540                         new_pri = fssproc->fss_umdpri;
2541                         ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
2542 
2543                         /*
2544                          * When the priority of a thread is changed, it may
2545                          * be necessary to adjust its position on a sleep queue
2546                          * or dispatch queue. The function thread_change_pri
2547                          * accomplishes this.
2548                          */
2549                         if (thread_change_pri(t, new_pri, 0)) {
2550                                 if ((t->t_schedflag & TS_LOAD) &&
2551                                     (lwp = t->t_lwp) &&
2552                                     lwp->lwp_state == LWP_USER)
2553                                         t->t_schedflag &= ~TS_DONT_SWAP;
2554                                 fssproc->fss_timeleft = fss_quantum;
2555                         } else {
2556                                 call_cpu_surrender = B_TRUE;
2557                         }
2558                 } else if (t->t_state == TS_ONPROC &&
2559                     t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2560                         /*
2561                          * If there is a higher-priority thread which is
2562                          * waiting for a processor, then thread surrenders
2563                          * the processor.
2564                          */
2565                         call_cpu_surrender = B_TRUE;
2566                 }
2567         }
2568 
2569         if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2570                 /*
2571                  * The thread used more than half of its quantum, so assume that
2572                  * it used the whole quantum.
2573                  *
2574                  * Update thread's priority just before putting it on the wait
2575                  * queue so that it gets charged for the CPU time from its
2576                  * quantum even before that quantum expires.
2577                  */
2578                 fss_newpri(fssproc, B_FALSE);
2579                 if (t->t_pri != fssproc->fss_umdpri)
2580                         fss_change_priority(t, fssproc);
2581 
2582                 /*
2583                  * We need to call cpu_surrender for this thread due to cpucaps
2584                  * enforcement, but fss_change_priority may have already done
2585                  * so. In this case FSSBACKQ is set and there is no need to call
2586                  * cpu-surrender again.
2587                  */
2588                 if (!(fssproc->fss_flags & FSSBACKQ))
2589                         call_cpu_surrender = B_TRUE;
2590         }
2591 
2592         if (call_cpu_surrender) {
2593                 fssproc->fss_flags |= FSSBACKQ;
2594                 cpu_surrender(t);
2595         }
2596 
2597         thread_unlock_nopreempt(t);     /* clock thread can't be preempted */
2598 }
2599 
2600 /*
2601  * Processes waking up go to the back of their queue.  We don't need to assign
2602  * a time quantum here because thread is still at a kernel mode priority and
2603  * the time slicing is not done for threads running in the kernel after
2604  * sleeping.  The proper time quantum will be assigned by fss_trapret before the
2605  * thread returns to user mode.
2606  */
2607 static void
2608 fss_wakeup(kthread_t *t)
2609 {
2610         fssproc_t *fssproc;
2611 
2612         ASSERT(THREAD_LOCK_HELD(t));
2613         ASSERT(t->t_state == TS_SLEEP);
2614 
2615         fss_active(t);
2616 
2617         t->t_stime = ddi_get_lbolt();                /* time stamp for the swapper */
2618         fssproc = FSSPROC(t);
2619         fssproc->fss_flags &= ~FSSBACKQ;
2620 
2621         if (fssproc->fss_flags & FSSKPRI) {
2622                 /*
2623                  * If we already have a kernel priority assigned, then we
2624                  * just use it.
2625                  */
2626                 setbackdq(t);
2627         } else if (t->t_kpri_req) {
2628                 /*
2629                  * Give thread a priority boost if we were asked.
2630                  */
2631                 fssproc->fss_flags |= FSSKPRI;
2632                 THREAD_CHANGE_PRI(t, minclsyspri);
2633                 setbackdq(t);
2634                 t->t_trapret = 1;    /* so that fss_trapret will run */
2635                 aston(t);
2636         } else {
2637                 /*
2638                  * Otherwise, we recalculate the priority.
2639                  */
2640                 if (t->t_disp_time == ddi_get_lbolt()) {
2641                         setfrontdq(t);
2642                 } else {
2643                         fssproc->fss_timeleft = fss_quantum;
2644                         THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2645                         setbackdq(t);
2646                 }
2647         }
2648 }
2649 
2650 /*
2651  * fss_donice() is called when a nice(1) command is issued on the thread to
2652  * alter the priority. The nice(1) command exists in Solaris for compatibility.
2653  * Thread priority adjustments should be done via priocntl(1).
2654  */
2655 static int
2656 fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2657 {
2658         int newnice;
2659         fssproc_t *fssproc = FSSPROC(t);
2660         fssparms_t fssparms;
2661 
2662         /*
2663          * If there is no change to priority, just return current setting.
2664          */
2665         if (incr == 0) {
2666                 if (retvalp)
2667                         *retvalp = fssproc->fss_nice - NZERO;
2668                 return (0);
2669         }
2670 
2671         if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0)
2672                 return (EPERM);
2673 
2674         /*
2675          * Specifying a nice increment greater than the upper limit of
2676          * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice
2677          * value being set to the upper limit.  We check for this before
2678          * computing the new value because otherwise we could get overflow
2679          * if a privileged user specified some ridiculous increment.
2680          */
2681         if (incr > FSS_NICE_MAX)
2682                 incr = FSS_NICE_MAX;
2683 
2684         newnice = fssproc->fss_nice + incr;
2685         if (newnice > FSS_NICE_MAX)
2686                 newnice = FSS_NICE_MAX;
2687         else if (newnice < FSS_NICE_MIN)
2688                 newnice = FSS_NICE_MIN;
2689 
2690         fssparms.fss_uprilim = fssparms.fss_upri =
2691             -((newnice - NZERO) * fss_maxupri) / NZERO;
2692 
2693         /*
2694          * Reset the uprilim and upri values of the thread.
2695          */
2696         (void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL);
2697 
2698         /*
2699          * Although fss_parmsset already reset fss_nice it may not have been
2700          * set to precisely the value calculated above because fss_parmsset
2701          * determines the nice value from the user priority and we may have
2702          * truncated during the integer conversion from nice value to user
2703          * priority and back. We reset fss_nice to the value we calculated
2704          * above.
2705          */
2706         fssproc->fss_nice = (char)newnice;
2707 
2708         if (retvalp)
2709                 *retvalp = newnice - NZERO;
2710         return (0);
2711 }
2712 
2713 /*
2714  * Increment the priority of the specified thread by incr and
2715  * return the new value in *retvalp.
2716  */
2717 static int
2718 fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2719 {
2720         int newpri;
2721         fssproc_t *fssproc = FSSPROC(t);
2722         fssparms_t fssparms;
2723 
2724         /*
2725          * If there is no change to priority, just return current setting.
2726          */
2727         if (incr == 0) {
2728                 *retvalp = fssproc->fss_upri;
2729                 return (0);
2730         }
2731 
2732         newpri = fssproc->fss_upri + incr;
2733         if (newpri > fss_maxupri || newpri < -fss_maxupri)
2734                 return (EINVAL);
2735 
2736         *retvalp = newpri;
2737         fssparms.fss_uprilim = fssparms.fss_upri = newpri;
2738 
2739         /*
2740          * Reset the uprilim and upri values of the thread.
2741          */
2742         return (fss_parmsset(t, &fssparms, (id_t)0, cr));
2743 }
2744 
2745 /*
2746  * Return the global scheduling priority that would be assigned to a thread
2747  * entering the fair-sharing class with the fss_upri.
2748  */
2749 /*ARGSUSED*/
2750 static pri_t
2751 fss_globpri(kthread_t *t)
2752 {
2753         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2754 
2755         return (fss_maxumdpri / 2);
2756 }
2757 
2758 /*
2759  * Called from the yield(2) system call when a thread is yielding (surrendering)
2760  * the processor. The kernel thread is placed at the back of a dispatch queue.
2761  */
2762 static void
2763 fss_yield(kthread_t *t)
2764 {
2765         fssproc_t *fssproc = FSSPROC(t);
2766 
2767         ASSERT(t == curthread);
2768         ASSERT(THREAD_LOCK_HELD(t));
2769 
2770         /*
2771          * Collect CPU usage spent before yielding
2772          */
2773         (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2774 
2775         /*
2776          * Clear the preemption control "yield" bit since the user is
2777          * doing a yield.
2778          */
2779         if (t->t_schedctl)
2780                 schedctl_set_yield(t, 0);
2781         /*
2782          * If fss_preempt() artifically increased the thread's priority
2783          * to avoid preemption, restore the original priority now.
2784          */
2785         if (fssproc->fss_flags & FSSRESTORE) {
2786                 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2787                 fssproc->fss_flags &= ~FSSRESTORE;
2788         }
2789         if (fssproc->fss_timeleft < 0) {
2790                 /*
2791                  * Time slice was artificially extended to avoid preemption,
2792                  * so pretend we're preempting it now.
2793                  */
2794                 DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft);
2795                 fssproc->fss_timeleft = fss_quantum;
2796         }
2797         fssproc->fss_flags &= ~FSSBACKQ;
2798         setbackdq(t);
2799 }
2800 
2801 void
2802 fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf,
2803     fssbuf_t *zonebuf)
2804 {
2805         kproject_t *kpj_new = kp;
2806         zone_t *zone = zp;
2807         fssproj_t *fssproj_old, *fssproj_new;
2808         fsspset_t *fsspset;
2809         kproject_t *kpj_old;
2810         fssproc_t *fssproc;
2811         fsszone_t *fsszone_old, *fsszone_new;
2812         int free = 0;
2813         int id;
2814 
2815         ASSERT(MUTEX_HELD(&cpu_lock));
2816         ASSERT(MUTEX_HELD(&pidlock));
2817         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2818 
2819         if (t->t_cid != fss_cid)
2820                 return;
2821 
2822         fssproc = FSSPROC(t);
2823         mutex_enter(&fsspsets_lock);
2824         fssproj_old = FSSPROC2FSSPROJ(fssproc);
2825         if (fssproj_old == NULL) {
2826                 mutex_exit(&fsspsets_lock);
2827                 return;
2828         }
2829 
2830         fsspset = FSSPROJ2FSSPSET(fssproj_old);
2831         mutex_enter(&fsspset->fssps_lock);
2832         kpj_old = FSSPROJ2KPROJ(fssproj_old);
2833         fsszone_old = fssproj_old->fssp_fsszone;
2834 
2835         ASSERT(t->t_cpupart == fsspset->fssps_cpupart);
2836 
2837         if (kpj_old == kpj_new) {
2838                 mutex_exit(&fsspset->fssps_lock);
2839                 mutex_exit(&fsspsets_lock);
2840                 return;
2841         }
2842 
2843         if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) {
2844                 /*
2845                  * If the zone for the new project is not currently active on
2846                  * the cpu partition we're on, get one of the pre-allocated
2847                  * buffers and link it in our per-pset zone list.  Such buffers
2848                  * should already exist.
2849                  */
2850                 for (id = 0; id < zonebuf->fssb_size; id++) {
2851                         if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2852                                 fss_insert_fsszone(fsspset, zone, fsszone_new);
2853                                 zonebuf->fssb_list[id] = NULL;
2854                                 break;
2855                         }
2856                 }
2857         }
2858         ASSERT(fsszone_new != NULL);
2859         if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) {
2860                 /*
2861                  * If our new project is not currently running
2862                  * on the cpu partition we're on, get one of the
2863                  * pre-allocated buffers and link it in our new cpu
2864                  * partition doubly linked list. Such buffers should already
2865                  * exist.
2866                  */
2867                 for (id = 0; id < projbuf->fssb_size; id++) {
2868                         if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2869                                 fss_insert_fssproj(fsspset, kpj_new,
2870                                     fsszone_new, fssproj_new);
2871                                 projbuf->fssb_list[id] = NULL;
2872                                 break;
2873                         }
2874                 }
2875         }
2876         ASSERT(fssproj_new != NULL);
2877 
2878         thread_lock(t);
2879         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2880             t->t_state == TS_WAIT)
2881                 fss_inactive(t);
2882         ASSERT(fssproj_old->fssp_threads > 0);
2883         if (--fssproj_old->fssp_threads == 0) {
2884                 fss_remove_fssproj(fsspset, fssproj_old);
2885                 free = 1;
2886         }
2887         fssproc->fss_proj = fssproj_new;
2888         fssproc->fss_fsspri = 0;
2889         fssproj_new->fssp_threads++;
2890         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2891             t->t_state == TS_WAIT)
2892                 fss_active(t);
2893         thread_unlock(t);
2894         if (free) {
2895                 if (fsszone_old->fssz_nproj == 0)
2896                         kmem_free(fsszone_old, sizeof (fsszone_t));
2897                 kmem_free(fssproj_old, sizeof (fssproj_t));
2898         }
2899 
2900         mutex_exit(&fsspset->fssps_lock);
2901         mutex_exit(&fsspsets_lock);
2902 }
2903 
2904 void
2905 fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf,
2906     fssbuf_t *zonebuf)
2907 {
2908         fsspset_t *fsspset_old, *fsspset_new;
2909         fssproj_t *fssproj_old, *fssproj_new;
2910         fsszone_t *fsszone_old, *fsszone_new;
2911         fssproc_t *fssproc;
2912         kproject_t *kpj;
2913         zone_t *zone;
2914         int id;
2915 
2916         ASSERT(MUTEX_HELD(&cpu_lock));
2917         ASSERT(MUTEX_HELD(&pidlock));
2918         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2919 
2920         if (t->t_cid != fss_cid)
2921                 return;
2922 
2923         fssproc = FSSPROC(t);
2924         zone = ttoproc(t)->p_zone;
2925         mutex_enter(&fsspsets_lock);
2926         fssproj_old = FSSPROC2FSSPROJ(fssproc);
2927         if (fssproj_old == NULL) {
2928                 mutex_exit(&fsspsets_lock);
2929                 return;
2930         }
2931         fsszone_old = fssproj_old->fssp_fsszone;
2932         fsspset_old = FSSPROJ2FSSPSET(fssproj_old);
2933         kpj = FSSPROJ2KPROJ(fssproj_old);
2934 
2935         if (fsspset_old->fssps_cpupart == newcp) {
2936                 mutex_exit(&fsspsets_lock);
2937                 return;
2938         }
2939 
2940         ASSERT(ttoproj(t) == kpj);
2941 
2942         fsspset_new = fss_find_fsspset(newcp);
2943 
2944         mutex_enter(&fsspset_new->fssps_lock);
2945         if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) {
2946                 for (id = 0; id < zonebuf->fssb_size; id++) {
2947                         if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2948                                 fss_insert_fsszone(fsspset_new, zone,
2949                                     fsszone_new);
2950                                 zonebuf->fssb_list[id] = NULL;
2951                                 break;
2952                         }
2953                 }
2954         }
2955         ASSERT(fsszone_new != NULL);
2956         if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) {
2957                 for (id = 0; id < projbuf->fssb_size; id++) {
2958                         if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2959                                 fss_insert_fssproj(fsspset_new, kpj,
2960                                     fsszone_new, fssproj_new);
2961                                 projbuf->fssb_list[id] = NULL;
2962                                 break;
2963                         }
2964                 }
2965         }
2966         ASSERT(fssproj_new != NULL);
2967 
2968         fssproj_new->fssp_threads++;
2969         thread_lock(t);
2970         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2971             t->t_state == TS_WAIT)
2972                 fss_inactive(t);
2973         fssproc->fss_proj = fssproj_new;
2974         fssproc->fss_fsspri = 0;
2975         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2976             t->t_state == TS_WAIT)
2977                 fss_active(t);
2978         thread_unlock(t);
2979         mutex_exit(&fsspset_new->fssps_lock);
2980 
2981         mutex_enter(&fsspset_old->fssps_lock);
2982         if (--fssproj_old->fssp_threads == 0) {
2983                 fss_remove_fssproj(fsspset_old, fssproj_old);
2984                 if (fsszone_old->fssz_nproj == 0)
2985                         kmem_free(fsszone_old, sizeof (fsszone_t));
2986                 kmem_free(fssproj_old, sizeof (fssproj_t));
2987         }
2988         mutex_exit(&fsspset_old->fssps_lock);
2989 
2990         mutex_exit(&fsspsets_lock);
2991 }