1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 /*
  26  * The System Duty Cycle (SDC) scheduling class
  27  * --------------------------------------------
  28  *
  29  * Background
  30  *
  31  * Kernel threads in Solaris have traditionally not been large consumers
  32  * of CPU time.  They typically wake up, perform a small amount of
  33  * work, then go back to sleep waiting for either a timeout or another
  34  * signal.  On the assumption that the small amount of work that they do
  35  * is important for the behavior of the whole system, these threads are
  36  * treated kindly by the dispatcher and the SYS scheduling class: they run
  37  * without preemption from anything other than real-time and interrupt
  38  * threads; when preempted, they are put at the front of the queue, so they
  39  * generally do not migrate between CPUs; and they are allowed to stay
  40  * running until they voluntarily give up the CPU.
  41  *
  42  * As Solaris has evolved, new workloads have emerged which require the
  43  * kernel to perform significant amounts of CPU-intensive work.  One
  44  * example of such a workload is ZFS's transaction group sync processing.
  45  * Each sync operation generates a large batch of I/Os, and each I/O
  46  * may need to be compressed and/or checksummed before it is written to
  47  * storage.  The taskq threads which perform the compression and checksums
  48  * will run nonstop as long as they have work to do; a large sync operation
  49  * on a compression-heavy dataset can keep them busy for seconds on end.
  50  * This causes human-time-scale dispatch latency bubbles for any other
  51  * threads which have the misfortune to share a CPU with the taskq threads.
  52  *
  53  * The SDC scheduling class is a solution to this problem.
  54  *
  55  *
  56  * Overview
  57  *
  58  * SDC is centered around the concept of a thread's duty cycle (DC):
  59  *
  60  *                            ONPROC time
  61  *      Duty Cycle =    ----------------------
  62  *                      ONPROC + Runnable time
  63  *
  64  * This is the ratio of the time that the thread spent running on a CPU
  65  * divided by the time it spent running or trying to run.  It is unaffected
  66  * by any time the thread spent sleeping, stopped, etc.
  67  *
  68  * A thread joining the SDC class specifies a "target" DC that it wants
  69  * to run at.  To implement this policy, the routine sysdc_update() scans
  70  * the list of active SDC threads every few ticks and uses each thread's
  71  * microstate data to compute the actual duty cycle that that thread
  72  * has experienced recently.  If the thread is under its target DC, its
  73  * priority is increased to the maximum available (sysdc_maxpri, which is
  74  * 99 by default).  If the thread is over its target DC, its priority is
  75  * reduced to the minimum available (sysdc_minpri, 0 by default).  This
  76  * is a fairly primitive approach, in that it doesn't use any of the
  77  * intermediate priorities, but it's not completely inappropriate.  Even
  78  * though threads in the SDC class might take a while to do their job, they
  79  * are by some definition important if they're running inside the kernel,
  80  * so it is reasonable that they should get to run at priority 99.
  81  *
  82  * If a thread is running when sysdc_update() calculates its actual duty
  83  * cycle, and there are other threads of equal or greater priority on its
  84  * CPU's dispatch queue, sysdc_update() preempts that thread.  The thread
  85  * acknowledges the preemption by calling sysdc_preempt(), which calls
  86  * setbackdq(), which gives other threads with the same priority a chance
  87  * to run.  This creates a de facto time quantum for threads in the SDC
  88  * scheduling class.
  89  *
  90  * An SDC thread which is assigned priority 0 can continue to run if
  91  * nothing else needs to use the CPU that it's running on.  Similarly, an
  92  * SDC thread at priority 99 might not get to run as much as it wants to
  93  * if there are other priority-99 or higher threads on its CPU.  These
  94  * situations would cause the thread to get ahead of or behind its target
  95  * DC; the longer the situations lasted, the further ahead or behind the
  96  * thread would get.  Rather than condemning a thread to a lifetime of
  97  * paying for its youthful indiscretions, SDC keeps "base" values for
  98  * ONPROC and Runnable times in each thread's sysdc data, and updates these
  99  * values periodically.  The duty cycle is then computed using the elapsed
 100  * amount of ONPROC and Runnable times since those base times.
 101  *
 102  * Since sysdc_update() scans SDC threads fairly frequently, it tries to
 103  * keep the list of "active" threads small by pruning out threads which
 104  * have been asleep for a brief time.  They are not pruned immediately upon
 105  * going to sleep, since some threads may bounce back and forth between
 106  * sleeping and being runnable.
 107  *
 108  *
 109  * Interfaces
 110  *
 111  * void sysdc_thread_enter(t, dc, flags)
 112  *
 113  *      Moves a kernel thread from the SYS scheduling class to the
 114  *      SDC class. t must have an associated LWP (created by calling
 115  *      lwp_kernel_create()).  The thread will have a target DC of dc.
 116  *      Flags should be either 0 or SYSDC_THREAD_BATCH.  If
 117  *      SYSDC_THREAD_BATCH is specified, the thread is expected to be
 118  *      doing large amounts of processing.
 119  *
 120  *
 121  * Complications
 122  *
 123  * - Run queue balancing
 124  *
 125  *      The Solaris dispatcher is biased towards letting a thread run
 126  *      on the same CPU which it last ran on, if no more than 3 ticks
 127  *      (i.e. rechoose_interval) have passed since the thread last ran.
 128  *      This helps to preserve cache warmth.  On the other hand, it also
 129  *      tries to keep the per-CPU run queues fairly balanced; if the CPU
 130  *      chosen for a runnable thread has a run queue which is three or
 131  *      more threads longer than a neighboring CPU's queue, the runnable
 132  *      thread is dispatched onto the neighboring CPU instead.
 133  *
 134  *      These policies work well for some workloads, but not for many SDC
 135  *      threads.  The taskq client of SDC, for example, has many discrete
 136  *      units of work to do.  The work units are largely independent, so
 137  *      cache warmth is not an important consideration.  It is important
 138  *      that the threads fan out quickly to different CPUs, since the
 139  *      amount of work these threads have to do (a few seconds worth at a
 140  *      time) doesn't leave much time to correct thread placement errors
 141  *      (i.e. two SDC threads being dispatched to the same CPU).
 142  *
 143  *      To fix this, SDC uses the TS_RUNQMATCH flag introduced for FSS.
 144  *      This tells the dispatcher to keep neighboring run queues' lengths
 145  *      more evenly matched, which allows SDC threads to migrate more
 146  *      easily.
 147  *
 148  * - LWPs and system processes
 149  *
 150  *      SDC can only be used for kernel threads.  Since SDC uses microstate
 151  *      accounting data to compute each thread's actual duty cycle, all
 152  *      threads entering the SDC class must have associated LWPs (which
 153  *      store the microstate data).  This means that the threads have to
 154  *      be associated with an SSYS process, i.e. one created by newproc().
 155  *      If the microstate accounting information is ever moved into the
 156  *      kthread_t, this restriction could be lifted.
 157  *
 158  * - Dealing with oversubscription
 159  *
 160  *      Since SDC duty cycles are per-thread, it is possible that the
 161  *      aggregate requested duty cycle of all SDC threads in a processor
 162  *      set could be greater than the total CPU time available in that set.
 163  *      The FSS scheduling class has an analogous situation, which it deals
 164  *      with by reducing each thread's allotted CPU time proportionally.
 165  *      Since SDC doesn't need to be as precise as FSS, it uses a simpler
 166  *      solution to the oversubscription problem.
 167  *
 168  *      sysdc_update() accumulates the amount of time that max-priority SDC
 169  *      threads have spent on-CPU in each processor set, and uses that sum
 170  *      to create an implied duty cycle for that processor set:
 171  *
 172  *                              accumulated CPU time
 173  *         pset DC =    -----------------------------------
 174  *                       (# CPUs) * time since last update
 175  *
 176  *      If this implied duty cycle is above a maximum pset duty cycle (90%
 177  *      by default), sysdc_update() sets the priority of all SDC threads
 178  *      in that processor set to sysdc_minpri for a "break" period.  After
 179  *      the break period, it waits for a "nobreak" period before trying to
 180  *      enforce the pset duty cycle limit again.
 181  *
 182  * - Processor sets
 183  *
 184  *      As the above implies, SDC is processor set aware, but it does not
 185  *      currently allow threads to change processor sets while in the SDC
 186  *      class.  Instead, those threads must join the desired processor set
 187  *      before entering SDC. [1]
 188  *
 189  * - Batch threads
 190  *
 191  *      A thread joining the SDC class can specify the SDC_THREAD_BATCH
 192  *      flag.  This flag currently has no effect, but marks threads which
 193  *      do bulk processing.
 194  *
 195  * - t_kpri_req
 196  *
 197  *      The TS and FSS scheduling classes pay attention to t_kpri_req,
 198  *      which provides a simple form of priority inheritance for
 199  *      synchronization primitives (such as rwlocks held as READER) which
 200  *      cannot be traced to a unique thread.  The SDC class does not honor
 201  *      t_kpri_req, for a few reasons:
 202  *
 203  *      1.  t_kpri_req is notoriously inaccurate.  A measure of its
 204  *          inaccuracy is that it needs to be cleared every time a thread
 205  *          returns to user mode, because it is frequently non-zero at that
 206  *          point.  This can happen because "ownership" of synchronization
 207  *          primitives that use t_kpri_req can be silently handed off,
 208  *          leaving no opportunity to will the t_kpri_req inheritance.
 209  *
 210  *      2.  Unlike in TS and FSS, threads in SDC *will* eventually run at
 211  *          kernel priority.  This means that even if an SDC thread
 212  *          is holding a synchronization primitive and running at low
 213  *          priority, its priority will eventually be raised above 60,
 214  *          allowing it to drive on and release the resource.
 215  *
 216  *      3.  The first consumer of SDC uses the taskq subsystem, which holds
 217  *          a reader lock for the duration of the task's execution.  This
 218  *          would mean that SDC threads would never drop below kernel
 219  *          priority in practice, which defeats one of the purposes of SDC.
 220  *
 221  * - Why not FSS?
 222  *
 223  *      It might seem that the existing FSS scheduling class could solve
 224  *      the problems that SDC is attempting to solve.  FSS's more precise
 225  *      solution to the oversubscription problem would hardly cause
 226  *      trouble, as long as it performed well.  SDC is implemented as
 227  *      a separate scheduling class for two main reasons: the initial
 228  *      consumer of SDC does not map well onto the "project" abstraction
 229  *      that is central to FSS, and FSS does not expect to run at kernel
 230  *      priorities.
 231  *
 232  *
 233  * Tunables
 234  *
 235  * - sysdc_update_interval_msec:  Number of milliseconds between
 236  *      consecutive thread priority updates.
 237  *
 238  * - sysdc_reset_interval_msec:  Number of milliseconds between
 239  *      consecutive resets of a thread's base ONPROC and Runnable
 240  *      times.
 241  *
 242  * - sysdc_prune_interval_msec:  Number of milliseconds of sleeping
 243  *      before a thread is pruned from the active list.
 244  *
 245  * - sysdc_max_pset_DC:  Allowable percentage of a processor set's
 246  *      CPU time which SDC can give to its high-priority threads.
 247  *
 248  * - sysdc_break_msec:  Number of milliseconds of "break" taken when
 249  *      sysdc_max_pset_DC is exceeded.
 250  *
 251  *
 252  * Future work (in SDC and related subsystems)
 253  *
 254  * - Per-thread rechoose interval (0 for SDC)
 255  *
 256  *      Allow each thread to specify its own rechoose interval.  SDC
 257  *      threads would specify an interval of zero, which would rechoose
 258  *      the CPU with the lowest priority once per update.
 259  *
 260  * - Allow threads to change processor sets after joining the SDC class
 261  *
 262  * - Thread groups and per-group DC
 263  *
 264  *      It might be nice to be able to specify a duty cycle which applies
 265  *      to a group of threads in aggregate.
 266  *
 267  * - Per-group DC callback to allow dynamic DC tuning
 268  *
 269  *      Currently, DCs are assigned when the thread joins SDC.  Some
 270  *      workloads could benefit from being able to tune their DC using
 271  *      subsystem-specific knowledge about the workload.
 272  *
 273  * - Finer-grained priority updates
 274  *
 275  * - More nuanced management of oversubscription
 276  *
 277  * - Moving other CPU-intensive threads into SDC
 278  *
 279  * - Move msacct data into kthread_t
 280  *
 281  *      This would allow kernel threads without LWPs to join SDC.
 282  *
 283  *
 284  * Footnotes
 285  *
 286  * [1] The details of doing so are left as an exercise for the reader.
 287  */
 288 
 289 #include <sys/types.h>
 290 #include <sys/sysdc.h>
 291 #include <sys/sysdc_impl.h>
 292 
 293 #include <sys/class.h>
 294 #include <sys/cmn_err.h>
 295 #include <sys/cpuvar.h>
 296 #include <sys/cpupart.h>
 297 #include <sys/debug.h>
 298 #include <sys/disp.h>
 299 #include <sys/errno.h>
 300 #include <sys/inline.h>
 301 #include <sys/kmem.h>
 302 #include <sys/modctl.h>
 303 #include <sys/schedctl.h>
 304 #include <sys/sdt.h>
 305 #include <sys/sunddi.h>
 306 #include <sys/sysmacros.h>
 307 #include <sys/systm.h>
 308 #include <sys/var.h>
 309 
 310 /*
 311  * Tunables - loaded into the internal state at module load time
 312  */
 313 uint_t          sysdc_update_interval_msec = 20;
 314 uint_t          sysdc_reset_interval_msec = 400;
 315 uint_t          sysdc_prune_interval_msec = 100;
 316 uint_t          sysdc_max_pset_DC = 90;
 317 uint_t          sysdc_break_msec = 80;
 318 
 319 /*
 320  * Internal state - constants set up by sysdc_initparam()
 321  */
 322 static clock_t  sysdc_update_ticks;     /* ticks between updates */
 323 static uint_t   sysdc_prune_updates;    /* updates asleep before pruning */
 324 static uint_t   sysdc_reset_updates;    /* # of updates before reset */
 325 static uint_t   sysdc_break_updates;    /* updates to break */
 326 static uint_t   sysdc_nobreak_updates;  /* updates to not check */
 327 static uint_t   sysdc_minDC;            /* minimum allowed DC */
 328 static uint_t   sysdc_maxDC;            /* maximum allowed DC */
 329 static pri_t    sysdc_minpri;           /* minimum allowed priority */
 330 static pri_t    sysdc_maxpri;           /* maximum allowed priority */
 331 
 332 /*
 333  * Internal state
 334  */
 335 static kmutex_t sysdc_pset_lock;        /* lock protecting pset data */
 336 static list_t   sysdc_psets;            /* list of psets with SDC threads */
 337 static uint_t   sysdc_param_init;       /* sysdc_initparam() has been called */
 338 static uint_t   sysdc_update_timeout_started; /* update timeout is active */
 339 static hrtime_t sysdc_last_update;      /* time of last sysdc_update() */
 340 static sysdc_t  sysdc_dummy;            /* used to terminate active lists */
 341 
 342 /*
 343  * Internal state - active hash table
 344  */
 345 #define SYSDC_NLISTS    8
 346 #define SYSDC_HASH(sdc) (((uintptr_t)(sdc) >> 6) & (SYSDC_NLISTS - 1))
 347 static sysdc_list_t     sysdc_active[SYSDC_NLISTS];
 348 #define SYSDC_LIST(sdc)         (&sysdc_active[SYSDC_HASH(sdc)])
 349 
 350 #ifdef DEBUG
 351 static struct {
 352         uint64_t        sysdc_update_times_asleep;
 353         uint64_t        sysdc_update_times_base_ran_backwards;
 354         uint64_t        sysdc_update_times_already_done;
 355         uint64_t        sysdc_update_times_cur_ran_backwards;
 356         uint64_t        sysdc_compute_pri_breaking;
 357         uint64_t        sysdc_activate_enter;
 358         uint64_t        sysdc_update_enter;
 359         uint64_t        sysdc_update_exited;
 360         uint64_t        sysdc_update_not_sdc;
 361         uint64_t        sysdc_update_idle;
 362         uint64_t        sysdc_update_take_break;
 363         uint64_t        sysdc_update_no_psets;
 364         uint64_t        sysdc_tick_not_sdc;
 365         uint64_t        sysdc_tick_quantum_expired;
 366         uint64_t        sysdc_thread_enter_enter;
 367 } sysdc_stats;
 368 
 369 #define SYSDC_INC_STAT(x)       (sysdc_stats.x++)
 370 #else
 371 #define SYSDC_INC_STAT(x)       ((void)0)
 372 #endif
 373 
 374 /* macros are UPPER CASE */
 375 #define HOWMANY(a, b)   howmany((a), (b))
 376 #define MSECTOTICKS(a)  HOWMANY((a) * 1000, usec_per_tick)
 377 
 378 static void
 379 sysdc_initparam(void)
 380 {
 381         uint_t sysdc_break_ticks;
 382 
 383         /* update / prune intervals */
 384         sysdc_update_ticks = MSECTOTICKS(sysdc_update_interval_msec);
 385 
 386         sysdc_prune_updates = HOWMANY(sysdc_prune_interval_msec,
 387             sysdc_update_interval_msec);
 388         sysdc_reset_updates = HOWMANY(sysdc_reset_interval_msec,
 389             sysdc_update_interval_msec);
 390 
 391         /* We must get at least a little time on CPU. */
 392         sysdc_minDC = 1;
 393         sysdc_maxDC = SYSDC_DC_MAX;
 394         sysdc_minpri = 0;
 395         sysdc_maxpri = maxclsyspri;
 396 
 397         /* break parameters */
 398         if (sysdc_max_pset_DC > SYSDC_DC_MAX) {
 399                 sysdc_max_pset_DC = SYSDC_DC_MAX;
 400         }
 401         sysdc_break_ticks = MSECTOTICKS(sysdc_break_msec);
 402         sysdc_break_updates = HOWMANY(sysdc_break_ticks, sysdc_update_ticks);
 403 
 404         /*
 405          * We want:
 406          *
 407          *      sysdc_max_pset_DC = (nobreak / (break + nobreak))
 408          *
 409          *      ==>    nobreak = sysdc_max_pset_DC * (break + nobreak)
 410          *
 411          *                          sysdc_max_pset_DC * break
 412          *      ==>    nobreak = -------------------------
 413          *                          1 - sysdc_max_pset_DC
 414          */
 415         sysdc_nobreak_updates =
 416             HOWMANY((uint64_t)sysdc_break_updates * sysdc_max_pset_DC,
 417             (SYSDC_DC_MAX - sysdc_max_pset_DC));
 418 
 419         sysdc_param_init = 1;
 420 }
 421 
 422 #undef HOWMANY
 423 #undef MSECTOTICKS
 424 
 425 #define SDC_UPDATE_INITIAL      0x1     /* for the initial update */
 426 #define SDC_UPDATE_TIMEOUT      0x2     /* from sysdc_update() */
 427 #define SDC_UPDATE_TICK         0x4     /* from sysdc_tick(), on expiry */
 428 
 429 /*
 430  * Updates the recorded times in the sdc, and returns the elapsed ONPROC
 431  * and Runnable times since the last reset.
 432  *
 433  * newO is the thread's actual ONPROC time; it's used during sysdc_update()
 434  * to track processor set usage.
 435  */
 436 static void
 437 sysdc_update_times(sysdc_t *sdc, uint_t flags,
 438     hrtime_t *O, hrtime_t *R, hrtime_t *newO)
 439 {
 440         kthread_t *const t = sdc->sdc_thread;
 441         const uint_t    initial = (flags & SDC_UPDATE_INITIAL);
 442         const uint_t    update = (flags & SDC_UPDATE_TIMEOUT);
 443         const clock_t   now = ddi_get_lbolt();
 444         uint_t          do_reset;
 445 
 446         ASSERT(THREAD_LOCK_HELD(t));
 447 
 448         *O = *R = 0;
 449 
 450         /* If we've been sleeping, we know we haven't had any ONPROC time. */
 451         if (sdc->sdc_sleep_updates != 0 &&
 452             sdc->sdc_sleep_updates != sdc->sdc_nupdates) {
 453                 *newO = sdc->sdc_last_base_O;
 454                 SYSDC_INC_STAT(sysdc_update_times_asleep);
 455                 return;
 456         }
 457 
 458         /*
 459          * If this is our first update, or we've hit the reset point,
 460          * we need to reset our base_{O,R}.  Once we've updated them, we
 461          * report O and R for the entire prior interval.
 462          */
 463         do_reset = initial;
 464         if (update) {
 465                 ++sdc->sdc_nupdates;
 466                 if ((sdc->sdc_nupdates % sysdc_reset_updates) == 0)
 467                         do_reset = 1;
 468         }
 469         if (do_reset) {
 470                 hrtime_t baseO, baseR;
 471                 if (initial) {
 472                         /*
 473                          * Start off our cycle count somewhere in the middle,
 474                          * to keep the resets from all happening at once.
 475                          *
 476                          * 4999 is a handy prime much larger than
 477                          * sysdc_reset_updates, so that we don't run into
 478                          * trouble if the resolution is a multiple of
 479                          * sysdc_reset_updates.
 480                          */
 481                         sdc->sdc_nupdates = (uint_t)((gethrtime() % 4999) %
 482                             sysdc_reset_updates);
 483                         baseO = baseR = 0;
 484                 } else {
 485                         baseO = sdc->sdc_base_O;
 486                         baseR = sdc->sdc_base_R;
 487                 }
 488 
 489                 mstate_systhread_times(t, &sdc->sdc_base_O, &sdc->sdc_base_R);
 490                 *newO = sdc->sdc_base_O;
 491 
 492                 sdc->sdc_reset = now;
 493                 sdc->sdc_pri_check = -1; /* force mismatch below */
 494 
 495                 /*
 496                  * See below for rationale.
 497                  */
 498                 if (baseO > sdc->sdc_base_O || baseR > sdc->sdc_base_R) {
 499                         SYSDC_INC_STAT(sysdc_update_times_base_ran_backwards);
 500                         baseO = sdc->sdc_base_O;
 501                         baseR = sdc->sdc_base_R;
 502                 }
 503 
 504                 /* compute based on the entire interval */
 505                 *O = (sdc->sdc_base_O - baseO);
 506                 *R = (sdc->sdc_base_R - baseR);
 507                 return;
 508         }
 509 
 510         /*
 511          * If we're called from sysdc_update(), we *must* return a value
 512          * for newO, so we always call mstate_systhread_times().
 513          *
 514          * Otherwise, if we've already done a pri check this tick,
 515          * we can skip it.
 516          */
 517         if (!update && sdc->sdc_pri_check == now) {
 518                 SYSDC_INC_STAT(sysdc_update_times_already_done);
 519                 return;
 520         }
 521 
 522         /* Get the current times from the thread */
 523         sdc->sdc_pri_check = now;
 524         mstate_systhread_times(t, &sdc->sdc_cur_O, &sdc->sdc_cur_R);
 525         *newO = sdc->sdc_cur_O;
 526 
 527         /*
 528          * The updating of microstate accounting is not done under a
 529          * consistent set of locks, particularly the t_waitrq field.  This
 530          * can lead to narrow windows in which we account for time in the
 531          * wrong bucket, which on the next read will be accounted for
 532          * correctly.
 533          *
 534          * If our sdc_base_* fields were affected by one of these blips, we
 535          * throw away the old data, and pretend this tick didn't happen.
 536          */
 537         if (sdc->sdc_cur_O < sdc->sdc_base_O ||
 538             sdc->sdc_cur_R < sdc->sdc_base_R) {
 539 
 540                 sdc->sdc_base_O = sdc->sdc_cur_O;
 541                 sdc->sdc_base_R = sdc->sdc_cur_R;
 542 
 543                 SYSDC_INC_STAT(sysdc_update_times_cur_ran_backwards);
 544                 return;
 545         }
 546 
 547         *O = sdc->sdc_cur_O - sdc->sdc_base_O;
 548         *R = sdc->sdc_cur_R - sdc->sdc_base_R;
 549 }
 550 
 551 /*
 552  * sysdc_compute_pri()
 553  *
 554  *      Recomputes the priority of the thread, leaving the result in
 555  *      sdc->sdc_epri.  Returns 1 if a priority update should occur
 556  *      (which will also trigger a cpu_surrender()), otherwise
 557  *      returns 0.
 558  */
 559 static uint_t
 560 sysdc_compute_pri(sysdc_t *sdc, uint_t flags)
 561 {
 562         kthread_t *const t = sdc->sdc_thread;
 563         const uint_t    update = (flags & SDC_UPDATE_TIMEOUT);
 564         const uint_t    tick = (flags & SDC_UPDATE_TICK);
 565 
 566         hrtime_t        O, R;
 567         hrtime_t        newO = -1;
 568 
 569         ASSERT(THREAD_LOCK_HELD(t));
 570 
 571         sysdc_update_times(sdc, flags, &O, &R, &newO);
 572         ASSERT(!update || newO != -1);
 573 
 574         /* If we have new data, recompute our priority. */
 575         if ((O + R) != 0) {
 576                 sdc->sdc_cur_DC = (O * SYSDC_DC_MAX) / (O + R);
 577 
 578                 /* Adjust our priority to move our DC closer to the target. */
 579                 if (sdc->sdc_cur_DC < sdc->sdc_target_DC)
 580                         sdc->sdc_pri = sdc->sdc_maxpri;
 581                 else
 582                         sdc->sdc_pri = sdc->sdc_minpri;
 583         }
 584 
 585         /*
 586          * If our per-pset duty cycle goes over the max, we will take a break.
 587          * This forces all sysdc threads in the pset to minimum priority, in
 588          * order to let everyone else have a chance at the CPU.
 589          */
 590         if (sdc->sdc_pset->sdp_need_break) {
 591                 SYSDC_INC_STAT(sysdc_compute_pri_breaking);
 592                 sdc->sdc_epri = sdc->sdc_minpri;
 593         } else {
 594                 sdc->sdc_epri = sdc->sdc_pri;
 595         }
 596 
 597         DTRACE_PROBE4(sysdc__compute__pri,
 598             kthread_t *, t, pri_t, sdc->sdc_epri, uint_t, sdc->sdc_cur_DC,
 599             uint_t, sdc->sdc_target_DC);
 600 
 601         /*
 602          * For sysdc_update(), we compute the ONPROC time for high-priority
 603          * threads, which is used to calculate the per-pset duty cycle.  We
 604          * will always tell our callers to update the thread's priority,
 605          * since we want to force a cpu_surrender().
 606          *
 607          * We reset sdc_update_ticks so that sysdc_tick() will only update
 608          * the thread's priority if our timeout is delayed by a tick or
 609          * more.
 610          */
 611         if (update) {
 612                 /* SDC threads are not allowed to change cpupart bindings. */
 613                 ASSERT(t->t_cpupart == sdc->sdc_pset->sdp_cpupart);
 614 
 615                 /* If we were at MAXPRI, account for our onproc time. */
 616                 if (t->t_pri == sdc->sdc_maxpri &&
 617                     sdc->sdc_last_base_O != 0 &&
 618                     sdc->sdc_last_base_O < newO) {
 619                         sdc->sdc_last_O = newO - sdc->sdc_last_base_O;
 620                         sdc->sdc_pset->sdp_onproc_time +=
 621                             (uint64_t)sdc->sdc_last_O;
 622                         sdc->sdc_pset->sdp_onproc_threads++;
 623                 } else {
 624                         sdc->sdc_last_O = 0;
 625                 }
 626                 sdc->sdc_last_base_O = newO;
 627 
 628                 sdc->sdc_update_ticks = sdc->sdc_ticks + sysdc_update_ticks + 1;
 629                 return (1);
 630         }
 631 
 632         /*
 633          * Like sysdc_update(), sysdc_tick() always wants to update the
 634          * thread's priority, so that the CPU is surrendered if necessary.
 635          * We reset sdc_update_ticks so that if the timeout continues to be
 636          * delayed, we'll update at the regular interval.
 637          */
 638         if (tick) {
 639                 ASSERT(sdc->sdc_ticks == sdc->sdc_update_ticks);
 640                 sdc->sdc_update_ticks = sdc->sdc_ticks + sysdc_update_ticks;
 641                 return (1);
 642         }
 643 
 644         /*
 645          * Otherwise, only tell our callers to update the priority if it has
 646          * changed.
 647          */
 648         return (sdc->sdc_epri != t->t_pri);
 649 }
 650 
 651 static void
 652 sysdc_update_pri(sysdc_t *sdc, uint_t flags)
 653 {
 654         kthread_t *t = sdc->sdc_thread;
 655 
 656         ASSERT(THREAD_LOCK_HELD(t));
 657 
 658         if (sysdc_compute_pri(sdc, flags)) {
 659                 if (!thread_change_pri(t, sdc->sdc_epri, 0)) {
 660                         cpu_surrender(t);
 661                 }
 662         }
 663 }
 664 
 665 /*
 666  * Add a thread onto the active list.  It will only be removed by
 667  * sysdc_update().
 668  */
 669 static void
 670 sysdc_activate(sysdc_t *sdc)
 671 {
 672         sysdc_t *volatile *headp = &SYSDC_LIST(sdc)->sdl_list;
 673         sysdc_t         *head;
 674         kthread_t       *t = sdc->sdc_thread;
 675 
 676         SYSDC_INC_STAT(sysdc_activate_enter);
 677 
 678         ASSERT(sdc->sdc_next == NULL);
 679         ASSERT(THREAD_LOCK_HELD(t));
 680 
 681         do {
 682                 head = *headp;
 683                 sdc->sdc_next = head;
 684         } while (atomic_cas_ptr(headp, head, sdc) != head);
 685 }
 686 
 687 /*
 688  * sysdc_update() has two jobs:
 689  *
 690  *      1. It updates the priorities of all active SDC threads on the system.
 691  *      2. It measures pset CPU usage and enforces sysdc_max_pset_DC.
 692  */
 693 static void
 694 sysdc_update(void *arg)
 695 {
 696         int             idx;
 697         sysdc_t         *freelist = NULL;
 698         sysdc_pset_t    *cur;
 699         hrtime_t        now, diff;
 700         uint_t          redeploy = 1;
 701 
 702         SYSDC_INC_STAT(sysdc_update_enter);
 703 
 704         ASSERT(sysdc_update_timeout_started);
 705 
 706         /*
 707          * If this is our first time through, diff will be gigantic, and
 708          * no breaks will be necessary.
 709          */
 710         now = gethrtime();
 711         diff = now - sysdc_last_update;
 712         sysdc_last_update = now;
 713 
 714         mutex_enter(&sysdc_pset_lock);
 715         for (cur = list_head(&sysdc_psets); cur != NULL;
 716             cur = list_next(&sysdc_psets, cur)) {
 717                 boolean_t breaking = (cur->sdp_should_break != 0);
 718 
 719                 if (cur->sdp_need_break != breaking) {
 720                         DTRACE_PROBE2(sdc__pset__break, sysdc_pset_t *, cur,
 721                             boolean_t, breaking);
 722                 }
 723                 cur->sdp_onproc_time = 0;
 724                 cur->sdp_onproc_threads = 0;
 725                 cur->sdp_need_break = breaking;
 726         }
 727         mutex_exit(&sysdc_pset_lock);
 728 
 729         for (idx = 0; idx < SYSDC_NLISTS; idx++) {
 730                 sysdc_list_t            *sdl = &sysdc_active[idx];
 731                 sysdc_t *volatile       *headp = &sdl->sdl_list;
 732                 sysdc_t                 *head, *tail;
 733                 sysdc_t                 **prevptr;
 734 
 735                 if (*headp == &sysdc_dummy)
 736                         continue;
 737 
 738                 /* Prevent any threads from exiting while we're poking them. */
 739                 mutex_enter(&sdl->sdl_lock);
 740 
 741                 /*
 742                  * Each sdl_list contains a singly-linked list of active
 743                  * threads. Threads which become active while we are
 744                  * processing the list will be added to sdl_list.  Since we
 745                  * don't want that to interfere with our own processing, we
 746                  * swap in an empty list.  Any newly active threads will
 747                  * go on to this empty list.  When finished, we'll put any
 748                  * such threads at the end of the processed list.
 749                  */
 750                 head = atomic_swap_ptr(headp, &sysdc_dummy);
 751                 prevptr = &head;
 752                 while (*prevptr != &sysdc_dummy) {
 753                         sysdc_t         *const  sdc = *prevptr;
 754                         kthread_t       *const  t = sdc->sdc_thread;
 755 
 756                         /*
 757                          * If the thread has exited, move its sysdc_t onto
 758                          * freelist, to be freed later.
 759                          */
 760                         if (t == NULL) {
 761                                 *prevptr = sdc->sdc_next;
 762                                 SYSDC_INC_STAT(sysdc_update_exited);
 763                                 sdc->sdc_next = freelist;
 764                                 freelist = sdc;
 765                                 continue;
 766                         }
 767 
 768                         thread_lock(t);
 769                         if (t->t_cid != sysdccid) {
 770                                 thread_unlock(t);
 771                                 prevptr = &sdc->sdc_next;
 772                                 SYSDC_INC_STAT(sysdc_update_not_sdc);
 773                                 continue;
 774                         }
 775                         ASSERT(t->t_cldata == sdc);
 776 
 777                         /*
 778                          * If the thread has been sleeping for longer
 779                          * than sysdc_prune_interval, make it inactive by
 780                          * removing it from the list.
 781                          */
 782                         if (!(t->t_state & (TS_RUN | TS_ONPROC)) &&
 783                             sdc->sdc_sleep_updates != 0 &&
 784                             (sdc->sdc_sleep_updates - sdc->sdc_nupdates) >
 785                             sysdc_prune_updates) {
 786                                 *prevptr = sdc->sdc_next;
 787                                 SYSDC_INC_STAT(sysdc_update_idle);
 788                                 sdc->sdc_next = NULL;
 789                                 thread_unlock(t);
 790                                 continue;
 791                         }
 792                         sysdc_update_pri(sdc, SDC_UPDATE_TIMEOUT);
 793                         thread_unlock(t);
 794 
 795                         prevptr = &sdc->sdc_next;
 796                 }
 797 
 798                 /*
 799                  * Add our list to the bucket, putting any new entries
 800                  * added while we were working at the tail of the list.
 801                  */
 802                 do {
 803                         tail = *headp;
 804                         *prevptr = tail;
 805                 } while (atomic_cas_ptr(headp, tail, head) != tail);
 806 
 807                 mutex_exit(&sdl->sdl_lock);
 808         }
 809 
 810         mutex_enter(&sysdc_pset_lock);
 811         for (cur = list_head(&sysdc_psets); cur != NULL;
 812             cur = list_next(&sysdc_psets, cur)) {
 813 
 814                 cur->sdp_vtime_last_interval =
 815                     diff * cur->sdp_cpupart->cp_ncpus;
 816                 cur->sdp_DC_last_interval =
 817                     (cur->sdp_onproc_time * SYSDC_DC_MAX) /
 818                     cur->sdp_vtime_last_interval;
 819 
 820                 if (cur->sdp_should_break > 0) {
 821                         cur->sdp_should_break--;     /* breaking */
 822                         continue;
 823                 }
 824                 if (cur->sdp_dont_break > 0) {
 825                         cur->sdp_dont_break--;       /* waiting before checking */
 826                         continue;
 827                 }
 828                 if (cur->sdp_DC_last_interval > sysdc_max_pset_DC) {
 829                         cur->sdp_should_break = sysdc_break_updates;
 830                         cur->sdp_dont_break = sysdc_nobreak_updates;
 831                         SYSDC_INC_STAT(sysdc_update_take_break);
 832                 }
 833         }
 834 
 835         /*
 836          * If there are no sysdc_psets, there can be no threads, so
 837          * we can stop doing our timeout.  Since we're holding the
 838          * sysdc_pset_lock, no new sysdc_psets can come in, which will
 839          * prevent anyone from racing with this and dropping our timeout
 840          * on the floor.
 841          */
 842         if (list_is_empty(&sysdc_psets)) {
 843                 SYSDC_INC_STAT(sysdc_update_no_psets);
 844                 ASSERT(sysdc_update_timeout_started);
 845                 sysdc_update_timeout_started = 0;
 846 
 847                 redeploy = 0;
 848         }
 849         mutex_exit(&sysdc_pset_lock);
 850 
 851         while (freelist != NULL) {
 852                 sysdc_t *cur = freelist;
 853                 freelist = cur->sdc_next;
 854                 kmem_free(cur, sizeof (*cur));
 855         }
 856 
 857         if (redeploy) {
 858                 (void) timeout(sysdc_update, arg, sysdc_update_ticks);
 859         }
 860 }
 861 
 862 static void
 863 sysdc_preempt(kthread_t *t)
 864 {
 865         ASSERT(t == curthread);
 866         ASSERT(THREAD_LOCK_HELD(t));
 867 
 868         setbackdq(t);           /* give others a chance to run */
 869 }
 870 
 871 static void
 872 sysdc_tick(kthread_t *t)
 873 {
 874         sysdc_t *sdc;
 875 
 876         thread_lock(t);
 877         if (t->t_cid != sysdccid) {
 878                 SYSDC_INC_STAT(sysdc_tick_not_sdc);
 879                 thread_unlock(t);
 880                 return;
 881         }
 882         sdc = t->t_cldata;
 883         if (t->t_state == TS_ONPROC &&
 884             t->t_pri < t->t_disp_queue->disp_maxrunpri) {
 885                 cpu_surrender(t);
 886         }
 887 
 888         if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
 889                 ASSERT(sdc->sdc_sleep_updates == 0);
 890         }
 891 
 892         ASSERT(sdc->sdc_ticks != sdc->sdc_update_ticks);
 893         sdc->sdc_ticks++;
 894         if (sdc->sdc_ticks == sdc->sdc_update_ticks) {
 895                 SYSDC_INC_STAT(sysdc_tick_quantum_expired);
 896                 sysdc_update_pri(sdc, SDC_UPDATE_TICK);
 897                 ASSERT(sdc->sdc_ticks != sdc->sdc_update_ticks);
 898         }
 899         thread_unlock(t);
 900 }
 901 
 902 static void
 903 sysdc_setrun(kthread_t *t)
 904 {
 905         sysdc_t *sdc = t->t_cldata;
 906 
 907         ASSERT(THREAD_LOCK_HELD(t));    /* t should be in transition */
 908 
 909         sdc->sdc_sleep_updates = 0;
 910 
 911         if (sdc->sdc_next == NULL) {
 912                 /*
 913                  * Since we're in transition, we don't want to use the
 914                  * full thread_update_pri().
 915                  */
 916                 if (sysdc_compute_pri(sdc, 0)) {
 917                         THREAD_CHANGE_PRI(t, sdc->sdc_epri);
 918                 }
 919                 sysdc_activate(sdc);
 920 
 921                 ASSERT(sdc->sdc_next != NULL);
 922         }
 923 
 924         setbackdq(t);
 925 }
 926 
 927 static void
 928 sysdc_wakeup(kthread_t *t)
 929 {
 930         sysdc_setrun(t);
 931 }
 932 
 933 static void
 934 sysdc_sleep(kthread_t *t)
 935 {
 936         sysdc_t *sdc = t->t_cldata;
 937 
 938         ASSERT(THREAD_LOCK_HELD(t));    /* t should be in transition */
 939 
 940         sdc->sdc_sleep_updates = sdc->sdc_nupdates;
 941 }
 942 
 943 /*ARGSUSED*/
 944 static int
 945 sysdc_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
 946     void *bufp)
 947 {
 948         cpupart_t *const cpupart = t->t_cpupart;
 949         sysdc_t *sdc = bufp;
 950         sysdc_params_t *sdpp = parmsp;
 951         sysdc_pset_t *newpset = sdc->sdc_pset;
 952         sysdc_pset_t *pset;
 953         int start_timeout;
 954 
 955         if (t->t_cid != syscid)
 956                 return (EPERM);
 957 
 958         ASSERT(ttolwp(t) != NULL);
 959         ASSERT(sdpp != NULL);
 960         ASSERT(newpset != NULL);
 961         ASSERT(sysdc_param_init);
 962 
 963         ASSERT(sdpp->sdp_minpri >= sysdc_minpri);
 964         ASSERT(sdpp->sdp_maxpri <= sysdc_maxpri);
 965         ASSERT(sdpp->sdp_DC >= sysdc_minDC);
 966         ASSERT(sdpp->sdp_DC <= sysdc_maxDC);
 967 
 968         sdc->sdc_thread = t;
 969         sdc->sdc_pri = sdpp->sdp_maxpri;  /* start off maximally */
 970         sdc->sdc_minpri = sdpp->sdp_minpri;
 971         sdc->sdc_maxpri = sdpp->sdp_maxpri;
 972         sdc->sdc_target_DC = sdpp->sdp_DC;
 973         sdc->sdc_ticks = 0;
 974         sdc->sdc_update_ticks = sysdc_update_ticks + 1;
 975 
 976         /* Assign ourselves to the appropriate pset. */
 977         sdc->sdc_pset = NULL;
 978         mutex_enter(&sysdc_pset_lock);
 979         for (pset = list_head(&sysdc_psets); pset != NULL;
 980             pset = list_next(&sysdc_psets, pset)) {
 981                 if (pset->sdp_cpupart == cpupart) {
 982                         break;
 983                 }
 984         }
 985         if (pset == NULL) {
 986                 pset = newpset;
 987                 newpset = NULL;
 988                 pset->sdp_cpupart = cpupart;
 989                 list_insert_tail(&sysdc_psets, pset);
 990         }
 991         pset->sdp_nthreads++;
 992         ASSERT(pset->sdp_nthreads > 0);
 993 
 994         sdc->sdc_pset = pset;
 995 
 996         start_timeout = (sysdc_update_timeout_started == 0);
 997         sysdc_update_timeout_started = 1;
 998         mutex_exit(&sysdc_pset_lock);
 999 
1000         if (newpset != NULL)
1001                 kmem_free(newpset, sizeof (*newpset));
1002 
1003         /* Update t's scheduling class and priority. */
1004         thread_lock(t);
1005         t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
1006         t->t_cid = cid;
1007         t->t_cldata = sdc;
1008         t->t_schedflag |= TS_RUNQMATCH;
1009 
1010         sysdc_update_pri(sdc, SDC_UPDATE_INITIAL);
1011         thread_unlock(t);
1012 
1013         /* Kick off the thread timeout if we're the first one in. */
1014         if (start_timeout) {
1015                 (void) timeout(sysdc_update, NULL, sysdc_update_ticks);
1016         }
1017 
1018         return (0);
1019 }
1020 
1021 static void
1022 sysdc_leave(sysdc_t *sdc)
1023 {
1024         sysdc_pset_t *sdp = sdc->sdc_pset;
1025         sysdc_list_t *sdl = SYSDC_LIST(sdc);
1026         uint_t freedc;
1027 
1028         mutex_enter(&sdl->sdl_lock);             /* block sysdc_update() */
1029         sdc->sdc_thread = NULL;
1030         freedc = (sdc->sdc_next == NULL);
1031         mutex_exit(&sdl->sdl_lock);
1032 
1033         mutex_enter(&sysdc_pset_lock);
1034         ASSERT(sdp != NULL);
1035         ASSERT(sdp->sdp_nthreads > 0);
1036         --sdp->sdp_nthreads;
1037         if (sdp->sdp_nthreads == 0) {
1038                 list_remove(&sysdc_psets, sdp);
1039         } else {
1040                 sdp = NULL;
1041         }
1042         mutex_exit(&sysdc_pset_lock);
1043 
1044         if (freedc)
1045                 kmem_free(sdc, sizeof (*sdc));
1046         if (sdp != NULL)
1047                 kmem_free(sdp, sizeof (*sdp));
1048 }
1049 
1050 static void
1051 sysdc_exitclass(void *buf)
1052 {
1053         sysdc_leave((sysdc_t *)buf);
1054 }
1055 
1056 /*ARGSUSED*/
1057 static int
1058 sysdc_canexit(kthread_t *t, cred_t *reqpcredp)
1059 {
1060         /* Threads cannot exit SDC once joined, except in a body bag. */
1061         return (EPERM);
1062 }
1063 
1064 static void
1065 sysdc_exit(kthread_t *t)
1066 {
1067         sysdc_t *sdc;
1068 
1069         /* We're exiting, so we just rejoin the SYS class. */
1070         thread_lock(t);
1071         ASSERT(t->t_cid == sysdccid);
1072         sdc = t->t_cldata;
1073         t->t_cid = syscid;
1074         t->t_cldata = NULL;
1075         t->t_clfuncs = &(sclass[syscid].cl_funcs->thread);
1076         (void) thread_change_pri(t, maxclsyspri, 0);
1077         t->t_schedflag &= ~TS_RUNQMATCH;
1078         thread_unlock_nopreempt(t);
1079 
1080         /* Unlink the sdc from everything. */
1081         sysdc_leave(sdc);
1082 }
1083 
1084 /*ARGSUSED*/
1085 static int
1086 sysdc_fork(kthread_t *t, kthread_t *ct, void *bufp)
1087 {
1088         /*
1089          * Threads cannot be created with SDC as their class; they must
1090          * be created as SYS and then added with sysdc_thread_enter().
1091          * Because of this restriction, sysdc_fork() should never be called.
1092          */
1093         panic("sysdc cannot be forked");
1094 
1095         return (ENOSYS);
1096 }
1097 
1098 /*ARGSUSED*/
1099 static void
1100 sysdc_forkret(kthread_t *t, kthread_t *ct)
1101 {
1102         /* SDC threads are part of system processes, which never fork. */
1103         panic("sysdc cannot be forked");
1104 }
1105 
1106 static pri_t
1107 sysdc_globpri(kthread_t *t)
1108 {
1109         return (t->t_epri);
1110 }
1111 
1112 /*ARGSUSED*/
1113 static pri_t
1114 sysdc_no_swap(kthread_t *t, int flags)
1115 {
1116         /* SDC threads cannot be swapped. */
1117         return (-1);
1118 }
1119 
1120 /*
1121  * Get maximum and minimum priorities enjoyed by SDC threads.
1122  */
1123 static int
1124 sysdc_getclpri(pcpri_t *pcprip)
1125 {
1126         pcprip->pc_clpmax = sysdc_maxpri;
1127         pcprip->pc_clpmin = sysdc_minpri;
1128         return (0);
1129 }
1130 
1131 /*ARGSUSED*/
1132 static int
1133 sysdc_getclinfo(void *arg)
1134 {
1135         return (0);             /* no class-specific info */
1136 }
1137 
1138 /*ARGSUSED*/
1139 static int
1140 sysdc_alloc(void **p, int flag)
1141 {
1142         sysdc_t *new;
1143 
1144         *p = NULL;
1145         if ((new = kmem_zalloc(sizeof (*new), flag)) == NULL) {
1146                 return (ENOMEM);
1147         }
1148         if ((new->sdc_pset = kmem_zalloc(sizeof (*new->sdc_pset), flag)) ==
1149             NULL) {
1150                 kmem_free(new, sizeof (*new));
1151                 return (ENOMEM);
1152         }
1153         *p = new;
1154         return (0);
1155 }
1156 
1157 static void
1158 sysdc_free(void *p)
1159 {
1160         sysdc_t *sdc = p;
1161 
1162         if (sdc != NULL) {
1163                 /*
1164                  * We must have failed CL_ENTERCLASS(), so our pset should be
1165                  * there and unused.
1166                  */
1167                 ASSERT(sdc->sdc_pset != NULL);
1168                 ASSERT(sdc->sdc_pset->sdp_cpupart == NULL);
1169                 kmem_free(sdc->sdc_pset, sizeof (*sdc->sdc_pset));
1170                 kmem_free(sdc, sizeof (*sdc));
1171         }
1172 }
1173 
1174 static int sysdc_enosys();      /* Boy, ANSI-C's K&R compatibility is weird. */
1175 static int sysdc_einval();
1176 static void sysdc_nullsys();
1177 
1178 static struct classfuncs sysdc_classfuncs = {
1179         /* messages to class manager */
1180         {
1181                 sysdc_enosys,   /* admin */
1182                 sysdc_getclinfo,
1183                 sysdc_enosys,   /* parmsin */
1184                 sysdc_enosys,   /* parmsout */
1185                 sysdc_enosys,   /* vaparmsin */
1186                 sysdc_enosys,   /* vaparmsout */
1187                 sysdc_getclpri,
1188                 sysdc_alloc,
1189                 sysdc_free,
1190         },
1191         /* operations on threads */
1192         {
1193                 sysdc_enterclass,
1194                 sysdc_exitclass,
1195                 sysdc_canexit,
1196                 sysdc_fork,
1197                 sysdc_forkret,
1198                 sysdc_nullsys,  /* parmsget */
1199                 sysdc_enosys,   /* parmsset */
1200                 sysdc_nullsys,  /* stop */
1201                 sysdc_exit,
1202                 sysdc_nullsys,  /* active */
1203                 sysdc_nullsys,  /* inactive */
1204                 sysdc_no_swap,  /* swapin */
1205                 sysdc_no_swap,  /* swapout */
1206                 sysdc_nullsys,  /* trapret */
1207                 sysdc_preempt,
1208                 sysdc_setrun,
1209                 sysdc_sleep,
1210                 sysdc_tick,
1211                 sysdc_wakeup,
1212                 sysdc_einval,   /* donice */
1213                 sysdc_globpri,
1214                 sysdc_nullsys,  /* set_process_group */
1215                 sysdc_nullsys,  /* yield */
1216                 sysdc_einval,   /* doprio */
1217         }
1218 };
1219 
1220 static int
1221 sysdc_enosys()
1222 {
1223         return (ENOSYS);
1224 }
1225 
1226 static int
1227 sysdc_einval()
1228 {
1229         return (EINVAL);
1230 }
1231 
1232 static void
1233 sysdc_nullsys()
1234 {
1235 }
1236 
1237 /*ARGSUSED*/
1238 static pri_t
1239 sysdc_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
1240 {
1241         int idx;
1242 
1243         list_create(&sysdc_psets, sizeof (sysdc_pset_t),
1244             offsetof(sysdc_pset_t, sdp_node));
1245 
1246         for (idx = 0; idx < SYSDC_NLISTS; idx++) {
1247                 sysdc_active[idx].sdl_list = &sysdc_dummy;
1248         }
1249 
1250         sysdc_initparam();
1251 
1252         sysdccid = cid;
1253         *clfuncspp = &sysdc_classfuncs;
1254 
1255         return ((pri_t)v.v_maxsyspri);
1256 }
1257 
1258 static struct sclass csw = {
1259         "SDC",
1260         sysdc_init,
1261         0
1262 };
1263 
1264 static struct modlsched modlsched = {
1265         &mod_schedops, "system duty cycle scheduling class", &csw
1266 };
1267 
1268 static struct modlinkage modlinkage = {
1269         MODREV_1, (void *)&modlsched, NULL
1270 };
1271 
1272 int
1273 _init()
1274 {
1275         return (mod_install(&modlinkage));
1276 }
1277 
1278 int
1279 _fini()
1280 {
1281         return (EBUSY);         /* can't unload for now */
1282 }
1283 
1284 int
1285 _info(struct modinfo *modinfop)
1286 {
1287         return (mod_info(&modlinkage, modinfop));
1288 }
1289 
1290 /* --- consolidation-private interfaces --- */
1291 void
1292 sysdc_thread_enter(kthread_t *t, uint_t dc, uint_t flags)
1293 {
1294         void *buf = NULL;
1295         sysdc_params_t sdp;
1296 
1297         SYSDC_INC_STAT(sysdc_thread_enter_enter);
1298 
1299         ASSERT(sysdc_param_init);
1300         ASSERT(sysdccid >= 0);
1301 
1302         ASSERT((flags & ~SYSDC_THREAD_BATCH) == 0);
1303 
1304         sdp.sdp_minpri = sysdc_minpri;
1305         sdp.sdp_maxpri = sysdc_maxpri;
1306         sdp.sdp_DC = MAX(MIN(dc, sysdc_maxDC), sysdc_minDC);
1307 
1308         VERIFY3U(CL_ALLOC(&buf, sysdccid, KM_SLEEP), ==, 0);
1309 
1310         ASSERT(t->t_lwp != NULL);
1311         ASSERT(t->t_cid == syscid);
1312         ASSERT(t->t_cldata == NULL);
1313         VERIFY3U(CL_CANEXIT(t, NULL), ==, 0);
1314         VERIFY3U(CL_ENTERCLASS(t, sysdccid, &sdp, kcred, buf), ==, 0);
1315         CL_EXITCLASS(syscid, NULL);
1316 }