1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27 #include <sys/types.h>
28 #include <sys/param.h>
29 #include <sys/sysmacros.h>
30 #include <sys/cred.h>
31 #include <sys/proc.h>
32 #include <sys/strsubr.h>
33 #include <sys/priocntl.h>
34 #include <sys/class.h>
35 #include <sys/disp.h>
36 #include <sys/procset.h>
37 #include <sys/debug.h>
38 #include <sys/kmem.h>
39 #include <sys/errno.h>
40 #include <sys/systm.h>
41 #include <sys/schedctl.h>
42 #include <sys/vmsystm.h>
43 #include <sys/atomic.h>
44 #include <sys/project.h>
45 #include <sys/modctl.h>
46 #include <sys/fss.h>
47 #include <sys/fsspriocntl.h>
48 #include <sys/cpupart.h>
49 #include <sys/zone.h>
50 #include <vm/rm.h>
51 #include <vm/seg_kmem.h>
52 #include <sys/tnf_probe.h>
53 #include <sys/policy.h>
54 #include <sys/sdt.h>
55 #include <sys/cpucaps.h>
56
57 /*
58 * The fair share scheduling class ensures that collections of processes
59 * (zones and projects) each get their configured share of CPU. This is in
60 * contrast to the TS class which considers individual processes.
61 *
62 * The FSS cpu-share is set on zones using the zone.cpu-shares rctl and on
63 * projects using the project.cpu-shares rctl. By default the value is 1
64 * and it can range from 0 - 64k. A value of 0 means that processes in the
65 * collection will only get CPU resources when there are no other processes
66 * that need CPU. The cpu-share is used as one of the inputs to calculate a
67 * thread's "user-mode" priority (umdpri) for the scheduler. The umdpri falls
68 * in the range 0-59. FSS calculates other, internal, priorities which are not
69 * visible outside of the FSS class.
70 *
71 * The FSS class should approximate TS behavior when there are excess CPU
72 * resources. When there is a backlog of runnable processes, then the share
73 * is used as input into the runnable process's priority calculation, where
74 * the final umdpri is used by the scheduler to determine when the process runs.
75 *
76 * Projects in a zone compete with each other for CPU time, receiving CPU
77 * allocation within a zone proportional to the project's share; at a higher
78 * level zones compete with each other, receiving allocation in a pset
79 * proportional to the zone's share.
80 *
81 * The FSS priority calculation consists of several parts.
82 *
83 * 1) Once per second the fss_update function runs. The first thing it does is
84 * call fss_decay_usage. This function does three things.
85 *
86 * a) fss_decay_usage first decays the maxfsspri value for the pset. This
87 * value is used in the per-process priority calculation described in step
88 * (2b). The maxfsspri is decayed using the following formula:
89 *
90 * maxfsspri * fss_nice_decay[NZERO])
91 * maxfsspri = ------------------------------------
92 * FSS_DECAY_BASE
93 *
94 *
95 * - NZERO is the default process priority (i.e. 20)
96 *
97 * The fss_nice_decay array is a fixed set of values used to adjust the
98 * decay rate of processes based on their nice value. Entries in this
99 * array are initialized in fss_init using the following formula:
100 *
101 * (FSS_DECAY_MAX - FSS_DECAY_MIN) * i
102 * FSS_DECAY_MIN + -------------------------------------
103 * FSS_NICE_RANGE - 1
104 *
105 * - FSS_DECAY_MIN is 82 = approximates 65% (82/128)
106 * - FSS_DECAY_MAX is 108 = approximates 85% (108/128)
107 * - FSS_NICE_RANGE is 40 (range is 0 - 39)
108 *
109 * b) The second thing fss_decay_usage does is update each project's "usage"
110 * for the last second and then recalculates the project's "share usage".
111 *
112 * The usage value is the recent CPU usage for all of the threads in the
113 * project. It is decayed and updated this way:
114 *
115 * (usage * FSS_DECAY_USG)
116 * usage = ------------------------- + ticks;
117 * FSS_DECAY_BASE
118 *
119 * - FSS_DECAY_BASE is 128 - used instead of 100 so we can shift vs divide
120 * - FSS_DECAY_USG is 96 - approximates 75% (96/128)
121 * - ticks is updated whenever a process in this project is running
122 * when the scheduler's tick processing fires. This is not a simple
123 * counter, the values are based on the entries in the fss_nice_tick
124 * array (see section 3 below). ticks is then reset to 0 so it can track
125 * the next seconds worth of nice-adjusted time for the project.
126 *
127 * c) The third thing fss_decay_usage does is update each project's "share
128 * usage" (shusage). This is the normalized usage value for the project and
129 * is calculated this way:
130 *
131 * pset_shares^2 zone_int_shares^2
132 * usage * ------------- * ------------------
133 * kpj_shares^2 zone_ext_shares^2
134 *
135 * - usage - see (1b) for more details
136 * - pset_shares is the total of all *active* zone shares in the pset (by
137 * default there is only one pset)
138 * - kpj_shares is the individual project's share (project.cpu-shares rctl)
139 * - zone_int_shares is the sum of shares of all active projects within the
140 * zone (the zone-internal total)
141 * - zone_ext_shares is the share value for the zone (zone.cpu-shares rctl)
142 *
143 * The shusage is used in step (2b) to calculate the thread's new internal
144 * priority. A larger shusage value leads to a lower priority.
145 *
146 * 2) The fss_update function then calls fss_update_list to update the priority
147 * of all threads. This does two things.
148 *
149 * a) First the thread's internal priority is decayed using the following
150 * formula:
151 *
152 * fsspri * fss_nice_decay[nice_value])
153 * fsspri = ------------------------------------
154 * FSS_DECAY_BASE
155 *
156 * - FSS_DECAY_BASE is 128 as described above
157 *
158 * b) Second, if the thread is runnable (TS_RUN or TS_WAIT) calls fss_newpri
159 * to update the user-mode priority (umdpri) of the runnable thread.
160 * Threads that are running (TS_ONPROC) or waiting for an event (TS_SLEEP)
161 * are not updated at this time. The updated user-mode priority can cause
162 * threads to change their position in the run queue.
163 *
164 * The process's new internal fsspri is calculated using the following
165 * formula. All runnable threads in the project will use the same shusage
166 * and nrunnable values in their calculation.
167 *
168 * fsspri += shusage * nrunnable * ticks
169 *
170 * - shusage is the project's share usage, calculated in (1c)
171 * - nrunnable is the number of runnable threads in the project
172 * - ticks is the number of ticks this thread ran since the last fss_newpri
173 * invocation.
174 *
175 * Finally the process's new user-mode priority is calculated using the
176 * following formula:
177 *
178 * (fsspri * umdprirange)
179 * umdpri = maxumdpri - ------------------------
180 * maxfsspri
181 *
182 * - maxumdpri is MINCLSYSPRI - 1 (i.e. 59)
183 * - umdprirange is maxumdpri - 1 (i.e. 58)
184 * - maxfsspri is the largest fsspri seen so far, as we're iterating all
185 * runnable processes
186 *
187 * Thus, a higher internal priority (fsspri) leads to a lower user-mode
188 * priority which means the thread runs less. The fsspri is higher when
189 * the project's normalized share usage is higher, when the project has
190 * more runnable threads, or when the thread has accumulated more run-time.
191 *
192 * This code has various checks to ensure the resulting umdpri is in the
193 * range 1-59. See fss_newpri for more details.
194 *
195 * To reiterate, the above processing is performed once per second to recompute
196 * the runnable thread user-mode priorities.
197 *
198 * 3) The final major component in the priority calculation is the tick
199 * processing which occurs on a thread that is running when the clock
200 * calls fss_tick.
201 *
202 * A thread can run continuously in user-land (compute-bound) for the
203 * fss_quantum (see "dispadmin -c FSS -g" for the configurable properties).
204 * The fss_quantum defaults to 11 (i.e. 11 ticks).
205 *
206 * Once the quantum has been consumed, the thread will call fss_newpri to
207 * recompute its umdpri priority, as described above in (2b). Threads that
208 * were T_ONPROC at the one second interval when runnable thread priorities
209 * were recalculated will have their umdpri priority recalculated when their
210 * quanta expires.
211 *
212 * To ensure that runnable threads within a project see the expected
213 * round-robin behavior, there is a special case in fss_newpri for a thread
214 * that has run for its quanta within the one second update interval. See
215 * the handling for the quanta_up parameter within fss_newpri.
216 *
217 * Also of interest, the fss_tick code increments the project's tick value
218 * using the fss_nice_tick array entry for the thread's nice value. The idea
219 * behind the fss_nice_tick array is that the cost of a tick is lower at
220 * positive nice values (so that it doesn't increase the project's usage
221 * as much as normal) with a 50% drop at the maximum level and a 50%
222 * increase at the minimum level. See (1b). The fss_nice_tick array is
223 * initialized in fss_init using the following formula:
224 *
225 * FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2) - i)
226 * --------------------------------------------------
227 * FSS_NICE_RANGE
228 *
229 * - FSS_TICK_COST is 1000, the tick cost for threads with nice level 0
230 *
231 * FSS Data Structures:
232 *
233 * fsszone
234 * ----- -----
235 * ----- | | | |
236 * | |-------->| |<------->| |<---->...
237 * | | ----- -----
238 * | | ^ ^ ^
239 * | |--- | \ \
240 * ----- | | \ \
241 * fsspset | | \ \
242 * | | \ \
243 * | ----- ----- -----
244 * -->| |<--->| |<--->| |
245 * | | | | | |
246 * ----- ----- -----
247 * fssproj
248 *
249 * That is, fsspsets contain a list of fsszone's that are currently active in
250 * the pset, and a list of fssproj's, corresponding to projects with runnable
251 * threads on the pset. fssproj's in turn point to the fsszone which they
252 * are a member of.
253 *
254 * An fssproj_t is removed when there are no threads in it.
255 *
256 * An fsszone_t is removed when there are no projects with threads in it.
257 */
258
259 static pri_t fss_init(id_t, int, classfuncs_t **);
260
261 static struct sclass fss = {
262 "FSS",
263 fss_init,
264 0
265 };
266
267 extern struct mod_ops mod_schedops;
268
269 /*
270 * Module linkage information for the kernel.
271 */
272 static struct modlsched modlsched = {
273 &mod_schedops, "fair share scheduling class", &fss
274 };
275
276 static struct modlinkage modlinkage = {
277 MODREV_1, (void *)&modlsched, NULL
278 };
279
280 #define FSS_MAXUPRI 60
281
282 /*
283 * The fssproc_t structures are kept in an array of circular doubly linked
284 * lists. A hash on the thread pointer is used to determine which list each
285 * thread should be placed in. Each list has a dummy "head" which is never
286 * removed, so the list is never empty. fss_update traverses these lists to
287 * update the priorities of threads that have been waiting on the run queue.
288 */
289 #define FSS_LISTS 16 /* number of lists, must be power of 2 */
290 #define FSS_LIST_HASH(t) (((uintptr_t)(t) >> 9) & (FSS_LISTS - 1))
291 #define FSS_LIST_NEXT(i) (((i) + 1) & (FSS_LISTS - 1))
292
293 #define FSS_LIST_INSERT(fssproc) \
294 { \
295 int index = FSS_LIST_HASH(fssproc->fss_tp); \
296 kmutex_t *lockp = &fss_listlock[index]; \
297 fssproc_t *headp = &fss_listhead[index]; \
298 mutex_enter(lockp); \
299 fssproc->fss_next = headp->fss_next; \
300 fssproc->fss_prev = headp; \
301 headp->fss_next->fss_prev = fssproc; \
302 headp->fss_next = fssproc; \
303 mutex_exit(lockp); \
304 }
305
306 #define FSS_LIST_DELETE(fssproc) \
307 { \
308 int index = FSS_LIST_HASH(fssproc->fss_tp); \
309 kmutex_t *lockp = &fss_listlock[index]; \
310 mutex_enter(lockp); \
311 fssproc->fss_prev->fss_next = fssproc->fss_next; \
312 fssproc->fss_next->fss_prev = fssproc->fss_prev; \
313 mutex_exit(lockp); \
314 }
315
316 #define FSS_TICK_COST 1000 /* tick cost for threads with nice level = 0 */
317
318 /*
319 * Decay rate percentages are based on n/128 rather than n/100 so that
320 * calculations can avoid having to do an integer divide by 100 (divide
321 * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift).
322 *
323 * FSS_DECAY_MIN = 83/128 ~= 65%
324 * FSS_DECAY_MAX = 108/128 ~= 85%
325 * FSS_DECAY_USG = 96/128 ~= 75%
326 */
327 #define FSS_DECAY_MIN 83 /* fsspri decay pct for threads w/ nice -20 */
328 #define FSS_DECAY_MAX 108 /* fsspri decay pct for threads w/ nice +19 */
329 #define FSS_DECAY_USG 96 /* fssusage decay pct for projects */
330 #define FSS_DECAY_BASE 128 /* base for decay percentages above */
331
332 #define FSS_NICE_MIN 0
333 #define FSS_NICE_MAX (2 * NZERO - 1)
334 #define FSS_NICE_RANGE (FSS_NICE_MAX - FSS_NICE_MIN + 1)
335
336 static int fss_nice_tick[FSS_NICE_RANGE];
337 static int fss_nice_decay[FSS_NICE_RANGE];
338
339 static pri_t fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
340 static pri_t fss_maxumdpri; /* maximum user mode fss priority */
341 static pri_t fss_maxglobpri; /* maximum global priority used by fss class */
342 static pri_t fss_minglobpri; /* minimum global priority */
343
344 static fssproc_t fss_listhead[FSS_LISTS];
345 static kmutex_t fss_listlock[FSS_LISTS];
346
347 static fsspset_t *fsspsets;
348 static kmutex_t fsspsets_lock; /* protects fsspsets */
349
350 static id_t fss_cid;
351
352 static time_t fss_minrun = 2; /* t_pri becomes 59 within 2 secs */
353 static time_t fss_minslp = 2; /* min time on sleep queue for hardswap */
354 static int fss_quantum = 11;
355
356 static void fss_newpri(fssproc_t *, boolean_t);
357 static void fss_update(void *);
358 static int fss_update_list(int);
359 static void fss_change_priority(kthread_t *, fssproc_t *);
360
361 static int fss_admin(caddr_t, cred_t *);
362 static int fss_getclinfo(void *);
363 static int fss_parmsin(void *);
364 static int fss_parmsout(void *, pc_vaparms_t *);
365 static int fss_vaparmsin(void *, pc_vaparms_t *);
366 static int fss_vaparmsout(void *, pc_vaparms_t *);
367 static int fss_getclpri(pcpri_t *);
368 static int fss_alloc(void **, int);
369 static void fss_free(void *);
370
371 static int fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
372 static void fss_exitclass(void *);
373 static int fss_canexit(kthread_t *, cred_t *);
374 static int fss_fork(kthread_t *, kthread_t *, void *);
375 static void fss_forkret(kthread_t *, kthread_t *);
376 static void fss_parmsget(kthread_t *, void *);
377 static int fss_parmsset(kthread_t *, void *, id_t, cred_t *);
378 static void fss_stop(kthread_t *, int, int);
379 static void fss_exit(kthread_t *);
380 static void fss_active(kthread_t *);
381 static void fss_inactive(kthread_t *);
382 static pri_t fss_swapin(kthread_t *, int);
383 static pri_t fss_swapout(kthread_t *, int);
384 static void fss_trapret(kthread_t *);
385 static void fss_preempt(kthread_t *);
386 static void fss_setrun(kthread_t *);
387 static void fss_sleep(kthread_t *);
388 static void fss_tick(kthread_t *);
389 static void fss_wakeup(kthread_t *);
390 static int fss_donice(kthread_t *, cred_t *, int, int *);
391 static int fss_doprio(kthread_t *, cred_t *, int, int *);
392 static pri_t fss_globpri(kthread_t *);
393 static void fss_yield(kthread_t *);
394 static void fss_nullsys();
395
396 static struct classfuncs fss_classfuncs = {
397 /* class functions */
398 fss_admin,
399 fss_getclinfo,
400 fss_parmsin,
401 fss_parmsout,
402 fss_vaparmsin,
403 fss_vaparmsout,
404 fss_getclpri,
405 fss_alloc,
406 fss_free,
407
408 /* thread functions */
409 fss_enterclass,
410 fss_exitclass,
411 fss_canexit,
412 fss_fork,
413 fss_forkret,
414 fss_parmsget,
415 fss_parmsset,
416 fss_stop,
417 fss_exit,
418 fss_active,
419 fss_inactive,
420 fss_swapin,
421 fss_swapout,
422 fss_trapret,
423 fss_preempt,
424 fss_setrun,
425 fss_sleep,
426 fss_tick,
427 fss_wakeup,
428 fss_donice,
429 fss_globpri,
430 fss_nullsys, /* set_process_group */
431 fss_yield,
432 fss_doprio,
433 };
434
435 int
436 _init()
437 {
438 return (mod_install(&modlinkage));
439 }
440
441 int
442 _fini()
443 {
444 return (EBUSY);
445 }
446
447 int
448 _info(struct modinfo *modinfop)
449 {
450 return (mod_info(&modlinkage, modinfop));
451 }
452
453 /*ARGSUSED*/
454 static int
455 fss_project_walker(kproject_t *kpj, void *buf)
456 {
457 return (0);
458 }
459
460 void *
461 fss_allocbuf(int op, int type)
462 {
463 fssbuf_t *fssbuf;
464 void **fsslist;
465 int cnt;
466 int i;
467 size_t size;
468
469 ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF);
470 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
471 ASSERT(MUTEX_HELD(&cpu_lock));
472
473 fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP);
474 switch (op) {
475 case FSS_NPSET_BUF:
476 cnt = cpupart_list(NULL, 0, CP_NONEMPTY);
477 break;
478 case FSS_NPROJ_BUF:
479 cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL);
480 break;
481 case FSS_ONE_BUF:
482 cnt = 1;
483 break;
484 }
485
486 switch (type) {
487 case FSS_ALLOC_PROJ:
488 size = sizeof (fssproj_t);
489 break;
490 case FSS_ALLOC_ZONE:
491 size = sizeof (fsszone_t);
492 break;
493 }
494 fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP);
495 fssbuf->fssb_size = cnt;
496 fssbuf->fssb_list = fsslist;
497 for (i = 0; i < cnt; i++)
498 fsslist[i] = kmem_zalloc(size, KM_SLEEP);
499 return (fssbuf);
500 }
501
502 void
503 fss_freebuf(fssbuf_t *fssbuf, int type)
504 {
505 void **fsslist;
506 int i;
507 size_t size;
508
509 ASSERT(fssbuf != NULL);
510 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
511 fsslist = fssbuf->fssb_list;
512
513 switch (type) {
514 case FSS_ALLOC_PROJ:
515 size = sizeof (fssproj_t);
516 break;
517 case FSS_ALLOC_ZONE:
518 size = sizeof (fsszone_t);
519 break;
520 }
521
522 for (i = 0; i < fssbuf->fssb_size; i++) {
523 if (fsslist[i] != NULL)
524 kmem_free(fsslist[i], size);
525 }
526 kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size);
527 kmem_free(fssbuf, sizeof (fssbuf_t));
528 }
529
530 static fsspset_t *
531 fss_find_fsspset(cpupart_t *cpupart)
532 {
533 int i;
534 fsspset_t *fsspset = NULL;
535 int found = 0;
536
537 ASSERT(cpupart != NULL);
538 ASSERT(MUTEX_HELD(&fsspsets_lock));
539
540 /*
541 * Search for the cpupart pointer in the array of fsspsets.
542 */
543 for (i = 0; i < max_ncpus; i++) {
544 fsspset = &fsspsets[i];
545 if (fsspset->fssps_cpupart == cpupart) {
546 ASSERT(fsspset->fssps_nproj > 0);
547 found = 1;
548 break;
549 }
550 }
551 if (found == 0) {
552 /*
553 * If we didn't find anything, then use the first
554 * available slot in the fsspsets array.
555 */
556 for (i = 0; i < max_ncpus; i++) {
557 fsspset = &fsspsets[i];
558 if (fsspset->fssps_cpupart == NULL) {
559 ASSERT(fsspset->fssps_nproj == 0);
560 found = 1;
561 break;
562 }
563 }
564 fsspset->fssps_cpupart = cpupart;
565 }
566 ASSERT(found == 1);
567 return (fsspset);
568 }
569
570 static void
571 fss_del_fsspset(fsspset_t *fsspset)
572 {
573 ASSERT(MUTEX_HELD(&fsspsets_lock));
574 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
575 ASSERT(fsspset->fssps_nproj == 0);
576 ASSERT(fsspset->fssps_list == NULL);
577 ASSERT(fsspset->fssps_zones == NULL);
578 fsspset->fssps_cpupart = NULL;
579 fsspset->fssps_maxfsspri = 0;
580 fsspset->fssps_shares = 0;
581 }
582
583 /*
584 * The following routine returns a pointer to the fsszone structure which
585 * belongs to zone "zone" and cpu partition fsspset, if such structure exists.
586 */
587 static fsszone_t *
588 fss_find_fsszone(fsspset_t *fsspset, zone_t *zone)
589 {
590 fsszone_t *fsszone;
591
592 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
593
594 if (fsspset->fssps_list != NULL) {
595 /*
596 * There are projects/zones active on this cpu partition
597 * already. Try to find our zone among them.
598 */
599 fsszone = fsspset->fssps_zones;
600 do {
601 if (fsszone->fssz_zone == zone) {
602 return (fsszone);
603 }
604 fsszone = fsszone->fssz_next;
605 } while (fsszone != fsspset->fssps_zones);
606 }
607 return (NULL);
608 }
609
610 /*
611 * The following routine links new fsszone structure into doubly linked list of
612 * zones active on the specified cpu partition.
613 */
614 static void
615 fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone)
616 {
617 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
618
619 fsszone->fssz_zone = zone;
620 fsszone->fssz_rshares = zone->zone_shares;
621
622 if (fsspset->fssps_zones == NULL) {
623 /*
624 * This will be the first fsszone for this fsspset
625 */
626 fsszone->fssz_next = fsszone->fssz_prev = fsszone;
627 fsspset->fssps_zones = fsszone;
628 } else {
629 /*
630 * Insert this fsszone to the doubly linked list.
631 */
632 fsszone_t *fssz_head = fsspset->fssps_zones;
633
634 fsszone->fssz_next = fssz_head;
635 fsszone->fssz_prev = fssz_head->fssz_prev;
636 fssz_head->fssz_prev->fssz_next = fsszone;
637 fssz_head->fssz_prev = fsszone;
638 fsspset->fssps_zones = fsszone;
639 }
640 }
641
642 /*
643 * The following routine removes a single fsszone structure from the doubly
644 * linked list of zones active on the specified cpu partition. Note that
645 * global fsspsets_lock must be held in case this fsszone structure is the last
646 * on the above mentioned list. Also note that the fsszone structure is not
647 * freed here, it is the responsibility of the caller to call kmem_free for it.
648 */
649 static void
650 fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone)
651 {
652 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
653 ASSERT(fsszone->fssz_nproj == 0);
654 ASSERT(fsszone->fssz_shares == 0);
655 ASSERT(fsszone->fssz_runnable == 0);
656
657 if (fsszone->fssz_next != fsszone) {
658 /*
659 * This is not the last zone in the list.
660 */
661 fsszone->fssz_prev->fssz_next = fsszone->fssz_next;
662 fsszone->fssz_next->fssz_prev = fsszone->fssz_prev;
663 if (fsspset->fssps_zones == fsszone)
664 fsspset->fssps_zones = fsszone->fssz_next;
665 } else {
666 /*
667 * This was the last zone active in this cpu partition.
668 */
669 fsspset->fssps_zones = NULL;
670 }
671 }
672
673 /*
674 * The following routine returns a pointer to the fssproj structure
675 * which belongs to project kpj and cpu partition fsspset, if such structure
676 * exists.
677 */
678 static fssproj_t *
679 fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj)
680 {
681 fssproj_t *fssproj;
682
683 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
684
685 if (fsspset->fssps_list != NULL) {
686 /*
687 * There are projects running on this cpu partition already.
688 * Try to find our project among them.
689 */
690 fssproj = fsspset->fssps_list;
691 do {
692 if (fssproj->fssp_proj == kpj) {
693 ASSERT(fssproj->fssp_pset == fsspset);
694 return (fssproj);
695 }
696 fssproj = fssproj->fssp_next;
697 } while (fssproj != fsspset->fssps_list);
698 }
699 return (NULL);
700 }
701
702 /*
703 * The following routine links new fssproj structure into doubly linked list
704 * of projects running on the specified cpu partition.
705 */
706 static void
707 fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone,
708 fssproj_t *fssproj)
709 {
710 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
711
712 fssproj->fssp_pset = fsspset;
713 fssproj->fssp_proj = kpj;
714 fssproj->fssp_shares = kpj->kpj_shares;
715
716 fsspset->fssps_nproj++;
717
718 if (fsspset->fssps_list == NULL) {
719 /*
720 * This will be the first fssproj for this fsspset
721 */
722 fssproj->fssp_next = fssproj->fssp_prev = fssproj;
723 fsspset->fssps_list = fssproj;
724 } else {
725 /*
726 * Insert this fssproj to the doubly linked list.
727 */
728 fssproj_t *fssp_head = fsspset->fssps_list;
729
730 fssproj->fssp_next = fssp_head;
731 fssproj->fssp_prev = fssp_head->fssp_prev;
732 fssp_head->fssp_prev->fssp_next = fssproj;
733 fssp_head->fssp_prev = fssproj;
734 fsspset->fssps_list = fssproj;
735 }
736 fssproj->fssp_fsszone = fsszone;
737 fsszone->fssz_nproj++;
738 ASSERT(fsszone->fssz_nproj != 0);
739 }
740
741 /*
742 * The following routine removes a single fssproj structure from the doubly
743 * linked list of projects running on the specified cpu partition. Note that
744 * global fsspsets_lock must be held in case if this fssproj structure is the
745 * last on the above mentioned list. Also note that the fssproj structure is
746 * not freed here, it is the responsibility of the caller to call kmem_free
747 * for it.
748 */
749 static void
750 fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj)
751 {
752 fsszone_t *fsszone;
753
754 ASSERT(MUTEX_HELD(&fsspsets_lock));
755 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
756 ASSERT(fssproj->fssp_runnable == 0);
757
758 fsspset->fssps_nproj--;
759
760 fsszone = fssproj->fssp_fsszone;
761 fsszone->fssz_nproj--;
762
763 if (fssproj->fssp_next != fssproj) {
764 /*
765 * This is not the last part in the list.
766 */
767 fssproj->fssp_prev->fssp_next = fssproj->fssp_next;
768 fssproj->fssp_next->fssp_prev = fssproj->fssp_prev;
769 if (fsspset->fssps_list == fssproj)
770 fsspset->fssps_list = fssproj->fssp_next;
771 if (fsszone->fssz_nproj == 0)
772 fss_remove_fsszone(fsspset, fsszone);
773 } else {
774 /*
775 * This was the last project part running
776 * at this cpu partition.
777 */
778 fsspset->fssps_list = NULL;
779 ASSERT(fsspset->fssps_nproj == 0);
780 ASSERT(fsszone->fssz_nproj == 0);
781 fss_remove_fsszone(fsspset, fsszone);
782 fss_del_fsspset(fsspset);
783 }
784 }
785
786 static void
787 fss_inactive(kthread_t *t)
788 {
789 fssproc_t *fssproc;
790 fssproj_t *fssproj;
791 fsspset_t *fsspset;
792 fsszone_t *fsszone;
793
794 ASSERT(THREAD_LOCK_HELD(t));
795 fssproc = FSSPROC(t);
796 fssproj = FSSPROC2FSSPROJ(fssproc);
797 if (fssproj == NULL) /* if this thread already exited */
798 return;
799 fsspset = FSSPROJ2FSSPSET(fssproj);
800 fsszone = fssproj->fssp_fsszone;
801 disp_lock_enter_high(&fsspset->fssps_displock);
802 ASSERT(fssproj->fssp_runnable > 0);
803 if (--fssproj->fssp_runnable == 0) {
804 fsszone->fssz_shares -= fssproj->fssp_shares;
805 if (--fsszone->fssz_runnable == 0)
806 fsspset->fssps_shares -= fsszone->fssz_rshares;
807 }
808 ASSERT(fssproc->fss_runnable == 1);
809 fssproc->fss_runnable = 0;
810 disp_lock_exit_high(&fsspset->fssps_displock);
811 }
812
813 static void
814 fss_active(kthread_t *t)
815 {
816 fssproc_t *fssproc;
817 fssproj_t *fssproj;
818 fsspset_t *fsspset;
819 fsszone_t *fsszone;
820
821 ASSERT(THREAD_LOCK_HELD(t));
822 fssproc = FSSPROC(t);
823 fssproj = FSSPROC2FSSPROJ(fssproc);
824 if (fssproj == NULL) /* if this thread already exited */
825 return;
826 fsspset = FSSPROJ2FSSPSET(fssproj);
827 fsszone = fssproj->fssp_fsszone;
828 disp_lock_enter_high(&fsspset->fssps_displock);
829 if (++fssproj->fssp_runnable == 1) {
830 fsszone->fssz_shares += fssproj->fssp_shares;
831 if (++fsszone->fssz_runnable == 1)
832 fsspset->fssps_shares += fsszone->fssz_rshares;
833 }
834 ASSERT(fssproc->fss_runnable == 0);
835 fssproc->fss_runnable = 1;
836 disp_lock_exit_high(&fsspset->fssps_displock);
837 }
838
839 /*
840 * Fair share scheduler initialization. Called by dispinit() at boot time.
841 * We can ignore clparmsz argument since we know that the smallest possible
842 * parameter buffer is big enough for us.
843 */
844 /*ARGSUSED*/
845 static pri_t
846 fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
847 {
848 int i;
849
850 ASSERT(MUTEX_HELD(&cpu_lock));
851
852 fss_cid = cid;
853 fss_maxumdpri = minclsyspri - 1;
854 fss_maxglobpri = minclsyspri;
855 fss_minglobpri = 0;
856 fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP);
857
858 /*
859 * Initialize the fssproc hash table.
860 */
861 for (i = 0; i < FSS_LISTS; i++)
862 fss_listhead[i].fss_next = fss_listhead[i].fss_prev =
863 &fss_listhead[i];
864
865 *clfuncspp = &fss_classfuncs;
866
867 /*
868 * Fill in fss_nice_tick and fss_nice_decay arrays:
869 * The cost of a tick is lower at positive nice values (so that it
870 * will not increase its project's usage as much as normal) with 50%
871 * drop at the maximum level and 50% increase at the minimum level.
872 * The fsspri decay is slower at positive nice values. fsspri values
873 * of processes with negative nice levels must decay faster to receive
874 * time slices more frequently than normal.
875 */
876 for (i = 0; i < FSS_NICE_RANGE; i++) {
877 fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2)
878 - i)) / FSS_NICE_RANGE;
879 fss_nice_decay[i] = FSS_DECAY_MIN +
880 ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) /
881 (FSS_NICE_RANGE - 1);
882 }
883
884 return (fss_maxglobpri);
885 }
886
887 /*
888 * Calculate the new fss_umdpri based on the usage, the normalized share usage
889 * and the number of active threads. Reset the tick counter for this thread.
890 *
891 * When calculating the new priority using the standard formula we can hit
892 * a scenario where we don't have good round-robin behavior. This would be
893 * most commonly seen when there is a zone with lots of runnable threads.
894 * In the bad scenario we will see the following behavior when using the
895 * standard formula and these conditions:
896 *
897 * - there are multiple runnable threads in the zone (project)
898 * - the fssps_maxfsspri is a very large value
899 * - (we also know all of these threads will use the project's
900 * fssp_shusage)
901 *
902 * Under these conditions, a thread with a low fss_fsspri value is chosen
903 * to run and the thread gets a high fss_umdpri. This thread can run for
904 * its full quanta (fss_timeleft) at which time fss_newpri is called to
905 * calculate the thread's new priority.
906 *
907 * In this case, because the newly calculated fsspri value is much smaller
908 * (orders of magnitude) than the fssps_maxfsspri value, if we used the
909 * standard formula the thread will still get a high fss_umdpri value and
910 * will run again for another quanta, even though there are other runnable
911 * threads in the project.
912 *
913 * For a thread that is runnable for a long time, the thread can continue
914 * to run for many quanta (totaling many seconds) before the thread's fsspri
915 * exceeds the fssps_maxfsspri and the thread's fss_umdpri is reset back
916 * down to 1. This behavior also keeps the fssps_maxfsspr at a high value,
917 * so that the next runnable thread might repeat this cycle.
918 *
919 * This leads to the case where we don't have round-robin behavior at quanta
920 * granularity, but instead, runnable threads within the project only run
921 * at several second intervals.
922 *
923 * To prevent this scenario from occuring, when a thread has consumed its
924 * quanta and there are multiple runnable threads in the project, we
925 * immediately cause the thread to hit fssps_maxfsspri so that it gets
926 * reset back to 1 and another runnable thread in the project can run.
927 */
928 static void
929 fss_newpri(fssproc_t *fssproc, boolean_t quanta_up)
930 {
931 kthread_t *tp;
932 fssproj_t *fssproj;
933 fsspset_t *fsspset;
934 fsszone_t *fsszone;
935 fsspri_t fsspri, maxfsspri;
936 uint32_t n_runnable;
937 pri_t invpri;
938 uint32_t ticks;
939
940 tp = fssproc->fss_tp;
941 ASSERT(tp != NULL);
942
943 if (tp->t_cid != fss_cid)
944 return;
945
946 ASSERT(THREAD_LOCK_HELD(tp));
947
948 fssproj = FSSPROC2FSSPROJ(fssproc);
949 fsszone = FSSPROJ2FSSZONE(fssproj);
950 if (fssproj == NULL)
951 /*
952 * No need to change priority of exited threads.
953 */
954 return;
955
956 fsspset = FSSPROJ2FSSPSET(fssproj);
957 disp_lock_enter_high(&fsspset->fssps_displock);
958
959 ticks = fssproc->fss_ticks;
960 fssproc->fss_ticks = 0;
961
962 if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) {
963 /*
964 * Special case: threads with no shares.
965 */
966 fssproc->fss_umdpri = fss_minglobpri;
967 disp_lock_exit_high(&fsspset->fssps_displock);
968 return;
969 }
970
971 maxfsspri = fsspset->fssps_maxfsspri;
972 n_runnable = fssproj->fssp_runnable;
973
974 if (quanta_up && n_runnable > 1) {
975 fsspri = maxfsspri;
976 } else {
977 /*
978 * fsspri += fssp_shusage * nrunnable * ticks
979 * If all three values are non-0, this typically calculates to
980 * a large number (sometimes > 1M, sometimes > 100B) due to
981 * fssp_shusage which can be > 1T.
982 */
983 fsspri = fssproc->fss_fsspri;
984 fsspri += fssproj->fssp_shusage * n_runnable * ticks;
985 }
986
987 fssproc->fss_fsspri = fsspri;
988
989 /*
990 * fss_maxumdpri is normally 59, since FSS priorities are 0-59.
991 * If the previous calculation resulted in 0 (e.g. was 0 and added 0
992 * because ticks == 0), then instead of 0, we use the largest priority,
993 * which is still small in comparison to the large numbers we typically
994 * see.
995 */
996 if (fsspri < fss_maxumdpri)
997 fsspri = fss_maxumdpri; /* so that maxfsspri is != 0 */
998
999 /*
1000 * The general priority formula:
1001 *
1002 * (fsspri * umdprirange)
1003 * pri = maxumdpri - ------------------------
1004 * maxfsspri
1005 *
1006 * If this thread's fsspri is greater than the previous largest
1007 * fsspri, then record it as the new high and priority for this
1008 * thread will be one (the lowest priority assigned to a thread
1009 * that has non-zero shares). Because of this check, maxfsspri can
1010 * change as this function is called via the
1011 * fss_update -> fss_update_list -> fss_newpri code path to update
1012 * all runnable threads. See the code in fss_update for how we
1013 * mitigate this issue.
1014 *
1015 * Note that this formula cannot produce out of bounds priority
1016 * values (0-59); if it is changed, additional checks may need to be
1017 * added.
1018 */
1019 if (fsspri >= maxfsspri) {
1020 fsspset->fssps_maxfsspri = fsspri;
1021 disp_lock_exit_high(&fsspset->fssps_displock);
1022 fssproc->fss_umdpri = 1;
1023 } else {
1024 disp_lock_exit_high(&fsspset->fssps_displock);
1025 invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri;
1026 fssproc->fss_umdpri = fss_maxumdpri - invpri;
1027 }
1028 }
1029
1030 /*
1031 * Decays usages of all running projects, resets their tick counters and
1032 * calcluates the projects normalized share usage. Called once per second from
1033 * fss_update().
1034 */
1035 static void
1036 fss_decay_usage()
1037 {
1038 uint32_t zone_ext_shares, zone_int_shares;
1039 uint32_t kpj_shares, pset_shares;
1040 fsspset_t *fsspset;
1041 fssproj_t *fssproj;
1042 fsszone_t *fsszone;
1043 fsspri_t maxfsspri;
1044 int psetid;
1045 struct zone *zp;
1046
1047 mutex_enter(&fsspsets_lock);
1048 /*
1049 * Go through all active processor sets and decay usages of projects
1050 * running on them.
1051 */
1052 for (psetid = 0; psetid < max_ncpus; psetid++) {
1053 fsspset = &fsspsets[psetid];
1054 mutex_enter(&fsspset->fssps_lock);
1055
1056 fsspset->fssps_gen++;
1057
1058 if (fsspset->fssps_cpupart == NULL ||
1059 (fssproj = fsspset->fssps_list) == NULL) {
1060 mutex_exit(&fsspset->fssps_lock);
1061 continue;
1062 }
1063
1064 /*
1065 * Decay maxfsspri for this cpu partition with the
1066 * fastest possible decay rate.
1067 */
1068 disp_lock_enter(&fsspset->fssps_displock);
1069
1070 pset_shares = fsspset->fssps_shares;
1071
1072 maxfsspri = (fsspset->fssps_maxfsspri *
1073 fss_nice_decay[NZERO]) / FSS_DECAY_BASE;
1074 if (maxfsspri < fss_maxumdpri)
1075 maxfsspri = fss_maxumdpri;
1076 fsspset->fssps_maxfsspri = maxfsspri;
1077
1078 do {
1079 fsszone = fssproj->fssp_fsszone;
1080 zp = fsszone->fssz_zone;
1081
1082 /*
1083 * Reset zone's FSS stats if they are from a
1084 * previous cycle.
1085 */
1086 if (fsspset->fssps_gen != zp->zone_fss_gen) {
1087 zp->zone_fss_gen = fsspset->fssps_gen;
1088 zp->zone_run_ticks = 0;
1089 }
1090
1091 /*
1092 * Decay project usage, then add in this cycle's
1093 * nice tick value.
1094 */
1095 fssproj->fssp_usage =
1096 (fssproj->fssp_usage * FSS_DECAY_USG) /
1097 FSS_DECAY_BASE +
1098 fssproj->fssp_ticks;
1099
1100 fssproj->fssp_ticks = 0;
1101 zp->zone_run_ticks += fssproj->fssp_tick_cnt;
1102 fssproj->fssp_tick_cnt = 0;
1103
1104 /*
1105 * Readjust the project's number of shares if it has
1106 * changed since we checked it last time.
1107 */
1108 kpj_shares = fssproj->fssp_proj->kpj_shares;
1109 if (fssproj->fssp_shares != kpj_shares) {
1110 if (fssproj->fssp_runnable != 0) {
1111 fsszone->fssz_shares -=
1112 fssproj->fssp_shares;
1113 fsszone->fssz_shares += kpj_shares;
1114 }
1115 fssproj->fssp_shares = kpj_shares;
1116 }
1117
1118 /*
1119 * Readjust the zone's number of shares if it
1120 * has changed since we checked it last time.
1121 */
1122 zone_ext_shares = zp->zone_shares;
1123 if (fsszone->fssz_rshares != zone_ext_shares) {
1124 if (fsszone->fssz_runnable != 0) {
1125 fsspset->fssps_shares -=
1126 fsszone->fssz_rshares;
1127 fsspset->fssps_shares +=
1128 zone_ext_shares;
1129 pset_shares = fsspset->fssps_shares;
1130 }
1131 fsszone->fssz_rshares = zone_ext_shares;
1132 }
1133 zone_int_shares = fsszone->fssz_shares;
1134
1135 /*
1136 * If anything is runnable in the project, track the
1137 * overall project share percent for monitoring useage.
1138 */
1139 if (fssproj->fssp_runnable > 0) {
1140 uint32_t zone_shr_pct;
1141 uint32_t int_shr_pct;
1142
1143 /*
1144 * Times 1000 to get tenths of a percent
1145 *
1146 * zone_ext_shares
1147 * zone_shr_pct = ---------------
1148 * pset_shares
1149 *
1150 * kpj_shares
1151 * int_shr_pct = ---------------
1152 * zone_int_shares
1153 */
1154 if (pset_shares == 0 || zone_int_shares == 0) {
1155 fssproj->fssp_shr_pct = 0;
1156 } else {
1157 zone_shr_pct =
1158 (zone_ext_shares * 1000) /
1159 pset_shares;
1160 int_shr_pct = (kpj_shares * 1000) /
1161 zone_int_shares;
1162 fssproj->fssp_shr_pct =
1163 (zone_shr_pct * int_shr_pct) /
1164 1000;
1165 }
1166 } else {
1167 DTRACE_PROBE1(fss__prj__norun, fssproj_t *,
1168 fssproj);
1169 }
1170
1171 /*
1172 * Calculate fssp_shusage value to be used
1173 * for fsspri increments for the next second.
1174 */
1175 if (kpj_shares == 0 || zone_ext_shares == 0) {
1176 fssproj->fssp_shusage = 0;
1177 } else if (FSSPROJ2KPROJ(fssproj) == proj0p) {
1178 uint32_t zone_shr_pct;
1179
1180 /*
1181 * Project 0 in the global zone has 50%
1182 * of its zone. See calculation above for
1183 * the zone's share percent.
1184 */
1185 if (pset_shares == 0)
1186 zone_shr_pct = 1000;
1187 else
1188 zone_shr_pct =
1189 (zone_ext_shares * 1000) /
1190 pset_shares;
1191
1192 fssproj->fssp_shr_pct = zone_shr_pct / 2;
1193
1194 fssproj->fssp_shusage = (fssproj->fssp_usage *
1195 zone_int_shares * zone_int_shares) /
1196 (zone_ext_shares * zone_ext_shares);
1197 } else {
1198 /*
1199 * Thread's priority is based on its project's
1200 * normalized usage (shusage) value which gets
1201 * calculated this way:
1202 *
1203 * pset_shares^2 zone_int_shares^2
1204 * usage * ------------- * ------------------
1205 * kpj_shares^2 zone_ext_shares^2
1206 *
1207 * Where zone_int_shares is the sum of shares
1208 * of all active projects within the zone (and
1209 * the pset), and zone_ext_shares is the number
1210 * of zone shares (ie, zone.cpu-shares).
1211 *
1212 * If there is only one zone active on the pset
1213 * the above reduces to:
1214 *
1215 * zone_int_shares^2
1216 * shusage = usage * ---------------------
1217 * kpj_shares^2
1218 *
1219 * If there's only one project active in the
1220 * zone this formula reduces to:
1221 *
1222 * pset_shares^2
1223 * shusage = usage * ----------------------
1224 * zone_ext_shares^2
1225 *
1226 * shusage is one input to calculating fss_pri
1227 * in fss_newpri(). Larger values tend toward
1228 * lower priorities for processes in the proj.
1229 */
1230 fssproj->fssp_shusage = fssproj->fssp_usage *
1231 pset_shares * zone_int_shares;
1232 fssproj->fssp_shusage /=
1233 kpj_shares * zone_ext_shares;
1234 fssproj->fssp_shusage *=
1235 pset_shares * zone_int_shares;
1236 fssproj->fssp_shusage /=
1237 kpj_shares * zone_ext_shares;
1238 }
1239 fssproj = fssproj->fssp_next;
1240 } while (fssproj != fsspset->fssps_list);
1241
1242 disp_lock_exit(&fsspset->fssps_displock);
1243 mutex_exit(&fsspset->fssps_lock);
1244 }
1245 mutex_exit(&fsspsets_lock);
1246 }
1247
1248 static void
1249 fss_change_priority(kthread_t *t, fssproc_t *fssproc)
1250 {
1251 pri_t new_pri;
1252
1253 ASSERT(THREAD_LOCK_HELD(t));
1254 new_pri = fssproc->fss_umdpri;
1255 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
1256
1257 t->t_cpri = fssproc->fss_upri;
1258 fssproc->fss_flags &= ~FSSRESTORE;
1259 if (t == curthread || t->t_state == TS_ONPROC) {
1260 /*
1261 * curthread is always onproc
1262 */
1263 cpu_t *cp = t->t_disp_queue->disp_cpu;
1264 THREAD_CHANGE_PRI(t, new_pri);
1265 if (t == cp->cpu_dispthread)
1266 cp->cpu_dispatch_pri = DISP_PRIO(t);
1267 if (DISP_MUST_SURRENDER(t)) {
1268 fssproc->fss_flags |= FSSBACKQ;
1269 cpu_surrender(t);
1270 } else {
1271 fssproc->fss_timeleft = fss_quantum;
1272 }
1273 } else {
1274 /*
1275 * When the priority of a thread is changed, it may be
1276 * necessary to adjust its position on a sleep queue or
1277 * dispatch queue. The function thread_change_pri accomplishes
1278 * this.
1279 */
1280 if (thread_change_pri(t, new_pri, 0)) {
1281 /*
1282 * The thread was on a run queue.
1283 */
1284 fssproc->fss_timeleft = fss_quantum;
1285 } else {
1286 fssproc->fss_flags |= FSSBACKQ;
1287 }
1288 }
1289 }
1290
1291 /*
1292 * Update priorities of all fair-sharing threads that are currently runnable
1293 * at a user mode priority based on the number of shares and current usage.
1294 * Called once per second via timeout which we reset here.
1295 *
1296 * There are several lists of fair-sharing threads broken up by a hash on the
1297 * thread pointer. Each list has its own lock. This avoids blocking all
1298 * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs.
1299 * fss_update traverses each list in turn.
1300 *
1301 * Each time we're run (once/second) we may start at the next list and iterate
1302 * through all of the lists. By starting with a different list, we mitigate any
1303 * effects we would see updating the fssps_maxfsspri value in fss_newpri.
1304 */
1305 static void
1306 fss_update(void *arg)
1307 {
1308 int i;
1309 int new_marker = -1;
1310 static int fss_update_marker;
1311
1312 /*
1313 * Decay and update usages for all projects.
1314 */
1315 fss_decay_usage();
1316
1317 /*
1318 * Start with the fss_update_marker list, then do the rest.
1319 */
1320 i = fss_update_marker;
1321
1322 /*
1323 * Go around all threads, set new priorities and decay
1324 * per-thread CPU usages.
1325 */
1326 do {
1327 /*
1328 * If this is the first list after the current marker to have
1329 * threads with priority updates, advance the marker to this
1330 * list for the next time fss_update runs.
1331 */
1332 if (fss_update_list(i) &&
1333 new_marker == -1 && i != fss_update_marker)
1334 new_marker = i;
1335 } while ((i = FSS_LIST_NEXT(i)) != fss_update_marker);
1336
1337 /*
1338 * Advance marker for the next fss_update call
1339 */
1340 if (new_marker != -1)
1341 fss_update_marker = new_marker;
1342
1343 (void) timeout(fss_update, arg, hz);
1344 }
1345
1346 /*
1347 * Updates priority for a list of threads. Returns 1 if the priority of one
1348 * of the threads was actually updated, 0 if none were for various reasons
1349 * (thread is no longer in the FSS class, is not runnable, has the preemption
1350 * control no-preempt bit set, etc.)
1351 */
1352 static int
1353 fss_update_list(int i)
1354 {
1355 fssproc_t *fssproc;
1356 fssproj_t *fssproj;
1357 fsspri_t fsspri;
1358 pri_t fss_umdpri;
1359 kthread_t *t;
1360 int updated = 0;
1361
1362 mutex_enter(&fss_listlock[i]);
1363 for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i];
1364 fssproc = fssproc->fss_next) {
1365 t = fssproc->fss_tp;
1366 /*
1367 * Lock the thread and verify the state.
1368 */
1369 thread_lock(t);
1370 /*
1371 * Skip the thread if it is no longer in the FSS class or
1372 * is running with kernel mode priority.
1373 */
1374 if (t->t_cid != fss_cid)
1375 goto next;
1376
1377 fssproj = FSSPROC2FSSPROJ(fssproc);
1378 if (fssproj == NULL)
1379 goto next;
1380
1381 if (fssproj->fssp_shares != 0) {
1382 /*
1383 * Decay fsspri value.
1384 */
1385 fsspri = fssproc->fss_fsspri;
1386 fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) /
1387 FSS_DECAY_BASE;
1388 fssproc->fss_fsspri = fsspri;
1389 }
1390
1391 if (t->t_schedctl && schedctl_get_nopreempt(t))
1392 goto next;
1393 if (t->t_state != TS_RUN && t->t_state != TS_WAIT) {
1394 /*
1395 * Make next syscall/trap call fss_trapret
1396 */
1397 t->t_trapret = 1;
1398 aston(t);
1399 if (t->t_state == TS_ONPROC)
1400 DTRACE_PROBE1(fss__onproc, fssproc_t *,
1401 fssproc);
1402 goto next;
1403 }
1404 fss_newpri(fssproc, B_FALSE);
1405 updated = 1;
1406
1407 fss_umdpri = fssproc->fss_umdpri;
1408
1409 /*
1410 * Only dequeue the thread if it needs to be moved; otherwise
1411 * it should just round-robin here.
1412 */
1413 if (t->t_pri != fss_umdpri)
1414 fss_change_priority(t, fssproc);
1415 next:
1416 thread_unlock(t);
1417 }
1418 mutex_exit(&fss_listlock[i]);
1419 return (updated);
1420 }
1421
1422 /*ARGSUSED*/
1423 static int
1424 fss_admin(caddr_t uaddr, cred_t *reqpcredp)
1425 {
1426 fssadmin_t fssadmin;
1427
1428 if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t)))
1429 return (EFAULT);
1430
1431 switch (fssadmin.fss_cmd) {
1432 case FSS_SETADMIN:
1433 if (secpolicy_dispadm(reqpcredp) != 0)
1434 return (EPERM);
1435 if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz)
1436 return (EINVAL);
1437 fss_quantum = fssadmin.fss_quantum;
1438 break;
1439 case FSS_GETADMIN:
1440 fssadmin.fss_quantum = fss_quantum;
1441 if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t)))
1442 return (EFAULT);
1443 break;
1444 default:
1445 return (EINVAL);
1446 }
1447 return (0);
1448 }
1449
1450 static int
1451 fss_getclinfo(void *infop)
1452 {
1453 fssinfo_t *fssinfo = (fssinfo_t *)infop;
1454 fssinfo->fss_maxupri = fss_maxupri;
1455 return (0);
1456 }
1457
1458 static int
1459 fss_parmsin(void *parmsp)
1460 {
1461 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1462
1463 /*
1464 * Check validity of parameters.
1465 */
1466 if ((fssparmsp->fss_uprilim > fss_maxupri ||
1467 fssparmsp->fss_uprilim < -fss_maxupri) &&
1468 fssparmsp->fss_uprilim != FSS_NOCHANGE)
1469 return (EINVAL);
1470
1471 if ((fssparmsp->fss_upri > fss_maxupri ||
1472 fssparmsp->fss_upri < -fss_maxupri) &&
1473 fssparmsp->fss_upri != FSS_NOCHANGE)
1474 return (EINVAL);
1475
1476 return (0);
1477 }
1478
1479 /*ARGSUSED*/
1480 static int
1481 fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1482 {
1483 return (0);
1484 }
1485
1486 static int
1487 fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp)
1488 {
1489 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1490 int priflag = 0;
1491 int limflag = 0;
1492 uint_t cnt;
1493 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1494
1495 /*
1496 * FSS_NOCHANGE (-32768) is outside of the range of values for
1497 * fss_uprilim and fss_upri. If the structure fssparms_t is changed,
1498 * FSS_NOCHANGE should be replaced by a flag word.
1499 */
1500 fssparmsp->fss_uprilim = FSS_NOCHANGE;
1501 fssparmsp->fss_upri = FSS_NOCHANGE;
1502
1503 /*
1504 * Get the varargs parameter and check validity of parameters.
1505 */
1506 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1507 return (EINVAL);
1508
1509 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1510 switch (vpp->pc_key) {
1511 case FSS_KY_UPRILIM:
1512 if (limflag++)
1513 return (EINVAL);
1514 fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm;
1515 if (fssparmsp->fss_uprilim > fss_maxupri ||
1516 fssparmsp->fss_uprilim < -fss_maxupri)
1517 return (EINVAL);
1518 break;
1519 case FSS_KY_UPRI:
1520 if (priflag++)
1521 return (EINVAL);
1522 fssparmsp->fss_upri = (pri_t)vpp->pc_parm;
1523 if (fssparmsp->fss_upri > fss_maxupri ||
1524 fssparmsp->fss_upri < -fss_maxupri)
1525 return (EINVAL);
1526 break;
1527 default:
1528 return (EINVAL);
1529 }
1530 }
1531
1532 if (vaparmsp->pc_vaparmscnt == 0) {
1533 /*
1534 * Use default parameters.
1535 */
1536 fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0;
1537 }
1538
1539 return (0);
1540 }
1541
1542 /*
1543 * Copy all selected fair-sharing class parameters to the user. The parameters
1544 * are specified by a key.
1545 */
1546 static int
1547 fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1548 {
1549 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1550 int priflag = 0;
1551 int limflag = 0;
1552 uint_t cnt;
1553 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1554
1555 ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));
1556
1557 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1558 return (EINVAL);
1559
1560 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1561 switch (vpp->pc_key) {
1562 case FSS_KY_UPRILIM:
1563 if (limflag++)
1564 return (EINVAL);
1565 if (copyout(&fssparmsp->fss_uprilim,
1566 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1567 return (EFAULT);
1568 break;
1569 case FSS_KY_UPRI:
1570 if (priflag++)
1571 return (EINVAL);
1572 if (copyout(&fssparmsp->fss_upri,
1573 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1574 return (EFAULT);
1575 break;
1576 default:
1577 return (EINVAL);
1578 }
1579 }
1580
1581 return (0);
1582 }
1583
1584 /*
1585 * Return the user mode scheduling priority range.
1586 */
1587 static int
1588 fss_getclpri(pcpri_t *pcprip)
1589 {
1590 pcprip->pc_clpmax = fss_maxupri;
1591 pcprip->pc_clpmin = -fss_maxupri;
1592 return (0);
1593 }
1594
1595 static int
1596 fss_alloc(void **p, int flag)
1597 {
1598 void *bufp;
1599
1600 if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) {
1601 return (ENOMEM);
1602 } else {
1603 *p = bufp;
1604 return (0);
1605 }
1606 }
1607
1608 static void
1609 fss_free(void *bufp)
1610 {
1611 if (bufp)
1612 kmem_free(bufp, sizeof (fssproc_t));
1613 }
1614
1615 /*
1616 * Thread functions
1617 */
1618 static int
1619 fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
1620 void *bufp)
1621 {
1622 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1623 fssproc_t *fssproc;
1624 pri_t reqfssuprilim;
1625 pri_t reqfssupri;
1626 static uint32_t fssexists = 0;
1627 fsspset_t *fsspset;
1628 fssproj_t *fssproj;
1629 fsszone_t *fsszone;
1630 kproject_t *kpj;
1631 zone_t *zone;
1632 int fsszone_allocated = 0;
1633
1634 fssproc = (fssproc_t *)bufp;
1635 ASSERT(fssproc != NULL);
1636
1637 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1638
1639 /*
1640 * Only root can move threads to FSS class.
1641 */
1642 if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0)
1643 return (EPERM);
1644 /*
1645 * Initialize the fssproc structure.
1646 */
1647 fssproc->fss_umdpri = fss_maxumdpri / 2;
1648
1649 if (fssparmsp == NULL) {
1650 /*
1651 * Use default values.
1652 */
1653 fssproc->fss_nice = NZERO;
1654 fssproc->fss_uprilim = fssproc->fss_upri = 0;
1655 } else {
1656 /*
1657 * Use supplied values.
1658 */
1659 if (fssparmsp->fss_uprilim == FSS_NOCHANGE) {
1660 reqfssuprilim = 0;
1661 } else {
1662 if (fssparmsp->fss_uprilim > 0 &&
1663 secpolicy_setpriority(reqpcredp) != 0)
1664 return (EPERM);
1665 reqfssuprilim = fssparmsp->fss_uprilim;
1666 }
1667 if (fssparmsp->fss_upri == FSS_NOCHANGE) {
1668 reqfssupri = reqfssuprilim;
1669 } else {
1670 if (fssparmsp->fss_upri > 0 &&
1671 secpolicy_setpriority(reqpcredp) != 0)
1672 return (EPERM);
1673 /*
1674 * Set the user priority to the requested value or
1675 * the upri limit, whichever is lower.
1676 */
1677 reqfssupri = fssparmsp->fss_upri;
1678 if (reqfssupri > reqfssuprilim)
1679 reqfssupri = reqfssuprilim;
1680 }
1681 fssproc->fss_uprilim = reqfssuprilim;
1682 fssproc->fss_upri = reqfssupri;
1683 fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri;
1684 if (fssproc->fss_nice > FSS_NICE_MAX)
1685 fssproc->fss_nice = FSS_NICE_MAX;
1686 }
1687
1688 fssproc->fss_timeleft = fss_quantum;
1689 fssproc->fss_tp = t;
1690 cpucaps_sc_init(&fssproc->fss_caps);
1691
1692 /*
1693 * Put a lock on our fsspset structure.
1694 */
1695 mutex_enter(&fsspsets_lock);
1696 fsspset = fss_find_fsspset(t->t_cpupart);
1697 mutex_enter(&fsspset->fssps_lock);
1698 mutex_exit(&fsspsets_lock);
1699
1700 zone = ttoproc(t)->p_zone;
1701 if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) {
1702 if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP))
1703 == NULL) {
1704 mutex_exit(&fsspset->fssps_lock);
1705 return (ENOMEM);
1706 } else {
1707 fsszone_allocated = 1;
1708 fss_insert_fsszone(fsspset, zone, fsszone);
1709 }
1710 }
1711 kpj = ttoproj(t);
1712 if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) {
1713 if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP))
1714 == NULL) {
1715 if (fsszone_allocated) {
1716 fss_remove_fsszone(fsspset, fsszone);
1717 kmem_free(fsszone, sizeof (fsszone_t));
1718 }
1719 mutex_exit(&fsspset->fssps_lock);
1720 return (ENOMEM);
1721 } else {
1722 fss_insert_fssproj(fsspset, kpj, fsszone, fssproj);
1723 }
1724 }
1725 fssproj->fssp_threads++;
1726 fssproc->fss_proj = fssproj;
1727
1728 /*
1729 * Reset priority. Process goes to a "user mode" priority here
1730 * regardless of whether or not it has slept since entering the kernel.
1731 */
1732 thread_lock(t);
1733 t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
1734 t->t_cid = cid;
1735 t->t_cldata = (void *)fssproc;
1736 t->t_schedflag |= TS_RUNQMATCH;
1737 fss_change_priority(t, fssproc);
1738 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
1739 t->t_state == TS_WAIT)
1740 fss_active(t);
1741 thread_unlock(t);
1742
1743 mutex_exit(&fsspset->fssps_lock);
1744
1745 /*
1746 * Link new structure into fssproc list.
1747 */
1748 FSS_LIST_INSERT(fssproc);
1749
1750 /*
1751 * If this is the first fair-sharing thread to occur since boot,
1752 * we set up the initial call to fss_update() here. Use an atomic
1753 * compare-and-swap since that's easier and faster than a mutex
1754 * (but check with an ordinary load first since most of the time
1755 * this will already be done).
1756 */
1757 if (fssexists == 0 && atomic_cas_32(&fssexists, 0, 1) == 0)
1758 (void) timeout(fss_update, NULL, hz);
1759
1760 return (0);
1761 }
1762
1763 /*
1764 * Remove fssproc_t from the list.
1765 */
1766 static void
1767 fss_exitclass(void *procp)
1768 {
1769 fssproc_t *fssproc = (fssproc_t *)procp;
1770 fssproj_t *fssproj;
1771 fsspset_t *fsspset;
1772 fsszone_t *fsszone;
1773 kthread_t *t = fssproc->fss_tp;
1774
1775 /*
1776 * We should be either getting this thread off the deathrow or
1777 * this thread has already moved to another scheduling class and
1778 * we're being called with its old cldata buffer pointer. In both
1779 * cases, the content of this buffer can not be changed while we're
1780 * here.
1781 */
1782 mutex_enter(&fsspsets_lock);
1783 thread_lock(t);
1784 if (t->t_cid != fss_cid) {
1785 /*
1786 * We're being called as a result of the priocntl() system
1787 * call -- someone is trying to move our thread to another
1788 * scheduling class. We can't call fss_inactive() here
1789 * because our thread's t_cldata pointer already points
1790 * to another scheduling class specific data.
1791 */
1792 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1793
1794 fssproj = FSSPROC2FSSPROJ(fssproc);
1795 fsspset = FSSPROJ2FSSPSET(fssproj);
1796 fsszone = fssproj->fssp_fsszone;
1797
1798 if (fssproc->fss_runnable) {
1799 disp_lock_enter_high(&fsspset->fssps_displock);
1800 if (--fssproj->fssp_runnable == 0) {
1801 fsszone->fssz_shares -= fssproj->fssp_shares;
1802 if (--fsszone->fssz_runnable == 0)
1803 fsspset->fssps_shares -=
1804 fsszone->fssz_rshares;
1805 }
1806 disp_lock_exit_high(&fsspset->fssps_displock);
1807 }
1808 thread_unlock(t);
1809
1810 mutex_enter(&fsspset->fssps_lock);
1811 if (--fssproj->fssp_threads == 0) {
1812 fss_remove_fssproj(fsspset, fssproj);
1813 if (fsszone->fssz_nproj == 0)
1814 kmem_free(fsszone, sizeof (fsszone_t));
1815 kmem_free(fssproj, sizeof (fssproj_t));
1816 }
1817 mutex_exit(&fsspset->fssps_lock);
1818
1819 } else {
1820 ASSERT(t->t_state == TS_FREE);
1821 /*
1822 * We're being called from thread_free() when our thread
1823 * is removed from the deathrow. There is nothing we need
1824 * do here since everything should've been done earlier
1825 * in fss_exit().
1826 */
1827 thread_unlock(t);
1828 }
1829 mutex_exit(&fsspsets_lock);
1830
1831 FSS_LIST_DELETE(fssproc);
1832 fss_free(fssproc);
1833 }
1834
1835 /*ARGSUSED*/
1836 static int
1837 fss_canexit(kthread_t *t, cred_t *credp)
1838 {
1839 /*
1840 * A thread is allowed to exit FSS only if we have sufficient
1841 * privileges.
1842 */
1843 if (credp != NULL && secpolicy_setpriority(credp) != 0)
1844 return (EPERM);
1845 else
1846 return (0);
1847 }
1848
1849 /*
1850 * Initialize fair-share class specific proc structure for a child.
1851 */
1852 static int
1853 fss_fork(kthread_t *pt, kthread_t *ct, void *bufp)
1854 {
1855 fssproc_t *pfssproc; /* ptr to parent's fssproc structure */
1856 fssproc_t *cfssproc; /* ptr to child's fssproc structure */
1857 fssproj_t *fssproj;
1858 fsspset_t *fsspset;
1859
1860 ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock));
1861 ASSERT(ct->t_state == TS_STOPPED);
1862
1863 cfssproc = (fssproc_t *)bufp;
1864 ASSERT(cfssproc != NULL);
1865 bzero(cfssproc, sizeof (fssproc_t));
1866
1867 thread_lock(pt);
1868 pfssproc = FSSPROC(pt);
1869 fssproj = FSSPROC2FSSPROJ(pfssproc);
1870 fsspset = FSSPROJ2FSSPSET(fssproj);
1871 thread_unlock(pt);
1872
1873 mutex_enter(&fsspset->fssps_lock);
1874 /*
1875 * Initialize child's fssproc structure.
1876 */
1877 thread_lock(pt);
1878 ASSERT(FSSPROJ(pt) == fssproj);
1879 cfssproc->fss_proj = fssproj;
1880 cfssproc->fss_timeleft = fss_quantum;
1881 cfssproc->fss_umdpri = pfssproc->fss_umdpri;
1882 cfssproc->fss_fsspri = 0;
1883 cfssproc->fss_uprilim = pfssproc->fss_uprilim;
1884 cfssproc->fss_upri = pfssproc->fss_upri;
1885 cfssproc->fss_tp = ct;
1886 cfssproc->fss_nice = pfssproc->fss_nice;
1887 cpucaps_sc_init(&cfssproc->fss_caps);
1888
1889 cfssproc->fss_flags =
1890 pfssproc->fss_flags & ~(FSSBACKQ | FSSRESTORE);
1891 ct->t_cldata = (void *)cfssproc;
1892 ct->t_schedflag |= TS_RUNQMATCH;
1893 thread_unlock(pt);
1894
1895 fssproj->fssp_threads++;
1896 mutex_exit(&fsspset->fssps_lock);
1897
1898 /*
1899 * Link new structure into fssproc hash table.
1900 */
1901 FSS_LIST_INSERT(cfssproc);
1902 return (0);
1903 }
1904
1905 /*
1906 * Child is placed at back of dispatcher queue and parent gives up processor
1907 * so that the child runs first after the fork. This allows the child
1908 * immediately execing to break the multiple use of copy on write pages with no
1909 * disk home. The parent will get to steal them back rather than uselessly
1910 * copying them.
1911 */
1912 static void
1913 fss_forkret(kthread_t *t, kthread_t *ct)
1914 {
1915 proc_t *pp = ttoproc(t);
1916 proc_t *cp = ttoproc(ct);
1917 fssproc_t *fssproc;
1918
1919 ASSERT(t == curthread);
1920 ASSERT(MUTEX_HELD(&pidlock));
1921
1922 /*
1923 * Grab the child's p_lock before dropping pidlock to ensure the
1924 * process does not disappear before we set it running.
1925 */
1926 mutex_enter(&cp->p_lock);
1927 continuelwps(cp);
1928 mutex_exit(&cp->p_lock);
1929
1930 mutex_enter(&pp->p_lock);
1931 mutex_exit(&pidlock);
1932 continuelwps(pp);
1933
1934 thread_lock(t);
1935
1936 fssproc = FSSPROC(t);
1937 fss_newpri(fssproc, B_FALSE);
1938 fssproc->fss_timeleft = fss_quantum;
1939 t->t_pri = fssproc->fss_umdpri;
1940 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1941 THREAD_TRANSITION(t);
1942
1943 /*
1944 * We don't want to call fss_setrun(t) here because it may call
1945 * fss_active, which we don't need.
1946 */
1947 fssproc->fss_flags &= ~FSSBACKQ;
1948
1949 if (t->t_disp_time != ddi_get_lbolt())
1950 setbackdq(t);
1951 else
1952 setfrontdq(t);
1953
1954 thread_unlock(t);
1955 /*
1956 * Safe to drop p_lock now since it is safe to change
1957 * the scheduling class after this point.
1958 */
1959 mutex_exit(&pp->p_lock);
1960
1961 swtch();
1962 }
1963
1964 /*
1965 * Get the fair-sharing parameters of the thread pointed to by fssprocp into
1966 * the buffer pointed by fssparmsp.
1967 */
1968 static void
1969 fss_parmsget(kthread_t *t, void *parmsp)
1970 {
1971 fssproc_t *fssproc = FSSPROC(t);
1972 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1973
1974 fssparmsp->fss_uprilim = fssproc->fss_uprilim;
1975 fssparmsp->fss_upri = fssproc->fss_upri;
1976 }
1977
1978 /*ARGSUSED*/
1979 static int
1980 fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1981 {
1982 char nice;
1983 pri_t reqfssuprilim;
1984 pri_t reqfssupri;
1985 fssproc_t *fssproc = FSSPROC(t);
1986 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1987
1988 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1989
1990 if (fssparmsp->fss_uprilim == FSS_NOCHANGE)
1991 reqfssuprilim = fssproc->fss_uprilim;
1992 else
1993 reqfssuprilim = fssparmsp->fss_uprilim;
1994
1995 if (fssparmsp->fss_upri == FSS_NOCHANGE)
1996 reqfssupri = fssproc->fss_upri;
1997 else
1998 reqfssupri = fssparmsp->fss_upri;
1999
2000 /*
2001 * Make sure the user priority doesn't exceed the upri limit.
2002 */
2003 if (reqfssupri > reqfssuprilim)
2004 reqfssupri = reqfssuprilim;
2005
2006 /*
2007 * Basic permissions enforced by generic kernel code for all classes
2008 * require that a thread attempting to change the scheduling parameters
2009 * of a target thread be privileged or have a real or effective UID
2010 * matching that of the target thread. We are not called unless these
2011 * basic permission checks have already passed. The fair-sharing class
2012 * requires in addition that the calling thread be privileged if it
2013 * is attempting to raise the upri limit above its current value.
2014 * This may have been checked previously but if our caller passed us
2015 * a non-NULL credential pointer we assume it hasn't and we check it
2016 * here.
2017 */
2018 if ((reqpcredp != NULL) &&
2019 (reqfssuprilim > fssproc->fss_uprilim) &&
2020 secpolicy_raisepriority(reqpcredp) != 0)
2021 return (EPERM);
2022
2023 /*
2024 * Set fss_nice to the nice value corresponding to the user priority we
2025 * are setting. Note that setting the nice field of the parameter
2026 * struct won't affect upri or nice.
2027 */
2028 nice = NZERO - (reqfssupri * NZERO) / fss_maxupri;
2029 if (nice > FSS_NICE_MAX)
2030 nice = FSS_NICE_MAX;
2031
2032 thread_lock(t);
2033
2034 fssproc->fss_uprilim = reqfssuprilim;
2035 fssproc->fss_upri = reqfssupri;
2036 fssproc->fss_nice = nice;
2037 fss_newpri(fssproc, B_FALSE);
2038
2039 fss_change_priority(t, fssproc);
2040 thread_unlock(t);
2041 return (0);
2042
2043 }
2044
2045 /*
2046 * The thread is being stopped.
2047 */
2048 /*ARGSUSED*/
2049 static void
2050 fss_stop(kthread_t *t, int why, int what)
2051 {
2052 ASSERT(THREAD_LOCK_HELD(t));
2053 ASSERT(t == curthread);
2054
2055 fss_inactive(t);
2056 }
2057
2058 /*
2059 * The current thread is exiting, do necessary adjustments to its project
2060 */
2061 static void
2062 fss_exit(kthread_t *t)
2063 {
2064 fsspset_t *fsspset;
2065 fssproj_t *fssproj;
2066 fssproc_t *fssproc;
2067 fsszone_t *fsszone;
2068 int free = 0;
2069
2070 /*
2071 * Thread t here is either a current thread (in which case we hold
2072 * its process' p_lock), or a thread being destroyed by forklwp_fail(),
2073 * in which case we hold pidlock and thread is no longer on the
2074 * thread list.
2075 */
2076 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock));
2077
2078 fssproc = FSSPROC(t);
2079 fssproj = FSSPROC2FSSPROJ(fssproc);
2080 fsspset = FSSPROJ2FSSPSET(fssproj);
2081 fsszone = fssproj->fssp_fsszone;
2082
2083 mutex_enter(&fsspsets_lock);
2084 mutex_enter(&fsspset->fssps_lock);
2085
2086 thread_lock(t);
2087 disp_lock_enter_high(&fsspset->fssps_displock);
2088 if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
2089 if (--fssproj->fssp_runnable == 0) {
2090 fsszone->fssz_shares -= fssproj->fssp_shares;
2091 if (--fsszone->fssz_runnable == 0)
2092 fsspset->fssps_shares -= fsszone->fssz_rshares;
2093 }
2094 ASSERT(fssproc->fss_runnable == 1);
2095 fssproc->fss_runnable = 0;
2096 }
2097 if (--fssproj->fssp_threads == 0) {
2098 fss_remove_fssproj(fsspset, fssproj);
2099 free = 1;
2100 }
2101 disp_lock_exit_high(&fsspset->fssps_displock);
2102 fssproc->fss_proj = NULL; /* mark this thread as already exited */
2103 thread_unlock(t);
2104
2105 if (free) {
2106 if (fsszone->fssz_nproj == 0)
2107 kmem_free(fsszone, sizeof (fsszone_t));
2108 kmem_free(fssproj, sizeof (fssproj_t));
2109 }
2110 mutex_exit(&fsspset->fssps_lock);
2111 mutex_exit(&fsspsets_lock);
2112
2113 /*
2114 * A thread could be exiting in between clock ticks, so we need to
2115 * calculate how much CPU time it used since it was charged last time.
2116 *
2117 * CPU caps are not enforced on exiting processes - it is usually
2118 * desirable to exit as soon as possible to free resources.
2119 */
2120 if (CPUCAPS_ON()) {
2121 thread_lock(t);
2122 fssproc = FSSPROC(t);
2123 (void) cpucaps_charge(t, &fssproc->fss_caps,
2124 CPUCAPS_CHARGE_ONLY);
2125 thread_unlock(t);
2126 }
2127 }
2128
2129 static void
2130 fss_nullsys()
2131 {
2132 }
2133
2134 /*
2135 * fss_swapin() returns -1 if the thread is loaded or is not eligible to be
2136 * swapped in. Otherwise, it returns the thread's effective priority based
2137 * on swapout time and size of process (0 <= epri <= 0 SHRT_MAX).
2138 */
2139 /*ARGSUSED*/
2140 static pri_t
2141 fss_swapin(kthread_t *t, int flags)
2142 {
2143 fssproc_t *fssproc = FSSPROC(t);
2144 long epri = -1;
2145 proc_t *pp = ttoproc(t);
2146
2147 ASSERT(THREAD_LOCK_HELD(t));
2148
2149 if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
2150 time_t swapout_time;
2151
2152 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
2153 if (INHERITED(t)) {
2154 epri = (long)DISP_PRIO(t) + swapout_time;
2155 } else {
2156 /*
2157 * Threads which have been out for a long time,
2158 * have high user mode priority and are associated
2159 * with a small address space are more deserving.
2160 */
2161 epri = fssproc->fss_umdpri;
2162 ASSERT(epri >= 0 && epri <= fss_maxumdpri);
2163 epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
2164 }
2165 /*
2166 * Scale epri so that SHRT_MAX / 2 represents zero priority.
2167 */
2168 epri += SHRT_MAX / 2;
2169 if (epri < 0)
2170 epri = 0;
2171 else if (epri > SHRT_MAX)
2172 epri = SHRT_MAX;
2173 }
2174 return ((pri_t)epri);
2175 }
2176
2177 /*
2178 * fss_swapout() returns -1 if the thread isn't loaded or is not eligible to
2179 * be swapped out. Otherwise, it returns the thread's effective priority
2180 * based on if the swapper is in softswap or hardswap mode.
2181 */
2182 static pri_t
2183 fss_swapout(kthread_t *t, int flags)
2184 {
2185 long epri = -1;
2186 proc_t *pp = ttoproc(t);
2187 time_t swapin_time;
2188
2189 ASSERT(THREAD_LOCK_HELD(t));
2190
2191 if (INHERITED(t) ||
2192 (t->t_proc_flag & TP_LWPEXIT) ||
2193 (t->t_state & (TS_ZOMB|TS_FREE|TS_STOPPED|TS_ONPROC|TS_WAIT)) ||
2194 !(t->t_schedflag & TS_LOAD) ||
2195 !(SWAP_OK(t)))
2196 return (-1);
2197
2198 ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
2199
2200 swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
2201
2202 if (flags == SOFTSWAP) {
2203 if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
2204 epri = 0;
2205 } else {
2206 return ((pri_t)epri);
2207 }
2208 } else {
2209 pri_t pri;
2210
2211 if ((t->t_state == TS_SLEEP && swapin_time > fss_minslp) ||
2212 (t->t_state == TS_RUN && swapin_time > fss_minrun)) {
2213 pri = fss_maxumdpri;
2214 epri = swapin_time -
2215 (rm_asrss(pp->p_as) / nz(maxpgio)/2) - (long)pri;
2216 } else {
2217 return ((pri_t)epri);
2218 }
2219 }
2220
2221 /*
2222 * Scale epri so that SHRT_MAX / 2 represents zero priority.
2223 */
2224 epri += SHRT_MAX / 2;
2225 if (epri < 0)
2226 epri = 0;
2227 else if (epri > SHRT_MAX)
2228 epri = SHRT_MAX;
2229
2230 return ((pri_t)epri);
2231 }
2232
2233 /*
2234 * Run swap-out checks when returning to userspace.
2235 */
2236 static void
2237 fss_trapret(kthread_t *t)
2238 {
2239 cpu_t *cp = CPU;
2240
2241 ASSERT(THREAD_LOCK_HELD(t));
2242 ASSERT(t == curthread);
2243 ASSERT(cp->cpu_dispthread == t);
2244 ASSERT(t->t_state == TS_ONPROC);
2245
2246 /*
2247 * Swapout lwp if the swapper is waiting for this thread to reach
2248 * a safe point.
2249 */
2250 if (t->t_schedflag & TS_SWAPENQ) {
2251 thread_unlock(t);
2252 swapout_lwp(ttolwp(t));
2253 thread_lock(t);
2254 }
2255 }
2256
2257 /*
2258 * Arrange for thread to be placed in appropriate location on dispatcher queue.
2259 * This is called with the current thread in TS_ONPROC and locked.
2260 */
2261 static void
2262 fss_preempt(kthread_t *t)
2263 {
2264 fssproc_t *fssproc = FSSPROC(t);
2265 klwp_t *lwp;
2266 uint_t flags;
2267
2268 ASSERT(t == curthread);
2269 ASSERT(THREAD_LOCK_HELD(curthread));
2270 ASSERT(t->t_state == TS_ONPROC);
2271
2272 /*
2273 * This thread may be placed on wait queue by CPU Caps. In this case we
2274 * do not need to do anything until it is removed from the wait queue.
2275 * Do not enforce CPU caps on threads running at a kernel priority
2276 */
2277 if (CPUCAPS_ON()) {
2278 (void) cpucaps_charge(t, &fssproc->fss_caps,
2279 CPUCAPS_CHARGE_ENFORCE);
2280
2281 if (CPUCAPS_ENFORCE(t))
2282 return;
2283 }
2284
2285 /*
2286 * If preempted in user-land mark the thread as swappable because it
2287 * cannot be holding any kernel locks.
2288 */
2289 ASSERT(t->t_schedflag & TS_DONT_SWAP);
2290 lwp = ttolwp(t);
2291 if (lwp != NULL && lwp->lwp_state == LWP_USER)
2292 t->t_schedflag &= ~TS_DONT_SWAP;
2293
2294 /*
2295 * Check to see if we're doing "preemption control" here. If
2296 * we are, and if the user has requested that this thread not
2297 * be preempted, and if preemptions haven't been put off for
2298 * too long, let the preemption happen here but try to make
2299 * sure the thread is rescheduled as soon as possible. We do
2300 * this by putting it on the front of the highest priority run
2301 * queue in the FSS class. If the preemption has been put off
2302 * for too long, clear the "nopreempt" bit and let the thread
2303 * be preempted.
2304 */
2305 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2306 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2307 DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
2308 /*
2309 * If not already remembered, remember current
2310 * priority for restoration in fss_yield().
2311 */
2312 if (!(fssproc->fss_flags & FSSRESTORE)) {
2313 fssproc->fss_scpri = t->t_pri;
2314 fssproc->fss_flags |= FSSRESTORE;
2315 }
2316 THREAD_CHANGE_PRI(t, fss_maxumdpri);
2317 t->t_schedflag |= TS_DONT_SWAP;
2318 schedctl_set_yield(t, 1);
2319 setfrontdq(t);
2320 return;
2321 } else {
2322 if (fssproc->fss_flags & FSSRESTORE) {
2323 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2324 fssproc->fss_flags &= ~FSSRESTORE;
2325 }
2326 schedctl_set_nopreempt(t, 0);
2327 DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
2328 /*
2329 * Fall through and be preempted below.
2330 */
2331 }
2332 }
2333
2334 flags = fssproc->fss_flags & FSSBACKQ;
2335
2336 if (flags == FSSBACKQ) {
2337 fssproc->fss_timeleft = fss_quantum;
2338 fssproc->fss_flags &= ~FSSBACKQ;
2339 setbackdq(t);
2340 } else {
2341 setfrontdq(t);
2342 }
2343 }
2344
2345 /*
2346 * Called when a thread is waking up and is to be placed on the run queue.
2347 */
2348 static void
2349 fss_setrun(kthread_t *t)
2350 {
2351 fssproc_t *fssproc = FSSPROC(t);
2352
2353 ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */
2354
2355 if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED)
2356 fss_active(t);
2357
2358 fssproc->fss_timeleft = fss_quantum;
2359
2360 fssproc->fss_flags &= ~FSSBACKQ;
2361 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2362
2363 if (t->t_disp_time != ddi_get_lbolt())
2364 setbackdq(t);
2365 else
2366 setfrontdq(t);
2367 }
2368
2369 /*
2370 * Prepare thread for sleep.
2371 */
2372 static void
2373 fss_sleep(kthread_t *t)
2374 {
2375 fssproc_t *fssproc = FSSPROC(t);
2376
2377 ASSERT(t == curthread);
2378 ASSERT(THREAD_LOCK_HELD(t));
2379
2380 ASSERT(t->t_state == TS_ONPROC);
2381
2382 /*
2383 * Account for time spent on CPU before going to sleep.
2384 */
2385 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2386
2387 fss_inactive(t);
2388 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
2389 }
2390
2391 /*
2392 * A tick interrupt has ocurrend on a running thread. Check to see if our
2393 * time slice has expired. We must also clear the TS_DONT_SWAP flag in
2394 * t_schedflag if the thread is eligible to be swapped out.
2395 */
2396 static void
2397 fss_tick(kthread_t *t)
2398 {
2399 fssproc_t *fssproc;
2400 fssproj_t *fssproj;
2401 klwp_t *lwp;
2402 boolean_t call_cpu_surrender = B_FALSE;
2403 boolean_t cpucaps_enforce = B_FALSE;
2404
2405 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
2406
2407 /*
2408 * It's safe to access fsspset and fssproj structures because we're
2409 * holding our p_lock here.
2410 */
2411 thread_lock(t);
2412 fssproc = FSSPROC(t);
2413 fssproj = FSSPROC2FSSPROJ(fssproc);
2414 if (fssproj != NULL) {
2415 fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
2416 disp_lock_enter_high(&fsspset->fssps_displock);
2417 fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2418 fssproj->fssp_tick_cnt++;
2419 fssproc->fss_ticks++;
2420 disp_lock_exit_high(&fsspset->fssps_displock);
2421 }
2422
2423 /*
2424 * Keep track of thread's project CPU usage. Note that projects
2425 * get charged even when threads are running in the kernel.
2426 * Do not surrender CPU if running in the SYS class.
2427 */
2428 if (CPUCAPS_ON()) {
2429 cpucaps_enforce = cpucaps_charge(t, &fssproc->fss_caps,
2430 CPUCAPS_CHARGE_ENFORCE);
2431 }
2432
2433 if (--fssproc->fss_timeleft <= 0) {
2434 pri_t new_pri;
2435
2436 /*
2437 * If we're doing preemption control and trying to avoid
2438 * preempting this thread, just note that the thread should
2439 * yield soon and let it keep running (unless it's been a
2440 * while).
2441 */
2442 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2443 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2444 DTRACE_SCHED1(schedctl__nopreempt,
2445 kthread_t *, t);
2446 schedctl_set_yield(t, 1);
2447 thread_unlock_nopreempt(t);
2448 return;
2449 }
2450 }
2451 fssproc->fss_flags &= ~FSSRESTORE;
2452
2453 fss_newpri(fssproc, B_TRUE);
2454 new_pri = fssproc->fss_umdpri;
2455 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
2456
2457 /*
2458 * When the priority of a thread is changed, it may be
2459 * necessary to adjust its position on a sleep queue or
2460 * dispatch queue. The function thread_change_pri accomplishes
2461 * this.
2462 */
2463 if (thread_change_pri(t, new_pri, 0)) {
2464 if ((t->t_schedflag & TS_LOAD) &&
2465 (lwp = t->t_lwp) &&
2466 lwp->lwp_state == LWP_USER)
2467 t->t_schedflag &= ~TS_DONT_SWAP;
2468 fssproc->fss_timeleft = fss_quantum;
2469 } else {
2470 call_cpu_surrender = B_TRUE;
2471 }
2472 } else if (t->t_state == TS_ONPROC &&
2473 t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2474 /*
2475 * If there is a higher-priority thread which is waiting for a
2476 * processor, then thread surrenders the processor.
2477 */
2478 call_cpu_surrender = B_TRUE;
2479 }
2480
2481 if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2482 /*
2483 * The thread used more than half of its quantum, so assume that
2484 * it used the whole quantum.
2485 *
2486 * Update thread's priority just before putting it on the wait
2487 * queue so that it gets charged for the CPU time from its
2488 * quantum even before that quantum expires.
2489 */
2490 fss_newpri(fssproc, B_FALSE);
2491 if (t->t_pri != fssproc->fss_umdpri)
2492 fss_change_priority(t, fssproc);
2493
2494 /*
2495 * We need to call cpu_surrender for this thread due to cpucaps
2496 * enforcement, but fss_change_priority may have already done
2497 * so. In this case FSSBACKQ is set and there is no need to call
2498 * cpu-surrender again.
2499 */
2500 if (!(fssproc->fss_flags & FSSBACKQ))
2501 call_cpu_surrender = B_TRUE;
2502 }
2503
2504 if (call_cpu_surrender) {
2505 fssproc->fss_flags |= FSSBACKQ;
2506 cpu_surrender(t);
2507 }
2508
2509 thread_unlock_nopreempt(t); /* clock thread can't be preempted */
2510 }
2511
2512 /*
2513 * Processes waking up go to the back of their queue. We don't need to assign
2514 * a time quantum here because thread is still at a kernel mode priority and
2515 * the time slicing is not done for threads running in the kernel after
2516 * sleeping. The proper time quantum will be assigned by fss_trapret before the
2517 * thread returns to user mode.
2518 */
2519 static void
2520 fss_wakeup(kthread_t *t)
2521 {
2522 fssproc_t *fssproc;
2523
2524 ASSERT(THREAD_LOCK_HELD(t));
2525 ASSERT(t->t_state == TS_SLEEP);
2526
2527 fss_active(t);
2528
2529 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
2530 fssproc = FSSPROC(t);
2531 fssproc->fss_flags &= ~FSSBACKQ;
2532
2533 /* Recalculate the priority. */
2534 if (t->t_disp_time == ddi_get_lbolt()) {
2535 setfrontdq(t);
2536 } else {
2537 fssproc->fss_timeleft = fss_quantum;
2538 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2539 setbackdq(t);
2540 }
2541 }
2542
2543 /*
2544 * fss_donice() is called when a nice(1) command is issued on the thread to
2545 * alter the priority. The nice(1) command exists in Solaris for compatibility.
2546 * Thread priority adjustments should be done via priocntl(1).
2547 */
2548 static int
2549 fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2550 {
2551 int newnice;
2552 fssproc_t *fssproc = FSSPROC(t);
2553 fssparms_t fssparms;
2554
2555 /*
2556 * If there is no change to priority, just return current setting.
2557 */
2558 if (incr == 0) {
2559 if (retvalp)
2560 *retvalp = fssproc->fss_nice - NZERO;
2561 return (0);
2562 }
2563
2564 if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0)
2565 return (EPERM);
2566
2567 /*
2568 * Specifying a nice increment greater than the upper limit of
2569 * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice
2570 * value being set to the upper limit. We check for this before
2571 * computing the new value because otherwise we could get overflow
2572 * if a privileged user specified some ridiculous increment.
2573 */
2574 if (incr > FSS_NICE_MAX)
2575 incr = FSS_NICE_MAX;
2576
2577 newnice = fssproc->fss_nice + incr;
2578 if (newnice > FSS_NICE_MAX)
2579 newnice = FSS_NICE_MAX;
2580 else if (newnice < FSS_NICE_MIN)
2581 newnice = FSS_NICE_MIN;
2582
2583 fssparms.fss_uprilim = fssparms.fss_upri =
2584 -((newnice - NZERO) * fss_maxupri) / NZERO;
2585
2586 /*
2587 * Reset the uprilim and upri values of the thread.
2588 */
2589 (void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL);
2590
2591 /*
2592 * Although fss_parmsset already reset fss_nice it may not have been
2593 * set to precisely the value calculated above because fss_parmsset
2594 * determines the nice value from the user priority and we may have
2595 * truncated during the integer conversion from nice value to user
2596 * priority and back. We reset fss_nice to the value we calculated
2597 * above.
2598 */
2599 fssproc->fss_nice = (char)newnice;
2600
2601 if (retvalp)
2602 *retvalp = newnice - NZERO;
2603 return (0);
2604 }
2605
2606 /*
2607 * Increment the priority of the specified thread by incr and
2608 * return the new value in *retvalp.
2609 */
2610 static int
2611 fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2612 {
2613 int newpri;
2614 fssproc_t *fssproc = FSSPROC(t);
2615 fssparms_t fssparms;
2616
2617 /*
2618 * If there is no change to priority, just return current setting.
2619 */
2620 if (incr == 0) {
2621 *retvalp = fssproc->fss_upri;
2622 return (0);
2623 }
2624
2625 newpri = fssproc->fss_upri + incr;
2626 if (newpri > fss_maxupri || newpri < -fss_maxupri)
2627 return (EINVAL);
2628
2629 *retvalp = newpri;
2630 fssparms.fss_uprilim = fssparms.fss_upri = newpri;
2631
2632 /*
2633 * Reset the uprilim and upri values of the thread.
2634 */
2635 return (fss_parmsset(t, &fssparms, (id_t)0, cr));
2636 }
2637
2638 /*
2639 * Return the global scheduling priority that would be assigned to a thread
2640 * entering the fair-sharing class with the fss_upri.
2641 */
2642 /*ARGSUSED*/
2643 static pri_t
2644 fss_globpri(kthread_t *t)
2645 {
2646 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2647
2648 return (fss_maxumdpri / 2);
2649 }
2650
2651 /*
2652 * Called from the yield(2) system call when a thread is yielding (surrendering)
2653 * the processor. The kernel thread is placed at the back of a dispatch queue.
2654 */
2655 static void
2656 fss_yield(kthread_t *t)
2657 {
2658 fssproc_t *fssproc = FSSPROC(t);
2659
2660 ASSERT(t == curthread);
2661 ASSERT(THREAD_LOCK_HELD(t));
2662
2663 /*
2664 * Collect CPU usage spent before yielding
2665 */
2666 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2667
2668 /*
2669 * Clear the preemption control "yield" bit since the user is
2670 * doing a yield.
2671 */
2672 if (t->t_schedctl)
2673 schedctl_set_yield(t, 0);
2674 /*
2675 * If fss_preempt() artifically increased the thread's priority
2676 * to avoid preemption, restore the original priority now.
2677 */
2678 if (fssproc->fss_flags & FSSRESTORE) {
2679 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2680 fssproc->fss_flags &= ~FSSRESTORE;
2681 }
2682 if (fssproc->fss_timeleft < 0) {
2683 /*
2684 * Time slice was artificially extended to avoid preemption,
2685 * so pretend we're preempting it now.
2686 */
2687 DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft);
2688 fssproc->fss_timeleft = fss_quantum;
2689 }
2690 fssproc->fss_flags &= ~FSSBACKQ;
2691 setbackdq(t);
2692 }
2693
2694 void
2695 fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf,
2696 fssbuf_t *zonebuf)
2697 {
2698 kproject_t *kpj_new = kp;
2699 zone_t *zone = zp;
2700 fssproj_t *fssproj_old, *fssproj_new;
2701 fsspset_t *fsspset;
2702 kproject_t *kpj_old;
2703 fssproc_t *fssproc;
2704 fsszone_t *fsszone_old, *fsszone_new;
2705 int free = 0;
2706 int id;
2707
2708 ASSERT(MUTEX_HELD(&cpu_lock));
2709 ASSERT(MUTEX_HELD(&pidlock));
2710 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2711
2712 if (t->t_cid != fss_cid)
2713 return;
2714
2715 fssproc = FSSPROC(t);
2716 mutex_enter(&fsspsets_lock);
2717 fssproj_old = FSSPROC2FSSPROJ(fssproc);
2718 if (fssproj_old == NULL) {
2719 mutex_exit(&fsspsets_lock);
2720 return;
2721 }
2722
2723 fsspset = FSSPROJ2FSSPSET(fssproj_old);
2724 mutex_enter(&fsspset->fssps_lock);
2725 kpj_old = FSSPROJ2KPROJ(fssproj_old);
2726 fsszone_old = fssproj_old->fssp_fsszone;
2727
2728 ASSERT(t->t_cpupart == fsspset->fssps_cpupart);
2729
2730 if (kpj_old == kpj_new) {
2731 mutex_exit(&fsspset->fssps_lock);
2732 mutex_exit(&fsspsets_lock);
2733 return;
2734 }
2735
2736 if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) {
2737 /*
2738 * If the zone for the new project is not currently active on
2739 * the cpu partition we're on, get one of the pre-allocated
2740 * buffers and link it in our per-pset zone list. Such buffers
2741 * should already exist.
2742 */
2743 for (id = 0; id < zonebuf->fssb_size; id++) {
2744 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2745 fss_insert_fsszone(fsspset, zone, fsszone_new);
2746 zonebuf->fssb_list[id] = NULL;
2747 break;
2748 }
2749 }
2750 }
2751 ASSERT(fsszone_new != NULL);
2752 if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) {
2753 /*
2754 * If our new project is not currently running
2755 * on the cpu partition we're on, get one of the
2756 * pre-allocated buffers and link it in our new cpu
2757 * partition doubly linked list. Such buffers should already
2758 * exist.
2759 */
2760 for (id = 0; id < projbuf->fssb_size; id++) {
2761 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2762 fss_insert_fssproj(fsspset, kpj_new,
2763 fsszone_new, fssproj_new);
2764 projbuf->fssb_list[id] = NULL;
2765 break;
2766 }
2767 }
2768 }
2769 ASSERT(fssproj_new != NULL);
2770
2771 thread_lock(t);
2772 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2773 t->t_state == TS_WAIT)
2774 fss_inactive(t);
2775 ASSERT(fssproj_old->fssp_threads > 0);
2776 if (--fssproj_old->fssp_threads == 0) {
2777 fss_remove_fssproj(fsspset, fssproj_old);
2778 free = 1;
2779 }
2780 fssproc->fss_proj = fssproj_new;
2781 fssproc->fss_fsspri = 0;
2782 fssproj_new->fssp_threads++;
2783 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2784 t->t_state == TS_WAIT)
2785 fss_active(t);
2786 thread_unlock(t);
2787 if (free) {
2788 if (fsszone_old->fssz_nproj == 0)
2789 kmem_free(fsszone_old, sizeof (fsszone_t));
2790 kmem_free(fssproj_old, sizeof (fssproj_t));
2791 }
2792
2793 mutex_exit(&fsspset->fssps_lock);
2794 mutex_exit(&fsspsets_lock);
2795 }
2796
2797 void
2798 fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf,
2799 fssbuf_t *zonebuf)
2800 {
2801 fsspset_t *fsspset_old, *fsspset_new;
2802 fssproj_t *fssproj_old, *fssproj_new;
2803 fsszone_t *fsszone_old, *fsszone_new;
2804 fssproc_t *fssproc;
2805 kproject_t *kpj;
2806 zone_t *zone;
2807 int id;
2808
2809 ASSERT(MUTEX_HELD(&cpu_lock));
2810 ASSERT(MUTEX_HELD(&pidlock));
2811 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2812
2813 if (t->t_cid != fss_cid)
2814 return;
2815
2816 fssproc = FSSPROC(t);
2817 zone = ttoproc(t)->p_zone;
2818 mutex_enter(&fsspsets_lock);
2819 fssproj_old = FSSPROC2FSSPROJ(fssproc);
2820 if (fssproj_old == NULL) {
2821 mutex_exit(&fsspsets_lock);
2822 return;
2823 }
2824 fsszone_old = fssproj_old->fssp_fsszone;
2825 fsspset_old = FSSPROJ2FSSPSET(fssproj_old);
2826 kpj = FSSPROJ2KPROJ(fssproj_old);
2827
2828 if (fsspset_old->fssps_cpupart == newcp) {
2829 mutex_exit(&fsspsets_lock);
2830 return;
2831 }
2832
2833 ASSERT(ttoproj(t) == kpj);
2834
2835 fsspset_new = fss_find_fsspset(newcp);
2836
2837 mutex_enter(&fsspset_new->fssps_lock);
2838 if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) {
2839 for (id = 0; id < zonebuf->fssb_size; id++) {
2840 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2841 fss_insert_fsszone(fsspset_new, zone,
2842 fsszone_new);
2843 zonebuf->fssb_list[id] = NULL;
2844 break;
2845 }
2846 }
2847 }
2848 ASSERT(fsszone_new != NULL);
2849 if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) {
2850 for (id = 0; id < projbuf->fssb_size; id++) {
2851 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2852 fss_insert_fssproj(fsspset_new, kpj,
2853 fsszone_new, fssproj_new);
2854 projbuf->fssb_list[id] = NULL;
2855 break;
2856 }
2857 }
2858 }
2859 ASSERT(fssproj_new != NULL);
2860
2861 fssproj_new->fssp_threads++;
2862 thread_lock(t);
2863 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2864 t->t_state == TS_WAIT)
2865 fss_inactive(t);
2866 fssproc->fss_proj = fssproj_new;
2867 fssproc->fss_fsspri = 0;
2868 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2869 t->t_state == TS_WAIT)
2870 fss_active(t);
2871 thread_unlock(t);
2872 mutex_exit(&fsspset_new->fssps_lock);
2873
2874 mutex_enter(&fsspset_old->fssps_lock);
2875 if (--fssproj_old->fssp_threads == 0) {
2876 fss_remove_fssproj(fsspset_old, fssproj_old);
2877 if (fsszone_old->fssz_nproj == 0)
2878 kmem_free(fsszone_old, sizeof (fsszone_t));
2879 kmem_free(fssproj_old, sizeof (fssproj_t));
2880 }
2881 mutex_exit(&fsspset_old->fssps_lock);
2882
2883 mutex_exit(&fsspsets_lock);
2884 }