1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2013, Joyent, Inc. All rights reserved. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/param.h> 29 #include <sys/sysmacros.h> 30 #include <sys/cred.h> 31 #include <sys/proc.h> 32 #include <sys/strsubr.h> 33 #include <sys/priocntl.h> 34 #include <sys/class.h> 35 #include <sys/disp.h> 36 #include <sys/procset.h> 37 #include <sys/debug.h> 38 #include <sys/kmem.h> 39 #include <sys/errno.h> 40 #include <sys/systm.h> 41 #include <sys/schedctl.h> 42 #include <sys/vmsystm.h> 43 #include <sys/atomic.h> 44 #include <sys/project.h> 45 #include <sys/modctl.h> 46 #include <sys/fss.h> 47 #include <sys/fsspriocntl.h> 48 #include <sys/cpupart.h> 49 #include <sys/zone.h> 50 #include <vm/rm.h> 51 #include <vm/seg_kmem.h> 52 #include <sys/tnf_probe.h> 53 #include <sys/policy.h> 54 #include <sys/sdt.h> 55 #include <sys/cpucaps.h> 56 57 /* 58 * The fair share scheduling class ensures that collections of processes 59 * (zones and projects) each get their configured share of CPU. This is in 60 * contrast to the TS class which considers individual processes. 61 * 62 * The FSS cpu-share is set on zones using the zone.cpu-shares rctl and on 63 * projects using the project.cpu-shares rctl. By default the value is 1 64 * and it can range from 0 - 64k. A value of 0 means that processes in the 65 * collection will only get CPU resources when there are no other processes 66 * that need CPU. The cpu-share is used as one of the inputs to calculate a 67 * thread's "user-mode" priority (umdpri) for the scheduler. The umdpri falls 68 * in the range 0-59. FSS calculates other, internal, priorities which are not 69 * visible outside of the FSS class. 70 * 71 * The FSS class should approximate TS behavior when there are excess CPU 72 * resources. When there is a backlog of runnable processes, then the share 73 * is used as input into the runnable process's priority calculation, where 74 * the final umdpri is used by the scheduler to determine when the process runs. 75 * 76 * Projects in a zone compete with each other for CPU time, receiving CPU 77 * allocation within a zone proportional to the project's share; at a higher 78 * level zones compete with each other, receiving allocation in a pset 79 * proportional to the zone's share. 80 * 81 * The FSS priority calculation consists of several parts. 82 * 83 * 1) Once per second the fss_update function runs. The first thing it does is 84 * call fss_decay_usage. This function does three things. 85 * 86 * a) fss_decay_usage first decays the maxfsspri value for the pset. This 87 * value is used in the per-process priority calculation described in step 88 * (2b). The maxfsspri is decayed using the following formula: 89 * 90 * maxfsspri * fss_nice_decay[NZERO]) 91 * maxfsspri = ------------------------------------ 92 * FSS_DECAY_BASE 93 * 94 * 95 * - NZERO is the default process priority (i.e. 20) 96 * 97 * The fss_nice_decay array is a fixed set of values used to adjust the 98 * decay rate of processes based on their nice value. Entries in this 99 * array are initialized in fss_init using the following formula: 100 * 101 * (FSS_DECAY_MAX - FSS_DECAY_MIN) * i 102 * FSS_DECAY_MIN + ------------------------------------- 103 * FSS_NICE_RANGE - 1 104 * 105 * - FSS_DECAY_MIN is 82 = approximates 65% (82/128) 106 * - FSS_DECAY_MAX is 108 = approximates 85% (108/128) 107 * - FSS_NICE_RANGE is 40 (range is 0 - 39) 108 * 109 * b) The second thing fss_decay_usage does is update each project's "usage" 110 * for the last second and then recalculates the project's "share usage". 111 * 112 * The usage value is the recent CPU usage for all of the threads in the 113 * project. It is decayed and updated this way: 114 * 115 * (usage * FSS_DECAY_USG) 116 * usage = ------------------------- + ticks; 117 * FSS_DECAY_BASE 118 * 119 * - FSS_DECAY_BASE is 128 - used instead of 100 so we can shift vs divide 120 * - FSS_DECAY_USG is 96 - approximates 75% (96/128) 121 * - ticks is updated whenever a process in this project is running 122 * when the scheduler's tick processing fires. This is not a simple 123 * counter, the values are based on the entries in the fss_nice_tick 124 * array (see section 3 below). ticks is then reset to 0 so it can track 125 * the next seconds worth of nice-adjusted time for the project. 126 * 127 * c) The third thing fss_decay_usage does is update each project's "share 128 * usage" (shusage). This is the normalized usage value for the project and 129 * is calculated this way: 130 * 131 * pset_shares^2 zone_int_shares^2 132 * usage * ------------- * ------------------ 133 * kpj_shares^2 zone_ext_shares^2 134 * 135 * - usage - see (1b) for more details 136 * - pset_shares is the total of all *active* zone shares in the pset (by 137 * default there is only one pset) 138 * - kpj_shares is the individual project's share (project.cpu-shares rctl) 139 * - zone_int_shares is the sum of shares of all active projects within the 140 * zone (the zone-internal total) 141 * - zone_ext_shares is the share value for the zone (zone.cpu-shares rctl) 142 * 143 * The shusage is used in step (2b) to calculate the thread's new internal 144 * priority. A larger shusage value leads to a lower priority. 145 * 146 * 2) The fss_update function then calls fss_update_list to update the priority 147 * of all threads. This does two things. 148 * 149 * a) First the thread's internal priority is decayed using the following 150 * formula: 151 * 152 * fsspri * fss_nice_decay[nice_value]) 153 * fsspri = ------------------------------------ 154 * FSS_DECAY_BASE 155 * 156 * - FSS_DECAY_BASE is 128 as described above 157 * 158 * b) Second, if the thread is runnable (TS_RUN or TS_WAIT) calls fss_newpri 159 * to update the user-mode priority (umdpri) of the runnable thread. 160 * Threads that are running (TS_ONPROC) or waiting for an event (TS_SLEEP) 161 * are not updated at this time. The updated user-mode priority can cause 162 * threads to change their position in the run queue. 163 * 164 * The process's new internal fsspri is calculated using the following 165 * formula. All runnable threads in the project will use the same shusage 166 * and nrunnable values in their calculation. 167 * 168 * fsspri += shusage * nrunnable * ticks 169 * 170 * - shusage is the project's share usage, calculated in (1c) 171 * - nrunnable is the number of runnable threads in the project 172 * - ticks is the number of ticks this thread ran since the last fss_newpri 173 * invocation. 174 * 175 * Finally the process's new user-mode priority is calculated using the 176 * following formula: 177 * 178 * (fsspri * umdprirange) 179 * umdpri = maxumdpri - ------------------------ 180 * maxfsspri 181 * 182 * - maxumdpri is MINCLSYSPRI - 1 (i.e. 59) 183 * - umdprirange is maxumdpri - 1 (i.e. 58) 184 * - maxfsspri is the largest fsspri seen so far, as we're iterating all 185 * runnable processes 186 * 187 * Thus, a higher internal priority (fsspri) leads to a lower user-mode 188 * priority which means the thread runs less. The fsspri is higher when 189 * the project's normalized share usage is higher, when the project has 190 * more runnable threads, or when the thread has accumulated more run-time. 191 * 192 * This code has various checks to ensure the resulting umdpri is in the 193 * range 1-59. See fss_newpri for more details. 194 * 195 * To reiterate, the above processing is performed once per second to recompute 196 * the runnable thread user-mode priorities. 197 * 198 * 3) The final major component in the priority calculation is the tick 199 * processing which occurs on a thread that is running when the clock 200 * calls fss_tick. 201 * 202 * A thread can run continuously in user-land (compute-bound) for the 203 * fss_quantum (see "dispadmin -c FSS -g" for the configurable properties). 204 * The fss_quantum defaults to 11 (i.e. 11 ticks). 205 * 206 * Once the quantum has been consumed, the thread will call fss_newpri to 207 * recompute its umdpri priority, as described above in (2b). Threads that 208 * were T_ONPROC at the one second interval when runnable thread priorities 209 * were recalculated will have their umdpri priority recalculated when their 210 * quanta expires. 211 * 212 * To ensure that runnable threads within a project see the expected 213 * round-robin behavior, there is a special case in fss_newpri for a thread 214 * that has run for its quanta within the one second update interval. See 215 * the handling for the quanta_up parameter within fss_newpri. 216 * 217 * Also of interest, the fss_tick code increments the project's tick value 218 * using the fss_nice_tick array entry for the thread's nice value. The idea 219 * behind the fss_nice_tick array is that the cost of a tick is lower at 220 * positive nice values (so that it doesn't increase the project's usage 221 * as much as normal) with a 50% drop at the maximum level and a 50% 222 * increase at the minimum level. See (1b). The fss_nice_tick array is 223 * initialized in fss_init using the following formula: 224 * 225 * FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2) - i) 226 * -------------------------------------------------- 227 * FSS_NICE_RANGE 228 * 229 * - FSS_TICK_COST is 1000, the tick cost for threads with nice level 0 230 * 231 * FSS Data Structures: 232 * 233 * fsszone 234 * ----- ----- 235 * ----- | | | | 236 * | |-------->| |<------->| |<---->... 237 * | | ----- ----- 238 * | | ^ ^ ^ 239 * | |--- | \ \ 240 * ----- | | \ \ 241 * fsspset | | \ \ 242 * | | \ \ 243 * | ----- ----- ----- 244 * -->| |<--->| |<--->| | 245 * | | | | | | 246 * ----- ----- ----- 247 * fssproj 248 * 249 * That is, fsspsets contain a list of fsszone's that are currently active in 250 * the pset, and a list of fssproj's, corresponding to projects with runnable 251 * threads on the pset. fssproj's in turn point to the fsszone which they 252 * are a member of. 253 * 254 * An fssproj_t is removed when there are no threads in it. 255 * 256 * An fsszone_t is removed when there are no projects with threads in it. 257 */ 258 259 static pri_t fss_init(id_t, int, classfuncs_t **); 260 261 static struct sclass fss = { 262 "FSS", 263 fss_init, 264 0 265 }; 266 267 extern struct mod_ops mod_schedops; 268 269 /* 270 * Module linkage information for the kernel. 271 */ 272 static struct modlsched modlsched = { 273 &mod_schedops, "fair share scheduling class", &fss 274 }; 275 276 static struct modlinkage modlinkage = { 277 MODREV_1, { (void *)&modlsched, NULL } 278 }; 279 280 #define FSS_MAXUPRI 60 281 282 /* 283 * The fssproc_t structures are kept in an array of circular doubly linked 284 * lists. A hash on the thread pointer is used to determine which list each 285 * thread should be placed in. Each list has a dummy "head" which is never 286 * removed, so the list is never empty. fss_update traverses these lists to 287 * update the priorities of threads that have been waiting on the run queue. 288 */ 289 #define FSS_LISTS 16 /* number of lists, must be power of 2 */ 290 #define FSS_LIST_HASH(t) (((uintptr_t)(t) >> 9) & (FSS_LISTS - 1)) 291 #define FSS_LIST_NEXT(i) (((i) + 1) & (FSS_LISTS - 1)) 292 293 #define FSS_LIST_INSERT(fssproc) \ 294 { \ 295 int index = FSS_LIST_HASH(fssproc->fss_tp); \ 296 kmutex_t *lockp = &fss_listlock[index]; \ 297 fssproc_t *headp = &fss_listhead[index]; \ 298 mutex_enter(lockp); \ 299 fssproc->fss_next = headp->fss_next; \ 300 fssproc->fss_prev = headp; \ 301 headp->fss_next->fss_prev = fssproc; \ 302 headp->fss_next = fssproc; \ 303 mutex_exit(lockp); \ 304 } 305 306 #define FSS_LIST_DELETE(fssproc) \ 307 { \ 308 int index = FSS_LIST_HASH(fssproc->fss_tp); \ 309 kmutex_t *lockp = &fss_listlock[index]; \ 310 mutex_enter(lockp); \ 311 fssproc->fss_prev->fss_next = fssproc->fss_next; \ 312 fssproc->fss_next->fss_prev = fssproc->fss_prev; \ 313 mutex_exit(lockp); \ 314 } 315 316 #define FSS_TICK_COST 1000 /* tick cost for threads with nice level = 0 */ 317 318 /* 319 * Decay rate percentages are based on n/128 rather than n/100 so that 320 * calculations can avoid having to do an integer divide by 100 (divide 321 * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift). 322 * 323 * FSS_DECAY_MIN = 83/128 ~= 65% 324 * FSS_DECAY_MAX = 108/128 ~= 85% 325 * FSS_DECAY_USG = 96/128 ~= 75% 326 */ 327 #define FSS_DECAY_MIN 83 /* fsspri decay pct for threads w/ nice -20 */ 328 #define FSS_DECAY_MAX 108 /* fsspri decay pct for threads w/ nice +19 */ 329 #define FSS_DECAY_USG 96 /* fssusage decay pct for projects */ 330 #define FSS_DECAY_BASE 128 /* base for decay percentages above */ 331 332 #define FSS_NICE_MIN 0 333 #define FSS_NICE_MAX (2 * NZERO - 1) 334 #define FSS_NICE_RANGE (FSS_NICE_MAX - FSS_NICE_MIN + 1) 335 336 static int fss_nice_tick[FSS_NICE_RANGE]; 337 static int fss_nice_decay[FSS_NICE_RANGE]; 338 339 static pri_t fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */ 340 static pri_t fss_maxumdpri; /* maximum user mode fss priority */ 341 static pri_t fss_maxglobpri; /* maximum global priority used by fss class */ 342 static pri_t fss_minglobpri; /* minimum global priority */ 343 344 static fssproc_t fss_listhead[FSS_LISTS]; 345 static kmutex_t fss_listlock[FSS_LISTS]; 346 347 static fsspset_t *fsspsets; 348 static kmutex_t fsspsets_lock; /* protects fsspsets */ 349 350 static id_t fss_cid; 351 352 static time_t fss_minrun = 2; /* t_pri becomes 59 within 2 secs */ 353 static time_t fss_minslp = 2; /* min time on sleep queue for hardswap */ 354 static int fss_quantum = 11; 355 356 static void fss_newpri(fssproc_t *, boolean_t); 357 static void fss_update(void *); 358 static int fss_update_list(int); 359 static void fss_change_priority(kthread_t *, fssproc_t *); 360 361 static int fss_admin(caddr_t, cred_t *); 362 static int fss_getclinfo(void *); 363 static int fss_parmsin(void *); 364 static int fss_parmsout(void *, pc_vaparms_t *); 365 static int fss_vaparmsin(void *, pc_vaparms_t *); 366 static int fss_vaparmsout(void *, pc_vaparms_t *); 367 static int fss_getclpri(pcpri_t *); 368 static int fss_alloc(void **, int); 369 static void fss_free(void *); 370 371 static int fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *); 372 static void fss_exitclass(void *); 373 static int fss_canexit(kthread_t *, cred_t *); 374 static int fss_fork(kthread_t *, kthread_t *, void *); 375 static void fss_forkret(kthread_t *, kthread_t *); 376 static void fss_parmsget(kthread_t *, void *); 377 static int fss_parmsset(kthread_t *, void *, id_t, cred_t *); 378 static void fss_stop(kthread_t *, int, int); 379 static void fss_exit(kthread_t *); 380 static void fss_active(kthread_t *); 381 static void fss_inactive(kthread_t *); 382 static pri_t fss_swapin(kthread_t *, int); 383 static pri_t fss_swapout(kthread_t *, int); 384 static void fss_trapret(kthread_t *); 385 static void fss_preempt(kthread_t *); 386 static void fss_setrun(kthread_t *); 387 static void fss_sleep(kthread_t *); 388 static void fss_tick(kthread_t *); 389 static void fss_wakeup(kthread_t *); 390 static int fss_donice(kthread_t *, cred_t *, int, int *); 391 static int fss_doprio(kthread_t *, cred_t *, int, int *); 392 static pri_t fss_globpri(kthread_t *); 393 static void fss_yield(kthread_t *); 394 static void fss_nullsys(); 395 396 static struct classfuncs fss_classfuncs = { 397 /* class functions */ 398 { fss_admin, 399 fss_getclinfo, 400 fss_parmsin, 401 fss_parmsout, 402 fss_vaparmsin, 403 fss_vaparmsout, 404 fss_getclpri, 405 fss_alloc, 406 fss_free }, 407 408 /* thread functions */ 409 { fss_enterclass, 410 fss_exitclass, 411 fss_canexit, 412 fss_fork, 413 fss_forkret, 414 fss_parmsget, 415 fss_parmsset, 416 fss_stop, 417 fss_exit, 418 fss_active, 419 fss_inactive, 420 fss_swapin, 421 fss_swapout, 422 fss_trapret, 423 fss_preempt, 424 fss_setrun, 425 fss_sleep, 426 fss_tick, 427 fss_wakeup, 428 fss_donice, 429 fss_globpri, 430 fss_nullsys, /* set_process_group */ 431 fss_yield, 432 fss_doprio }, 433 }; 434 435 int 436 _init() 437 { 438 return (mod_install(&modlinkage)); 439 } 440 441 int 442 _fini() 443 { 444 return (EBUSY); 445 } 446 447 int 448 _info(struct modinfo *modinfop) 449 { 450 return (mod_info(&modlinkage, modinfop)); 451 } 452 453 /*ARGSUSED*/ 454 static int 455 fss_project_walker(kproject_t *kpj, void *buf) 456 { 457 return (0); 458 } 459 460 void * 461 fss_allocbuf(int op, int type) 462 { 463 fssbuf_t *fssbuf; 464 void **fsslist; 465 int cnt; 466 int i; 467 size_t size; 468 469 ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF); 470 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE); 471 ASSERT(MUTEX_HELD(&cpu_lock)); 472 473 fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP); 474 switch (op) { 475 case FSS_NPSET_BUF: 476 cnt = cpupart_list(NULL, 0, CP_NONEMPTY); 477 break; 478 case FSS_NPROJ_BUF: 479 cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL); 480 break; 481 case FSS_ONE_BUF: 482 cnt = 1; 483 break; 484 } 485 486 switch (type) { 487 case FSS_ALLOC_PROJ: 488 size = sizeof (fssproj_t); 489 break; 490 case FSS_ALLOC_ZONE: 491 size = sizeof (fsszone_t); 492 break; 493 } 494 fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP); 495 fssbuf->fssb_size = cnt; 496 fssbuf->fssb_list = fsslist; 497 for (i = 0; i < cnt; i++) 498 fsslist[i] = kmem_zalloc(size, KM_SLEEP); 499 return (fssbuf); 500 } 501 502 void 503 fss_freebuf(fssbuf_t *fssbuf, int type) 504 { 505 void **fsslist; 506 int i; 507 size_t size; 508 509 ASSERT(fssbuf != NULL); 510 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE); 511 fsslist = fssbuf->fssb_list; 512 513 switch (type) { 514 case FSS_ALLOC_PROJ: 515 size = sizeof (fssproj_t); 516 break; 517 case FSS_ALLOC_ZONE: 518 size = sizeof (fsszone_t); 519 break; 520 } 521 522 for (i = 0; i < fssbuf->fssb_size; i++) { 523 if (fsslist[i] != NULL) 524 kmem_free(fsslist[i], size); 525 } 526 kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size); 527 kmem_free(fssbuf, sizeof (fssbuf_t)); 528 } 529 530 static fsspset_t * 531 fss_find_fsspset(cpupart_t *cpupart) 532 { 533 int i; 534 fsspset_t *fsspset = NULL; 535 int found = 0; 536 537 ASSERT(cpupart != NULL); 538 ASSERT(MUTEX_HELD(&fsspsets_lock)); 539 540 /* 541 * Search for the cpupart pointer in the array of fsspsets. 542 */ 543 for (i = 0; i < max_ncpus; i++) { 544 fsspset = &fsspsets[i]; 545 if (fsspset->fssps_cpupart == cpupart) { 546 ASSERT(fsspset->fssps_nproj > 0); 547 found = 1; 548 break; 549 } 550 } 551 if (found == 0) { 552 /* 553 * If we didn't find anything, then use the first 554 * available slot in the fsspsets array. 555 */ 556 for (i = 0; i < max_ncpus; i++) { 557 fsspset = &fsspsets[i]; 558 if (fsspset->fssps_cpupart == NULL) { 559 ASSERT(fsspset->fssps_nproj == 0); 560 found = 1; 561 break; 562 } 563 } 564 fsspset->fssps_cpupart = cpupart; 565 } 566 ASSERT(found == 1); 567 return (fsspset); 568 } 569 570 static void 571 fss_del_fsspset(fsspset_t *fsspset) 572 { 573 ASSERT(MUTEX_HELD(&fsspsets_lock)); 574 ASSERT(MUTEX_HELD(&fsspset->fssps_lock)); 575 ASSERT(fsspset->fssps_nproj == 0); 576 ASSERT(fsspset->fssps_list == NULL); 577 ASSERT(fsspset->fssps_zones == NULL); 578 fsspset->fssps_cpupart = NULL; 579 fsspset->fssps_maxfsspri = 0; 580 fsspset->fssps_shares = 0; 581 } 582 583 /* 584 * The following routine returns a pointer to the fsszone structure which 585 * belongs to zone "zone" and cpu partition fsspset, if such structure exists. 586 */ 587 static fsszone_t * 588 fss_find_fsszone(fsspset_t *fsspset, zone_t *zone) 589 { 590 fsszone_t *fsszone; 591 592 ASSERT(MUTEX_HELD(&fsspset->fssps_lock)); 593 594 if (fsspset->fssps_list != NULL) { 595 /* 596 * There are projects/zones active on this cpu partition 597 * already. Try to find our zone among them. 598 */ 599 fsszone = fsspset->fssps_zones; 600 do { 601 if (fsszone->fssz_zone == zone) { 602 return (fsszone); 603 } 604 fsszone = fsszone->fssz_next; 605 } while (fsszone != fsspset->fssps_zones); 606 } 607 return (NULL); 608 } 609 610 /* 611 * The following routine links new fsszone structure into doubly linked list of 612 * zones active on the specified cpu partition. 613 */ 614 static void 615 fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone) 616 { 617 ASSERT(MUTEX_HELD(&fsspset->fssps_lock)); 618 619 fsszone->fssz_zone = zone; 620 fsszone->fssz_rshares = zone->zone_shares; 621 622 if (fsspset->fssps_zones == NULL) { 623 /* 624 * This will be the first fsszone for this fsspset 625 */ 626 fsszone->fssz_next = fsszone->fssz_prev = fsszone; 627 fsspset->fssps_zones = fsszone; 628 } else { 629 /* 630 * Insert this fsszone to the doubly linked list. 631 */ 632 fsszone_t *fssz_head = fsspset->fssps_zones; 633 634 fsszone->fssz_next = fssz_head; 635 fsszone->fssz_prev = fssz_head->fssz_prev; 636 fssz_head->fssz_prev->fssz_next = fsszone; 637 fssz_head->fssz_prev = fsszone; 638 fsspset->fssps_zones = fsszone; 639 } 640 } 641 642 /* 643 * The following routine removes a single fsszone structure from the doubly 644 * linked list of zones active on the specified cpu partition. Note that 645 * global fsspsets_lock must be held in case this fsszone structure is the last 646 * on the above mentioned list. Also note that the fsszone structure is not 647 * freed here, it is the responsibility of the caller to call kmem_free for it. 648 */ 649 static void 650 fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone) 651 { 652 ASSERT(MUTEX_HELD(&fsspset->fssps_lock)); 653 ASSERT(fsszone->fssz_nproj == 0); 654 ASSERT(fsszone->fssz_shares == 0); 655 ASSERT(fsszone->fssz_runnable == 0); 656 657 if (fsszone->fssz_next != fsszone) { 658 /* 659 * This is not the last zone in the list. 660 */ 661 fsszone->fssz_prev->fssz_next = fsszone->fssz_next; 662 fsszone->fssz_next->fssz_prev = fsszone->fssz_prev; 663 if (fsspset->fssps_zones == fsszone) 664 fsspset->fssps_zones = fsszone->fssz_next; 665 } else { 666 /* 667 * This was the last zone active in this cpu partition. 668 */ 669 fsspset->fssps_zones = NULL; 670 } 671 } 672 673 /* 674 * The following routine returns a pointer to the fssproj structure 675 * which belongs to project kpj and cpu partition fsspset, if such structure 676 * exists. 677 */ 678 static fssproj_t * 679 fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj) 680 { 681 fssproj_t *fssproj; 682 683 ASSERT(MUTEX_HELD(&fsspset->fssps_lock)); 684 685 if (fsspset->fssps_list != NULL) { 686 /* 687 * There are projects running on this cpu partition already. 688 * Try to find our project among them. 689 */ 690 fssproj = fsspset->fssps_list; 691 do { 692 if (fssproj->fssp_proj == kpj) { 693 ASSERT(fssproj->fssp_pset == fsspset); 694 return (fssproj); 695 } 696 fssproj = fssproj->fssp_next; 697 } while (fssproj != fsspset->fssps_list); 698 } 699 return (NULL); 700 } 701 702 /* 703 * The following routine links new fssproj structure into doubly linked list 704 * of projects running on the specified cpu partition. 705 */ 706 static void 707 fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone, 708 fssproj_t *fssproj) 709 { 710 ASSERT(MUTEX_HELD(&fsspset->fssps_lock)); 711 712 fssproj->fssp_pset = fsspset; 713 fssproj->fssp_proj = kpj; 714 fssproj->fssp_shares = kpj->kpj_shares; 715 716 fsspset->fssps_nproj++; 717 718 if (fsspset->fssps_list == NULL) { 719 /* 720 * This will be the first fssproj for this fsspset 721 */ 722 fssproj->fssp_next = fssproj->fssp_prev = fssproj; 723 fsspset->fssps_list = fssproj; 724 } else { 725 /* 726 * Insert this fssproj to the doubly linked list. 727 */ 728 fssproj_t *fssp_head = fsspset->fssps_list; 729 730 fssproj->fssp_next = fssp_head; 731 fssproj->fssp_prev = fssp_head->fssp_prev; 732 fssp_head->fssp_prev->fssp_next = fssproj; 733 fssp_head->fssp_prev = fssproj; 734 fsspset->fssps_list = fssproj; 735 } 736 fssproj->fssp_fsszone = fsszone; 737 fsszone->fssz_nproj++; 738 ASSERT(fsszone->fssz_nproj != 0); 739 } 740 741 /* 742 * The following routine removes a single fssproj structure from the doubly 743 * linked list of projects running on the specified cpu partition. Note that 744 * global fsspsets_lock must be held in case if this fssproj structure is the 745 * last on the above mentioned list. Also note that the fssproj structure is 746 * not freed here, it is the responsibility of the caller to call kmem_free 747 * for it. 748 */ 749 static void 750 fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj) 751 { 752 fsszone_t *fsszone; 753 754 ASSERT(MUTEX_HELD(&fsspsets_lock)); 755 ASSERT(MUTEX_HELD(&fsspset->fssps_lock)); 756 ASSERT(fssproj->fssp_runnable == 0); 757 758 fsspset->fssps_nproj--; 759 760 fsszone = fssproj->fssp_fsszone; 761 fsszone->fssz_nproj--; 762 763 if (fssproj->fssp_next != fssproj) { 764 /* 765 * This is not the last part in the list. 766 */ 767 fssproj->fssp_prev->fssp_next = fssproj->fssp_next; 768 fssproj->fssp_next->fssp_prev = fssproj->fssp_prev; 769 if (fsspset->fssps_list == fssproj) 770 fsspset->fssps_list = fssproj->fssp_next; 771 if (fsszone->fssz_nproj == 0) 772 fss_remove_fsszone(fsspset, fsszone); 773 } else { 774 /* 775 * This was the last project part running 776 * at this cpu partition. 777 */ 778 fsspset->fssps_list = NULL; 779 ASSERT(fsspset->fssps_nproj == 0); 780 ASSERT(fsszone->fssz_nproj == 0); 781 fss_remove_fsszone(fsspset, fsszone); 782 fss_del_fsspset(fsspset); 783 } 784 } 785 786 static void 787 fss_inactive(kthread_t *t) 788 { 789 fssproc_t *fssproc; 790 fssproj_t *fssproj; 791 fsspset_t *fsspset; 792 fsszone_t *fsszone; 793 794 ASSERT(THREAD_LOCK_HELD(t)); 795 fssproc = FSSPROC(t); 796 fssproj = FSSPROC2FSSPROJ(fssproc); 797 if (fssproj == NULL) /* if this thread already exited */ 798 return; 799 fsspset = FSSPROJ2FSSPSET(fssproj); 800 fsszone = fssproj->fssp_fsszone; 801 disp_lock_enter_high(&fsspset->fssps_displock); 802 ASSERT(fssproj->fssp_runnable > 0); 803 if (--fssproj->fssp_runnable == 0) { 804 fsszone->fssz_shares -= fssproj->fssp_shares; 805 if (--fsszone->fssz_runnable == 0) 806 fsspset->fssps_shares -= fsszone->fssz_rshares; 807 } 808 ASSERT(fssproc->fss_runnable == 1); 809 fssproc->fss_runnable = 0; 810 disp_lock_exit_high(&fsspset->fssps_displock); 811 } 812 813 static void 814 fss_active(kthread_t *t) 815 { 816 fssproc_t *fssproc; 817 fssproj_t *fssproj; 818 fsspset_t *fsspset; 819 fsszone_t *fsszone; 820 821 ASSERT(THREAD_LOCK_HELD(t)); 822 fssproc = FSSPROC(t); 823 fssproj = FSSPROC2FSSPROJ(fssproc); 824 if (fssproj == NULL) /* if this thread already exited */ 825 return; 826 fsspset = FSSPROJ2FSSPSET(fssproj); 827 fsszone = fssproj->fssp_fsszone; 828 disp_lock_enter_high(&fsspset->fssps_displock); 829 if (++fssproj->fssp_runnable == 1) { 830 fsszone->fssz_shares += fssproj->fssp_shares; 831 if (++fsszone->fssz_runnable == 1) 832 fsspset->fssps_shares += fsszone->fssz_rshares; 833 } 834 ASSERT(fssproc->fss_runnable == 0); 835 fssproc->fss_runnable = 1; 836 disp_lock_exit_high(&fsspset->fssps_displock); 837 } 838 839 /* 840 * Fair share scheduler initialization. Called by dispinit() at boot time. 841 * We can ignore clparmsz argument since we know that the smallest possible 842 * parameter buffer is big enough for us. 843 */ 844 /*ARGSUSED*/ 845 static pri_t 846 fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp) 847 { 848 int i; 849 850 ASSERT(MUTEX_HELD(&cpu_lock)); 851 852 fss_cid = cid; 853 fss_maxumdpri = minclsyspri - 1; 854 fss_maxglobpri = minclsyspri; 855 fss_minglobpri = 0; 856 fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP); 857 858 /* 859 * Initialize the fssproc hash table. 860 */ 861 for (i = 0; i < FSS_LISTS; i++) 862 fss_listhead[i].fss_next = fss_listhead[i].fss_prev = 863 &fss_listhead[i]; 864 865 *clfuncspp = &fss_classfuncs; 866 867 /* 868 * Fill in fss_nice_tick and fss_nice_decay arrays: 869 * The cost of a tick is lower at positive nice values (so that it 870 * will not increase its project's usage as much as normal) with 50% 871 * drop at the maximum level and 50% increase at the minimum level. 872 * The fsspri decay is slower at positive nice values. fsspri values 873 * of processes with negative nice levels must decay faster to receive 874 * time slices more frequently than normal. 875 */ 876 for (i = 0; i < FSS_NICE_RANGE; i++) { 877 fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2) 878 - i)) / FSS_NICE_RANGE; 879 fss_nice_decay[i] = FSS_DECAY_MIN + 880 ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) / 881 (FSS_NICE_RANGE - 1); 882 } 883 884 return (fss_maxglobpri); 885 } 886 887 /* 888 * Calculate the new fss_umdpri based on the usage, the normalized share usage 889 * and the number of active threads. Reset the tick counter for this thread. 890 * 891 * When calculating the new priority using the standard formula we can hit 892 * a scenario where we don't have good round-robin behavior. This would be 893 * most commonly seen when there is a zone with lots of runnable threads. 894 * In the bad scenario we will see the following behavior when using the 895 * standard formula and these conditions: 896 * 897 * - there are multiple runnable threads in the zone (project) 898 * - the fssps_maxfsspri is a very large value 899 * - (we also know all of these threads will use the project's 900 * fssp_shusage) 901 * 902 * Under these conditions, a thread with a low fss_fsspri value is chosen 903 * to run and the thread gets a high fss_umdpri. This thread can run for 904 * its full quanta (fss_timeleft) at which time fss_newpri is called to 905 * calculate the thread's new priority. 906 * 907 * In this case, because the newly calculated fsspri value is much smaller 908 * (orders of magnitude) than the fssps_maxfsspri value, if we used the 909 * standard formula the thread will still get a high fss_umdpri value and 910 * will run again for another quanta, even though there are other runnable 911 * threads in the project. 912 * 913 * For a thread that is runnable for a long time, the thread can continue 914 * to run for many quanta (totaling many seconds) before the thread's fsspri 915 * exceeds the fssps_maxfsspri and the thread's fss_umdpri is reset back 916 * down to 1. This behavior also keeps the fssps_maxfsspr at a high value, 917 * so that the next runnable thread might repeat this cycle. 918 * 919 * This leads to the case where we don't have round-robin behavior at quanta 920 * granularity, but instead, runnable threads within the project only run 921 * at several second intervals. 922 * 923 * To prevent this scenario from occuring, when a thread has consumed its 924 * quanta and there are multiple runnable threads in the project, we 925 * immediately cause the thread to hit fssps_maxfsspri so that it gets 926 * reset back to 1 and another runnable thread in the project can run. 927 */ 928 static void 929 fss_newpri(fssproc_t *fssproc, boolean_t quanta_up) 930 { 931 kthread_t *tp; 932 fssproj_t *fssproj; 933 fsspset_t *fsspset; 934 fsszone_t *fsszone; 935 fsspri_t fsspri, maxfsspri; 936 uint32_t n_runnable; 937 pri_t invpri; 938 uint32_t ticks; 939 940 tp = fssproc->fss_tp; 941 ASSERT(tp != NULL); 942 943 if (tp->t_cid != fss_cid) 944 return; 945 946 ASSERT(THREAD_LOCK_HELD(tp)); 947 948 fssproj = FSSPROC2FSSPROJ(fssproc); 949 fsszone = FSSPROJ2FSSZONE(fssproj); 950 if (fssproj == NULL) 951 /* 952 * No need to change priority of exited threads. 953 */ 954 return; 955 956 fsspset = FSSPROJ2FSSPSET(fssproj); 957 disp_lock_enter_high(&fsspset->fssps_displock); 958 959 ticks = fssproc->fss_ticks; 960 fssproc->fss_ticks = 0; 961 962 if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) { 963 /* 964 * Special case: threads with no shares. 965 */ 966 fssproc->fss_umdpri = fss_minglobpri; 967 disp_lock_exit_high(&fsspset->fssps_displock); 968 return; 969 } 970 971 maxfsspri = fsspset->fssps_maxfsspri; 972 n_runnable = fssproj->fssp_runnable; 973 974 if (quanta_up && n_runnable > 1) { 975 fsspri = maxfsspri; 976 } else { 977 /* 978 * fsspri += fssp_shusage * nrunnable * ticks 979 * If all three values are non-0, this typically calculates to 980 * a large number (sometimes > 1M, sometimes > 100B) due to 981 * fssp_shusage which can be > 1T. 982 */ 983 fsspri = fssproc->fss_fsspri; 984 fsspri += fssproj->fssp_shusage * n_runnable * ticks; 985 } 986 987 fssproc->fss_fsspri = fsspri; 988 989 /* 990 * fss_maxumdpri is normally 59, since FSS priorities are 0-59. 991 * If the previous calculation resulted in 0 (e.g. was 0 and added 0 992 * because ticks == 0), then instead of 0, we use the largest priority, 993 * which is still small in comparison to the large numbers we typically 994 * see. 995 */ 996 if (fsspri < fss_maxumdpri) 997 fsspri = fss_maxumdpri; /* so that maxfsspri is != 0 */ 998 999 /* 1000 * The general priority formula: 1001 * 1002 * (fsspri * umdprirange) 1003 * pri = maxumdpri - ------------------------ 1004 * maxfsspri 1005 * 1006 * If this thread's fsspri is greater than the previous largest 1007 * fsspri, then record it as the new high and priority for this 1008 * thread will be one (the lowest priority assigned to a thread 1009 * that has non-zero shares). Because of this check, maxfsspri can 1010 * change as this function is called via the 1011 * fss_update -> fss_update_list -> fss_newpri code path to update 1012 * all runnable threads. See the code in fss_update for how we 1013 * mitigate this issue. 1014 * 1015 * Note that this formula cannot produce out of bounds priority 1016 * values (0-59); if it is changed, additional checks may need to be 1017 * added. 1018 */ 1019 if (fsspri >= maxfsspri) { 1020 fsspset->fssps_maxfsspri = fsspri; 1021 disp_lock_exit_high(&fsspset->fssps_displock); 1022 fssproc->fss_umdpri = 1; 1023 } else { 1024 disp_lock_exit_high(&fsspset->fssps_displock); 1025 invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri; 1026 fssproc->fss_umdpri = fss_maxumdpri - invpri; 1027 } 1028 } 1029 1030 /* 1031 * Decays usages of all running projects, resets their tick counters and 1032 * calcluates the projects normalized share usage. Called once per second from 1033 * fss_update(). 1034 */ 1035 static void 1036 fss_decay_usage() 1037 { 1038 uint32_t zone_ext_shares, zone_int_shares; 1039 uint32_t kpj_shares, pset_shares; 1040 fsspset_t *fsspset; 1041 fssproj_t *fssproj; 1042 fsszone_t *fsszone; 1043 fsspri_t maxfsspri; 1044 int psetid; 1045 struct zone *zp; 1046 1047 mutex_enter(&fsspsets_lock); 1048 /* 1049 * Go through all active processor sets and decay usages of projects 1050 * running on them. 1051 */ 1052 for (psetid = 0; psetid < max_ncpus; psetid++) { 1053 fsspset = &fsspsets[psetid]; 1054 mutex_enter(&fsspset->fssps_lock); 1055 1056 fsspset->fssps_gen++; 1057 1058 if (fsspset->fssps_cpupart == NULL || 1059 (fssproj = fsspset->fssps_list) == NULL) { 1060 mutex_exit(&fsspset->fssps_lock); 1061 continue; 1062 } 1063 1064 /* 1065 * Decay maxfsspri for this cpu partition with the 1066 * fastest possible decay rate. 1067 */ 1068 disp_lock_enter(&fsspset->fssps_displock); 1069 1070 pset_shares = fsspset->fssps_shares; 1071 1072 maxfsspri = (fsspset->fssps_maxfsspri * 1073 fss_nice_decay[NZERO]) / FSS_DECAY_BASE; 1074 if (maxfsspri < fss_maxumdpri) 1075 maxfsspri = fss_maxumdpri; 1076 fsspset->fssps_maxfsspri = maxfsspri; 1077 1078 do { 1079 fsszone = fssproj->fssp_fsszone; 1080 zp = fsszone->fssz_zone; 1081 1082 /* 1083 * Reset zone's FSS stats if they are from a 1084 * previous cycle. 1085 */ 1086 if (fsspset->fssps_gen != zp->zone_fss_gen) { 1087 zp->zone_fss_gen = fsspset->fssps_gen; 1088 zp->zone_run_ticks = 0; 1089 } 1090 1091 /* 1092 * Decay project usage, then add in this cycle's 1093 * nice tick value. 1094 */ 1095 fssproj->fssp_usage = 1096 (fssproj->fssp_usage * FSS_DECAY_USG) / 1097 FSS_DECAY_BASE + 1098 fssproj->fssp_ticks; 1099 1100 fssproj->fssp_ticks = 0; 1101 zp->zone_run_ticks += fssproj->fssp_tick_cnt; 1102 fssproj->fssp_tick_cnt = 0; 1103 1104 /* 1105 * Readjust the project's number of shares if it has 1106 * changed since we checked it last time. 1107 */ 1108 kpj_shares = fssproj->fssp_proj->kpj_shares; 1109 if (fssproj->fssp_shares != kpj_shares) { 1110 if (fssproj->fssp_runnable != 0) { 1111 fsszone->fssz_shares -= 1112 fssproj->fssp_shares; 1113 fsszone->fssz_shares += kpj_shares; 1114 } 1115 fssproj->fssp_shares = kpj_shares; 1116 } 1117 1118 /* 1119 * Readjust the zone's number of shares if it 1120 * has changed since we checked it last time. 1121 */ 1122 zone_ext_shares = zp->zone_shares; 1123 if (fsszone->fssz_rshares != zone_ext_shares) { 1124 if (fsszone->fssz_runnable != 0) { 1125 fsspset->fssps_shares -= 1126 fsszone->fssz_rshares; 1127 fsspset->fssps_shares += 1128 zone_ext_shares; 1129 pset_shares = fsspset->fssps_shares; 1130 } 1131 fsszone->fssz_rshares = zone_ext_shares; 1132 } 1133 zone_int_shares = fsszone->fssz_shares; 1134 1135 /* 1136 * If anything is runnable in the project, track the 1137 * overall project share percent for monitoring useage. 1138 */ 1139 if (fssproj->fssp_runnable > 0) { 1140 uint32_t zone_shr_pct; 1141 uint32_t int_shr_pct; 1142 1143 /* 1144 * Times 1000 to get tenths of a percent 1145 * 1146 * zone_ext_shares 1147 * zone_shr_pct = --------------- 1148 * pset_shares 1149 * 1150 * kpj_shares 1151 * int_shr_pct = --------------- 1152 * zone_int_shares 1153 */ 1154 if (pset_shares == 0 || zone_int_shares == 0) { 1155 fssproj->fssp_shr_pct = 0; 1156 } else { 1157 zone_shr_pct = 1158 (zone_ext_shares * 1000) / 1159 pset_shares; 1160 int_shr_pct = (kpj_shares * 1000) / 1161 zone_int_shares; 1162 fssproj->fssp_shr_pct = 1163 (zone_shr_pct * int_shr_pct) / 1164 1000; 1165 } 1166 } else { 1167 DTRACE_PROBE1(fss__prj__norun, fssproj_t *, 1168 fssproj); 1169 } 1170 1171 /* 1172 * Calculate fssp_shusage value to be used 1173 * for fsspri increments for the next second. 1174 */ 1175 if (kpj_shares == 0 || zone_ext_shares == 0) { 1176 fssproj->fssp_shusage = 0; 1177 } else if (FSSPROJ2KPROJ(fssproj) == proj0p) { 1178 uint32_t zone_shr_pct; 1179 1180 /* 1181 * Project 0 in the global zone has 50% 1182 * of its zone. See calculation above for 1183 * the zone's share percent. 1184 */ 1185 if (pset_shares == 0) 1186 zone_shr_pct = 1000; 1187 else 1188 zone_shr_pct = 1189 (zone_ext_shares * 1000) / 1190 pset_shares; 1191 1192 fssproj->fssp_shr_pct = zone_shr_pct / 2; 1193 1194 fssproj->fssp_shusage = (fssproj->fssp_usage * 1195 zone_int_shares * zone_int_shares) / 1196 (zone_ext_shares * zone_ext_shares); 1197 } else { 1198 /* 1199 * Thread's priority is based on its project's 1200 * normalized usage (shusage) value which gets 1201 * calculated this way: 1202 * 1203 * pset_shares^2 zone_int_shares^2 1204 * usage * ------------- * ------------------ 1205 * kpj_shares^2 zone_ext_shares^2 1206 * 1207 * Where zone_int_shares is the sum of shares 1208 * of all active projects within the zone (and 1209 * the pset), and zone_ext_shares is the number 1210 * of zone shares (ie, zone.cpu-shares). 1211 * 1212 * If there is only one zone active on the pset 1213 * the above reduces to: 1214 * 1215 * zone_int_shares^2 1216 * shusage = usage * --------------------- 1217 * kpj_shares^2 1218 * 1219 * If there's only one project active in the 1220 * zone this formula reduces to: 1221 * 1222 * pset_shares^2 1223 * shusage = usage * ---------------------- 1224 * zone_ext_shares^2 1225 * 1226 * shusage is one input to calculating fss_pri 1227 * in fss_newpri(). Larger values tend toward 1228 * lower priorities for processes in the proj. 1229 */ 1230 fssproj->fssp_shusage = fssproj->fssp_usage * 1231 pset_shares * zone_int_shares; 1232 fssproj->fssp_shusage /= 1233 kpj_shares * zone_ext_shares; 1234 fssproj->fssp_shusage *= 1235 pset_shares * zone_int_shares; 1236 fssproj->fssp_shusage /= 1237 kpj_shares * zone_ext_shares; 1238 } 1239 fssproj = fssproj->fssp_next; 1240 } while (fssproj != fsspset->fssps_list); 1241 1242 disp_lock_exit(&fsspset->fssps_displock); 1243 mutex_exit(&fsspset->fssps_lock); 1244 } 1245 mutex_exit(&fsspsets_lock); 1246 } 1247 1248 static void 1249 fss_change_priority(kthread_t *t, fssproc_t *fssproc) 1250 { 1251 pri_t new_pri; 1252 1253 ASSERT(THREAD_LOCK_HELD(t)); 1254 new_pri = fssproc->fss_umdpri; 1255 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri); 1256 1257 t->t_cpri = fssproc->fss_upri; 1258 fssproc->fss_flags &= ~FSSRESTORE; 1259 if (t == curthread || t->t_state == TS_ONPROC) { 1260 /* 1261 * curthread is always onproc 1262 */ 1263 cpu_t *cp = t->t_disp_queue->disp_cpu; 1264 THREAD_CHANGE_PRI(t, new_pri); 1265 if (t == cp->cpu_dispthread) 1266 cp->cpu_dispatch_pri = DISP_PRIO(t); 1267 if (DISP_MUST_SURRENDER(t)) { 1268 fssproc->fss_flags |= FSSBACKQ; 1269 cpu_surrender(t); 1270 } else { 1271 fssproc->fss_timeleft = fss_quantum; 1272 } 1273 } else { 1274 /* 1275 * When the priority of a thread is changed, it may be 1276 * necessary to adjust its position on a sleep queue or 1277 * dispatch queue. The function thread_change_pri accomplishes 1278 * this. 1279 */ 1280 if (thread_change_pri(t, new_pri, 0)) { 1281 /* 1282 * The thread was on a run queue. 1283 */ 1284 fssproc->fss_timeleft = fss_quantum; 1285 } else { 1286 fssproc->fss_flags |= FSSBACKQ; 1287 } 1288 } 1289 } 1290 1291 /* 1292 * Update priorities of all fair-sharing threads that are currently runnable 1293 * at a user mode priority based on the number of shares and current usage. 1294 * Called once per second via timeout which we reset here. 1295 * 1296 * There are several lists of fair-sharing threads broken up by a hash on the 1297 * thread pointer. Each list has its own lock. This avoids blocking all 1298 * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs. 1299 * fss_update traverses each list in turn. 1300 * 1301 * Each time we're run (once/second) we may start at the next list and iterate 1302 * through all of the lists. By starting with a different list, we mitigate any 1303 * effects we would see updating the fssps_maxfsspri value in fss_newpri. 1304 */ 1305 static void 1306 fss_update(void *arg) 1307 { 1308 int i; 1309 int new_marker = -1; 1310 static int fss_update_marker; 1311 1312 /* 1313 * Decay and update usages for all projects. 1314 */ 1315 fss_decay_usage(); 1316 1317 /* 1318 * Start with the fss_update_marker list, then do the rest. 1319 */ 1320 i = fss_update_marker; 1321 1322 /* 1323 * Go around all threads, set new priorities and decay 1324 * per-thread CPU usages. 1325 */ 1326 do { 1327 /* 1328 * If this is the first list after the current marker to have 1329 * threads with priority updates, advance the marker to this 1330 * list for the next time fss_update runs. 1331 */ 1332 if (fss_update_list(i) && 1333 new_marker == -1 && i != fss_update_marker) 1334 new_marker = i; 1335 } while ((i = FSS_LIST_NEXT(i)) != fss_update_marker); 1336 1337 /* 1338 * Advance marker for the next fss_update call 1339 */ 1340 if (new_marker != -1) 1341 fss_update_marker = new_marker; 1342 1343 (void) timeout(fss_update, arg, hz); 1344 } 1345 1346 /* 1347 * Updates priority for a list of threads. Returns 1 if the priority of one 1348 * of the threads was actually updated, 0 if none were for various reasons 1349 * (thread is no longer in the FSS class, is not runnable, has the preemption 1350 * control no-preempt bit set, etc.) 1351 */ 1352 static int 1353 fss_update_list(int i) 1354 { 1355 fssproc_t *fssproc; 1356 fssproj_t *fssproj; 1357 fsspri_t fsspri; 1358 pri_t fss_umdpri; 1359 kthread_t *t; 1360 int updated = 0; 1361 1362 mutex_enter(&fss_listlock[i]); 1363 for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i]; 1364 fssproc = fssproc->fss_next) { 1365 t = fssproc->fss_tp; 1366 /* 1367 * Lock the thread and verify the state. 1368 */ 1369 thread_lock(t); 1370 /* 1371 * Skip the thread if it is no longer in the FSS class or 1372 * is running with kernel mode priority. 1373 */ 1374 if (t->t_cid != fss_cid) 1375 goto next; 1376 if ((fssproc->fss_flags & FSSKPRI) != 0) 1377 goto next; 1378 1379 fssproj = FSSPROC2FSSPROJ(fssproc); 1380 if (fssproj == NULL) 1381 goto next; 1382 1383 if (fssproj->fssp_shares != 0) { 1384 /* 1385 * Decay fsspri value. 1386 */ 1387 fsspri = fssproc->fss_fsspri; 1388 fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) / 1389 FSS_DECAY_BASE; 1390 fssproc->fss_fsspri = fsspri; 1391 } 1392 1393 if (t->t_schedctl && schedctl_get_nopreempt(t)) 1394 goto next; 1395 if (t->t_state != TS_RUN && t->t_state != TS_WAIT) { 1396 /* 1397 * Make next syscall/trap call fss_trapret 1398 */ 1399 t->t_trapret = 1; 1400 aston(t); 1401 if (t->t_state == TS_ONPROC) 1402 DTRACE_PROBE1(fss__onproc, fssproc_t *, 1403 fssproc); 1404 goto next; 1405 } 1406 fss_newpri(fssproc, B_FALSE); 1407 updated = 1; 1408 1409 fss_umdpri = fssproc->fss_umdpri; 1410 1411 /* 1412 * Only dequeue the thread if it needs to be moved; otherwise 1413 * it should just round-robin here. 1414 */ 1415 if (t->t_pri != fss_umdpri) 1416 fss_change_priority(t, fssproc); 1417 next: 1418 thread_unlock(t); 1419 } 1420 mutex_exit(&fss_listlock[i]); 1421 return (updated); 1422 } 1423 1424 /*ARGSUSED*/ 1425 static int 1426 fss_admin(caddr_t uaddr, cred_t *reqpcredp) 1427 { 1428 fssadmin_t fssadmin; 1429 1430 if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t))) 1431 return (EFAULT); 1432 1433 switch (fssadmin.fss_cmd) { 1434 case FSS_SETADMIN: 1435 if (secpolicy_dispadm(reqpcredp) != 0) 1436 return (EPERM); 1437 if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz) 1438 return (EINVAL); 1439 fss_quantum = fssadmin.fss_quantum; 1440 break; 1441 case FSS_GETADMIN: 1442 fssadmin.fss_quantum = fss_quantum; 1443 if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t))) 1444 return (EFAULT); 1445 break; 1446 default: 1447 return (EINVAL); 1448 } 1449 return (0); 1450 } 1451 1452 static int 1453 fss_getclinfo(void *infop) 1454 { 1455 fssinfo_t *fssinfo = (fssinfo_t *)infop; 1456 fssinfo->fss_maxupri = fss_maxupri; 1457 return (0); 1458 } 1459 1460 static int 1461 fss_parmsin(void *parmsp) 1462 { 1463 fssparms_t *fssparmsp = (fssparms_t *)parmsp; 1464 1465 /* 1466 * Check validity of parameters. 1467 */ 1468 if ((fssparmsp->fss_uprilim > fss_maxupri || 1469 fssparmsp->fss_uprilim < -fss_maxupri) && 1470 fssparmsp->fss_uprilim != FSS_NOCHANGE) 1471 return (EINVAL); 1472 1473 if ((fssparmsp->fss_upri > fss_maxupri || 1474 fssparmsp->fss_upri < -fss_maxupri) && 1475 fssparmsp->fss_upri != FSS_NOCHANGE) 1476 return (EINVAL); 1477 1478 return (0); 1479 } 1480 1481 /*ARGSUSED*/ 1482 static int 1483 fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp) 1484 { 1485 return (0); 1486 } 1487 1488 static int 1489 fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp) 1490 { 1491 fssparms_t *fssparmsp = (fssparms_t *)parmsp; 1492 int priflag = 0; 1493 int limflag = 0; 1494 uint_t cnt; 1495 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0]; 1496 1497 /* 1498 * FSS_NOCHANGE (-32768) is outside of the range of values for 1499 * fss_uprilim and fss_upri. If the structure fssparms_t is changed, 1500 * FSS_NOCHANGE should be replaced by a flag word. 1501 */ 1502 fssparmsp->fss_uprilim = FSS_NOCHANGE; 1503 fssparmsp->fss_upri = FSS_NOCHANGE; 1504 1505 /* 1506 * Get the varargs parameter and check validity of parameters. 1507 */ 1508 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT) 1509 return (EINVAL); 1510 1511 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) { 1512 switch (vpp->pc_key) { 1513 case FSS_KY_UPRILIM: 1514 if (limflag++) 1515 return (EINVAL); 1516 fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm; 1517 if (fssparmsp->fss_uprilim > fss_maxupri || 1518 fssparmsp->fss_uprilim < -fss_maxupri) 1519 return (EINVAL); 1520 break; 1521 case FSS_KY_UPRI: 1522 if (priflag++) 1523 return (EINVAL); 1524 fssparmsp->fss_upri = (pri_t)vpp->pc_parm; 1525 if (fssparmsp->fss_upri > fss_maxupri || 1526 fssparmsp->fss_upri < -fss_maxupri) 1527 return (EINVAL); 1528 break; 1529 default: 1530 return (EINVAL); 1531 } 1532 } 1533 1534 if (vaparmsp->pc_vaparmscnt == 0) { 1535 /* 1536 * Use default parameters. 1537 */ 1538 fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0; 1539 } 1540 1541 return (0); 1542 } 1543 1544 /* 1545 * Copy all selected fair-sharing class parameters to the user. The parameters 1546 * are specified by a key. 1547 */ 1548 static int 1549 fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp) 1550 { 1551 fssparms_t *fssparmsp = (fssparms_t *)parmsp; 1552 int priflag = 0; 1553 int limflag = 0; 1554 uint_t cnt; 1555 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0]; 1556 1557 ASSERT(MUTEX_NOT_HELD(&curproc->p_lock)); 1558 1559 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT) 1560 return (EINVAL); 1561 1562 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) { 1563 switch (vpp->pc_key) { 1564 case FSS_KY_UPRILIM: 1565 if (limflag++) 1566 return (EINVAL); 1567 if (copyout(&fssparmsp->fss_uprilim, 1568 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t))) 1569 return (EFAULT); 1570 break; 1571 case FSS_KY_UPRI: 1572 if (priflag++) 1573 return (EINVAL); 1574 if (copyout(&fssparmsp->fss_upri, 1575 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t))) 1576 return (EFAULT); 1577 break; 1578 default: 1579 return (EINVAL); 1580 } 1581 } 1582 1583 return (0); 1584 } 1585 1586 /* 1587 * Return the user mode scheduling priority range. 1588 */ 1589 static int 1590 fss_getclpri(pcpri_t *pcprip) 1591 { 1592 pcprip->pc_clpmax = fss_maxupri; 1593 pcprip->pc_clpmin = -fss_maxupri; 1594 return (0); 1595 } 1596 1597 static int 1598 fss_alloc(void **p, int flag) 1599 { 1600 void *bufp; 1601 1602 if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) { 1603 return (ENOMEM); 1604 } else { 1605 *p = bufp; 1606 return (0); 1607 } 1608 } 1609 1610 static void 1611 fss_free(void *bufp) 1612 { 1613 if (bufp) 1614 kmem_free(bufp, sizeof (fssproc_t)); 1615 } 1616 1617 /* 1618 * Thread functions 1619 */ 1620 static int 1621 fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp, 1622 void *bufp) 1623 { 1624 fssparms_t *fssparmsp = (fssparms_t *)parmsp; 1625 fssproc_t *fssproc; 1626 pri_t reqfssuprilim; 1627 pri_t reqfssupri; 1628 static uint32_t fssexists = 0; 1629 fsspset_t *fsspset; 1630 fssproj_t *fssproj; 1631 fsszone_t *fsszone; 1632 kproject_t *kpj; 1633 zone_t *zone; 1634 int fsszone_allocated = 0; 1635 1636 fssproc = (fssproc_t *)bufp; 1637 ASSERT(fssproc != NULL); 1638 1639 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)); 1640 1641 /* 1642 * Only root can move threads to FSS class. 1643 */ 1644 if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0) 1645 return (EPERM); 1646 /* 1647 * Initialize the fssproc structure. 1648 */ 1649 fssproc->fss_umdpri = fss_maxumdpri / 2; 1650 1651 if (fssparmsp == NULL) { 1652 /* 1653 * Use default values. 1654 */ 1655 fssproc->fss_nice = NZERO; 1656 fssproc->fss_uprilim = fssproc->fss_upri = 0; 1657 } else { 1658 /* 1659 * Use supplied values. 1660 */ 1661 if (fssparmsp->fss_uprilim == FSS_NOCHANGE) { 1662 reqfssuprilim = 0; 1663 } else { 1664 if (fssparmsp->fss_uprilim > 0 && 1665 secpolicy_setpriority(reqpcredp) != 0) 1666 return (EPERM); 1667 reqfssuprilim = fssparmsp->fss_uprilim; 1668 } 1669 if (fssparmsp->fss_upri == FSS_NOCHANGE) { 1670 reqfssupri = reqfssuprilim; 1671 } else { 1672 if (fssparmsp->fss_upri > 0 && 1673 secpolicy_setpriority(reqpcredp) != 0) 1674 return (EPERM); 1675 /* 1676 * Set the user priority to the requested value or 1677 * the upri limit, whichever is lower. 1678 */ 1679 reqfssupri = fssparmsp->fss_upri; 1680 if (reqfssupri > reqfssuprilim) 1681 reqfssupri = reqfssuprilim; 1682 } 1683 fssproc->fss_uprilim = reqfssuprilim; 1684 fssproc->fss_upri = reqfssupri; 1685 fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri; 1686 if (fssproc->fss_nice > FSS_NICE_MAX) 1687 fssproc->fss_nice = FSS_NICE_MAX; 1688 } 1689 1690 fssproc->fss_timeleft = fss_quantum; 1691 fssproc->fss_tp = t; 1692 cpucaps_sc_init(&fssproc->fss_caps); 1693 1694 /* 1695 * Put a lock on our fsspset structure. 1696 */ 1697 mutex_enter(&fsspsets_lock); 1698 fsspset = fss_find_fsspset(t->t_cpupart); 1699 mutex_enter(&fsspset->fssps_lock); 1700 mutex_exit(&fsspsets_lock); 1701 1702 zone = ttoproc(t)->p_zone; 1703 if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) { 1704 if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP)) 1705 == NULL) { 1706 mutex_exit(&fsspset->fssps_lock); 1707 return (ENOMEM); 1708 } else { 1709 fsszone_allocated = 1; 1710 fss_insert_fsszone(fsspset, zone, fsszone); 1711 } 1712 } 1713 kpj = ttoproj(t); 1714 if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) { 1715 if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP)) 1716 == NULL) { 1717 if (fsszone_allocated) { 1718 fss_remove_fsszone(fsspset, fsszone); 1719 kmem_free(fsszone, sizeof (fsszone_t)); 1720 } 1721 mutex_exit(&fsspset->fssps_lock); 1722 return (ENOMEM); 1723 } else { 1724 fss_insert_fssproj(fsspset, kpj, fsszone, fssproj); 1725 } 1726 } 1727 fssproj->fssp_threads++; 1728 fssproc->fss_proj = fssproj; 1729 1730 /* 1731 * Reset priority. Process goes to a "user mode" priority here 1732 * regardless of whether or not it has slept since entering the kernel. 1733 */ 1734 thread_lock(t); 1735 t->t_clfuncs = &(sclass[cid].cl_funcs->thread); 1736 t->t_cid = cid; 1737 t->t_cldata = (void *)fssproc; 1738 t->t_schedflag |= TS_RUNQMATCH; 1739 fss_change_priority(t, fssproc); 1740 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC || 1741 t->t_state == TS_WAIT) 1742 fss_active(t); 1743 thread_unlock(t); 1744 1745 mutex_exit(&fsspset->fssps_lock); 1746 1747 /* 1748 * Link new structure into fssproc list. 1749 */ 1750 FSS_LIST_INSERT(fssproc); 1751 1752 /* 1753 * If this is the first fair-sharing thread to occur since boot, 1754 * we set up the initial call to fss_update() here. Use an atomic 1755 * compare-and-swap since that's easier and faster than a mutex 1756 * (but check with an ordinary load first since most of the time 1757 * this will already be done). 1758 */ 1759 if (fssexists == 0 && atomic_cas_32(&fssexists, 0, 1) == 0) 1760 (void) timeout(fss_update, NULL, hz); 1761 1762 return (0); 1763 } 1764 1765 /* 1766 * Remove fssproc_t from the list. 1767 */ 1768 static void 1769 fss_exitclass(void *procp) 1770 { 1771 fssproc_t *fssproc = (fssproc_t *)procp; 1772 fssproj_t *fssproj; 1773 fsspset_t *fsspset; 1774 fsszone_t *fsszone; 1775 kthread_t *t = fssproc->fss_tp; 1776 1777 /* 1778 * We should be either getting this thread off the deathrow or 1779 * this thread has already moved to another scheduling class and 1780 * we're being called with its old cldata buffer pointer. In both 1781 * cases, the content of this buffer can not be changed while we're 1782 * here. 1783 */ 1784 mutex_enter(&fsspsets_lock); 1785 thread_lock(t); 1786 if (t->t_cid != fss_cid) { 1787 /* 1788 * We're being called as a result of the priocntl() system 1789 * call -- someone is trying to move our thread to another 1790 * scheduling class. We can't call fss_inactive() here 1791 * because our thread's t_cldata pointer already points 1792 * to another scheduling class specific data. 1793 */ 1794 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)); 1795 1796 fssproj = FSSPROC2FSSPROJ(fssproc); 1797 fsspset = FSSPROJ2FSSPSET(fssproj); 1798 fsszone = fssproj->fssp_fsszone; 1799 1800 if (fssproc->fss_runnable) { 1801 disp_lock_enter_high(&fsspset->fssps_displock); 1802 if (--fssproj->fssp_runnable == 0) { 1803 fsszone->fssz_shares -= fssproj->fssp_shares; 1804 if (--fsszone->fssz_runnable == 0) 1805 fsspset->fssps_shares -= 1806 fsszone->fssz_rshares; 1807 } 1808 disp_lock_exit_high(&fsspset->fssps_displock); 1809 } 1810 thread_unlock(t); 1811 1812 mutex_enter(&fsspset->fssps_lock); 1813 if (--fssproj->fssp_threads == 0) { 1814 fss_remove_fssproj(fsspset, fssproj); 1815 if (fsszone->fssz_nproj == 0) 1816 kmem_free(fsszone, sizeof (fsszone_t)); 1817 kmem_free(fssproj, sizeof (fssproj_t)); 1818 } 1819 mutex_exit(&fsspset->fssps_lock); 1820 1821 } else { 1822 ASSERT(t->t_state == TS_FREE); 1823 /* 1824 * We're being called from thread_free() when our thread 1825 * is removed from the deathrow. There is nothing we need 1826 * do here since everything should've been done earlier 1827 * in fss_exit(). 1828 */ 1829 thread_unlock(t); 1830 } 1831 mutex_exit(&fsspsets_lock); 1832 1833 FSS_LIST_DELETE(fssproc); 1834 fss_free(fssproc); 1835 } 1836 1837 /*ARGSUSED*/ 1838 static int 1839 fss_canexit(kthread_t *t, cred_t *credp) 1840 { 1841 /* 1842 * A thread is allowed to exit FSS only if we have sufficient 1843 * privileges. 1844 */ 1845 if (credp != NULL && secpolicy_setpriority(credp) != 0) 1846 return (EPERM); 1847 else 1848 return (0); 1849 } 1850 1851 /* 1852 * Initialize fair-share class specific proc structure for a child. 1853 */ 1854 static int 1855 fss_fork(kthread_t *pt, kthread_t *ct, void *bufp) 1856 { 1857 fssproc_t *pfssproc; /* ptr to parent's fssproc structure */ 1858 fssproc_t *cfssproc; /* ptr to child's fssproc structure */ 1859 fssproj_t *fssproj; 1860 fsspset_t *fsspset; 1861 1862 ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock)); 1863 ASSERT(ct->t_state == TS_STOPPED); 1864 1865 cfssproc = (fssproc_t *)bufp; 1866 ASSERT(cfssproc != NULL); 1867 bzero(cfssproc, sizeof (fssproc_t)); 1868 1869 thread_lock(pt); 1870 pfssproc = FSSPROC(pt); 1871 fssproj = FSSPROC2FSSPROJ(pfssproc); 1872 fsspset = FSSPROJ2FSSPSET(fssproj); 1873 thread_unlock(pt); 1874 1875 mutex_enter(&fsspset->fssps_lock); 1876 /* 1877 * Initialize child's fssproc structure. 1878 */ 1879 thread_lock(pt); 1880 ASSERT(FSSPROJ(pt) == fssproj); 1881 cfssproc->fss_proj = fssproj; 1882 cfssproc->fss_timeleft = fss_quantum; 1883 cfssproc->fss_umdpri = pfssproc->fss_umdpri; 1884 cfssproc->fss_fsspri = 0; 1885 cfssproc->fss_uprilim = pfssproc->fss_uprilim; 1886 cfssproc->fss_upri = pfssproc->fss_upri; 1887 cfssproc->fss_tp = ct; 1888 cfssproc->fss_nice = pfssproc->fss_nice; 1889 cpucaps_sc_init(&cfssproc->fss_caps); 1890 1891 cfssproc->fss_flags = 1892 pfssproc->fss_flags & ~(FSSKPRI | FSSBACKQ | FSSRESTORE); 1893 ct->t_cldata = (void *)cfssproc; 1894 ct->t_schedflag |= TS_RUNQMATCH; 1895 thread_unlock(pt); 1896 1897 fssproj->fssp_threads++; 1898 mutex_exit(&fsspset->fssps_lock); 1899 1900 /* 1901 * Link new structure into fssproc hash table. 1902 */ 1903 FSS_LIST_INSERT(cfssproc); 1904 return (0); 1905 } 1906 1907 /* 1908 * Child is placed at back of dispatcher queue and parent gives up processor 1909 * so that the child runs first after the fork. This allows the child 1910 * immediately execing to break the multiple use of copy on write pages with no 1911 * disk home. The parent will get to steal them back rather than uselessly 1912 * copying them. 1913 */ 1914 static void 1915 fss_forkret(kthread_t *t, kthread_t *ct) 1916 { 1917 proc_t *pp = ttoproc(t); 1918 proc_t *cp = ttoproc(ct); 1919 fssproc_t *fssproc; 1920 1921 ASSERT(t == curthread); 1922 ASSERT(MUTEX_HELD(&pidlock)); 1923 1924 /* 1925 * Grab the child's p_lock before dropping pidlock to ensure the 1926 * process does not disappear before we set it running. 1927 */ 1928 mutex_enter(&cp->p_lock); 1929 continuelwps(cp); 1930 mutex_exit(&cp->p_lock); 1931 1932 mutex_enter(&pp->p_lock); 1933 mutex_exit(&pidlock); 1934 continuelwps(pp); 1935 1936 thread_lock(t); 1937 1938 fssproc = FSSPROC(t); 1939 fss_newpri(fssproc, B_FALSE); 1940 fssproc->fss_timeleft = fss_quantum; 1941 t->t_pri = fssproc->fss_umdpri; 1942 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri); 1943 fssproc->fss_flags &= ~FSSKPRI; 1944 THREAD_TRANSITION(t); 1945 1946 /* 1947 * We don't want to call fss_setrun(t) here because it may call 1948 * fss_active, which we don't need. 1949 */ 1950 fssproc->fss_flags &= ~FSSBACKQ; 1951 1952 if (t->t_disp_time != ddi_get_lbolt()) 1953 setbackdq(t); 1954 else 1955 setfrontdq(t); 1956 1957 thread_unlock(t); 1958 /* 1959 * Safe to drop p_lock now since it is safe to change 1960 * the scheduling class after this point. 1961 */ 1962 mutex_exit(&pp->p_lock); 1963 1964 swtch(); 1965 } 1966 1967 /* 1968 * Get the fair-sharing parameters of the thread pointed to by fssprocp into 1969 * the buffer pointed by fssparmsp. 1970 */ 1971 static void 1972 fss_parmsget(kthread_t *t, void *parmsp) 1973 { 1974 fssproc_t *fssproc = FSSPROC(t); 1975 fssparms_t *fssparmsp = (fssparms_t *)parmsp; 1976 1977 fssparmsp->fss_uprilim = fssproc->fss_uprilim; 1978 fssparmsp->fss_upri = fssproc->fss_upri; 1979 } 1980 1981 /*ARGSUSED*/ 1982 static int 1983 fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp) 1984 { 1985 char nice; 1986 pri_t reqfssuprilim; 1987 pri_t reqfssupri; 1988 fssproc_t *fssproc = FSSPROC(t); 1989 fssparms_t *fssparmsp = (fssparms_t *)parmsp; 1990 1991 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock)); 1992 1993 if (fssparmsp->fss_uprilim == FSS_NOCHANGE) 1994 reqfssuprilim = fssproc->fss_uprilim; 1995 else 1996 reqfssuprilim = fssparmsp->fss_uprilim; 1997 1998 if (fssparmsp->fss_upri == FSS_NOCHANGE) 1999 reqfssupri = fssproc->fss_upri; 2000 else 2001 reqfssupri = fssparmsp->fss_upri; 2002 2003 /* 2004 * Make sure the user priority doesn't exceed the upri limit. 2005 */ 2006 if (reqfssupri > reqfssuprilim) 2007 reqfssupri = reqfssuprilim; 2008 2009 /* 2010 * Basic permissions enforced by generic kernel code for all classes 2011 * require that a thread attempting to change the scheduling parameters 2012 * of a target thread be privileged or have a real or effective UID 2013 * matching that of the target thread. We are not called unless these 2014 * basic permission checks have already passed. The fair-sharing class 2015 * requires in addition that the calling thread be privileged if it 2016 * is attempting to raise the upri limit above its current value. 2017 * This may have been checked previously but if our caller passed us 2018 * a non-NULL credential pointer we assume it hasn't and we check it 2019 * here. 2020 */ 2021 if ((reqpcredp != NULL) && 2022 (reqfssuprilim > fssproc->fss_uprilim) && 2023 secpolicy_raisepriority(reqpcredp) != 0) 2024 return (EPERM); 2025 2026 /* 2027 * Set fss_nice to the nice value corresponding to the user priority we 2028 * are setting. Note that setting the nice field of the parameter 2029 * struct won't affect upri or nice. 2030 */ 2031 nice = NZERO - (reqfssupri * NZERO) / fss_maxupri; 2032 if (nice > FSS_NICE_MAX) 2033 nice = FSS_NICE_MAX; 2034 2035 thread_lock(t); 2036 2037 fssproc->fss_uprilim = reqfssuprilim; 2038 fssproc->fss_upri = reqfssupri; 2039 fssproc->fss_nice = nice; 2040 fss_newpri(fssproc, B_FALSE); 2041 2042 if ((fssproc->fss_flags & FSSKPRI) != 0) { 2043 thread_unlock(t); 2044 return (0); 2045 } 2046 2047 fss_change_priority(t, fssproc); 2048 thread_unlock(t); 2049 return (0); 2050 2051 } 2052 2053 /* 2054 * The thread is being stopped. 2055 */ 2056 /*ARGSUSED*/ 2057 static void 2058 fss_stop(kthread_t *t, int why, int what) 2059 { 2060 ASSERT(THREAD_LOCK_HELD(t)); 2061 ASSERT(t == curthread); 2062 2063 fss_inactive(t); 2064 } 2065 2066 /* 2067 * The current thread is exiting, do necessary adjustments to its project 2068 */ 2069 static void 2070 fss_exit(kthread_t *t) 2071 { 2072 fsspset_t *fsspset; 2073 fssproj_t *fssproj; 2074 fssproc_t *fssproc; 2075 fsszone_t *fsszone; 2076 int free = 0; 2077 2078 /* 2079 * Thread t here is either a current thread (in which case we hold 2080 * its process' p_lock), or a thread being destroyed by forklwp_fail(), 2081 * in which case we hold pidlock and thread is no longer on the 2082 * thread list. 2083 */ 2084 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock)); 2085 2086 fssproc = FSSPROC(t); 2087 fssproj = FSSPROC2FSSPROJ(fssproc); 2088 fsspset = FSSPROJ2FSSPSET(fssproj); 2089 fsszone = fssproj->fssp_fsszone; 2090 2091 mutex_enter(&fsspsets_lock); 2092 mutex_enter(&fsspset->fssps_lock); 2093 2094 thread_lock(t); 2095 disp_lock_enter_high(&fsspset->fssps_displock); 2096 if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) { 2097 if (--fssproj->fssp_runnable == 0) { 2098 fsszone->fssz_shares -= fssproj->fssp_shares; 2099 if (--fsszone->fssz_runnable == 0) 2100 fsspset->fssps_shares -= fsszone->fssz_rshares; 2101 } 2102 ASSERT(fssproc->fss_runnable == 1); 2103 fssproc->fss_runnable = 0; 2104 } 2105 if (--fssproj->fssp_threads == 0) { 2106 fss_remove_fssproj(fsspset, fssproj); 2107 free = 1; 2108 } 2109 disp_lock_exit_high(&fsspset->fssps_displock); 2110 fssproc->fss_proj = NULL; /* mark this thread as already exited */ 2111 thread_unlock(t); 2112 2113 if (free) { 2114 if (fsszone->fssz_nproj == 0) 2115 kmem_free(fsszone, sizeof (fsszone_t)); 2116 kmem_free(fssproj, sizeof (fssproj_t)); 2117 } 2118 mutex_exit(&fsspset->fssps_lock); 2119 mutex_exit(&fsspsets_lock); 2120 2121 /* 2122 * A thread could be exiting in between clock ticks, so we need to 2123 * calculate how much CPU time it used since it was charged last time. 2124 * 2125 * CPU caps are not enforced on exiting processes - it is usually 2126 * desirable to exit as soon as possible to free resources. 2127 */ 2128 if (CPUCAPS_ON()) { 2129 thread_lock(t); 2130 fssproc = FSSPROC(t); 2131 (void) cpucaps_charge(t, &fssproc->fss_caps, 2132 CPUCAPS_CHARGE_ONLY); 2133 thread_unlock(t); 2134 } 2135 } 2136 2137 static void 2138 fss_nullsys() 2139 { 2140 } 2141 2142 /* 2143 * fss_swapin() returns -1 if the thread is loaded or is not eligible to be 2144 * swapped in. Otherwise, it returns the thread's effective priority based 2145 * on swapout time and size of process (0 <= epri <= 0 SHRT_MAX). 2146 */ 2147 /*ARGSUSED*/ 2148 static pri_t 2149 fss_swapin(kthread_t *t, int flags) 2150 { 2151 fssproc_t *fssproc = FSSPROC(t); 2152 long epri = -1; 2153 proc_t *pp = ttoproc(t); 2154 2155 ASSERT(THREAD_LOCK_HELD(t)); 2156 2157 if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) { 2158 time_t swapout_time; 2159 2160 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz; 2161 if (INHERITED(t) || (fssproc->fss_flags & FSSKPRI)) { 2162 epri = (long)DISP_PRIO(t) + swapout_time; 2163 } else { 2164 /* 2165 * Threads which have been out for a long time, 2166 * have high user mode priority and are associated 2167 * with a small address space are more deserving. 2168 */ 2169 epri = fssproc->fss_umdpri; 2170 ASSERT(epri >= 0 && epri <= fss_maxumdpri); 2171 epri += swapout_time - pp->p_swrss / nz(maxpgio)/2; 2172 } 2173 /* 2174 * Scale epri so that SHRT_MAX / 2 represents zero priority. 2175 */ 2176 epri += SHRT_MAX / 2; 2177 if (epri < 0) 2178 epri = 0; 2179 else if (epri > SHRT_MAX) 2180 epri = SHRT_MAX; 2181 } 2182 return ((pri_t)epri); 2183 } 2184 2185 /* 2186 * fss_swapout() returns -1 if the thread isn't loaded or is not eligible to 2187 * be swapped out. Otherwise, it returns the thread's effective priority 2188 * based on if the swapper is in softswap or hardswap mode. 2189 */ 2190 static pri_t 2191 fss_swapout(kthread_t *t, int flags) 2192 { 2193 fssproc_t *fssproc = FSSPROC(t); 2194 long epri = -1; 2195 proc_t *pp = ttoproc(t); 2196 time_t swapin_time; 2197 2198 ASSERT(THREAD_LOCK_HELD(t)); 2199 2200 if (INHERITED(t) || 2201 (fssproc->fss_flags & FSSKPRI) || 2202 (t->t_proc_flag & TP_LWPEXIT) || 2203 (t->t_state & (TS_ZOMB|TS_FREE|TS_STOPPED|TS_ONPROC|TS_WAIT)) || 2204 !(t->t_schedflag & TS_LOAD) || 2205 !(SWAP_OK(t))) 2206 return (-1); 2207 2208 ASSERT(t->t_state & (TS_SLEEP | TS_RUN)); 2209 2210 swapin_time = (ddi_get_lbolt() - t->t_stime) / hz; 2211 2212 if (flags == SOFTSWAP) { 2213 if (t->t_state == TS_SLEEP && swapin_time > maxslp) { 2214 epri = 0; 2215 } else { 2216 return ((pri_t)epri); 2217 } 2218 } else { 2219 pri_t pri; 2220 2221 if ((t->t_state == TS_SLEEP && swapin_time > fss_minslp) || 2222 (t->t_state == TS_RUN && swapin_time > fss_minrun)) { 2223 pri = fss_maxumdpri; 2224 epri = swapin_time - 2225 (rm_asrss(pp->p_as) / nz(maxpgio)/2) - (long)pri; 2226 } else { 2227 return ((pri_t)epri); 2228 } 2229 } 2230 2231 /* 2232 * Scale epri so that SHRT_MAX / 2 represents zero priority. 2233 */ 2234 epri += SHRT_MAX / 2; 2235 if (epri < 0) 2236 epri = 0; 2237 else if (epri > SHRT_MAX) 2238 epri = SHRT_MAX; 2239 2240 return ((pri_t)epri); 2241 } 2242 2243 /* 2244 * If thread is currently at a kernel mode priority (has slept) and is 2245 * returning to the userland we assign it the appropriate user mode priority 2246 * and time quantum here. If we're lowering the thread's priority below that 2247 * of other runnable threads then we will set runrun via cpu_surrender() to 2248 * cause preemption. 2249 */ 2250 static void 2251 fss_trapret(kthread_t *t) 2252 { 2253 fssproc_t *fssproc = FSSPROC(t); 2254 cpu_t *cp = CPU; 2255 2256 ASSERT(THREAD_LOCK_HELD(t)); 2257 ASSERT(t == curthread); 2258 ASSERT(cp->cpu_dispthread == t); 2259 ASSERT(t->t_state == TS_ONPROC); 2260 2261 t->t_kpri_req = 0; 2262 if (fssproc->fss_flags & FSSKPRI) { 2263 /* 2264 * If thread has blocked in the kernel 2265 */ 2266 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri); 2267 cp->cpu_dispatch_pri = DISP_PRIO(t); 2268 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri); 2269 fssproc->fss_flags &= ~FSSKPRI; 2270 2271 if (DISP_MUST_SURRENDER(t)) 2272 cpu_surrender(t); 2273 } 2274 2275 /* 2276 * Swapout lwp if the swapper is waiting for this thread to reach 2277 * a safe point. 2278 */ 2279 if (t->t_schedflag & TS_SWAPENQ) { 2280 thread_unlock(t); 2281 swapout_lwp(ttolwp(t)); 2282 thread_lock(t); 2283 } 2284 } 2285 2286 /* 2287 * Arrange for thread to be placed in appropriate location on dispatcher queue. 2288 * This is called with the current thread in TS_ONPROC and locked. 2289 */ 2290 static void 2291 fss_preempt(kthread_t *t) 2292 { 2293 fssproc_t *fssproc = FSSPROC(t); 2294 klwp_t *lwp; 2295 uint_t flags; 2296 2297 ASSERT(t == curthread); 2298 ASSERT(THREAD_LOCK_HELD(curthread)); 2299 ASSERT(t->t_state == TS_ONPROC); 2300 2301 /* 2302 * If preempted in the kernel, make sure the thread has a kernel 2303 * priority if needed. 2304 */ 2305 lwp = curthread->t_lwp; 2306 if (!(fssproc->fss_flags & FSSKPRI) && lwp != NULL && t->t_kpri_req) { 2307 fssproc->fss_flags |= FSSKPRI; 2308 THREAD_CHANGE_PRI(t, minclsyspri); 2309 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri); 2310 t->t_trapret = 1; /* so that fss_trapret will run */ 2311 aston(t); 2312 } 2313 2314 /* 2315 * This thread may be placed on wait queue by CPU Caps. In this case we 2316 * do not need to do anything until it is removed from the wait queue. 2317 * Do not enforce CPU caps on threads running at a kernel priority 2318 */ 2319 if (CPUCAPS_ON()) { 2320 (void) cpucaps_charge(t, &fssproc->fss_caps, 2321 CPUCAPS_CHARGE_ENFORCE); 2322 2323 if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t)) 2324 return; 2325 } 2326 2327 /* 2328 * If preempted in user-land mark the thread as swappable because it 2329 * cannot be holding any kernel locks. 2330 */ 2331 ASSERT(t->t_schedflag & TS_DONT_SWAP); 2332 if (lwp != NULL && lwp->lwp_state == LWP_USER) 2333 t->t_schedflag &= ~TS_DONT_SWAP; 2334 2335 /* 2336 * Check to see if we're doing "preemption control" here. If 2337 * we are, and if the user has requested that this thread not 2338 * be preempted, and if preemptions haven't been put off for 2339 * too long, let the preemption happen here but try to make 2340 * sure the thread is rescheduled as soon as possible. We do 2341 * this by putting it on the front of the highest priority run 2342 * queue in the FSS class. If the preemption has been put off 2343 * for too long, clear the "nopreempt" bit and let the thread 2344 * be preempted. 2345 */ 2346 if (t->t_schedctl && schedctl_get_nopreempt(t)) { 2347 if (fssproc->fss_timeleft > -SC_MAX_TICKS) { 2348 DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t); 2349 if (!(fssproc->fss_flags & FSSKPRI)) { 2350 /* 2351 * If not already remembered, remember current 2352 * priority for restoration in fss_yield(). 2353 */ 2354 if (!(fssproc->fss_flags & FSSRESTORE)) { 2355 fssproc->fss_scpri = t->t_pri; 2356 fssproc->fss_flags |= FSSRESTORE; 2357 } 2358 THREAD_CHANGE_PRI(t, fss_maxumdpri); 2359 t->t_schedflag |= TS_DONT_SWAP; 2360 } 2361 schedctl_set_yield(t, 1); 2362 setfrontdq(t); 2363 return; 2364 } else { 2365 if (fssproc->fss_flags & FSSRESTORE) { 2366 THREAD_CHANGE_PRI(t, fssproc->fss_scpri); 2367 fssproc->fss_flags &= ~FSSRESTORE; 2368 } 2369 schedctl_set_nopreempt(t, 0); 2370 DTRACE_SCHED1(schedctl__preempt, kthread_t *, t); 2371 /* 2372 * Fall through and be preempted below. 2373 */ 2374 } 2375 } 2376 2377 flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI); 2378 2379 if (flags == FSSBACKQ) { 2380 fssproc->fss_timeleft = fss_quantum; 2381 fssproc->fss_flags &= ~FSSBACKQ; 2382 setbackdq(t); 2383 } else if (flags == (FSSBACKQ | FSSKPRI)) { 2384 fssproc->fss_flags &= ~FSSBACKQ; 2385 setbackdq(t); 2386 } else { 2387 setfrontdq(t); 2388 } 2389 } 2390 2391 /* 2392 * Called when a thread is waking up and is to be placed on the run queue. 2393 */ 2394 static void 2395 fss_setrun(kthread_t *t) 2396 { 2397 fssproc_t *fssproc = FSSPROC(t); 2398 2399 ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */ 2400 2401 if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED) 2402 fss_active(t); 2403 2404 fssproc->fss_timeleft = fss_quantum; 2405 2406 fssproc->fss_flags &= ~FSSBACKQ; 2407 /* 2408 * If previously were running at the kernel priority then keep that 2409 * priority and the fss_timeleft doesn't matter. 2410 */ 2411 if ((fssproc->fss_flags & FSSKPRI) == 0) 2412 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri); 2413 2414 if (t->t_disp_time != ddi_get_lbolt()) 2415 setbackdq(t); 2416 else 2417 setfrontdq(t); 2418 } 2419 2420 /* 2421 * Prepare thread for sleep. We reset the thread priority so it will run at the 2422 * kernel priority level when it wakes up. 2423 */ 2424 static void 2425 fss_sleep(kthread_t *t) 2426 { 2427 fssproc_t *fssproc = FSSPROC(t); 2428 2429 ASSERT(t == curthread); 2430 ASSERT(THREAD_LOCK_HELD(t)); 2431 2432 ASSERT(t->t_state == TS_ONPROC); 2433 2434 /* 2435 * Account for time spent on CPU before going to sleep. 2436 */ 2437 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE); 2438 2439 fss_inactive(t); 2440 2441 /* 2442 * Assign a system priority to the thread and arrange for it to be 2443 * retained when the thread is next placed on the run queue (i.e., 2444 * when it wakes up) instead of being given a new pri. Also arrange 2445 * for trapret processing as the thread leaves the system call so it 2446 * will drop back to normal priority range. 2447 */ 2448 if (t->t_kpri_req) { 2449 THREAD_CHANGE_PRI(t, minclsyspri); 2450 fssproc->fss_flags |= FSSKPRI; 2451 t->t_trapret = 1; /* so that fss_trapret will run */ 2452 aston(t); 2453 } else if (fssproc->fss_flags & FSSKPRI) { 2454 /* 2455 * The thread has done a THREAD_KPRI_REQUEST(), slept, then 2456 * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again), 2457 * then slept again all without finishing the current system 2458 * call so trapret won't have cleared FSSKPRI 2459 */ 2460 fssproc->fss_flags &= ~FSSKPRI; 2461 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri); 2462 if (DISP_MUST_SURRENDER(curthread)) 2463 cpu_surrender(t); 2464 } 2465 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */ 2466 } 2467 2468 /* 2469 * A tick interrupt has ocurrend on a running thread. Check to see if our 2470 * time slice has expired. We must also clear the TS_DONT_SWAP flag in 2471 * t_schedflag if the thread is eligible to be swapped out. 2472 */ 2473 static void 2474 fss_tick(kthread_t *t) 2475 { 2476 fssproc_t *fssproc; 2477 fssproj_t *fssproj; 2478 klwp_t *lwp; 2479 boolean_t call_cpu_surrender = B_FALSE; 2480 boolean_t cpucaps_enforce = B_FALSE; 2481 2482 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock)); 2483 2484 /* 2485 * It's safe to access fsspset and fssproj structures because we're 2486 * holding our p_lock here. 2487 */ 2488 thread_lock(t); 2489 fssproc = FSSPROC(t); 2490 fssproj = FSSPROC2FSSPROJ(fssproc); 2491 if (fssproj != NULL) { 2492 fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj); 2493 disp_lock_enter_high(&fsspset->fssps_displock); 2494 fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice]; 2495 fssproj->fssp_tick_cnt++; 2496 fssproc->fss_ticks++; 2497 disp_lock_exit_high(&fsspset->fssps_displock); 2498 } 2499 2500 /* 2501 * Keep track of thread's project CPU usage. Note that projects 2502 * get charged even when threads are running in the kernel. 2503 * Do not surrender CPU if running in the SYS class. 2504 */ 2505 if (CPUCAPS_ON()) { 2506 cpucaps_enforce = cpucaps_charge(t, 2507 &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE) && 2508 !(fssproc->fss_flags & FSSKPRI); 2509 } 2510 2511 /* 2512 * A thread's execution time for threads running in the SYS class 2513 * is not tracked. 2514 */ 2515 if ((fssproc->fss_flags & FSSKPRI) == 0) { 2516 /* 2517 * If thread is not in kernel mode, decrement its fss_timeleft 2518 */ 2519 if (--fssproc->fss_timeleft <= 0) { 2520 pri_t new_pri; 2521 2522 /* 2523 * If we're doing preemption control and trying to 2524 * avoid preempting this thread, just note that the 2525 * thread should yield soon and let it keep running 2526 * (unless it's been a while). 2527 */ 2528 if (t->t_schedctl && schedctl_get_nopreempt(t)) { 2529 if (fssproc->fss_timeleft > -SC_MAX_TICKS) { 2530 DTRACE_SCHED1(schedctl__nopreempt, 2531 kthread_t *, t); 2532 schedctl_set_yield(t, 1); 2533 thread_unlock_nopreempt(t); 2534 return; 2535 } 2536 } 2537 fssproc->fss_flags &= ~FSSRESTORE; 2538 2539 fss_newpri(fssproc, B_TRUE); 2540 new_pri = fssproc->fss_umdpri; 2541 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri); 2542 2543 /* 2544 * When the priority of a thread is changed, it may 2545 * be necessary to adjust its position on a sleep queue 2546 * or dispatch queue. The function thread_change_pri 2547 * accomplishes this. 2548 */ 2549 if (thread_change_pri(t, new_pri, 0)) { 2550 if ((t->t_schedflag & TS_LOAD) && 2551 (lwp = t->t_lwp) && 2552 lwp->lwp_state == LWP_USER) 2553 t->t_schedflag &= ~TS_DONT_SWAP; 2554 fssproc->fss_timeleft = fss_quantum; 2555 } else { 2556 call_cpu_surrender = B_TRUE; 2557 } 2558 } else if (t->t_state == TS_ONPROC && 2559 t->t_pri < t->t_disp_queue->disp_maxrunpri) { 2560 /* 2561 * If there is a higher-priority thread which is 2562 * waiting for a processor, then thread surrenders 2563 * the processor. 2564 */ 2565 call_cpu_surrender = B_TRUE; 2566 } 2567 } 2568 2569 if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) { 2570 /* 2571 * The thread used more than half of its quantum, so assume that 2572 * it used the whole quantum. 2573 * 2574 * Update thread's priority just before putting it on the wait 2575 * queue so that it gets charged for the CPU time from its 2576 * quantum even before that quantum expires. 2577 */ 2578 fss_newpri(fssproc, B_FALSE); 2579 if (t->t_pri != fssproc->fss_umdpri) 2580 fss_change_priority(t, fssproc); 2581 2582 /* 2583 * We need to call cpu_surrender for this thread due to cpucaps 2584 * enforcement, but fss_change_priority may have already done 2585 * so. In this case FSSBACKQ is set and there is no need to call 2586 * cpu-surrender again. 2587 */ 2588 if (!(fssproc->fss_flags & FSSBACKQ)) 2589 call_cpu_surrender = B_TRUE; 2590 } 2591 2592 if (call_cpu_surrender) { 2593 fssproc->fss_flags |= FSSBACKQ; 2594 cpu_surrender(t); 2595 } 2596 2597 thread_unlock_nopreempt(t); /* clock thread can't be preempted */ 2598 } 2599 2600 /* 2601 * Processes waking up go to the back of their queue. We don't need to assign 2602 * a time quantum here because thread is still at a kernel mode priority and 2603 * the time slicing is not done for threads running in the kernel after 2604 * sleeping. The proper time quantum will be assigned by fss_trapret before the 2605 * thread returns to user mode. 2606 */ 2607 static void 2608 fss_wakeup(kthread_t *t) 2609 { 2610 fssproc_t *fssproc; 2611 2612 ASSERT(THREAD_LOCK_HELD(t)); 2613 ASSERT(t->t_state == TS_SLEEP); 2614 2615 fss_active(t); 2616 2617 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */ 2618 fssproc = FSSPROC(t); 2619 fssproc->fss_flags &= ~FSSBACKQ; 2620 2621 if (fssproc->fss_flags & FSSKPRI) { 2622 /* 2623 * If we already have a kernel priority assigned, then we 2624 * just use it. 2625 */ 2626 setbackdq(t); 2627 } else if (t->t_kpri_req) { 2628 /* 2629 * Give thread a priority boost if we were asked. 2630 */ 2631 fssproc->fss_flags |= FSSKPRI; 2632 THREAD_CHANGE_PRI(t, minclsyspri); 2633 setbackdq(t); 2634 t->t_trapret = 1; /* so that fss_trapret will run */ 2635 aston(t); 2636 } else { 2637 /* 2638 * Otherwise, we recalculate the priority. 2639 */ 2640 if (t->t_disp_time == ddi_get_lbolt()) { 2641 setfrontdq(t); 2642 } else { 2643 fssproc->fss_timeleft = fss_quantum; 2644 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri); 2645 setbackdq(t); 2646 } 2647 } 2648 } 2649 2650 /* 2651 * fss_donice() is called when a nice(1) command is issued on the thread to 2652 * alter the priority. The nice(1) command exists in Solaris for compatibility. 2653 * Thread priority adjustments should be done via priocntl(1). 2654 */ 2655 static int 2656 fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp) 2657 { 2658 int newnice; 2659 fssproc_t *fssproc = FSSPROC(t); 2660 fssparms_t fssparms; 2661 2662 /* 2663 * If there is no change to priority, just return current setting. 2664 */ 2665 if (incr == 0) { 2666 if (retvalp) 2667 *retvalp = fssproc->fss_nice - NZERO; 2668 return (0); 2669 } 2670 2671 if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0) 2672 return (EPERM); 2673 2674 /* 2675 * Specifying a nice increment greater than the upper limit of 2676 * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice 2677 * value being set to the upper limit. We check for this before 2678 * computing the new value because otherwise we could get overflow 2679 * if a privileged user specified some ridiculous increment. 2680 */ 2681 if (incr > FSS_NICE_MAX) 2682 incr = FSS_NICE_MAX; 2683 2684 newnice = fssproc->fss_nice + incr; 2685 if (newnice > FSS_NICE_MAX) 2686 newnice = FSS_NICE_MAX; 2687 else if (newnice < FSS_NICE_MIN) 2688 newnice = FSS_NICE_MIN; 2689 2690 fssparms.fss_uprilim = fssparms.fss_upri = 2691 -((newnice - NZERO) * fss_maxupri) / NZERO; 2692 2693 /* 2694 * Reset the uprilim and upri values of the thread. 2695 */ 2696 (void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL); 2697 2698 /* 2699 * Although fss_parmsset already reset fss_nice it may not have been 2700 * set to precisely the value calculated above because fss_parmsset 2701 * determines the nice value from the user priority and we may have 2702 * truncated during the integer conversion from nice value to user 2703 * priority and back. We reset fss_nice to the value we calculated 2704 * above. 2705 */ 2706 fssproc->fss_nice = (char)newnice; 2707 2708 if (retvalp) 2709 *retvalp = newnice - NZERO; 2710 return (0); 2711 } 2712 2713 /* 2714 * Increment the priority of the specified thread by incr and 2715 * return the new value in *retvalp. 2716 */ 2717 static int 2718 fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp) 2719 { 2720 int newpri; 2721 fssproc_t *fssproc = FSSPROC(t); 2722 fssparms_t fssparms; 2723 2724 /* 2725 * If there is no change to priority, just return current setting. 2726 */ 2727 if (incr == 0) { 2728 *retvalp = fssproc->fss_upri; 2729 return (0); 2730 } 2731 2732 newpri = fssproc->fss_upri + incr; 2733 if (newpri > fss_maxupri || newpri < -fss_maxupri) 2734 return (EINVAL); 2735 2736 *retvalp = newpri; 2737 fssparms.fss_uprilim = fssparms.fss_upri = newpri; 2738 2739 /* 2740 * Reset the uprilim and upri values of the thread. 2741 */ 2742 return (fss_parmsset(t, &fssparms, (id_t)0, cr)); 2743 } 2744 2745 /* 2746 * Return the global scheduling priority that would be assigned to a thread 2747 * entering the fair-sharing class with the fss_upri. 2748 */ 2749 /*ARGSUSED*/ 2750 static pri_t 2751 fss_globpri(kthread_t *t) 2752 { 2753 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)); 2754 2755 return (fss_maxumdpri / 2); 2756 } 2757 2758 /* 2759 * Called from the yield(2) system call when a thread is yielding (surrendering) 2760 * the processor. The kernel thread is placed at the back of a dispatch queue. 2761 */ 2762 static void 2763 fss_yield(kthread_t *t) 2764 { 2765 fssproc_t *fssproc = FSSPROC(t); 2766 2767 ASSERT(t == curthread); 2768 ASSERT(THREAD_LOCK_HELD(t)); 2769 2770 /* 2771 * Collect CPU usage spent before yielding 2772 */ 2773 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE); 2774 2775 /* 2776 * Clear the preemption control "yield" bit since the user is 2777 * doing a yield. 2778 */ 2779 if (t->t_schedctl) 2780 schedctl_set_yield(t, 0); 2781 /* 2782 * If fss_preempt() artifically increased the thread's priority 2783 * to avoid preemption, restore the original priority now. 2784 */ 2785 if (fssproc->fss_flags & FSSRESTORE) { 2786 THREAD_CHANGE_PRI(t, fssproc->fss_scpri); 2787 fssproc->fss_flags &= ~FSSRESTORE; 2788 } 2789 if (fssproc->fss_timeleft < 0) { 2790 /* 2791 * Time slice was artificially extended to avoid preemption, 2792 * so pretend we're preempting it now. 2793 */ 2794 DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft); 2795 fssproc->fss_timeleft = fss_quantum; 2796 } 2797 fssproc->fss_flags &= ~FSSBACKQ; 2798 setbackdq(t); 2799 } 2800 2801 void 2802 fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf, 2803 fssbuf_t *zonebuf) 2804 { 2805 kproject_t *kpj_new = kp; 2806 zone_t *zone = zp; 2807 fssproj_t *fssproj_old, *fssproj_new; 2808 fsspset_t *fsspset; 2809 kproject_t *kpj_old; 2810 fssproc_t *fssproc; 2811 fsszone_t *fsszone_old, *fsszone_new; 2812 int free = 0; 2813 int id; 2814 2815 ASSERT(MUTEX_HELD(&cpu_lock)); 2816 ASSERT(MUTEX_HELD(&pidlock)); 2817 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)); 2818 2819 if (t->t_cid != fss_cid) 2820 return; 2821 2822 fssproc = FSSPROC(t); 2823 mutex_enter(&fsspsets_lock); 2824 fssproj_old = FSSPROC2FSSPROJ(fssproc); 2825 if (fssproj_old == NULL) { 2826 mutex_exit(&fsspsets_lock); 2827 return; 2828 } 2829 2830 fsspset = FSSPROJ2FSSPSET(fssproj_old); 2831 mutex_enter(&fsspset->fssps_lock); 2832 kpj_old = FSSPROJ2KPROJ(fssproj_old); 2833 fsszone_old = fssproj_old->fssp_fsszone; 2834 2835 ASSERT(t->t_cpupart == fsspset->fssps_cpupart); 2836 2837 if (kpj_old == kpj_new) { 2838 mutex_exit(&fsspset->fssps_lock); 2839 mutex_exit(&fsspsets_lock); 2840 return; 2841 } 2842 2843 if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) { 2844 /* 2845 * If the zone for the new project is not currently active on 2846 * the cpu partition we're on, get one of the pre-allocated 2847 * buffers and link it in our per-pset zone list. Such buffers 2848 * should already exist. 2849 */ 2850 for (id = 0; id < zonebuf->fssb_size; id++) { 2851 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) { 2852 fss_insert_fsszone(fsspset, zone, fsszone_new); 2853 zonebuf->fssb_list[id] = NULL; 2854 break; 2855 } 2856 } 2857 } 2858 ASSERT(fsszone_new != NULL); 2859 if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) { 2860 /* 2861 * If our new project is not currently running 2862 * on the cpu partition we're on, get one of the 2863 * pre-allocated buffers and link it in our new cpu 2864 * partition doubly linked list. Such buffers should already 2865 * exist. 2866 */ 2867 for (id = 0; id < projbuf->fssb_size; id++) { 2868 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) { 2869 fss_insert_fssproj(fsspset, kpj_new, 2870 fsszone_new, fssproj_new); 2871 projbuf->fssb_list[id] = NULL; 2872 break; 2873 } 2874 } 2875 } 2876 ASSERT(fssproj_new != NULL); 2877 2878 thread_lock(t); 2879 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC || 2880 t->t_state == TS_WAIT) 2881 fss_inactive(t); 2882 ASSERT(fssproj_old->fssp_threads > 0); 2883 if (--fssproj_old->fssp_threads == 0) { 2884 fss_remove_fssproj(fsspset, fssproj_old); 2885 free = 1; 2886 } 2887 fssproc->fss_proj = fssproj_new; 2888 fssproc->fss_fsspri = 0; 2889 fssproj_new->fssp_threads++; 2890 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC || 2891 t->t_state == TS_WAIT) 2892 fss_active(t); 2893 thread_unlock(t); 2894 if (free) { 2895 if (fsszone_old->fssz_nproj == 0) 2896 kmem_free(fsszone_old, sizeof (fsszone_t)); 2897 kmem_free(fssproj_old, sizeof (fssproj_t)); 2898 } 2899 2900 mutex_exit(&fsspset->fssps_lock); 2901 mutex_exit(&fsspsets_lock); 2902 } 2903 2904 void 2905 fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf, 2906 fssbuf_t *zonebuf) 2907 { 2908 fsspset_t *fsspset_old, *fsspset_new; 2909 fssproj_t *fssproj_old, *fssproj_new; 2910 fsszone_t *fsszone_old, *fsszone_new; 2911 fssproc_t *fssproc; 2912 kproject_t *kpj; 2913 zone_t *zone; 2914 int id; 2915 2916 ASSERT(MUTEX_HELD(&cpu_lock)); 2917 ASSERT(MUTEX_HELD(&pidlock)); 2918 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)); 2919 2920 if (t->t_cid != fss_cid) 2921 return; 2922 2923 fssproc = FSSPROC(t); 2924 zone = ttoproc(t)->p_zone; 2925 mutex_enter(&fsspsets_lock); 2926 fssproj_old = FSSPROC2FSSPROJ(fssproc); 2927 if (fssproj_old == NULL) { 2928 mutex_exit(&fsspsets_lock); 2929 return; 2930 } 2931 fsszone_old = fssproj_old->fssp_fsszone; 2932 fsspset_old = FSSPROJ2FSSPSET(fssproj_old); 2933 kpj = FSSPROJ2KPROJ(fssproj_old); 2934 2935 if (fsspset_old->fssps_cpupart == newcp) { 2936 mutex_exit(&fsspsets_lock); 2937 return; 2938 } 2939 2940 ASSERT(ttoproj(t) == kpj); 2941 2942 fsspset_new = fss_find_fsspset(newcp); 2943 2944 mutex_enter(&fsspset_new->fssps_lock); 2945 if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) { 2946 for (id = 0; id < zonebuf->fssb_size; id++) { 2947 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) { 2948 fss_insert_fsszone(fsspset_new, zone, 2949 fsszone_new); 2950 zonebuf->fssb_list[id] = NULL; 2951 break; 2952 } 2953 } 2954 } 2955 ASSERT(fsszone_new != NULL); 2956 if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) { 2957 for (id = 0; id < projbuf->fssb_size; id++) { 2958 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) { 2959 fss_insert_fssproj(fsspset_new, kpj, 2960 fsszone_new, fssproj_new); 2961 projbuf->fssb_list[id] = NULL; 2962 break; 2963 } 2964 } 2965 } 2966 ASSERT(fssproj_new != NULL); 2967 2968 fssproj_new->fssp_threads++; 2969 thread_lock(t); 2970 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC || 2971 t->t_state == TS_WAIT) 2972 fss_inactive(t); 2973 fssproc->fss_proj = fssproj_new; 2974 fssproc->fss_fsspri = 0; 2975 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC || 2976 t->t_state == TS_WAIT) 2977 fss_active(t); 2978 thread_unlock(t); 2979 mutex_exit(&fsspset_new->fssps_lock); 2980 2981 mutex_enter(&fsspset_old->fssps_lock); 2982 if (--fssproj_old->fssp_threads == 0) { 2983 fss_remove_fssproj(fsspset_old, fssproj_old); 2984 if (fsszone_old->fssz_nproj == 0) 2985 kmem_free(fsszone_old, sizeof (fsszone_t)); 2986 kmem_free(fssproj_old, sizeof (fssproj_t)); 2987 } 2988 mutex_exit(&fsspset_old->fssps_lock); 2989 2990 mutex_exit(&fsspsets_lock); 2991 }