1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Basic NUMA support in terms of locality groups
28 *
29 * Solaris needs to know which CPUs, memory, etc. are near each other to
30 * provide good performance on NUMA machines by optimizing for locality.
31 * In order to do this, a new abstraction called a "locality group (lgroup)"
32 * has been introduced to keep track of which CPU-like and memory-like hardware
33 * resources are close to each other. Currently, latency is the only measure
34 * used to determine how to group hardware resources into lgroups, but this
35 * does not limit the groupings to be based solely on latency. Other factors
36 * may be used to determine the groupings in the future.
37 *
38 * Lgroups are organized into a hieararchy or topology that represents the
39 * latency topology of the machine. There is always at least a root lgroup in
40 * the system. It represents all the hardware resources in the machine at a
41 * latency big enough that any hardware resource can at least access any other
42 * hardware resource within that latency. A Uniform Memory Access (UMA)
43 * machine is represented with one lgroup (the root). In contrast, a NUMA
44 * machine is represented at least by the root lgroup and some number of leaf
45 * lgroups where the leaf lgroups contain the hardware resources within the
46 * least latency of each other and the root lgroup still contains all the
47 * resources in the machine. Some number of intermediate lgroups may exist
48 * which represent more levels of locality than just the local latency of the
49 * leaf lgroups and the system latency of the root lgroup. Non-leaf lgroups
50 * (eg. root and intermediate lgroups) contain the next nearest resources to
51 * its children lgroups. Thus, the lgroup hierarchy from a given leaf lgroup
52 * to the root lgroup shows the hardware resources from closest to farthest
53 * from the leaf lgroup such that each successive ancestor lgroup contains
54 * the next nearest resources at the next level of locality from the previous.
55 *
56 * The kernel uses the lgroup abstraction to know how to allocate resources
57 * near a given process/thread. At fork() and lwp/thread_create() time, a
58 * "home" lgroup is chosen for a thread. This is done by picking the lgroup
59 * with the lowest load average. Binding to a processor or processor set will
60 * change the home lgroup for a thread. The scheduler has been modified to try
61 * to dispatch a thread on a CPU in its home lgroup. Physical memory
62 * allocation is lgroup aware too, so memory will be allocated from the current
63 * thread's home lgroup if possible. If the desired resources are not
64 * available, the kernel traverses the lgroup hierarchy going to the parent
65 * lgroup to find resources at the next level of locality until it reaches the
66 * root lgroup.
67 */
68
69 #include <sys/lgrp.h>
70 #include <sys/lgrp_user.h>
71 #include <sys/types.h>
72 #include <sys/mman.h>
73 #include <sys/param.h>
74 #include <sys/var.h>
75 #include <sys/thread.h>
76 #include <sys/cpuvar.h>
77 #include <sys/cpupart.h>
78 #include <sys/kmem.h>
79 #include <vm/seg.h>
80 #include <vm/seg_kmem.h>
81 #include <vm/seg_spt.h>
82 #include <vm/seg_vn.h>
83 #include <vm/as.h>
84 #include <sys/atomic.h>
85 #include <sys/systm.h>
86 #include <sys/errno.h>
87 #include <sys/cmn_err.h>
88 #include <sys/kstat.h>
89 #include <sys/sysmacros.h>
90 #include <sys/pg.h>
91 #include <sys/promif.h>
92 #include <sys/sdt.h>
93
94 lgrp_gen_t lgrp_gen = 0; /* generation of lgroup hierarchy */
95 lgrp_t *lgrp_table[NLGRPS_MAX]; /* table of all initialized lgrp_t structs */
96 /* indexed by lgrp_id */
97 int nlgrps; /* number of lgroups in machine */
98 int lgrp_alloc_hint = -1; /* hint for where to try to allocate next */
99 int lgrp_alloc_max = 0; /* max lgroup ID allocated so far */
100
101 /*
102 * Kstat data for lgroups.
103 *
104 * Actual kstat data is collected in lgrp_stats array.
105 * The lgrp_kstat_data array of named kstats is used to extract data from
106 * lgrp_stats and present it to kstat framework. It is protected from partallel
107 * modifications by lgrp_kstat_mutex. This may cause some contention when
108 * several kstat commands run in parallel but this is not the
109 * performance-critical path.
110 */
111 extern struct lgrp_stats lgrp_stats[]; /* table of per-lgrp stats */
112
113 /*
114 * Declare kstat names statically for enums as defined in the header file.
115 */
116 LGRP_KSTAT_NAMES;
117
118 static void lgrp_kstat_init(void);
119 static int lgrp_kstat_extract(kstat_t *, int);
120 static void lgrp_kstat_reset(lgrp_id_t);
121
122 static struct kstat_named lgrp_kstat_data[LGRP_NUM_STATS];
123 static kmutex_t lgrp_kstat_mutex;
124
125
126 /*
127 * max number of lgroups supported by the platform
128 */
129 int nlgrpsmax = 0;
130
131 /*
132 * The root lgroup. Represents the set of resources at the system wide
133 * level of locality.
134 */
135 lgrp_t *lgrp_root = NULL;
136
137 /*
138 * During system bootstrap cp_default does not contain the list of lgrp load
139 * averages (cp_lgrploads). The list is allocated after the first CPU is brought
140 * on-line when cp_default is initialized by cpupart_initialize_default().
141 * Configuring CPU0 may create a two-level topology with root and one leaf node
142 * containing CPU0. This topology is initially constructed in a special
143 * statically allocated 2-element lpl list lpl_bootstrap_list and later cloned
144 * to cp_default when cp_default is initialized. The lpl_bootstrap_list is used
145 * for all lpl operations until cp_default is fully constructed.
146 *
147 * The lpl_bootstrap_list is maintained by the code in lgrp.c. Every other
148 * consumer who needs default lpl should use lpl_bootstrap which is a pointer to
149 * the first element of lpl_bootstrap_list.
150 *
151 * CPUs that are added to the system, but have not yet been assigned to an
152 * lgrp will use lpl_bootstrap as a default lpl. This is necessary because
153 * on some architectures (x86) it's possible for the slave CPU startup thread
154 * to enter the dispatcher or allocate memory before calling lgrp_cpu_init().
155 */
156 #define LPL_BOOTSTRAP_SIZE 2
157 static lpl_t lpl_bootstrap_list[LPL_BOOTSTRAP_SIZE];
158 lpl_t *lpl_bootstrap;
159 static lpl_t *lpl_bootstrap_rset[LPL_BOOTSTRAP_SIZE];
160 static int lpl_bootstrap_id2rset[LPL_BOOTSTRAP_SIZE];
161
162 /*
163 * If cp still references the bootstrap lpl, it has not yet been added to
164 * an lgrp. lgrp_mem_choose() uses this macro to detect the case where
165 * a thread is trying to allocate memory close to a CPU that has no lgrp.
166 */
167 #define LGRP_CPU_HAS_NO_LGRP(cp) ((cp)->cpu_lpl == lpl_bootstrap)
168
169 static lgrp_t lroot;
170
171 /*
172 * Size, in bytes, beyond which random memory allocation policy is applied
173 * to non-shared memory. Default is the maximum size, so random memory
174 * allocation won't be used for non-shared memory by default.
175 */
176 size_t lgrp_privm_random_thresh = (size_t)(-1);
177
178 /* the maximum effect that a single thread can have on it's lgroup's load */
179 #define LGRP_LOADAVG_MAX_EFFECT(ncpu) \
180 ((lgrp_loadavg_max_effect) / (ncpu))
181 uint32_t lgrp_loadavg_max_effect = LGRP_LOADAVG_THREAD_MAX;
182
183
184 /*
185 * Size, in bytes, beyond which random memory allocation policy is applied to
186 * shared memory. Default is 8MB (2 ISM pages).
187 */
188 size_t lgrp_shm_random_thresh = 8*1024*1024;
189
190 /*
191 * Whether to do processor set aware memory allocation by default
192 */
193 int lgrp_mem_pset_aware = 0;
194
195 /*
196 * Set the default memory allocation policy for root lgroup
197 */
198 lgrp_mem_policy_t lgrp_mem_policy_root = LGRP_MEM_POLICY_RANDOM;
199
200 /*
201 * Set the default memory allocation policy. For most platforms,
202 * next touch is sufficient, but some platforms may wish to override
203 * this.
204 */
205 lgrp_mem_policy_t lgrp_mem_default_policy = LGRP_MEM_POLICY_NEXT;
206
207
208 /*
209 * lgroup CPU event handlers
210 */
211 static void lgrp_cpu_init(struct cpu *);
212 static void lgrp_cpu_fini(struct cpu *, lgrp_id_t);
213 static lgrp_t *lgrp_cpu_to_lgrp(struct cpu *);
214
215 /*
216 * lgroup memory event handlers
217 */
218 static void lgrp_mem_init(int, lgrp_handle_t, boolean_t);
219 static void lgrp_mem_fini(int, lgrp_handle_t, boolean_t);
220 static void lgrp_mem_rename(int, lgrp_handle_t, lgrp_handle_t);
221
222 /*
223 * lgroup CPU partition event handlers
224 */
225 static void lgrp_part_add_cpu(struct cpu *, lgrp_id_t);
226 static void lgrp_part_del_cpu(struct cpu *);
227
228 /*
229 * lgroup framework initialization
230 */
231 static void lgrp_main_init(void);
232 static void lgrp_main_mp_init(void);
233 static void lgrp_root_init(void);
234 static void lgrp_setup(void);
235
236 /*
237 * lpl topology
238 */
239 static void lpl_init(lpl_t *, lpl_t *, lgrp_t *);
240 static void lpl_clear(lpl_t *);
241 static void lpl_leaf_insert(lpl_t *, struct cpupart *);
242 static void lpl_leaf_remove(lpl_t *, struct cpupart *);
243 static void lpl_rset_add(lpl_t *, lpl_t *);
244 static void lpl_rset_del(lpl_t *, lpl_t *);
245 static int lpl_rset_contains(lpl_t *, lpl_t *);
246 static void lpl_cpu_adjcnt(lpl_act_t, struct cpu *);
247 static void lpl_child_update(lpl_t *, struct cpupart *);
248 static int lpl_pick(lpl_t *, lpl_t *);
249 static void lpl_verify_wrapper(struct cpupart *);
250
251 /*
252 * defines for lpl topology verifier return codes
253 */
254
255 #define LPL_TOPO_CORRECT 0
256 #define LPL_TOPO_PART_HAS_NO_LPL -1
257 #define LPL_TOPO_CPUS_NOT_EMPTY -2
258 #define LPL_TOPO_LGRP_MISMATCH -3
259 #define LPL_TOPO_MISSING_PARENT -4
260 #define LPL_TOPO_PARENT_MISMATCH -5
261 #define LPL_TOPO_BAD_CPUCNT -6
262 #define LPL_TOPO_RSET_MISMATCH -7
263 #define LPL_TOPO_LPL_ORPHANED -8
264 #define LPL_TOPO_LPL_BAD_NCPU -9
265 #define LPL_TOPO_RSET_MSSNG_LF -10
266 #define LPL_TOPO_CPU_HAS_BAD_LPL -11
267 #define LPL_TOPO_NONLEAF_HAS_CPUS -12
268 #define LPL_TOPO_LGRP_NOT_LEAF -13
269 #define LPL_TOPO_BAD_RSETCNT -14
270
271 /*
272 * Return whether lgroup optimizations should be enabled on this system
273 */
274 int
275 lgrp_optimizations(void)
276 {
277 /*
278 * System must have more than 2 lgroups to enable lgroup optimizations
279 *
280 * XXX This assumes that a 2 lgroup system has an empty root lgroup
281 * with one child lgroup containing all the resources. A 2 lgroup
282 * system with a root lgroup directly containing CPUs or memory might
283 * need lgroup optimizations with its child lgroup, but there
284 * isn't such a machine for now....
285 */
286 if (nlgrps > 2)
287 return (1);
288
289 return (0);
290 }
291
292 /*
293 * Setup root lgroup
294 */
295 static void
296 lgrp_root_init(void)
297 {
298 lgrp_handle_t hand;
299 int i;
300 lgrp_id_t id;
301
302 /*
303 * Create the "root" lgroup
304 */
305 ASSERT(nlgrps == 0);
306 id = nlgrps++;
307
308 lgrp_root = &lroot;
309
310 lgrp_root->lgrp_cpu = NULL;
311 lgrp_root->lgrp_mnodes = 0;
312 lgrp_root->lgrp_nmnodes = 0;
313 hand = lgrp_plat_root_hand();
314 lgrp_root->lgrp_plathand = hand;
315
316 lgrp_root->lgrp_id = id;
317 lgrp_root->lgrp_cpucnt = 0;
318 lgrp_root->lgrp_childcnt = 0;
319 klgrpset_clear(lgrp_root->lgrp_children);
320 klgrpset_clear(lgrp_root->lgrp_leaves);
321 lgrp_root->lgrp_parent = NULL;
322 lgrp_root->lgrp_latency = lgrp_plat_latency(hand, hand);
323
324 for (i = 0; i < LGRP_RSRC_COUNT; i++)
325 klgrpset_clear(lgrp_root->lgrp_set[i]);
326
327 lgrp_root->lgrp_kstat = NULL;
328
329 lgrp_table[id] = lgrp_root;
330
331 /*
332 * Setup initial lpl list for CPU0 and initial t0 home.
333 * The only lpl space we have so far is lpl_bootstrap. It is used for
334 * all topology operations until cp_default is initialized at which
335 * point t0.t_lpl will be updated.
336 */
337 lpl_bootstrap = lpl_bootstrap_list;
338 t0.t_lpl = lpl_bootstrap;
339 cp_default.cp_nlgrploads = LPL_BOOTSTRAP_SIZE;
340 lpl_bootstrap_list[1].lpl_lgrpid = 1;
341
342 /*
343 * Set up the bootstrap rset
344 * Since the bootstrap toplogy has just the root, and a leaf,
345 * the rset contains just the leaf, and both lpls can use the same rset
346 */
347 lpl_bootstrap_rset[0] = &lpl_bootstrap_list[1];
348 lpl_bootstrap_list[0].lpl_rset_sz = 1;
349 lpl_bootstrap_list[0].lpl_rset = lpl_bootstrap_rset;
350 lpl_bootstrap_list[0].lpl_id2rset = lpl_bootstrap_id2rset;
351
352 lpl_bootstrap_list[1].lpl_rset_sz = 1;
353 lpl_bootstrap_list[1].lpl_rset = lpl_bootstrap_rset;
354 lpl_bootstrap_list[1].lpl_id2rset = lpl_bootstrap_id2rset;
355
356 cp_default.cp_lgrploads = lpl_bootstrap;
357 }
358
359 /*
360 * Initialize the lgroup framework and allow the platform to do the same
361 *
362 * This happens in stages during boot and is all funnelled through this routine
363 * (see definition of lgrp_init_stages_t to see what happens at each stage and
364 * when)
365 */
366 void
367 lgrp_init(lgrp_init_stages_t stage)
368 {
369 /*
370 * Initialize the platform
371 */
372 lgrp_plat_init(stage);
373
374 switch (stage) {
375 case LGRP_INIT_STAGE1:
376 /*
377 * Set max number of lgroups supported on this platform which
378 * must be less than the max number of lgroups supported by the
379 * common lgroup framework (eg. NLGRPS_MAX is max elements in
380 * lgrp_table[], etc.)
381 */
382 nlgrpsmax = lgrp_plat_max_lgrps();
383 ASSERT(nlgrpsmax <= NLGRPS_MAX);
384 break;
385
386 case LGRP_INIT_STAGE2:
387 lgrp_setup();
388 break;
389
390 case LGRP_INIT_STAGE4:
391 lgrp_main_init();
392 break;
393
394 case LGRP_INIT_STAGE5:
395 lgrp_main_mp_init();
396 break;
397
398 default:
399 break;
400 }
401 }
402
403 /*
404 * Create the root and cpu0's lgroup, and set t0's home.
405 */
406 static void
407 lgrp_setup(void)
408 {
409 /*
410 * Setup the root lgroup
411 */
412 lgrp_root_init();
413
414 /*
415 * Add cpu0 to an lgroup
416 */
417 lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)CPU, 0);
418 lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)CPU, 0);
419 }
420
421 /*
422 * true when lgrp initialization has been completed.
423 */
424 int lgrp_initialized = 0;
425
426 /*
427 * True when lgrp topology is constructed.
428 */
429 int lgrp_topo_initialized = 0;
430
431 /*
432 * Init routine called after startup(), /etc/system has been processed,
433 * and cpu0 has been added to an lgroup.
434 */
435 static void
436 lgrp_main_init(void)
437 {
438 cpu_t *cp = CPU;
439 lgrp_id_t lgrpid;
440 int i;
441 extern void pg_cpu0_reinit();
442
443 /*
444 * Enforce a valid lgrp_mem_default_policy
445 */
446 if ((lgrp_mem_default_policy <= LGRP_MEM_POLICY_DEFAULT) ||
447 (lgrp_mem_default_policy >= LGRP_NUM_MEM_POLICIES) ||
448 (lgrp_mem_default_policy == LGRP_MEM_POLICY_NEXT_SEG))
449 lgrp_mem_default_policy = LGRP_MEM_POLICY_NEXT;
450
451 /*
452 * See if mpo should be disabled.
453 * This may happen in the case of null proc LPA on Starcat.
454 * The platform won't be able to detect null proc LPA until after
455 * cpu0 and memory have already been added to lgroups.
456 * When and if it is detected, the Starcat platform will return
457 * a different platform handle for cpu0 which is what we check for
458 * here. If mpo should be disabled move cpu0 to it's rightful place
459 * (the root), and destroy the remaining lgroups. This effectively
460 * provides an UMA lgroup topology.
461 */
462 lgrpid = cp->cpu_lpl->lpl_lgrpid;
463 if (lgrp_table[lgrpid]->lgrp_plathand !=
464 lgrp_plat_cpu_to_hand(cp->cpu_id)) {
465 lgrp_part_del_cpu(cp);
466 lgrp_cpu_fini(cp, lgrpid);
467
468 lgrp_cpu_init(cp);
469 lgrp_part_add_cpu(cp, cp->cpu_lpl->lpl_lgrpid);
470
471 ASSERT(cp->cpu_lpl->lpl_lgrpid == LGRP_ROOTID);
472
473 /*
474 * Notify the PG subsystem that the CPU's lgrp
475 * association has changed
476 */
477 pg_cpu0_reinit();
478
479 /*
480 * Destroy all lgroups except for root
481 */
482 for (i = 0; i <= lgrp_alloc_max; i++) {
483 if (LGRP_EXISTS(lgrp_table[i]) &&
484 lgrp_table[i] != lgrp_root)
485 lgrp_destroy(lgrp_table[i]);
486 }
487
488 /*
489 * Fix up root to point at itself for leaves and resources
490 * and not have any children
491 */
492 lgrp_root->lgrp_childcnt = 0;
493 klgrpset_clear(lgrp_root->lgrp_children);
494 klgrpset_clear(lgrp_root->lgrp_leaves);
495 klgrpset_add(lgrp_root->lgrp_leaves, LGRP_ROOTID);
496 klgrpset_clear(lgrp_root->lgrp_set[LGRP_RSRC_MEM]);
497 klgrpset_add(lgrp_root->lgrp_set[LGRP_RSRC_MEM], LGRP_ROOTID);
498 }
499
500 /*
501 * Initialize kstats framework.
502 */
503 lgrp_kstat_init();
504 /*
505 * cpu0 is finally where it should be, so create it's lgroup's kstats
506 */
507 mutex_enter(&cpu_lock);
508 lgrp_kstat_create(cp);
509 mutex_exit(&cpu_lock);
510
511 lgrp_initialized = 1;
512 }
513
514 /*
515 * Finish lgrp initialization after all CPUS are brought on-line.
516 * This routine is called after start_other_cpus().
517 */
518 static void
519 lgrp_main_mp_init(void)
520 {
521 klgrpset_t changed;
522
523 /*
524 * Update lgroup topology (if necessary)
525 */
526 klgrpset_clear(changed);
527 (void) lgrp_topo_update(lgrp_table, lgrp_alloc_max + 1, &changed);
528 lgrp_topo_initialized = 1;
529 }
530
531 /*
532 * Change latency of lgroup with specified lgroup platform handle (if one is
533 * given) or change all lgroups with old latency to new latency
534 */
535 void
536 lgrp_latency_change(lgrp_handle_t hand, u_longlong_t oldtime,
537 u_longlong_t newtime)
538 {
539 lgrp_t *lgrp;
540 int i;
541
542 for (i = 0; i <= lgrp_alloc_max; i++) {
543 lgrp = lgrp_table[i];
544
545 if (!LGRP_EXISTS(lgrp))
546 continue;
547
548 if ((hand == LGRP_NULL_HANDLE &&
549 lgrp->lgrp_latency == oldtime) ||
550 (hand != LGRP_NULL_HANDLE && lgrp->lgrp_plathand == hand))
551 lgrp->lgrp_latency = (int)newtime;
552 }
553 }
554
555 /*
556 * Handle lgroup (re)configuration events (eg. addition of CPU, etc.)
557 */
558 void
559 lgrp_config(lgrp_config_flag_t event, uintptr_t resource, uintptr_t where)
560 {
561 klgrpset_t changed;
562 cpu_t *cp;
563 lgrp_id_t id;
564 int rc;
565
566 switch (event) {
567 /*
568 * The following (re)configuration events are common code
569 * initiated. lgrp_plat_config() is called here to inform the
570 * platform of the reconfiguration event.
571 */
572 case LGRP_CONFIG_CPU_ADD:
573 cp = (cpu_t *)resource;
574
575 /*
576 * Initialize the new CPU's lgrp related next/prev
577 * links, and give it a bootstrap lpl so that it can
578 * survive should it need to enter the dispatcher.
579 */
580 cp->cpu_next_lpl = cp;
581 cp->cpu_prev_lpl = cp;
582 cp->cpu_next_lgrp = cp;
583 cp->cpu_prev_lgrp = cp;
584 cp->cpu_lpl = lpl_bootstrap;
585
586 lgrp_plat_config(event, resource);
587 atomic_inc_32(&lgrp_gen);
588
589 break;
590 case LGRP_CONFIG_CPU_DEL:
591 lgrp_plat_config(event, resource);
592 atomic_inc_32(&lgrp_gen);
593
594 break;
595 case LGRP_CONFIG_CPU_ONLINE:
596 cp = (cpu_t *)resource;
597 lgrp_cpu_init(cp);
598 lgrp_part_add_cpu(cp, cp->cpu_lpl->lpl_lgrpid);
599 rc = lpl_topo_verify(cp->cpu_part);
600 if (rc != LPL_TOPO_CORRECT) {
601 panic("lpl_topo_verify failed: %d", rc);
602 }
603 lgrp_plat_config(event, resource);
604 atomic_inc_32(&lgrp_gen);
605
606 break;
607 case LGRP_CONFIG_CPU_OFFLINE:
608 cp = (cpu_t *)resource;
609 id = cp->cpu_lpl->lpl_lgrpid;
610 lgrp_part_del_cpu(cp);
611 lgrp_cpu_fini(cp, id);
612 rc = lpl_topo_verify(cp->cpu_part);
613 if (rc != LPL_TOPO_CORRECT) {
614 panic("lpl_topo_verify failed: %d", rc);
615 }
616 lgrp_plat_config(event, resource);
617 atomic_inc_32(&lgrp_gen);
618
619 break;
620 case LGRP_CONFIG_CPUPART_ADD:
621 cp = (cpu_t *)resource;
622 lgrp_part_add_cpu((cpu_t *)resource, (lgrp_id_t)where);
623 rc = lpl_topo_verify(cp->cpu_part);
624 if (rc != LPL_TOPO_CORRECT) {
625 panic("lpl_topo_verify failed: %d", rc);
626 }
627 lgrp_plat_config(event, resource);
628
629 break;
630 case LGRP_CONFIG_CPUPART_DEL:
631 cp = (cpu_t *)resource;
632 lgrp_part_del_cpu((cpu_t *)resource);
633 rc = lpl_topo_verify(cp->cpu_part);
634 if (rc != LPL_TOPO_CORRECT) {
635 panic("lpl_topo_verify failed: %d", rc);
636 }
637 lgrp_plat_config(event, resource);
638
639 break;
640 /*
641 * The following events are initiated by the memnode
642 * subsystem.
643 */
644 case LGRP_CONFIG_MEM_ADD:
645 lgrp_mem_init((int)resource, where, B_FALSE);
646 atomic_inc_32(&lgrp_gen);
647
648 break;
649 case LGRP_CONFIG_MEM_DEL:
650 lgrp_mem_fini((int)resource, where, B_FALSE);
651 atomic_inc_32(&lgrp_gen);
652
653 break;
654 case LGRP_CONFIG_MEM_RENAME: {
655 lgrp_config_mem_rename_t *ren_arg =
656 (lgrp_config_mem_rename_t *)where;
657
658 lgrp_mem_rename((int)resource,
659 ren_arg->lmem_rename_from,
660 ren_arg->lmem_rename_to);
661 atomic_inc_32(&lgrp_gen);
662
663 break;
664 }
665 case LGRP_CONFIG_GEN_UPDATE:
666 atomic_inc_32(&lgrp_gen);
667
668 break;
669 case LGRP_CONFIG_FLATTEN:
670 if (where == 0)
671 lgrp_topo_levels = (int)resource;
672 else
673 (void) lgrp_topo_flatten(resource,
674 lgrp_table, lgrp_alloc_max, &changed);
675
676 break;
677 /*
678 * Update any lgroups with old latency to new latency
679 */
680 case LGRP_CONFIG_LAT_CHANGE_ALL:
681 lgrp_latency_change(LGRP_NULL_HANDLE, (u_longlong_t)resource,
682 (u_longlong_t)where);
683
684 break;
685 /*
686 * Update lgroup with specified lgroup platform handle to have
687 * new latency
688 */
689 case LGRP_CONFIG_LAT_CHANGE:
690 lgrp_latency_change((lgrp_handle_t)resource, 0,
691 (u_longlong_t)where);
692
693 break;
694 case LGRP_CONFIG_NOP:
695
696 break;
697 default:
698 break;
699 }
700
701 }
702
703 /*
704 * Called to add lgrp info into cpu structure from cpu_add_unit;
705 * do not assume cpu is in cpu[] yet!
706 *
707 * CPUs are brought online with all other CPUs paused so we can't
708 * allocate memory or we could deadlock the system, so we rely on
709 * the platform to statically allocate as much space as we need
710 * for the lgrp structs and stats.
711 */
712 static void
713 lgrp_cpu_init(struct cpu *cp)
714 {
715 klgrpset_t changed;
716 int count;
717 lgrp_handle_t hand;
718 int first_cpu;
719 lgrp_t *my_lgrp;
720 lgrp_id_t lgrpid;
721 struct cpu *cptr;
722
723 /*
724 * This is the first time through if the resource set
725 * for the root lgroup is empty. After cpu0 has been
726 * initially added to an lgroup, the root's CPU resource
727 * set can never be empty, since the system's last CPU
728 * cannot be offlined.
729 */
730 if (klgrpset_isempty(lgrp_root->lgrp_set[LGRP_RSRC_CPU])) {
731 /*
732 * First time through.
733 */
734 first_cpu = 1;
735 } else {
736 /*
737 * If cpu0 needs to move lgroups, we may come
738 * through here again, at which time cpu_lock won't
739 * be held, and lgrp_initialized will be false.
740 */
741 ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized);
742 ASSERT(cp->cpu_part != NULL);
743 first_cpu = 0;
744 }
745
746 hand = lgrp_plat_cpu_to_hand(cp->cpu_id);
747 my_lgrp = lgrp_hand_to_lgrp(hand);
748
749 if (my_lgrp == NULL) {
750 /*
751 * Create new lgrp and add it to lgroup topology
752 */
753 my_lgrp = lgrp_create();
754 my_lgrp->lgrp_plathand = hand;
755 my_lgrp->lgrp_latency = lgrp_plat_latency(hand, hand);
756 lgrpid = my_lgrp->lgrp_id;
757 klgrpset_add(my_lgrp->lgrp_leaves, lgrpid);
758 klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
759
760 count = 0;
761 klgrpset_clear(changed);
762 count += lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1,
763 &changed);
764 /*
765 * May have added new intermediate lgroups, so need to add
766 * resources other than CPUs which are added below
767 */
768 (void) lgrp_mnode_update(changed, NULL);
769 } else if (my_lgrp->lgrp_latency == 0 && lgrp_plat_latency(hand, hand)
770 > 0) {
771 /*
772 * Leaf lgroup was created, but latency wasn't available
773 * then. So, set latency for it and fill in rest of lgroup
774 * topology now that we know how far it is from other leaf
775 * lgroups.
776 */
777 lgrpid = my_lgrp->lgrp_id;
778 klgrpset_clear(changed);
779 if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_CPU],
780 lgrpid))
781 klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
782 count = lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1,
783 &changed);
784
785 /*
786 * May have added new intermediate lgroups, so need to add
787 * resources other than CPUs which are added below
788 */
789 (void) lgrp_mnode_update(changed, NULL);
790 } else if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_CPU],
791 my_lgrp->lgrp_id)) {
792 int i;
793
794 /*
795 * Update existing lgroup and lgroups containing it with CPU
796 * resource
797 */
798 lgrpid = my_lgrp->lgrp_id;
799 klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
800 for (i = 0; i <= lgrp_alloc_max; i++) {
801 lgrp_t *lgrp;
802
803 lgrp = lgrp_table[i];
804 if (!LGRP_EXISTS(lgrp) ||
805 !lgrp_rsets_member(lgrp->lgrp_set, lgrpid))
806 continue;
807
808 klgrpset_add(lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
809 }
810 }
811
812 lgrpid = my_lgrp->lgrp_id;
813 cp->cpu_lpl = &cp->cpu_part->cp_lgrploads[lgrpid];
814
815 /*
816 * For multi-lgroup systems, need to setup lpl for CPU0 or CPU0 will
817 * end up in lpl for lgroup 0 whether it is supposed to be in there or
818 * not since none of lgroup IDs in the lpl's have been set yet.
819 */
820 if (first_cpu && nlgrpsmax > 1 && lgrpid != cp->cpu_lpl->lpl_lgrpid)
821 cp->cpu_lpl->lpl_lgrpid = lgrpid;
822
823 /*
824 * link the CPU into the lgrp's CPU list
825 */
826 if (my_lgrp->lgrp_cpucnt == 0) {
827 my_lgrp->lgrp_cpu = cp;
828 cp->cpu_next_lgrp = cp->cpu_prev_lgrp = cp;
829 } else {
830 cptr = my_lgrp->lgrp_cpu;
831 cp->cpu_next_lgrp = cptr;
832 cp->cpu_prev_lgrp = cptr->cpu_prev_lgrp;
833 cptr->cpu_prev_lgrp->cpu_next_lgrp = cp;
834 cptr->cpu_prev_lgrp = cp;
835 }
836 my_lgrp->lgrp_cpucnt++;
837 }
838
839 lgrp_t *
840 lgrp_create(void)
841 {
842 lgrp_t *my_lgrp;
843 lgrp_id_t lgrpid;
844 int i;
845
846 ASSERT(!lgrp_initialized || MUTEX_HELD(&cpu_lock));
847
848 /*
849 * Find an open slot in the lgroup table and recycle unused lgroup
850 * left there if any
851 */
852 my_lgrp = NULL;
853 if (lgrp_alloc_hint == -1)
854 /*
855 * Allocate from end when hint not set yet because no lgroups
856 * have been deleted yet
857 */
858 lgrpid = nlgrps++;
859 else {
860 /*
861 * Start looking for next open slot from hint and leave hint
862 * at slot allocated
863 */
864 for (i = lgrp_alloc_hint; i < nlgrpsmax; i++) {
865 my_lgrp = lgrp_table[i];
866 if (!LGRP_EXISTS(my_lgrp)) {
867 lgrpid = i;
868 nlgrps++;
869 break;
870 }
871 }
872 lgrp_alloc_hint = lgrpid;
873 }
874
875 /*
876 * Keep track of max lgroup ID allocated so far to cut down on searches
877 */
878 if (lgrpid > lgrp_alloc_max)
879 lgrp_alloc_max = lgrpid;
880
881 /*
882 * Need to allocate new lgroup if next open slot didn't have one
883 * for recycling
884 */
885 if (my_lgrp == NULL)
886 my_lgrp = lgrp_plat_alloc(lgrpid);
887
888 if (nlgrps > nlgrpsmax || my_lgrp == NULL)
889 panic("Too many lgrps for platform (%d)", nlgrps);
890
891 my_lgrp->lgrp_id = lgrpid;
892 my_lgrp->lgrp_latency = 0;
893 my_lgrp->lgrp_plathand = LGRP_NULL_HANDLE;
894 my_lgrp->lgrp_parent = NULL;
895 my_lgrp->lgrp_childcnt = 0;
896 my_lgrp->lgrp_mnodes = (mnodeset_t)0;
897 my_lgrp->lgrp_nmnodes = 0;
898 klgrpset_clear(my_lgrp->lgrp_children);
899 klgrpset_clear(my_lgrp->lgrp_leaves);
900 for (i = 0; i < LGRP_RSRC_COUNT; i++)
901 klgrpset_clear(my_lgrp->lgrp_set[i]);
902
903 my_lgrp->lgrp_cpu = NULL;
904 my_lgrp->lgrp_cpucnt = 0;
905
906 if (my_lgrp->lgrp_kstat != NULL)
907 lgrp_kstat_reset(lgrpid);
908
909 lgrp_table[my_lgrp->lgrp_id] = my_lgrp;
910
911 return (my_lgrp);
912 }
913
914 void
915 lgrp_destroy(lgrp_t *lgrp)
916 {
917 int i;
918
919 /*
920 * Unless this lgroup is being destroyed on behalf of
921 * the boot CPU, cpu_lock must be held
922 */
923 ASSERT(!lgrp_initialized || MUTEX_HELD(&cpu_lock));
924
925 if (nlgrps == 1)
926 cmn_err(CE_PANIC, "Can't destroy only lgroup!");
927
928 if (!LGRP_EXISTS(lgrp))
929 return;
930
931 /*
932 * Set hint to lgroup being deleted and try to keep lower numbered
933 * hints to facilitate finding empty slots
934 */
935 if (lgrp_alloc_hint == -1 || lgrp->lgrp_id < lgrp_alloc_hint)
936 lgrp_alloc_hint = lgrp->lgrp_id;
937
938 /*
939 * Mark this lgroup to be recycled by setting its lgroup ID to
940 * LGRP_NONE and clear relevant fields
941 */
942 lgrp->lgrp_id = LGRP_NONE;
943 lgrp->lgrp_latency = 0;
944 lgrp->lgrp_plathand = LGRP_NULL_HANDLE;
945 lgrp->lgrp_parent = NULL;
946 lgrp->lgrp_childcnt = 0;
947
948 klgrpset_clear(lgrp->lgrp_children);
949 klgrpset_clear(lgrp->lgrp_leaves);
950 for (i = 0; i < LGRP_RSRC_COUNT; i++)
951 klgrpset_clear(lgrp->lgrp_set[i]);
952
953 lgrp->lgrp_mnodes = (mnodeset_t)0;
954 lgrp->lgrp_nmnodes = 0;
955
956 lgrp->lgrp_cpu = NULL;
957 lgrp->lgrp_cpucnt = 0;
958
959 nlgrps--;
960 }
961
962 /*
963 * Initialize kstat data. Called from lgrp intialization code.
964 */
965 static void
966 lgrp_kstat_init(void)
967 {
968 lgrp_stat_t stat;
969
970 mutex_init(&lgrp_kstat_mutex, NULL, MUTEX_DEFAULT, NULL);
971
972 for (stat = 0; stat < LGRP_NUM_STATS; stat++)
973 kstat_named_init(&lgrp_kstat_data[stat],
974 lgrp_kstat_names[stat], KSTAT_DATA_INT64);
975 }
976
977 /*
978 * initialize an lgrp's kstats if needed
979 * called with cpu_lock held but not with cpus paused.
980 * we don't tear these down now because we don't know about
981 * memory leaving the lgrp yet...
982 */
983
984 void
985 lgrp_kstat_create(cpu_t *cp)
986 {
987 kstat_t *lgrp_kstat;
988 lgrp_id_t lgrpid;
989 lgrp_t *my_lgrp;
990
991 ASSERT(MUTEX_HELD(&cpu_lock));
992
993 lgrpid = cp->cpu_lpl->lpl_lgrpid;
994 my_lgrp = lgrp_table[lgrpid];
995
996 if (my_lgrp->lgrp_kstat != NULL)
997 return; /* already initialized */
998
999 lgrp_kstat = kstat_create("lgrp", lgrpid, NULL, "misc",
1000 KSTAT_TYPE_NAMED, LGRP_NUM_STATS,
1001 KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE);
1002
1003 if (lgrp_kstat != NULL) {
1004 lgrp_kstat->ks_lock = &lgrp_kstat_mutex;
1005 lgrp_kstat->ks_private = my_lgrp;
1006 lgrp_kstat->ks_data = &lgrp_kstat_data;
1007 lgrp_kstat->ks_update = lgrp_kstat_extract;
1008 my_lgrp->lgrp_kstat = lgrp_kstat;
1009 kstat_install(lgrp_kstat);
1010 }
1011 }
1012
1013 /*
1014 * this will do something when we manage to remove now unused lgrps
1015 */
1016
1017 /* ARGSUSED */
1018 void
1019 lgrp_kstat_destroy(cpu_t *cp)
1020 {
1021 ASSERT(MUTEX_HELD(&cpu_lock));
1022 }
1023
1024 /*
1025 * Called when a CPU is off-lined.
1026 */
1027 static void
1028 lgrp_cpu_fini(struct cpu *cp, lgrp_id_t lgrpid)
1029 {
1030 lgrp_t *my_lgrp;
1031 struct cpu *prev;
1032 struct cpu *next;
1033
1034 ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized);
1035
1036 prev = cp->cpu_prev_lgrp;
1037 next = cp->cpu_next_lgrp;
1038
1039 prev->cpu_next_lgrp = next;
1040 next->cpu_prev_lgrp = prev;
1041
1042 /*
1043 * just because I'm paranoid doesn't mean...
1044 */
1045
1046 cp->cpu_next_lgrp = cp->cpu_prev_lgrp = NULL;
1047
1048 my_lgrp = lgrp_table[lgrpid];
1049 my_lgrp->lgrp_cpucnt--;
1050
1051 /*
1052 * Removing last CPU in lgroup, so update lgroup topology
1053 */
1054 if (my_lgrp->lgrp_cpucnt == 0) {
1055 klgrpset_t changed;
1056 int count;
1057 int i;
1058
1059 my_lgrp->lgrp_cpu = NULL;
1060
1061 /*
1062 * Remove this lgroup from its lgroup CPU resources and remove
1063 * lgroup from lgroup topology if it doesn't have any more
1064 * resources in it now
1065 */
1066 klgrpset_del(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
1067 if (lgrp_rsets_empty(my_lgrp->lgrp_set)) {
1068 count = 0;
1069 klgrpset_clear(changed);
1070 count += lgrp_leaf_delete(my_lgrp, lgrp_table,
1071 lgrp_alloc_max + 1, &changed);
1072 return;
1073 }
1074
1075 /*
1076 * This lgroup isn't empty, so just remove it from CPU
1077 * resources of any lgroups that contain it as such
1078 */
1079 for (i = 0; i <= lgrp_alloc_max; i++) {
1080 lgrp_t *lgrp;
1081
1082 lgrp = lgrp_table[i];
1083 if (!LGRP_EXISTS(lgrp) ||
1084 !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_CPU],
1085 lgrpid))
1086 continue;
1087
1088 klgrpset_del(lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
1089 }
1090 return;
1091 }
1092
1093 if (my_lgrp->lgrp_cpu == cp)
1094 my_lgrp->lgrp_cpu = next;
1095
1096 }
1097
1098 /*
1099 * Update memory nodes in target lgroups and return ones that get changed
1100 */
1101 int
1102 lgrp_mnode_update(klgrpset_t target, klgrpset_t *changed)
1103 {
1104 int count;
1105 int i;
1106 int j;
1107 lgrp_t *lgrp;
1108 lgrp_t *lgrp_rsrc;
1109
1110 count = 0;
1111 if (changed)
1112 klgrpset_clear(*changed);
1113
1114 if (klgrpset_isempty(target))
1115 return (0);
1116
1117 /*
1118 * Find each lgroup in target lgroups
1119 */
1120 for (i = 0; i <= lgrp_alloc_max; i++) {
1121 /*
1122 * Skip any lgroups that don't exist or aren't in target group
1123 */
1124 lgrp = lgrp_table[i];
1125 if (!klgrpset_ismember(target, i) || !LGRP_EXISTS(lgrp)) {
1126 continue;
1127 }
1128
1129 /*
1130 * Initialize memnodes for intermediate lgroups to 0
1131 * and update them from scratch since they may have completely
1132 * changed
1133 */
1134 if (lgrp->lgrp_childcnt && lgrp != lgrp_root) {
1135 lgrp->lgrp_mnodes = (mnodeset_t)0;
1136 lgrp->lgrp_nmnodes = 0;
1137 }
1138
1139 /*
1140 * Update memory nodes of of target lgroup with memory nodes
1141 * from each lgroup in its lgroup memory resource set
1142 */
1143 for (j = 0; j <= lgrp_alloc_max; j++) {
1144 int k;
1145
1146 /*
1147 * Skip any lgroups that don't exist or aren't in
1148 * memory resources of target lgroup
1149 */
1150 lgrp_rsrc = lgrp_table[j];
1151 if (!LGRP_EXISTS(lgrp_rsrc) ||
1152 !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM],
1153 j))
1154 continue;
1155
1156 /*
1157 * Update target lgroup's memnodes to include memnodes
1158 * of this lgroup
1159 */
1160 for (k = 0; k < sizeof (mnodeset_t) * NBBY; k++) {
1161 mnodeset_t mnode_mask;
1162
1163 mnode_mask = (mnodeset_t)1 << k;
1164 if ((lgrp_rsrc->lgrp_mnodes & mnode_mask) &&
1165 !(lgrp->lgrp_mnodes & mnode_mask)) {
1166 lgrp->lgrp_mnodes |= mnode_mask;
1167 lgrp->lgrp_nmnodes++;
1168 }
1169 }
1170 count++;
1171 if (changed)
1172 klgrpset_add(*changed, lgrp->lgrp_id);
1173 }
1174 }
1175
1176 return (count);
1177 }
1178
1179 /*
1180 * Memory copy-rename. Called when the "mnode" containing the kernel cage memory
1181 * is moved from one board to another. The "from" and "to" arguments specify the
1182 * source and the destination of the move.
1183 *
1184 * See plat_lgrp_config() for a detailed description of the copy-rename
1185 * semantics.
1186 *
1187 * The lgrp_mem_rename() is called by the platform copy-rename code to update
1188 * the lgroup topology which is changing as memory moves from one lgroup to
1189 * another. It removes the mnode from the source lgroup and re-inserts it in the
1190 * target lgroup.
1191 *
1192 * The lgrp_mem_rename() function passes a flag to lgrp_mem_init() and
1193 * lgrp_mem_fini() telling that the insertion and deleteion are part of a DR
1194 * copy-rename operation.
1195 *
1196 * There is one case which requires special handling. If the system contains
1197 * only two boards (mnodes), the lgrp_mem_fini() removes the only mnode from the
1198 * lgroup hierarchy. This mnode is soon re-inserted back in the hierarchy by
1199 * lgrp_mem_init), but there is a window when the system has no memory in the
1200 * lgroup hierarchy. If another thread tries to allocate memory during this
1201 * window, the allocation will fail, although the system has physical memory.
1202 * This may cause a system panic or a deadlock (some sleeping memory allocations
1203 * happen with cpu_lock held which prevents lgrp_mem_init() from re-inserting
1204 * the mnode back).
1205 *
1206 * The lgrp_memnode_choose() function walks the lgroup hierarchy looking for the
1207 * lgrp with non-empty lgrp_mnodes. To deal with the special case above,
1208 * lgrp_mem_fini() does not remove the last mnode from the lroot->lgrp_mnodes,
1209 * but it updates the rest of the lgroup topology as if the mnode was actually
1210 * removed. The lgrp_mem_init() function recognizes that the mnode being
1211 * inserted represents such a special case and updates the topology
1212 * appropriately.
1213 */
1214 void
1215 lgrp_mem_rename(int mnode, lgrp_handle_t from, lgrp_handle_t to)
1216 {
1217 /*
1218 * Remove the memory from the source node and add it to the destination
1219 * node.
1220 */
1221 lgrp_mem_fini(mnode, from, B_TRUE);
1222 lgrp_mem_init(mnode, to, B_TRUE);
1223 }
1224
1225 /*
1226 * Called to indicate that the lgrp with platform handle "hand" now
1227 * contains the memory identified by "mnode".
1228 *
1229 * LOCKING for this routine is a bit tricky. Usually it is called without
1230 * cpu_lock and it must must grab cpu_lock here to prevent racing with other
1231 * callers. During DR of the board containing the caged memory it may be called
1232 * with cpu_lock already held and CPUs paused.
1233 *
1234 * If the insertion is part of the DR copy-rename and the inserted mnode (and
1235 * only this mnode) is already present in the lgrp_root->lgrp_mnodes set, we are
1236 * dealing with the special case of DR copy-rename described in
1237 * lgrp_mem_rename().
1238 */
1239 void
1240 lgrp_mem_init(int mnode, lgrp_handle_t hand, boolean_t is_copy_rename)
1241 {
1242 klgrpset_t changed;
1243 int count;
1244 int i;
1245 lgrp_t *my_lgrp;
1246 lgrp_id_t lgrpid;
1247 mnodeset_t mnodes_mask = ((mnodeset_t)1 << mnode);
1248 boolean_t drop_lock = B_FALSE;
1249 boolean_t need_synch = B_FALSE;
1250
1251 /*
1252 * Grab CPU lock (if we haven't already)
1253 */
1254 if (!MUTEX_HELD(&cpu_lock)) {
1255 mutex_enter(&cpu_lock);
1256 drop_lock = B_TRUE;
1257 }
1258
1259 /*
1260 * This routine may be called from a context where we already
1261 * hold cpu_lock, and have already paused cpus.
1262 */
1263 if (!cpus_paused())
1264 need_synch = B_TRUE;
1265
1266 /*
1267 * Check if this mnode is already configured and return immediately if
1268 * it is.
1269 *
1270 * NOTE: in special case of copy-rename of the only remaining mnode,
1271 * lgrp_mem_fini() refuses to remove the last mnode from the root, so we
1272 * recognize this case and continue as usual, but skip the update to
1273 * the lgrp_mnodes and the lgrp_nmnodes. This restores the inconsistency
1274 * in topology, temporarily introduced by lgrp_mem_fini().
1275 */
1276 if (! (is_copy_rename && (lgrp_root->lgrp_mnodes == mnodes_mask)) &&
1277 lgrp_root->lgrp_mnodes & mnodes_mask) {
1278 if (drop_lock)
1279 mutex_exit(&cpu_lock);
1280 return;
1281 }
1282
1283 /*
1284 * Update lgroup topology with new memory resources, keeping track of
1285 * which lgroups change
1286 */
1287 count = 0;
1288 klgrpset_clear(changed);
1289 my_lgrp = lgrp_hand_to_lgrp(hand);
1290 if (my_lgrp == NULL) {
1291 /* new lgrp */
1292 my_lgrp = lgrp_create();
1293 lgrpid = my_lgrp->lgrp_id;
1294 my_lgrp->lgrp_plathand = hand;
1295 my_lgrp->lgrp_latency = lgrp_plat_latency(hand, hand);
1296 klgrpset_add(my_lgrp->lgrp_leaves, lgrpid);
1297 klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
1298
1299 if (need_synch)
1300 pause_cpus(NULL, NULL);
1301 count = lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1,
1302 &changed);
1303 if (need_synch)
1304 start_cpus();
1305 } else if (my_lgrp->lgrp_latency == 0 && lgrp_plat_latency(hand, hand)
1306 > 0) {
1307 /*
1308 * Leaf lgroup was created, but latency wasn't available
1309 * then. So, set latency for it and fill in rest of lgroup
1310 * topology now that we know how far it is from other leaf
1311 * lgroups.
1312 */
1313 klgrpset_clear(changed);
1314 lgrpid = my_lgrp->lgrp_id;
1315 if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_MEM],
1316 lgrpid))
1317 klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
1318 if (need_synch)
1319 pause_cpus(NULL, NULL);
1320 count = lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1,
1321 &changed);
1322 if (need_synch)
1323 start_cpus();
1324 } else if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_MEM],
1325 my_lgrp->lgrp_id)) {
1326 /*
1327 * Add new lgroup memory resource to existing lgroup
1328 */
1329 lgrpid = my_lgrp->lgrp_id;
1330 klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
1331 klgrpset_add(changed, lgrpid);
1332 count++;
1333 for (i = 0; i <= lgrp_alloc_max; i++) {
1334 lgrp_t *lgrp;
1335
1336 lgrp = lgrp_table[i];
1337 if (!LGRP_EXISTS(lgrp) ||
1338 !lgrp_rsets_member(lgrp->lgrp_set, lgrpid))
1339 continue;
1340
1341 klgrpset_add(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
1342 klgrpset_add(changed, lgrp->lgrp_id);
1343 count++;
1344 }
1345 }
1346
1347 /*
1348 * Add memory node to lgroup and remove lgroup from ones that need
1349 * to be updated
1350 */
1351 if (!(my_lgrp->lgrp_mnodes & mnodes_mask)) {
1352 my_lgrp->lgrp_mnodes |= mnodes_mask;
1353 my_lgrp->lgrp_nmnodes++;
1354 }
1355 klgrpset_del(changed, lgrpid);
1356
1357 /*
1358 * Update memory node information for all lgroups that changed and
1359 * contain new memory node as a resource
1360 */
1361 if (count)
1362 (void) lgrp_mnode_update(changed, NULL);
1363
1364 if (drop_lock)
1365 mutex_exit(&cpu_lock);
1366 }
1367
1368 /*
1369 * Called to indicate that the lgroup associated with the platform
1370 * handle "hand" no longer contains given memory node
1371 *
1372 * LOCKING for this routine is a bit tricky. Usually it is called without
1373 * cpu_lock and it must must grab cpu_lock here to prevent racing with other
1374 * callers. During DR of the board containing the caged memory it may be called
1375 * with cpu_lock already held and CPUs paused.
1376 *
1377 * If the deletion is part of the DR copy-rename and the deleted mnode is the
1378 * only one present in the lgrp_root->lgrp_mnodes, all the topology is updated,
1379 * but lgrp_root->lgrp_mnodes is left intact. Later, lgrp_mem_init() will insert
1380 * the same mnode back into the topology. See lgrp_mem_rename() and
1381 * lgrp_mem_init() for additional details.
1382 */
1383 void
1384 lgrp_mem_fini(int mnode, lgrp_handle_t hand, boolean_t is_copy_rename)
1385 {
1386 klgrpset_t changed;
1387 int count;
1388 int i;
1389 lgrp_t *my_lgrp;
1390 lgrp_id_t lgrpid;
1391 mnodeset_t mnodes_mask;
1392 boolean_t drop_lock = B_FALSE;
1393 boolean_t need_synch = B_FALSE;
1394
1395 /*
1396 * Grab CPU lock (if we haven't already)
1397 */
1398 if (!MUTEX_HELD(&cpu_lock)) {
1399 mutex_enter(&cpu_lock);
1400 drop_lock = B_TRUE;
1401 }
1402
1403 /*
1404 * This routine may be called from a context where we already
1405 * hold cpu_lock and have already paused cpus.
1406 */
1407 if (!cpus_paused())
1408 need_synch = B_TRUE;
1409
1410 my_lgrp = lgrp_hand_to_lgrp(hand);
1411
1412 /*
1413 * The lgrp *must* be pre-existing
1414 */
1415 ASSERT(my_lgrp != NULL);
1416
1417 /*
1418 * Delete memory node from lgroups which contain it
1419 */
1420 mnodes_mask = ((mnodeset_t)1 << mnode);
1421 for (i = 0; i <= lgrp_alloc_max; i++) {
1422 lgrp_t *lgrp = lgrp_table[i];
1423 /*
1424 * Skip any non-existent lgroups and any lgroups that don't
1425 * contain leaf lgroup of memory as a memory resource
1426 */
1427 if (!LGRP_EXISTS(lgrp) ||
1428 !(lgrp->lgrp_mnodes & mnodes_mask))
1429 continue;
1430
1431 /*
1432 * Avoid removing the last mnode from the root in the DR
1433 * copy-rename case. See lgrp_mem_rename() for details.
1434 */
1435 if (is_copy_rename &&
1436 (lgrp == lgrp_root) && (lgrp->lgrp_mnodes == mnodes_mask))
1437 continue;
1438
1439 /*
1440 * Remove memory node from lgroup.
1441 */
1442 lgrp->lgrp_mnodes &= ~mnodes_mask;
1443 lgrp->lgrp_nmnodes--;
1444 ASSERT(lgrp->lgrp_nmnodes >= 0);
1445 }
1446 ASSERT(lgrp_root->lgrp_nmnodes > 0);
1447
1448 /*
1449 * Don't need to update lgroup topology if this lgroup still has memory.
1450 *
1451 * In the special case of DR copy-rename with the only mnode being
1452 * removed, the lgrp_mnodes for the root is always non-zero, but we
1453 * still need to update the lgroup topology.
1454 */
1455 if ((my_lgrp->lgrp_nmnodes > 0) &&
1456 !(is_copy_rename && (my_lgrp == lgrp_root) &&
1457 (my_lgrp->lgrp_mnodes == mnodes_mask))) {
1458 if (drop_lock)
1459 mutex_exit(&cpu_lock);
1460 return;
1461 }
1462
1463 /*
1464 * This lgroup does not contain any memory now
1465 */
1466 klgrpset_clear(my_lgrp->lgrp_set[LGRP_RSRC_MEM]);
1467
1468 /*
1469 * Remove this lgroup from lgroup topology if it does not contain any
1470 * resources now
1471 */
1472 lgrpid = my_lgrp->lgrp_id;
1473 count = 0;
1474 klgrpset_clear(changed);
1475 if (lgrp_rsets_empty(my_lgrp->lgrp_set)) {
1476 /*
1477 * Delete lgroup when no more resources
1478 */
1479 if (need_synch)
1480 pause_cpus(NULL, NULL);
1481 count = lgrp_leaf_delete(my_lgrp, lgrp_table,
1482 lgrp_alloc_max + 1, &changed);
1483 ASSERT(count > 0);
1484 if (need_synch)
1485 start_cpus();
1486 } else {
1487 /*
1488 * Remove lgroup from memory resources of any lgroups that
1489 * contain it as such
1490 */
1491 for (i = 0; i <= lgrp_alloc_max; i++) {
1492 lgrp_t *lgrp;
1493
1494 lgrp = lgrp_table[i];
1495 if (!LGRP_EXISTS(lgrp) ||
1496 !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM],
1497 lgrpid))
1498 continue;
1499
1500 klgrpset_del(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
1501 }
1502 }
1503 if (drop_lock)
1504 mutex_exit(&cpu_lock);
1505 }
1506
1507 /*
1508 * Return lgroup with given platform handle
1509 */
1510 lgrp_t *
1511 lgrp_hand_to_lgrp(lgrp_handle_t hand)
1512 {
1513 int i;
1514 lgrp_t *lgrp;
1515
1516 if (hand == LGRP_NULL_HANDLE)
1517 return (NULL);
1518
1519 for (i = 0; i <= lgrp_alloc_max; i++) {
1520 lgrp = lgrp_table[i];
1521 if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand)
1522 return (lgrp);
1523 }
1524 return (NULL);
1525 }
1526
1527 /*
1528 * Return the home lgroup of the current thread.
1529 * We must do this with kernel preemption disabled, since we don't want our
1530 * thread to be re-homed while we're poking around with its lpl, and the lpl
1531 * should never be NULL.
1532 *
1533 * NOTE: Can't guarantee that lgroup will be valid once kernel preemption
1534 * is enabled because of DR. Callers can use disable kernel preemption
1535 * around this call to guarantee that the lgroup will be valid beyond this
1536 * routine, since kernel preemption can be recursive.
1537 */
1538 lgrp_t *
1539 lgrp_home_lgrp(void)
1540 {
1541 lgrp_t *lgrp;
1542 lpl_t *lpl;
1543
1544 kpreempt_disable();
1545
1546 lpl = curthread->t_lpl;
1547 ASSERT(lpl != NULL);
1548 ASSERT(lpl->lpl_lgrpid >= 0 && lpl->lpl_lgrpid <= lgrp_alloc_max);
1549 ASSERT(LGRP_EXISTS(lgrp_table[lpl->lpl_lgrpid]));
1550 lgrp = lgrp_table[lpl->lpl_lgrpid];
1551
1552 kpreempt_enable();
1553
1554 return (lgrp);
1555 }
1556
1557 /*
1558 * Return ID of home lgroup for given thread
1559 * (See comments for lgrp_home_lgrp() for special care and handling
1560 * instructions)
1561 */
1562 lgrp_id_t
1563 lgrp_home_id(kthread_t *t)
1564 {
1565 lgrp_id_t lgrp;
1566 lpl_t *lpl;
1567
1568 ASSERT(t != NULL);
1569 /*
1570 * We'd like to ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)), but we
1571 * cannot since the HAT layer can call into this routine to
1572 * determine the locality for its data structures in the context
1573 * of a page fault.
1574 */
1575
1576 kpreempt_disable();
1577
1578 lpl = t->t_lpl;
1579 ASSERT(lpl != NULL);
1580 ASSERT(lpl->lpl_lgrpid >= 0 && lpl->lpl_lgrpid <= lgrp_alloc_max);
1581 lgrp = lpl->lpl_lgrpid;
1582
1583 kpreempt_enable();
1584
1585 return (lgrp);
1586 }
1587
1588 /*
1589 * Return lgroup containing the physical memory for the given page frame number
1590 */
1591 lgrp_t *
1592 lgrp_pfn_to_lgrp(pfn_t pfn)
1593 {
1594 lgrp_handle_t hand;
1595 int i;
1596 lgrp_t *lgrp;
1597
1598 hand = lgrp_plat_pfn_to_hand(pfn);
1599 if (hand != LGRP_NULL_HANDLE)
1600 for (i = 0; i <= lgrp_alloc_max; i++) {
1601 lgrp = lgrp_table[i];
1602 if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand)
1603 return (lgrp);
1604 }
1605 return (NULL);
1606 }
1607
1608 /*
1609 * Return lgroup containing the physical memory for the given page frame number
1610 */
1611 lgrp_t *
1612 lgrp_phys_to_lgrp(u_longlong_t physaddr)
1613 {
1614 lgrp_handle_t hand;
1615 int i;
1616 lgrp_t *lgrp;
1617 pfn_t pfn;
1618
1619 pfn = btop(physaddr);
1620 hand = lgrp_plat_pfn_to_hand(pfn);
1621 if (hand != LGRP_NULL_HANDLE)
1622 for (i = 0; i <= lgrp_alloc_max; i++) {
1623 lgrp = lgrp_table[i];
1624 if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand)
1625 return (lgrp);
1626 }
1627 return (NULL);
1628 }
1629
1630 /*
1631 * Return the leaf lgroup containing the given CPU
1632 *
1633 * The caller needs to take precautions necessary to prevent
1634 * "cpu", and it's lpl from going away across a call to this function.
1635 * hint: kpreempt_disable()/kpreempt_enable()
1636 */
1637 static lgrp_t *
1638 lgrp_cpu_to_lgrp(cpu_t *cpu)
1639 {
1640 return (cpu->cpu_lpl->lpl_lgrp);
1641 }
1642
1643 /*
1644 * Return the sum of the partition loads in an lgrp divided by
1645 * the number of CPUs in the lgrp. This is our best approximation
1646 * of an 'lgroup load average' for a useful per-lgroup kstat.
1647 */
1648 static uint64_t
1649 lgrp_sum_loadavgs(lgrp_t *lgrp)
1650 {
1651 cpu_t *cpu;
1652 int ncpu;
1653 uint64_t loads = 0;
1654
1655 mutex_enter(&cpu_lock);
1656
1657 cpu = lgrp->lgrp_cpu;
1658 ncpu = lgrp->lgrp_cpucnt;
1659
1660 if (cpu == NULL || ncpu == 0) {
1661 mutex_exit(&cpu_lock);
1662 return (0ull);
1663 }
1664
1665 do {
1666 loads += cpu->cpu_lpl->lpl_loadavg;
1667 cpu = cpu->cpu_next_lgrp;
1668 } while (cpu != lgrp->lgrp_cpu);
1669
1670 mutex_exit(&cpu_lock);
1671
1672 return (loads / ncpu);
1673 }
1674
1675 void
1676 lgrp_stat_add(lgrp_id_t lgrpid, lgrp_stat_t stat, int64_t val)
1677 {
1678 struct lgrp_stats *pstats;
1679
1680 /*
1681 * Verify that the caller isn't trying to add to
1682 * a statistic for an lgroup that has gone away
1683 */
1684 if (lgrpid < 0 || lgrpid > lgrp_alloc_max)
1685 return;
1686
1687 pstats = &lgrp_stats[lgrpid];
1688 atomic_add_64((uint64_t *)LGRP_STAT_WRITE_PTR(pstats, stat), val);
1689 }
1690
1691 int64_t
1692 lgrp_stat_read(lgrp_id_t lgrpid, lgrp_stat_t stat)
1693 {
1694 uint64_t val;
1695 struct lgrp_stats *pstats;
1696
1697 if (lgrpid < 0 || lgrpid > lgrp_alloc_max)
1698 return ((int64_t)0);
1699
1700 pstats = &lgrp_stats[lgrpid];
1701 LGRP_STAT_READ(pstats, stat, val);
1702 return (val);
1703 }
1704
1705 /*
1706 * Reset all kstats for lgrp specified by its lgrpid.
1707 */
1708 static void
1709 lgrp_kstat_reset(lgrp_id_t lgrpid)
1710 {
1711 lgrp_stat_t stat;
1712
1713 if (lgrpid < 0 || lgrpid > lgrp_alloc_max)
1714 return;
1715
1716 for (stat = 0; stat < LGRP_NUM_COUNTER_STATS; stat++) {
1717 LGRP_STAT_RESET(&lgrp_stats[lgrpid], stat);
1718 }
1719 }
1720
1721 /*
1722 * Collect all per-lgrp statistics for the lgrp associated with this
1723 * kstat, and store them in the ks_data array.
1724 *
1725 * The superuser can reset all the running counter statistics for an
1726 * lgrp by writing to any of the lgrp's stats.
1727 */
1728 static int
1729 lgrp_kstat_extract(kstat_t *ksp, int rw)
1730 {
1731 lgrp_stat_t stat;
1732 struct kstat_named *ksd;
1733 lgrp_t *lgrp;
1734 lgrp_id_t lgrpid;
1735
1736 lgrp = (lgrp_t *)ksp->ks_private;
1737
1738 ksd = (struct kstat_named *)ksp->ks_data;
1739 ASSERT(ksd == (struct kstat_named *)&lgrp_kstat_data);
1740
1741 lgrpid = lgrp->lgrp_id;
1742
1743 if (lgrpid == LGRP_NONE) {
1744 /*
1745 * Return all zeroes as stats for freed lgrp.
1746 */
1747 for (stat = 0; stat < LGRP_NUM_COUNTER_STATS; stat++) {
1748 ksd[stat].value.i64 = 0;
1749 }
1750 ksd[stat + LGRP_NUM_CPUS].value.i64 = 0;
1751 ksd[stat + LGRP_NUM_PG_INSTALL].value.i64 = 0;
1752 ksd[stat + LGRP_NUM_PG_AVAIL].value.i64 = 0;
1753 ksd[stat + LGRP_NUM_PG_FREE].value.i64 = 0;
1754 ksd[stat + LGRP_LOADAVG].value.i64 = 0;
1755 } else if (rw != KSTAT_WRITE) {
1756 /*
1757 * Handle counter stats
1758 */
1759 for (stat = 0; stat < LGRP_NUM_COUNTER_STATS; stat++) {
1760 ksd[stat].value.i64 = lgrp_stat_read(lgrpid, stat);
1761 }
1762
1763 /*
1764 * Handle kernel data snapshot stats
1765 */
1766 ksd[stat + LGRP_NUM_CPUS].value.i64 = lgrp->lgrp_cpucnt;
1767 ksd[stat + LGRP_NUM_PG_INSTALL].value.i64 =
1768 lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_INSTALL);
1769 ksd[stat + LGRP_NUM_PG_AVAIL].value.i64 =
1770 lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_AVAIL);
1771 ksd[stat + LGRP_NUM_PG_FREE].value.i64 =
1772 lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_FREE);
1773 ksd[stat + LGRP_LOADAVG].value.i64 = lgrp_sum_loadavgs(lgrp);
1774 ksd[stat + LGRP_LOADAVG_SCALE].value.i64 =
1775 lgrp_loadavg_max_effect;
1776 } else {
1777 lgrp_kstat_reset(lgrpid);
1778 }
1779
1780 return (0);
1781 }
1782
1783 int
1784 lgrp_query_cpu(processorid_t id, lgrp_id_t *lp)
1785 {
1786 cpu_t *cp;
1787
1788 mutex_enter(&cpu_lock);
1789
1790 if ((cp = cpu_get(id)) == NULL) {
1791 mutex_exit(&cpu_lock);
1792 return (EINVAL);
1793 }
1794
1795 if (cpu_is_offline(cp) || cpu_is_poweredoff(cp)) {
1796 mutex_exit(&cpu_lock);
1797 return (EINVAL);
1798 }
1799
1800 ASSERT(cp->cpu_lpl != NULL);
1801
1802 *lp = cp->cpu_lpl->lpl_lgrpid;
1803
1804 mutex_exit(&cpu_lock);
1805
1806 return (0);
1807 }
1808
1809 int
1810 lgrp_query_load(processorid_t id, lgrp_load_t *lp)
1811 {
1812 cpu_t *cp;
1813
1814 mutex_enter(&cpu_lock);
1815
1816 if ((cp = cpu_get(id)) == NULL) {
1817 mutex_exit(&cpu_lock);
1818 return (EINVAL);
1819 }
1820
1821 ASSERT(cp->cpu_lpl != NULL);
1822
1823 *lp = cp->cpu_lpl->lpl_loadavg;
1824
1825 mutex_exit(&cpu_lock);
1826
1827 return (0);
1828 }
1829
1830 /*
1831 * Add a resource named by lpl_leaf to rset of lpl_target
1832 *
1833 * This routine also adjusts ncpu and nrset if the call succeeds in adding a
1834 * resource. It is adjusted here, as this is presently the only place that we
1835 * can be certain a resource addition has succeeded.
1836 *
1837 * We keep the list of rsets sorted so that the dispatcher can quickly walk the
1838 * list in order until it reaches a NULL. (This list is required to be NULL
1839 * terminated, too). This is done so that we can mark start pos + 1, so that
1840 * each lpl is traversed sequentially, but in a different order. We hope this
1841 * will improve performance a bit. (Hopefully, less read-to-own traffic...)
1842 */
1843
1844 void
1845 lpl_rset_add(lpl_t *lpl_target, lpl_t *lpl_leaf)
1846 {
1847 int i;
1848 int entry_slot = 0;
1849
1850 /* return if leaf is already present */
1851 for (i = 0; i < lpl_target->lpl_nrset; i++) {
1852 if (lpl_target->lpl_rset[i] == lpl_leaf) {
1853 return;
1854 }
1855
1856 if (lpl_target->lpl_rset[i]->lpl_lgrpid >
1857 lpl_leaf->lpl_lgrpid) {
1858 break;
1859 }
1860 }
1861
1862 /* insert leaf, update counts */
1863 entry_slot = i;
1864 i = lpl_target->lpl_nrset++;
1865
1866 /*
1867 * Start at the end of the rset array and work backwards towards the
1868 * slot into which the new lpl will be inserted. This effectively
1869 * preserves the current ordering by scooting everybody over one entry,
1870 * and placing the new entry into the space created.
1871 */
1872 while (i-- > entry_slot) {
1873 lpl_target->lpl_rset[i + 1] = lpl_target->lpl_rset[i];
1874 lpl_target->lpl_id2rset[lpl_target->lpl_rset[i]->lpl_lgrpid] =
1875 i + 1;
1876 }
1877
1878 lpl_target->lpl_rset[entry_slot] = lpl_leaf;
1879 lpl_target->lpl_id2rset[lpl_leaf->lpl_lgrpid] = entry_slot;
1880
1881 lpl_target->lpl_ncpu += lpl_leaf->lpl_ncpu;
1882 }
1883
1884 /*
1885 * Update each of lpl_parent's children with a reference to their parent.
1886 * The lgrp topology is used as the reference since it is fully
1887 * consistent and correct at this point.
1888 * This should be called after any potential change in lpl_parent's
1889 * rset.
1890 */
1891 static void
1892 lpl_child_update(lpl_t *lpl_parent, struct cpupart *cp)
1893 {
1894 klgrpset_t children;
1895 int i;
1896
1897 children = lgrp_table[lpl_parent->lpl_lgrpid]->lgrp_children;
1898 if (klgrpset_isempty(children))
1899 return; /* nothing to do */
1900
1901 for (i = 0; i <= lgrp_alloc_max; i++) {
1902 if (klgrpset_ismember(children, i)) {
1903 /*
1904 * (Re)set the parent. It may be incorrect if
1905 * lpl_parent is new in the topology.
1906 */
1907 cp->cp_lgrploads[i].lpl_parent = lpl_parent;
1908 }
1909 }
1910 }
1911
1912 /*
1913 * Delete resource lpl_leaf from rset of lpl_target, assuming it's there.
1914 *
1915 * This routine also adjusts ncpu and nrset if the call succeeds in deleting a
1916 * resource. The values are adjusted here, as this is the only place that we can
1917 * be certain a resource was successfully deleted.
1918 */
1919 void
1920 lpl_rset_del(lpl_t *lpl_target, lpl_t *lpl_leaf)
1921 {
1922 int i;
1923 lpl_t *leaf;
1924
1925 if (lpl_target->lpl_nrset == 0)
1926 return;
1927
1928 /* find leaf in intermediate node */
1929 for (i = 0; i < lpl_target->lpl_nrset; i++) {
1930 if (lpl_target->lpl_rset[i] == lpl_leaf)
1931 break;
1932 }
1933
1934 /* return if leaf not found */
1935 if (lpl_target->lpl_rset[i] != lpl_leaf)
1936 return;
1937
1938 /* prune leaf, compress array */
1939 lpl_target->lpl_rset[lpl_target->lpl_nrset--] = NULL;
1940 lpl_target->lpl_id2rset[lpl_leaf->lpl_lgrpid] = -1;
1941 lpl_target->lpl_ncpu--;
1942 do {
1943 lpl_target->lpl_rset[i] = lpl_target->lpl_rset[i + 1];
1944 /*
1945 * Update the lgrp id <=> rset mapping
1946 */
1947 if ((leaf = lpl_target->lpl_rset[i]) != NULL) {
1948 lpl_target->lpl_id2rset[leaf->lpl_lgrpid] = i;
1949 }
1950 } while (i++ < lpl_target->lpl_nrset);
1951 }
1952
1953 /*
1954 * Check to see if the resource set of the target lpl contains the
1955 * supplied leaf lpl. This returns 1 if the lpl is found, 0 if it is not.
1956 */
1957
1958 int
1959 lpl_rset_contains(lpl_t *lpl_target, lpl_t *lpl_leaf)
1960 {
1961 int i;
1962
1963 for (i = 0; i < lpl_target->lpl_nrset; i++) {
1964 if (lpl_target->lpl_rset[i] == lpl_leaf)
1965 return (1);
1966 }
1967
1968 return (0);
1969 }
1970
1971 /*
1972 * Called when we change cpu lpl membership. This increments or decrements the
1973 * per-cpu counter in every lpl in which our leaf appears.
1974 */
1975 void
1976 lpl_cpu_adjcnt(lpl_act_t act, cpu_t *cp)
1977 {
1978 cpupart_t *cpupart;
1979 lgrp_t *lgrp_leaf;
1980 lgrp_t *lgrp_cur;
1981 lpl_t *lpl_leaf;
1982 lpl_t *lpl_cur;
1983 int i;
1984
1985 ASSERT(act == LPL_DECREMENT || act == LPL_INCREMENT);
1986
1987 cpupart = cp->cpu_part;
1988 lpl_leaf = cp->cpu_lpl;
1989 lgrp_leaf = lgrp_table[lpl_leaf->lpl_lgrpid];
1990
1991 for (i = 0; i <= lgrp_alloc_max; i++) {
1992 lgrp_cur = lgrp_table[i];
1993
1994 /*
1995 * Don't adjust if the lgrp isn't there, if we're the leaf lpl
1996 * for the cpu in question, or if the current lgrp and leaf
1997 * don't share the same resources.
1998 */
1999
2000 if (!LGRP_EXISTS(lgrp_cur) || (lgrp_cur == lgrp_leaf) ||
2001 !klgrpset_intersects(lgrp_leaf->lgrp_set[LGRP_RSRC_CPU],
2002 lgrp_cur->lgrp_set[LGRP_RSRC_CPU]))
2003 continue;
2004
2005
2006 lpl_cur = &cpupart->cp_lgrploads[lgrp_cur->lgrp_id];
2007
2008 if (lpl_cur->lpl_nrset > 0) {
2009 if (act == LPL_INCREMENT) {
2010 lpl_cur->lpl_ncpu++;
2011 } else if (act == LPL_DECREMENT) {
2012 lpl_cur->lpl_ncpu--;
2013 }
2014 }
2015 }
2016 }
2017
2018 /*
2019 * Initialize lpl with given resources and specified lgrp
2020 */
2021 void
2022 lpl_init(lpl_t *lpl, lpl_t *lpl_leaf, lgrp_t *lgrp)
2023 {
2024 lpl->lpl_lgrpid = lgrp->lgrp_id;
2025 lpl->lpl_loadavg = 0;
2026 if (lpl == lpl_leaf)
2027 lpl->lpl_ncpu = 1;
2028 else
2029 lpl->lpl_ncpu = lpl_leaf->lpl_ncpu;
2030 lpl->lpl_nrset = 1;
2031 lpl->lpl_rset[0] = lpl_leaf;
2032 lpl->lpl_id2rset[lpl_leaf->lpl_lgrpid] = 0;
2033 lpl->lpl_lgrp = lgrp;
2034 lpl->lpl_parent = NULL; /* set by lpl_leaf_insert() */
2035 lpl->lpl_cpus = NULL; /* set by lgrp_part_add_cpu() */
2036 }
2037
2038 /*
2039 * Clear an unused lpl
2040 */
2041 void
2042 lpl_clear(lpl_t *lpl)
2043 {
2044 /*
2045 * Clear out all fields in the lpl except:
2046 * lpl_lgrpid - to facilitate debugging
2047 * lpl_rset, lpl_rset_sz, lpl_id2rset - rset array references / size
2048 *
2049 * Note that the lpl's rset and id2rset mapping are cleared as well.
2050 */
2051 lpl->lpl_loadavg = 0;
2052 lpl->lpl_ncpu = 0;
2053 lpl->lpl_lgrp = NULL;
2054 lpl->lpl_parent = NULL;
2055 lpl->lpl_cpus = NULL;
2056 lpl->lpl_nrset = 0;
2057 lpl->lpl_homed_time = 0;
2058 bzero(lpl->lpl_rset, sizeof (lpl->lpl_rset[0]) * lpl->lpl_rset_sz);
2059 bzero(lpl->lpl_id2rset,
2060 sizeof (lpl->lpl_id2rset[0]) * lpl->lpl_rset_sz);
2061 }
2062
2063 /*
2064 * Given a CPU-partition, verify that the lpl topology in the CPU-partition
2065 * is in sync with the lgroup toplogy in the system. The lpl topology may not
2066 * make full use of all of the lgroup topology, but this checks to make sure
2067 * that for the parts that it does use, it has correctly understood the
2068 * relationships that exist. This function returns
2069 * 0 if the topology is correct, and a non-zero error code, for non-debug
2070 * kernels if incorrect. Asserts are spread throughout the code to aid in
2071 * debugging on a DEBUG kernel.
2072 */
2073 int
2074 lpl_topo_verify(cpupart_t *cpupart)
2075 {
2076 lgrp_t *lgrp;
2077 lpl_t *lpl;
2078 klgrpset_t rset;
2079 klgrpset_t cset;
2080 cpu_t *cpu;
2081 cpu_t *cp_start;
2082 int i;
2083 int j;
2084 int sum;
2085
2086 /* topology can't be incorrect if it doesn't exist */
2087 if (!lgrp_topo_initialized || !lgrp_initialized)
2088 return (LPL_TOPO_CORRECT);
2089
2090 ASSERT(cpupart != NULL);
2091
2092 for (i = 0; i <= lgrp_alloc_max; i++) {
2093 lgrp = lgrp_table[i];
2094 lpl = NULL;
2095 /* make sure lpls are allocated */
2096 ASSERT(cpupart->cp_lgrploads);
2097 if (!cpupart->cp_lgrploads)
2098 return (LPL_TOPO_PART_HAS_NO_LPL);
2099
2100 lpl = &cpupart->cp_lgrploads[i];
2101 /* make sure our index is good */
2102 ASSERT(i < cpupart->cp_nlgrploads);
2103
2104 /* if lgroup doesn't exist, make sure lpl is empty */
2105 if (!LGRP_EXISTS(lgrp)) {
2106 ASSERT(lpl->lpl_ncpu == 0);
2107 if (lpl->lpl_ncpu > 0) {
2108 return (LPL_TOPO_CPUS_NOT_EMPTY);
2109 } else {
2110 continue;
2111 }
2112 }
2113
2114 /* verify that lgroup and lpl are identically numbered */
2115 ASSERT(lgrp->lgrp_id == lpl->lpl_lgrpid);
2116
2117 /* if lgroup isn't in our partition, make sure lpl is empty */
2118 if (!klgrpset_intersects(lgrp->lgrp_leaves,
2119 cpupart->cp_lgrpset)) {
2120 ASSERT(lpl->lpl_ncpu == 0);
2121 if (lpl->lpl_ncpu > 0) {
2122 return (LPL_TOPO_CPUS_NOT_EMPTY);
2123 }
2124 /*
2125 * lpl is empty, and lgroup isn't in partition. verify
2126 * that lpl doesn't show up in anyone else's rsets (in
2127 * this partition, anyway)
2128 */
2129 for (j = 0; j < cpupart->cp_nlgrploads; j++) {
2130 lpl_t *i_lpl; /* lpl we're iterating over */
2131
2132 i_lpl = &cpupart->cp_lgrploads[j];
2133
2134 ASSERT(!lpl_rset_contains(i_lpl, lpl));
2135 if (lpl_rset_contains(i_lpl, lpl)) {
2136 return (LPL_TOPO_LPL_ORPHANED);
2137 }
2138 }
2139 /* lgroup is empty, and everything is ok. continue */
2140 continue;
2141 }
2142
2143
2144 /* lgroup is in this partition, now check it against lpl */
2145
2146 /* do both have matching lgrps? */
2147 ASSERT(lgrp == lpl->lpl_lgrp);
2148 if (lgrp != lpl->lpl_lgrp) {
2149 return (LPL_TOPO_LGRP_MISMATCH);
2150 }
2151
2152 /* do the parent lgroups exist and do they match? */
2153 if (lgrp->lgrp_parent) {
2154 ASSERT(lpl->lpl_parent);
2155 ASSERT(lgrp->lgrp_parent->lgrp_id ==
2156 lpl->lpl_parent->lpl_lgrpid);
2157
2158 if (!lpl->lpl_parent) {
2159 return (LPL_TOPO_MISSING_PARENT);
2160 } else if (lgrp->lgrp_parent->lgrp_id !=
2161 lpl->lpl_parent->lpl_lgrpid) {
2162 return (LPL_TOPO_PARENT_MISMATCH);
2163 }
2164 }
2165
2166 /* only leaf lgroups keep a cpucnt, only check leaves */
2167 if ((lpl->lpl_nrset == 1) && (lpl == lpl->lpl_rset[0])) {
2168
2169 /* verify that lgrp is also a leaf */
2170 ASSERT((lgrp->lgrp_childcnt == 0) &&
2171 (klgrpset_ismember(lgrp->lgrp_leaves,
2172 lpl->lpl_lgrpid)));
2173
2174 if ((lgrp->lgrp_childcnt > 0) ||
2175 (!klgrpset_ismember(lgrp->lgrp_leaves,
2176 lpl->lpl_lgrpid))) {
2177 return (LPL_TOPO_LGRP_NOT_LEAF);
2178 }
2179
2180 ASSERT((lgrp->lgrp_cpucnt >= lpl->lpl_ncpu) &&
2181 (lpl->lpl_ncpu > 0));
2182 if ((lgrp->lgrp_cpucnt < lpl->lpl_ncpu) ||
2183 (lpl->lpl_ncpu <= 0)) {
2184 return (LPL_TOPO_BAD_CPUCNT);
2185 }
2186
2187 /*
2188 * Check that lpl_ncpu also matches the number of
2189 * cpus in the lpl's linked list. This only exists in
2190 * leaves, but they should always match.
2191 */
2192 j = 0;
2193 cpu = cp_start = lpl->lpl_cpus;
2194 while (cpu != NULL) {
2195 j++;
2196
2197 /* check to make sure cpu's lpl is leaf lpl */
2198 ASSERT(cpu->cpu_lpl == lpl);
2199 if (cpu->cpu_lpl != lpl) {
2200 return (LPL_TOPO_CPU_HAS_BAD_LPL);
2201 }
2202
2203 /* check next cpu */
2204 if ((cpu = cpu->cpu_next_lpl) != cp_start) {
2205 continue;
2206 } else {
2207 cpu = NULL;
2208 }
2209 }
2210
2211 ASSERT(j == lpl->lpl_ncpu);
2212 if (j != lpl->lpl_ncpu) {
2213 return (LPL_TOPO_LPL_BAD_NCPU);
2214 }
2215
2216 /*
2217 * Also, check that leaf lpl is contained in all
2218 * intermediate lpls that name the leaf as a descendant
2219 */
2220 for (j = 0; j <= lgrp_alloc_max; j++) {
2221 klgrpset_t intersect;
2222 lgrp_t *lgrp_cand;
2223 lpl_t *lpl_cand;
2224
2225 lgrp_cand = lgrp_table[j];
2226 intersect = klgrpset_intersects(
2227 lgrp_cand->lgrp_set[LGRP_RSRC_CPU],
2228 cpupart->cp_lgrpset);
2229
2230 if (!LGRP_EXISTS(lgrp_cand) ||
2231 !klgrpset_intersects(lgrp_cand->lgrp_leaves,
2232 cpupart->cp_lgrpset) ||
2233 (intersect == 0))
2234 continue;
2235
2236 lpl_cand =
2237 &cpupart->cp_lgrploads[lgrp_cand->lgrp_id];
2238
2239 if (klgrpset_ismember(intersect,
2240 lgrp->lgrp_id)) {
2241 ASSERT(lpl_rset_contains(lpl_cand,
2242 lpl));
2243
2244 if (!lpl_rset_contains(lpl_cand, lpl)) {
2245 return (LPL_TOPO_RSET_MSSNG_LF);
2246 }
2247 }
2248 }
2249
2250 } else { /* non-leaf specific checks */
2251
2252 /*
2253 * Non-leaf lpls should have lpl_cpus == NULL
2254 * verify that this is so
2255 */
2256 ASSERT(lpl->lpl_cpus == NULL);
2257 if (lpl->lpl_cpus != NULL) {
2258 return (LPL_TOPO_NONLEAF_HAS_CPUS);
2259 }
2260
2261 /*
2262 * verify that the sum of the cpus in the leaf resources
2263 * is equal to the total ncpu in the intermediate
2264 */
2265 for (j = sum = 0; j < lpl->lpl_nrset; j++) {
2266 sum += lpl->lpl_rset[j]->lpl_ncpu;
2267 }
2268
2269 ASSERT(sum == lpl->lpl_ncpu);
2270 if (sum != lpl->lpl_ncpu) {
2271 return (LPL_TOPO_LPL_BAD_NCPU);
2272 }
2273 }
2274
2275 /*
2276 * Check the rset of the lpl in question. Make sure that each
2277 * rset contains a subset of the resources in
2278 * lgrp_set[LGRP_RSRC_CPU] and in cp_lgrpset. This also makes
2279 * sure that each rset doesn't include resources that are
2280 * outside of that set. (Which would be resources somehow not
2281 * accounted for).
2282 */
2283 klgrpset_clear(rset);
2284 for (j = 0; j < lpl->lpl_nrset; j++) {
2285 klgrpset_add(rset, lpl->lpl_rset[j]->lpl_lgrpid);
2286 }
2287 klgrpset_copy(cset, rset);
2288 /* make sure lpl rset matches lgrp rset */
2289 klgrpset_diff(rset, lgrp->lgrp_set[LGRP_RSRC_CPU]);
2290 /* make sure rset is contained with in partition, too */
2291 klgrpset_diff(cset, cpupart->cp_lgrpset);
2292
2293 ASSERT(klgrpset_isempty(rset) && klgrpset_isempty(cset));
2294 if (!klgrpset_isempty(rset) || !klgrpset_isempty(cset)) {
2295 return (LPL_TOPO_RSET_MISMATCH);
2296 }
2297
2298 /*
2299 * check to make sure lpl_nrset matches the number of rsets
2300 * contained in the lpl
2301 */
2302 for (j = 0; j < lpl->lpl_nrset; j++) {
2303 if (lpl->lpl_rset[j] == NULL)
2304 break;
2305 }
2306
2307 ASSERT(j == lpl->lpl_nrset);
2308 if (j != lpl->lpl_nrset) {
2309 return (LPL_TOPO_BAD_RSETCNT);
2310 }
2311
2312 }
2313 return (LPL_TOPO_CORRECT);
2314 }
2315
2316 /*
2317 * Flatten lpl topology to given number of levels. This is presently only
2318 * implemented for a flatten to 2 levels, which will prune out the intermediates
2319 * and home the leaf lpls to the root lpl.
2320 */
2321 int
2322 lpl_topo_flatten(int levels)
2323 {
2324 int i;
2325 uint_t sum;
2326 lgrp_t *lgrp_cur;
2327 lpl_t *lpl_cur;
2328 lpl_t *lpl_root;
2329 cpupart_t *cp;
2330
2331 if (levels != 2)
2332 return (0);
2333
2334 /* called w/ cpus paused - grab no locks! */
2335 ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
2336 !lgrp_initialized);
2337
2338 cp = cp_list_head;
2339 do {
2340 lpl_root = &cp->cp_lgrploads[lgrp_root->lgrp_id];
2341 ASSERT(LGRP_EXISTS(lgrp_root) && (lpl_root->lpl_ncpu > 0));
2342
2343 for (i = 0; i <= lgrp_alloc_max; i++) {
2344 lgrp_cur = lgrp_table[i];
2345 lpl_cur = &cp->cp_lgrploads[i];
2346
2347 if ((lgrp_cur == lgrp_root) ||
2348 (!LGRP_EXISTS(lgrp_cur) &&
2349 (lpl_cur->lpl_ncpu == 0)))
2350 continue;
2351
2352 if (!LGRP_EXISTS(lgrp_cur) && (lpl_cur->lpl_ncpu > 0)) {
2353 /*
2354 * this should be a deleted intermediate, so
2355 * clear it
2356 */
2357 lpl_clear(lpl_cur);
2358 } else if ((lpl_cur->lpl_nrset == 1) &&
2359 (lpl_cur->lpl_rset[0] == lpl_cur) &&
2360 ((lpl_cur->lpl_parent->lpl_ncpu == 0) ||
2361 (!LGRP_EXISTS(lpl_cur->lpl_parent->lpl_lgrp)))) {
2362 /*
2363 * this is a leaf whose parent was deleted, or
2364 * whose parent had their lgrp deleted. (And
2365 * whose parent will soon be deleted). Point
2366 * this guy back to the root lpl.
2367 */
2368 lpl_cur->lpl_parent = lpl_root;
2369 lpl_rset_add(lpl_root, lpl_cur);
2370 }
2371
2372 }
2373
2374 /*
2375 * Now that we're done, make sure the count on the root lpl is
2376 * correct, and update the hints of the children for the sake of
2377 * thoroughness
2378 */
2379 for (i = sum = 0; i < lpl_root->lpl_nrset; i++) {
2380 sum += lpl_root->lpl_rset[i]->lpl_ncpu;
2381 }
2382 lpl_root->lpl_ncpu = sum;
2383 lpl_child_update(lpl_root, cp);
2384
2385 cp = cp->cp_next;
2386 } while (cp != cp_list_head);
2387
2388 return (levels);
2389 }
2390
2391 /*
2392 * Insert a lpl into the resource hierarchy and create any additional lpls that
2393 * are necessary to represent the varying states of locality for the cpu
2394 * resoruces newly added to the partition.
2395 *
2396 * This routine is clever enough that it can correctly add resources from the
2397 * new leaf into both direct and indirect resource sets in the hierarchy. (Ie,
2398 * those for which the lpl is a leaf as opposed to simply a named equally local
2399 * resource). The one special case that needs additional processing is when a
2400 * new intermediate lpl is introduced. Since the main loop only traverses
2401 * looking to add the leaf resource where it does not yet exist, additional work
2402 * is necessary to add other leaf resources that may need to exist in the newly
2403 * created intermediate. This is performed by the second inner loop, and is
2404 * only done when the check for more than one overlapping resource succeeds.
2405 */
2406
2407 void
2408 lpl_leaf_insert(lpl_t *lpl_leaf, cpupart_t *cpupart)
2409 {
2410 int i;
2411 int j;
2412 int rset_num_intersect;
2413 lgrp_t *lgrp_cur;
2414 lpl_t *lpl_cur;
2415 lpl_t *lpl_parent;
2416 lgrp_id_t parent_id;
2417 klgrpset_t rset_intersect; /* resources in cpupart and lgrp */
2418
2419 for (i = 0; i <= lgrp_alloc_max; i++) {
2420 lgrp_cur = lgrp_table[i];
2421
2422 /*
2423 * Don't insert if the lgrp isn't there, if the leaf isn't
2424 * contained within the current lgrp, or if the current lgrp has
2425 * no leaves in this partition
2426 */
2427
2428 if (!LGRP_EXISTS(lgrp_cur) ||
2429 !klgrpset_ismember(lgrp_cur->lgrp_set[LGRP_RSRC_CPU],
2430 lpl_leaf->lpl_lgrpid) ||
2431 !klgrpset_intersects(lgrp_cur->lgrp_leaves,
2432 cpupart->cp_lgrpset))
2433 continue;
2434
2435 lpl_cur = &cpupart->cp_lgrploads[lgrp_cur->lgrp_id];
2436 if (lgrp_cur->lgrp_parent != NULL) {
2437 /* if lgrp has a parent, assign it properly */
2438 parent_id = lgrp_cur->lgrp_parent->lgrp_id;
2439 lpl_parent = &cpupart->cp_lgrploads[parent_id];
2440 } else {
2441 /* if not, make sure parent ptr gets set to null */
2442 lpl_parent = NULL;
2443 }
2444
2445 if (lpl_cur == lpl_leaf) {
2446 /*
2447 * Almost all leaf state was initialized elsewhere. The
2448 * only thing left to do is to set the parent.
2449 */
2450 lpl_cur->lpl_parent = lpl_parent;
2451 continue;
2452 }
2453
2454 lpl_clear(lpl_cur);
2455 lpl_init(lpl_cur, lpl_leaf, lgrp_cur);
2456
2457 lpl_cur->lpl_parent = lpl_parent;
2458
2459 /* does new lpl need to be populated with other resources? */
2460 rset_intersect =
2461 klgrpset_intersects(lgrp_cur->lgrp_set[LGRP_RSRC_CPU],
2462 cpupart->cp_lgrpset);
2463 klgrpset_nlgrps(rset_intersect, rset_num_intersect);
2464
2465 if (rset_num_intersect > 1) {
2466 /*
2467 * If so, figure out what lpls have resources that
2468 * intersect this one, and add them.
2469 */
2470 for (j = 0; j <= lgrp_alloc_max; j++) {
2471 lgrp_t *lgrp_cand; /* candidate lgrp */
2472 lpl_t *lpl_cand; /* candidate lpl */
2473
2474 lgrp_cand = lgrp_table[j];
2475 if (!LGRP_EXISTS(lgrp_cand) ||
2476 !klgrpset_ismember(rset_intersect,
2477 lgrp_cand->lgrp_id))
2478 continue;
2479 lpl_cand =
2480 &cpupart->cp_lgrploads[lgrp_cand->lgrp_id];
2481 lpl_rset_add(lpl_cur, lpl_cand);
2482 }
2483 }
2484 /*
2485 * This lpl's rset has changed. Update the hint in it's
2486 * children.
2487 */
2488 lpl_child_update(lpl_cur, cpupart);
2489 }
2490 }
2491
2492 /*
2493 * remove a lpl from the hierarchy of resources, clearing its state when
2494 * finished. If the lpls at the intermediate levels of the hierarchy have no
2495 * remaining resources, or no longer name a leaf resource in the cpu-partition,
2496 * delete them as well.
2497 */
2498
2499 void
2500 lpl_leaf_remove(lpl_t *lpl_leaf, cpupart_t *cpupart)
2501 {
2502 int i;
2503 lgrp_t *lgrp_cur;
2504 lpl_t *lpl_cur;
2505 klgrpset_t leaf_intersect; /* intersection of leaves */
2506
2507 for (i = 0; i <= lgrp_alloc_max; i++) {
2508 lgrp_cur = lgrp_table[i];
2509
2510 /*
2511 * Don't attempt to remove from lgrps that aren't there, that
2512 * don't contain our leaf, or from the leaf itself. (We do that
2513 * later)
2514 */
2515
2516 if (!LGRP_EXISTS(lgrp_cur))
2517 continue;
2518
2519 lpl_cur = &cpupart->cp_lgrploads[lgrp_cur->lgrp_id];
2520
2521 if (!klgrpset_ismember(lgrp_cur->lgrp_set[LGRP_RSRC_CPU],
2522 lpl_leaf->lpl_lgrpid) ||
2523 (lpl_cur == lpl_leaf)) {
2524 continue;
2525 }
2526
2527 /*
2528 * This is a slightly sleazy simplification in that we have
2529 * already marked the cp_lgrpset as no longer containing the
2530 * leaf we've deleted. Any lpls that pass the above checks
2531 * based upon lgrp membership but not necessarily cpu-part
2532 * membership also get cleared by the checks below. Currently
2533 * this is harmless, as the lpls should be empty anyway.
2534 *
2535 * In particular, we want to preserve lpls that have additional
2536 * leaf resources, even though we don't yet have a processor
2537 * architecture that represents resources this way.
2538 */
2539
2540 leaf_intersect = klgrpset_intersects(lgrp_cur->lgrp_leaves,
2541 cpupart->cp_lgrpset);
2542
2543 lpl_rset_del(lpl_cur, lpl_leaf);
2544 if ((lpl_cur->lpl_nrset == 0) || (!leaf_intersect)) {
2545 lpl_clear(lpl_cur);
2546 } else {
2547 /*
2548 * Update this lpl's children
2549 */
2550 lpl_child_update(lpl_cur, cpupart);
2551 }
2552 }
2553 lpl_clear(lpl_leaf);
2554 }
2555
2556 /*
2557 * add a cpu to a partition in terms of lgrp load avg bookeeping
2558 *
2559 * The lpl (cpu partition load average information) is now arranged in a
2560 * hierarchical fashion whereby resources that are closest, ie. most local, to
2561 * the cpu in question are considered to be leaves in a tree of resources.
2562 * There are two general cases for cpu additon:
2563 *
2564 * 1. A lpl structure that contains resources already in the hierarchy tree.
2565 * In this case, all of the associated lpl relationships have been defined, and
2566 * all that is necessary is that we link the new cpu into the per-lpl list of
2567 * cpus, and increment the ncpu count of all places where this cpu resource will
2568 * be accounted for. lpl_cpu_adjcnt updates the cpu count, and the cpu pointer
2569 * pushing is accomplished by this routine.
2570 *
2571 * 2. The lpl to contain the resources in this cpu-partition for this lgrp does
2572 * not exist yet. In this case, it is necessary to build the leaf lpl, and
2573 * construct the hierarchy of state necessary to name it's more distant
2574 * resources, if they should exist. The leaf structure is initialized by this
2575 * routine, as is the cpu-partition state for the lgrp membership. This routine
2576 * also calls lpl_leaf_insert() which inserts the named lpl into the hierarchy
2577 * and builds all of the "ancestoral" state necessary to identify resources at
2578 * differing levels of locality.
2579 */
2580 void
2581 lgrp_part_add_cpu(cpu_t *cp, lgrp_id_t lgrpid)
2582 {
2583 cpupart_t *cpupart;
2584 lgrp_t *lgrp_leaf;
2585 lpl_t *lpl_leaf;
2586
2587 /* called sometimes w/ cpus paused - grab no locks */
2588 ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized);
2589
2590 cpupart = cp->cpu_part;
2591 lgrp_leaf = lgrp_table[lgrpid];
2592
2593 /* don't add non-existent lgrp */
2594 ASSERT(LGRP_EXISTS(lgrp_leaf));
2595 lpl_leaf = &cpupart->cp_lgrploads[lgrpid];
2596 cp->cpu_lpl = lpl_leaf;
2597
2598 /* only leaf lpls contain cpus */
2599
2600 if (lpl_leaf->lpl_ncpu++ == 0) {
2601 lpl_init(lpl_leaf, lpl_leaf, lgrp_leaf);
2602 klgrpset_add(cpupart->cp_lgrpset, lgrpid);
2603 lpl_leaf_insert(lpl_leaf, cpupart);
2604 } else {
2605 /*
2606 * the lpl should already exist in the parent, so just update
2607 * the count of available CPUs
2608 */
2609 lpl_cpu_adjcnt(LPL_INCREMENT, cp);
2610 }
2611
2612 /* link cpu into list of cpus in lpl */
2613
2614 if (lpl_leaf->lpl_cpus) {
2615 cp->cpu_next_lpl = lpl_leaf->lpl_cpus;
2616 cp->cpu_prev_lpl = lpl_leaf->lpl_cpus->cpu_prev_lpl;
2617 lpl_leaf->lpl_cpus->cpu_prev_lpl->cpu_next_lpl = cp;
2618 lpl_leaf->lpl_cpus->cpu_prev_lpl = cp;
2619 } else {
2620 /*
2621 * We increment ncpu immediately after we create a new leaf
2622 * lpl, so assert that ncpu == 1 for the case where we don't
2623 * have any cpu pointers yet.
2624 */
2625 ASSERT(lpl_leaf->lpl_ncpu == 1);
2626 lpl_leaf->lpl_cpus = cp->cpu_next_lpl = cp->cpu_prev_lpl = cp;
2627 }
2628
2629 }
2630
2631
2632 /*
2633 * remove a cpu from a partition in terms of lgrp load avg bookeeping
2634 *
2635 * The lpl (cpu partition load average information) is now arranged in a
2636 * hierarchical fashion whereby resources that are closest, ie. most local, to
2637 * the cpu in question are considered to be leaves in a tree of resources.
2638 * There are two removal cases in question:
2639 *
2640 * 1. Removal of the resource in the leaf leaves other resources remaining in
2641 * that leaf. (Another cpu still exists at this level of locality). In this
2642 * case, the count of available cpus is decremented in all assocated lpls by
2643 * calling lpl_adj_cpucnt(), and the pointer to the removed cpu is pruned
2644 * from the per-cpu lpl list.
2645 *
2646 * 2. Removal of the resource results in the lpl containing no resources. (It's
2647 * empty) In this case, all of what has occurred for the first step must take
2648 * place; however, additionally we must remove the lpl structure itself, prune
2649 * out any stranded lpls that do not directly name a leaf resource, and mark the
2650 * cpu partition in question as no longer containing resources from the lgrp of
2651 * the lpl that has been delted. Cpu-partition changes are handled by this
2652 * method, but the lpl_leaf_remove function deals with the details of pruning
2653 * out the empty lpl and any of its orphaned direct ancestors.
2654 */
2655 void
2656 lgrp_part_del_cpu(cpu_t *cp)
2657 {
2658 lpl_t *lpl;
2659 lpl_t *leaf_lpl;
2660 lgrp_t *lgrp_leaf;
2661
2662 /* called sometimes w/ cpus paused - grab no locks */
2663
2664 ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized);
2665
2666 lpl = leaf_lpl = cp->cpu_lpl;
2667 lgrp_leaf = leaf_lpl->lpl_lgrp;
2668
2669 /* don't delete a leaf that isn't there */
2670 ASSERT(LGRP_EXISTS(lgrp_leaf));
2671
2672 /* no double-deletes */
2673 ASSERT(lpl->lpl_ncpu);
2674 if (--lpl->lpl_ncpu == 0) {
2675 /*
2676 * This was the last cpu in this lgroup for this partition,
2677 * clear its bit in the partition's lgroup bitmask
2678 */
2679 klgrpset_del(cp->cpu_part->cp_lgrpset, lpl->lpl_lgrpid);
2680
2681 /* eliminate remaning lpl link pointers in cpu, lpl */
2682 lpl->lpl_cpus = cp->cpu_next_lpl = cp->cpu_prev_lpl = NULL;
2683
2684 lpl_leaf_remove(leaf_lpl, cp->cpu_part);
2685 } else {
2686
2687 /* unlink cpu from lists of cpus in lpl */
2688 cp->cpu_prev_lpl->cpu_next_lpl = cp->cpu_next_lpl;
2689 cp->cpu_next_lpl->cpu_prev_lpl = cp->cpu_prev_lpl;
2690 if (lpl->lpl_cpus == cp) {
2691 lpl->lpl_cpus = cp->cpu_next_lpl;
2692 }
2693
2694 /*
2695 * Update the cpu count in the lpls associated with parent
2696 * lgroups.
2697 */
2698 lpl_cpu_adjcnt(LPL_DECREMENT, cp);
2699
2700 }
2701 /* clear cpu's lpl ptr when we're all done */
2702 cp->cpu_lpl = NULL;
2703 }
2704
2705 /*
2706 * Recompute load average for the specified partition/lgrp fragment.
2707 *
2708 * We rely on the fact that this routine is called from the clock thread
2709 * at a point before the clock thread can block (i.e. before its first
2710 * lock request). Since the clock thread can not be preempted (since it
2711 * runs at highest priority), we know that cpu partitions can not change
2712 * (since doing so would require either the repartition requester or the
2713 * cpu_pause thread to run on this cpu), so we can update the cpu's load
2714 * without grabbing cpu_lock.
2715 */
2716 void
2717 lgrp_loadavg(lpl_t *lpl, uint_t nrcpus, int ageflag)
2718 {
2719 uint_t ncpu;
2720 int64_t old, new, f;
2721
2722 /*
2723 * 1 - exp(-1/(20 * ncpu)) << 13 = 400 for 1 cpu...
2724 */
2725 static short expval[] = {
2726 0, 3196, 1618, 1083,
2727 814, 652, 543, 466,
2728 408, 363, 326, 297,
2729 272, 251, 233, 218,
2730 204, 192, 181, 172,
2731 163, 155, 148, 142,
2732 136, 130, 125, 121,
2733 116, 112, 109, 105
2734 };
2735
2736 /* ASSERT (called from clock level) */
2737
2738 if ((lpl == NULL) || /* we're booting - this is easiest for now */
2739 ((ncpu = lpl->lpl_ncpu) == 0)) {
2740 return;
2741 }
2742
2743 for (;;) {
2744
2745 if (ncpu >= sizeof (expval) / sizeof (expval[0]))
2746 f = expval[1]/ncpu; /* good approx. for large ncpu */
2747 else
2748 f = expval[ncpu];
2749
2750 /*
2751 * Modify the load average atomically to avoid losing
2752 * anticipatory load updates (see lgrp_move_thread()).
2753 */
2754 if (ageflag) {
2755 /*
2756 * We're supposed to both update and age the load.
2757 * This happens 10 times/sec. per cpu. We do a
2758 * little hoop-jumping to avoid integer overflow.
2759 */
2760 int64_t q, r;
2761
2762 do {
2763 old = new = lpl->lpl_loadavg;
2764 q = (old >> 16) << 7;
2765 r = (old & 0xffff) << 7;
2766 new += ((long long)(nrcpus - q) * f -
2767 ((r * f) >> 16)) >> 7;
2768
2769 /*
2770 * Check for overflow
2771 */
2772 if (new > LGRP_LOADAVG_MAX)
2773 new = LGRP_LOADAVG_MAX;
2774 else if (new < 0)
2775 new = 0;
2776 } while (atomic_cas_32((lgrp_load_t *)&lpl->lpl_loadavg,
2777 old, new) != old);
2778 } else {
2779 /*
2780 * We're supposed to update the load, but not age it.
2781 * This option is used to update the load (which either
2782 * has already been aged in this 1/10 sec. interval or
2783 * soon will be) to account for a remotely executing
2784 * thread.
2785 */
2786 do {
2787 old = new = lpl->lpl_loadavg;
2788 new += f;
2789 /*
2790 * Check for overflow
2791 * Underflow not possible here
2792 */
2793 if (new < old)
2794 new = LGRP_LOADAVG_MAX;
2795 } while (atomic_cas_32((lgrp_load_t *)&lpl->lpl_loadavg,
2796 old, new) != old);
2797 }
2798
2799 /*
2800 * Do the same for this lpl's parent
2801 */
2802 if ((lpl = lpl->lpl_parent) == NULL)
2803 break;
2804 ncpu = lpl->lpl_ncpu;
2805 }
2806 }
2807
2808 /*
2809 * Initialize lpl topology in the target based on topology currently present in
2810 * lpl_bootstrap.
2811 *
2812 * lpl_topo_bootstrap is only called once from cpupart_initialize_default() to
2813 * initialize cp_default list of lpls. Up to this point all topology operations
2814 * were performed using lpl_bootstrap. Now cp_default has its own list of lpls
2815 * and all subsequent lpl operations should use it instead of lpl_bootstrap. The
2816 * `target' points to the list of lpls in cp_default and `size' is the size of
2817 * this list.
2818 *
2819 * This function walks the lpl topology in lpl_bootstrap and does for things:
2820 *
2821 * 1) Copies all fields from lpl_bootstrap to the target.
2822 *
2823 * 2) Sets CPU0 lpl pointer to the correct element of the target list.
2824 *
2825 * 3) Updates lpl_parent pointers to point to the lpls in the target list
2826 * instead of lpl_bootstrap.
2827 *
2828 * 4) Updates pointers in the resource list of the target to point to the lpls
2829 * in the target list instead of lpl_bootstrap.
2830 *
2831 * After lpl_topo_bootstrap() completes, target contains the same information
2832 * that would be present there if it were used during boot instead of
2833 * lpl_bootstrap. There is no need in information in lpl_bootstrap after this
2834 * and it is bzeroed.
2835 */
2836 void
2837 lpl_topo_bootstrap(lpl_t *target, int size)
2838 {
2839 lpl_t *lpl = lpl_bootstrap;
2840 lpl_t *target_lpl = target;
2841 lpl_t **rset;
2842 int *id2rset;
2843 int sz;
2844 int howmany;
2845 int id;
2846 int i;
2847
2848 /*
2849 * The only target that should be passed here is cp_default lpl list.
2850 */
2851 ASSERT(target == cp_default.cp_lgrploads);
2852 ASSERT(size == cp_default.cp_nlgrploads);
2853 ASSERT(!lgrp_topo_initialized);
2854 ASSERT(ncpus == 1);
2855
2856 howmany = MIN(LPL_BOOTSTRAP_SIZE, size);
2857 for (i = 0; i < howmany; i++, lpl++, target_lpl++) {
2858 /*
2859 * Copy all fields from lpl, except for the rset,
2860 * lgrp id <=> rset mapping storage,
2861 * and amount of storage
2862 */
2863 rset = target_lpl->lpl_rset;
2864 id2rset = target_lpl->lpl_id2rset;
2865 sz = target_lpl->lpl_rset_sz;
2866
2867 *target_lpl = *lpl;
2868
2869 target_lpl->lpl_rset_sz = sz;
2870 target_lpl->lpl_rset = rset;
2871 target_lpl->lpl_id2rset = id2rset;
2872
2873 /*
2874 * Substitute CPU0 lpl pointer with one relative to target.
2875 */
2876 if (lpl->lpl_cpus == CPU) {
2877 ASSERT(CPU->cpu_lpl == lpl);
2878 CPU->cpu_lpl = target_lpl;
2879 }
2880
2881 /*
2882 * Substitute parent information with parent relative to target.
2883 */
2884 if (lpl->lpl_parent != NULL)
2885 target_lpl->lpl_parent = (lpl_t *)
2886 (((uintptr_t)lpl->lpl_parent -
2887 (uintptr_t)lpl_bootstrap) +
2888 (uintptr_t)target);
2889
2890 /*
2891 * Walk over resource set substituting pointers relative to
2892 * lpl_bootstrap's rset to pointers relative to target's
2893 */
2894 ASSERT(lpl->lpl_nrset <= 1);
2895
2896 for (id = 0; id < lpl->lpl_nrset; id++) {
2897 if (lpl->lpl_rset[id] != NULL) {
2898 target_lpl->lpl_rset[id] = (lpl_t *)
2899 (((uintptr_t)lpl->lpl_rset[id] -
2900 (uintptr_t)lpl_bootstrap) +
2901 (uintptr_t)target);
2902 }
2903 target_lpl->lpl_id2rset[id] =
2904 lpl->lpl_id2rset[id];
2905 }
2906 }
2907
2908 /*
2909 * Clean up the bootstrap lpls since we have switched over to the
2910 * actual lpl array in the default cpu partition.
2911 *
2912 * We still need to keep one empty lpl around for newly starting
2913 * slave CPUs to reference should they need to make it through the
2914 * dispatcher prior to their lgrp/lpl initialization.
2915 *
2916 * The lpl related dispatcher code has been designed to work properly
2917 * (and without extra checks) for this special case of a zero'ed
2918 * bootstrap lpl. Such an lpl appears to the dispatcher as an lpl
2919 * with lgrpid 0 and an empty resource set. Iteration over the rset
2920 * array by the dispatcher is also NULL terminated for this reason.
2921 *
2922 * This provides the desired behaviour for an uninitialized CPU.
2923 * It shouldn't see any other CPU to either dispatch to or steal
2924 * from until it is properly initialized.
2925 */
2926 bzero(lpl_bootstrap_list, sizeof (lpl_bootstrap_list));
2927 bzero(lpl_bootstrap_id2rset, sizeof (lpl_bootstrap_id2rset));
2928 bzero(lpl_bootstrap_rset, sizeof (lpl_bootstrap_rset));
2929
2930 lpl_bootstrap_list[0].lpl_rset = lpl_bootstrap_rset;
2931 lpl_bootstrap_list[0].lpl_id2rset = lpl_bootstrap_id2rset;
2932 }
2933
2934 /*
2935 * If the lowest load among the lgroups a process' threads are currently
2936 * spread across is greater than lgrp_expand_proc_thresh, we'll consider
2937 * expanding the process to a new lgroup.
2938 */
2939 #define LGRP_EXPAND_PROC_THRESH_DEFAULT 62250
2940 lgrp_load_t lgrp_expand_proc_thresh = LGRP_EXPAND_PROC_THRESH_DEFAULT;
2941
2942 #define LGRP_EXPAND_PROC_THRESH(ncpu) \
2943 ((lgrp_expand_proc_thresh) / (ncpu))
2944
2945 /*
2946 * A process will be expanded to a new lgroup only if the difference between
2947 * the lowest load on the lgroups the process' thread's are currently spread
2948 * across and the lowest load on the other lgroups in the process' partition
2949 * is greater than lgrp_expand_proc_diff.
2950 */
2951 #define LGRP_EXPAND_PROC_DIFF_DEFAULT 60000
2952 lgrp_load_t lgrp_expand_proc_diff = LGRP_EXPAND_PROC_DIFF_DEFAULT;
2953
2954 #define LGRP_EXPAND_PROC_DIFF(ncpu) \
2955 ((lgrp_expand_proc_diff) / (ncpu))
2956
2957 /*
2958 * The loadavg tolerance accounts for "noise" inherent in the load, which may
2959 * be present due to impreciseness of the load average decay algorithm.
2960 *
2961 * The default tolerance is lgrp_loadavg_max_effect. Note that the tunable
2962 * tolerance is scaled by the number of cpus in the lgroup just like
2963 * lgrp_loadavg_max_effect. For example, if lgrp_loadavg_tolerance = 0x10000,
2964 * and ncpu = 4, then lgrp_choose will consider differences in lgroup loads
2965 * of: 0x10000 / 4 => 0x4000 or greater to be significant.
2966 */
2967 uint32_t lgrp_loadavg_tolerance = LGRP_LOADAVG_THREAD_MAX;
2968 #define LGRP_LOADAVG_TOLERANCE(ncpu) \
2969 ((lgrp_loadavg_tolerance) / ncpu)
2970
2971 /*
2972 * lgrp_choose() will choose root lgroup as home when lowest lgroup load
2973 * average is above this threshold
2974 */
2975 uint32_t lgrp_load_thresh = UINT32_MAX;
2976
2977 /*
2978 * lgrp_choose() will try to skip any lgroups with less memory
2979 * than this free when choosing a home lgroup
2980 */
2981 pgcnt_t lgrp_mem_free_thresh = 0;
2982
2983 /*
2984 * When choosing between similarly loaded lgroups, lgrp_choose() will pick
2985 * one based on one of the following policies:
2986 * - Random selection
2987 * - Pseudo round robin placement
2988 * - Longest time since a thread was last placed
2989 */
2990 #define LGRP_CHOOSE_RANDOM 1
2991 #define LGRP_CHOOSE_RR 2
2992 #define LGRP_CHOOSE_TIME 3
2993
2994 int lgrp_choose_policy = LGRP_CHOOSE_TIME;
2995
2996 /*
2997 * Choose a suitable leaf lgroup for a kthread. The kthread is assumed not to
2998 * be bound to a CPU or processor set.
2999 *
3000 * Arguments:
3001 * t The thread
3002 * cpupart The partition the thread belongs to.
3003 *
3004 * NOTE: Should at least be called with the cpu_lock held, kernel preemption
3005 * disabled, or thread_lock held (at splhigh) to protect against the CPU
3006 * partitions changing out from under us and assumes that given thread is
3007 * protected. Also, called sometimes w/ cpus paused or kernel preemption
3008 * disabled, so don't grab any locks because we should never block under
3009 * those conditions.
3010 */
3011 lpl_t *
3012 lgrp_choose(kthread_t *t, cpupart_t *cpupart)
3013 {
3014 lgrp_load_t bestload, bestrload;
3015 int lgrpid_offset, lgrp_count;
3016 lgrp_id_t lgrpid, lgrpid_start;
3017 lpl_t *lpl, *bestlpl, *bestrlpl;
3018 klgrpset_t lgrpset;
3019 proc_t *p;
3020
3021 ASSERT(t != NULL);
3022 ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
3023 THREAD_LOCK_HELD(t));
3024 ASSERT(cpupart != NULL);
3025
3026 p = t->t_procp;
3027
3028 /* A process should always be in an active partition */
3029 ASSERT(!klgrpset_isempty(cpupart->cp_lgrpset));
3030
3031 bestlpl = bestrlpl = NULL;
3032 bestload = bestrload = LGRP_LOADAVG_MAX;
3033 lgrpset = cpupart->cp_lgrpset;
3034
3035 switch (lgrp_choose_policy) {
3036 case LGRP_CHOOSE_RR:
3037 lgrpid = cpupart->cp_lgrp_hint;
3038 do {
3039 if (++lgrpid > lgrp_alloc_max)
3040 lgrpid = 0;
3041 } while (!klgrpset_ismember(lgrpset, lgrpid));
3042
3043 break;
3044 default:
3045 case LGRP_CHOOSE_TIME:
3046 case LGRP_CHOOSE_RANDOM:
3047 klgrpset_nlgrps(lgrpset, lgrp_count);
3048 lgrpid_offset =
3049 (((ushort_t)(gethrtime() >> 4)) % lgrp_count) + 1;
3050 for (lgrpid = 0; ; lgrpid++) {
3051 if (klgrpset_ismember(lgrpset, lgrpid)) {
3052 if (--lgrpid_offset == 0)
3053 break;
3054 }
3055 }
3056 break;
3057 }
3058
3059 lgrpid_start = lgrpid;
3060
3061 DTRACE_PROBE2(lgrp_choose_start, lgrp_id_t, lgrpid_start,
3062 lgrp_id_t, cpupart->cp_lgrp_hint);
3063
3064 /*
3065 * Use lgroup affinities (if any) to choose best lgroup
3066 *
3067 * NOTE: Assumes that thread is protected from going away and its
3068 * lgroup affinities won't change (ie. p_lock, or
3069 * thread_lock() being held and/or CPUs paused)
3070 */
3071 if (t->t_lgrp_affinity) {
3072 lpl = lgrp_affinity_best(t, cpupart, lgrpid_start, B_FALSE);
3073 if (lpl != NULL)
3074 return (lpl);
3075 }
3076
3077 ASSERT(klgrpset_ismember(lgrpset, lgrpid_start));
3078
3079 do {
3080 pgcnt_t npgs;
3081
3082 /*
3083 * Skip any lgroups outside of thread's pset
3084 */
3085 if (!klgrpset_ismember(lgrpset, lgrpid)) {
3086 if (++lgrpid > lgrp_alloc_max)
3087 lgrpid = 0; /* wrap the search */
3088 continue;
3089 }
3090
3091 /*
3092 * Skip any non-leaf lgroups
3093 */
3094 if (lgrp_table[lgrpid]->lgrp_childcnt != 0)
3095 continue;
3096
3097 /*
3098 * Skip any lgroups without enough free memory
3099 * (when threshold set to nonzero positive value)
3100 */
3101 if (lgrp_mem_free_thresh > 0) {
3102 npgs = lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_FREE);
3103 if (npgs < lgrp_mem_free_thresh) {
3104 if (++lgrpid > lgrp_alloc_max)
3105 lgrpid = 0; /* wrap the search */
3106 continue;
3107 }
3108 }
3109
3110 lpl = &cpupart->cp_lgrploads[lgrpid];
3111 if (klgrpset_isempty(p->p_lgrpset) ||
3112 klgrpset_ismember(p->p_lgrpset, lgrpid)) {
3113 /*
3114 * Either this is a new process or the process already
3115 * has threads on this lgrp, so this is a preferred
3116 * lgroup for the thread.
3117 */
3118 if (bestlpl == NULL ||
3119 lpl_pick(lpl, bestlpl)) {
3120 bestload = lpl->lpl_loadavg;
3121 bestlpl = lpl;
3122 }
3123 } else {
3124 /*
3125 * The process doesn't have any threads on this lgrp,
3126 * but we're willing to consider this lgrp if the load
3127 * difference is big enough to justify splitting up
3128 * the process' threads.
3129 */
3130 if (bestrlpl == NULL ||
3131 lpl_pick(lpl, bestrlpl)) {
3132 bestrload = lpl->lpl_loadavg;
3133 bestrlpl = lpl;
3134 }
3135 }
3136 if (++lgrpid > lgrp_alloc_max)
3137 lgrpid = 0; /* wrap the search */
3138 } while (lgrpid != lgrpid_start);
3139
3140 /*
3141 * Return root lgroup if threshold isn't set to maximum value and
3142 * lowest lgroup load average more than a certain threshold
3143 */
3144 if (lgrp_load_thresh != UINT32_MAX &&
3145 bestload >= lgrp_load_thresh && bestrload >= lgrp_load_thresh)
3146 return (&cpupart->cp_lgrploads[lgrp_root->lgrp_id]);
3147
3148 /*
3149 * If all the lgroups over which the thread's process is spread are
3150 * heavily loaded, or otherwise undesirable, we'll consider placing
3151 * the thread on one of the other leaf lgroups in the thread's
3152 * partition.
3153 */
3154 if ((bestlpl == NULL) ||
3155 ((bestload > LGRP_EXPAND_PROC_THRESH(bestlpl->lpl_ncpu)) &&
3156 (bestrload < bestload) && /* paranoid about wraparound */
3157 (bestrload + LGRP_EXPAND_PROC_DIFF(bestrlpl->lpl_ncpu) <
3158 bestload))) {
3159 bestlpl = bestrlpl;
3160 }
3161
3162 if (bestlpl == NULL) {
3163 /*
3164 * No lgroup looked particularly good, but we still
3165 * have to pick something. Go with the randomly selected
3166 * legal lgroup we started with above.
3167 */
3168 bestlpl = &cpupart->cp_lgrploads[lgrpid_start];
3169 }
3170
3171 cpupart->cp_lgrp_hint = bestlpl->lpl_lgrpid;
3172 bestlpl->lpl_homed_time = gethrtime_unscaled();
3173
3174 ASSERT(bestlpl->lpl_ncpu > 0);
3175 return (bestlpl);
3176 }
3177
3178 /*
3179 * Decide if lpl1 is a better candidate than lpl2 for lgrp homing.
3180 * Returns non-zero if lpl1 is a better candidate, and 0 otherwise.
3181 */
3182 static int
3183 lpl_pick(lpl_t *lpl1, lpl_t *lpl2)
3184 {
3185 lgrp_load_t l1, l2;
3186 lgrp_load_t tolerance = LGRP_LOADAVG_TOLERANCE(lpl1->lpl_ncpu);
3187
3188 l1 = lpl1->lpl_loadavg;
3189 l2 = lpl2->lpl_loadavg;
3190
3191 if ((l1 + tolerance < l2) && (l1 < l2)) {
3192 /* lpl1 is significantly less loaded than lpl2 */
3193 return (1);
3194 }
3195
3196 if (lgrp_choose_policy == LGRP_CHOOSE_TIME &&
3197 l1 + tolerance >= l2 && l1 < l2 &&
3198 lpl1->lpl_homed_time < lpl2->lpl_homed_time) {
3199 /*
3200 * lpl1's load is within the tolerance of lpl2. We're
3201 * willing to consider it be to better however if
3202 * it has been longer since we last homed a thread there
3203 */
3204 return (1);
3205 }
3206
3207 return (0);
3208 }
3209
3210 /*
3211 * lgrp_trthr_moves counts the number of times main thread (t_tid = 1) of a
3212 * process that uses text replication changed home lgrp. This info is used by
3213 * segvn asyncronous thread to detect if it needs to recheck what lgrps
3214 * should be used for text replication.
3215 */
3216 static uint64_t lgrp_trthr_moves = 0;
3217
3218 uint64_t
3219 lgrp_get_trthr_migrations(void)
3220 {
3221 return (lgrp_trthr_moves);
3222 }
3223
3224 void
3225 lgrp_update_trthr_migrations(uint64_t incr)
3226 {
3227 atomic_add_64(&lgrp_trthr_moves, incr);
3228 }
3229
3230 /*
3231 * An LWP is expected to be assigned to an lgroup for at least this long
3232 * for its anticipatory load to be justified. NOTE that this value should
3233 * not be set extremely huge (say, larger than 100 years), to avoid problems
3234 * with overflow in the calculation that uses it.
3235 */
3236 #define LGRP_MIN_NSEC (NANOSEC / 10) /* 1/10 of a second */
3237 hrtime_t lgrp_min_nsec = LGRP_MIN_NSEC;
3238
3239 /*
3240 * Routine to change a thread's lgroup affiliation. This routine updates
3241 * the thread's kthread_t struct and its process' proc_t struct to note the
3242 * thread's new lgroup affiliation, and its lgroup affinities.
3243 *
3244 * Note that this is the only routine that modifies a thread's t_lpl field,
3245 * and that adds in or removes anticipatory load.
3246 *
3247 * If the thread is exiting, newlpl is NULL.
3248 *
3249 * Locking:
3250 * The following lock must be held on entry:
3251 * cpu_lock, kpreempt_disable(), or thread_lock -- to assure t's new lgrp
3252 * doesn't get removed from t's partition
3253 *
3254 * This routine is not allowed to grab any locks, since it may be called
3255 * with cpus paused (such as from cpu_offline).
3256 */
3257 void
3258 lgrp_move_thread(kthread_t *t, lpl_t *newlpl, int do_lgrpset_delete)
3259 {
3260 proc_t *p;
3261 lpl_t *lpl, *oldlpl;
3262 lgrp_id_t oldid;
3263 kthread_t *tp;
3264 uint_t ncpu;
3265 lgrp_load_t old, new;
3266
3267 ASSERT(t);
3268 ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
3269 THREAD_LOCK_HELD(t));
3270
3271 /*
3272 * If not changing lpls, just return
3273 */
3274 if ((oldlpl = t->t_lpl) == newlpl)
3275 return;
3276
3277 /*
3278 * Make sure the thread's lwp hasn't exited (if so, this thread is now
3279 * associated with process 0 rather than with its original process).
3280 */
3281 if (t->t_proc_flag & TP_LWPEXIT) {
3282 if (newlpl != NULL) {
3283 t->t_lpl = newlpl;
3284 }
3285 return;
3286 }
3287
3288 p = ttoproc(t);
3289
3290 /*
3291 * If the thread had a previous lgroup, update its process' p_lgrpset
3292 * to account for it being moved from its old lgroup.
3293 */
3294 if ((oldlpl != NULL) && /* thread had a previous lgroup */
3295 (p->p_tlist != NULL)) {
3296 oldid = oldlpl->lpl_lgrpid;
3297
3298 if (newlpl != NULL)
3299 lgrp_stat_add(oldid, LGRP_NUM_MIGR, 1);
3300
3301 if ((do_lgrpset_delete) &&
3302 (klgrpset_ismember(p->p_lgrpset, oldid))) {
3303 for (tp = p->p_tlist->t_forw; ; tp = tp->t_forw) {
3304 /*
3305 * Check if a thread other than the thread
3306 * that's moving is assigned to the same
3307 * lgroup as the thread that's moving. Note
3308 * that we have to compare lgroup IDs, rather
3309 * than simply comparing t_lpl's, since the
3310 * threads may belong to different partitions
3311 * but be assigned to the same lgroup.
3312 */
3313 ASSERT(tp->t_lpl != NULL);
3314
3315 if ((tp != t) &&
3316 (tp->t_lpl->lpl_lgrpid == oldid)) {
3317 /*
3318 * Another thread is assigned to the
3319 * same lgroup as the thread that's
3320 * moving, p_lgrpset doesn't change.
3321 */
3322 break;
3323 } else if (tp == p->p_tlist) {
3324 /*
3325 * No other thread is assigned to the
3326 * same lgroup as the exiting thread,
3327 * clear the lgroup's bit in p_lgrpset.
3328 */
3329 klgrpset_del(p->p_lgrpset, oldid);
3330 break;
3331 }
3332 }
3333 }
3334
3335 /*
3336 * If this thread was assigned to its old lgroup for such a
3337 * short amount of time that the anticipatory load that was
3338 * added on its behalf has aged very little, remove that
3339 * anticipatory load.
3340 */
3341 if ((t->t_anttime + lgrp_min_nsec > gethrtime()) &&
3342 ((ncpu = oldlpl->lpl_ncpu) > 0)) {
3343 lpl = oldlpl;
3344 for (;;) {
3345 do {
3346 old = new = lpl->lpl_loadavg;
3347 new -= LGRP_LOADAVG_MAX_EFFECT(ncpu);
3348 if (new > old) {
3349 /*
3350 * this can happen if the load
3351 * average was aged since we
3352 * added in the anticipatory
3353 * load
3354 */
3355 new = 0;
3356 }
3357 } while (atomic_cas_32(
3358 (lgrp_load_t *)&lpl->lpl_loadavg, old,
3359 new) != old);
3360
3361 lpl = lpl->lpl_parent;
3362 if (lpl == NULL)
3363 break;
3364
3365 ncpu = lpl->lpl_ncpu;
3366 ASSERT(ncpu > 0);
3367 }
3368 }
3369 }
3370 /*
3371 * If the thread has a new lgroup (i.e. it's not exiting), update its
3372 * t_lpl and its process' p_lgrpset, and apply an anticipatory load
3373 * to its new lgroup to account for its move to its new lgroup.
3374 */
3375 if (newlpl != NULL) {
3376 /*
3377 * This thread is moving to a new lgroup
3378 */
3379 t->t_lpl = newlpl;
3380 if (t->t_tid == 1 && p->p_t1_lgrpid != newlpl->lpl_lgrpid) {
3381 p->p_t1_lgrpid = newlpl->lpl_lgrpid;
3382 membar_producer();
3383 if (p->p_tr_lgrpid != LGRP_NONE &&
3384 p->p_tr_lgrpid != p->p_t1_lgrpid) {
3385 lgrp_update_trthr_migrations(1);
3386 }
3387 }
3388
3389 /*
3390 * Reflect move in load average of new lgroup
3391 * unless it is root lgroup
3392 */
3393 if (lgrp_table[newlpl->lpl_lgrpid] == lgrp_root)
3394 return;
3395
3396 if (!klgrpset_ismember(p->p_lgrpset, newlpl->lpl_lgrpid)) {
3397 klgrpset_add(p->p_lgrpset, newlpl->lpl_lgrpid);
3398 }
3399
3400 /*
3401 * It'll take some time for the load on the new lgroup
3402 * to reflect this thread's placement on it. We'd
3403 * like not, however, to have all threads between now
3404 * and then also piling on to this lgroup. To avoid
3405 * this pileup, we anticipate the load this thread
3406 * will generate on its new lgroup. The goal is to
3407 * make the lgroup's load appear as though the thread
3408 * had been there all along. We're very conservative
3409 * in calculating this anticipatory load, we assume
3410 * the worst case case (100% CPU-bound thread). This
3411 * may be modified in the future to be more accurate.
3412 */
3413 lpl = newlpl;
3414 for (;;) {
3415 ncpu = lpl->lpl_ncpu;
3416 ASSERT(ncpu > 0);
3417 do {
3418 old = new = lpl->lpl_loadavg;
3419 new += LGRP_LOADAVG_MAX_EFFECT(ncpu);
3420 /*
3421 * Check for overflow
3422 * Underflow not possible here
3423 */
3424 if (new < old)
3425 new = UINT32_MAX;
3426 } while (atomic_cas_32((lgrp_load_t *)&lpl->lpl_loadavg,
3427 old, new) != old);
3428
3429 lpl = lpl->lpl_parent;
3430 if (lpl == NULL)
3431 break;
3432 }
3433 t->t_anttime = gethrtime();
3434 }
3435 }
3436
3437 /*
3438 * Return lgroup memory allocation policy given advice from madvise(3C)
3439 */
3440 lgrp_mem_policy_t
3441 lgrp_madv_to_policy(uchar_t advice, size_t size, int type)
3442 {
3443 switch (advice) {
3444 case MADV_ACCESS_LWP:
3445 return (LGRP_MEM_POLICY_NEXT);
3446 case MADV_ACCESS_MANY:
3447 return (LGRP_MEM_POLICY_RANDOM);
3448 default:
3449 return (lgrp_mem_policy_default(size, type));
3450 }
3451 }
3452
3453 /*
3454 * Figure out default policy
3455 */
3456 lgrp_mem_policy_t
3457 lgrp_mem_policy_default(size_t size, int type)
3458 {
3459 cpupart_t *cp;
3460 lgrp_mem_policy_t policy;
3461 size_t pset_mem_size;
3462
3463 /*
3464 * Randomly allocate memory across lgroups for shared memory
3465 * beyond a certain threshold
3466 */
3467 if ((type != MAP_SHARED && size > lgrp_privm_random_thresh) ||
3468 (type == MAP_SHARED && size > lgrp_shm_random_thresh)) {
3469 /*
3470 * Get total memory size of current thread's pset
3471 */
3472 kpreempt_disable();
3473 cp = curthread->t_cpupart;
3474 klgrpset_totalsize(cp->cp_lgrpset, pset_mem_size);
3475 kpreempt_enable();
3476
3477 /*
3478 * Choose policy to randomly allocate memory across
3479 * lgroups in pset if it will fit and is not default
3480 * partition. Otherwise, allocate memory randomly
3481 * across machine.
3482 */
3483 if (lgrp_mem_pset_aware && size < pset_mem_size)
3484 policy = LGRP_MEM_POLICY_RANDOM_PSET;
3485 else
3486 policy = LGRP_MEM_POLICY_RANDOM;
3487 } else
3488 /*
3489 * Apply default policy for private memory and
3490 * shared memory under the respective random
3491 * threshold.
3492 */
3493 policy = lgrp_mem_default_policy;
3494
3495 return (policy);
3496 }
3497
3498 /*
3499 * Get memory allocation policy for this segment
3500 */
3501 lgrp_mem_policy_info_t *
3502 lgrp_mem_policy_get(struct seg *seg, caddr_t vaddr)
3503 {
3504 lgrp_mem_policy_info_t *policy_info;
3505 extern struct seg_ops segspt_ops;
3506 extern struct seg_ops segspt_shmops;
3507
3508 /*
3509 * This is for binary compatibility to protect against third party
3510 * segment drivers which haven't recompiled to allow for
3511 * SEGOP_GETPOLICY()
3512 */
3513 if (seg->s_ops != &segvn_ops && seg->s_ops != &segspt_ops &&
3514 seg->s_ops != &segspt_shmops)
3515 return (NULL);
3516
3517 policy_info = NULL;
3518 if (seg->s_ops->getpolicy != NULL)
3519 policy_info = SEGOP_GETPOLICY(seg, vaddr);
3520
3521 return (policy_info);
3522 }
3523
3524 /*
3525 * Set policy for allocating private memory given desired policy, policy info,
3526 * size in bytes of memory that policy is being applied.
3527 * Return 0 if policy wasn't set already and 1 if policy was set already
3528 */
3529 int
3530 lgrp_privm_policy_set(lgrp_mem_policy_t policy,
3531 lgrp_mem_policy_info_t *policy_info, size_t size)
3532 {
3533
3534 ASSERT(policy_info != NULL);
3535
3536 if (policy == LGRP_MEM_POLICY_DEFAULT)
3537 policy = lgrp_mem_policy_default(size, MAP_PRIVATE);
3538
3539 /*
3540 * Policy set already?
3541 */
3542 if (policy == policy_info->mem_policy)
3543 return (1);
3544
3545 /*
3546 * Set policy
3547 */
3548 policy_info->mem_policy = policy;
3549 policy_info->mem_lgrpid = LGRP_NONE;
3550
3551 return (0);
3552 }
3553
3554
3555 /*
3556 * Get shared memory allocation policy with given tree and offset
3557 */
3558 lgrp_mem_policy_info_t *
3559 lgrp_shm_policy_get(struct anon_map *amp, ulong_t anon_index, vnode_t *vp,
3560 u_offset_t vn_off)
3561 {
3562 u_offset_t off;
3563 lgrp_mem_policy_info_t *policy_info;
3564 lgrp_shm_policy_seg_t *policy_seg;
3565 lgrp_shm_locality_t *shm_locality;
3566 avl_tree_t *tree;
3567 avl_index_t where;
3568
3569 /*
3570 * Get policy segment tree from anon_map or vnode and use specified
3571 * anon index or vnode offset as offset
3572 *
3573 * Assume that no lock needs to be held on anon_map or vnode, since
3574 * they should be protected by their reference count which must be
3575 * nonzero for an existing segment
3576 */
3577 if (amp) {
3578 ASSERT(amp->refcnt != 0);
3579 shm_locality = amp->locality;
3580 if (shm_locality == NULL)
3581 return (NULL);
3582 tree = shm_locality->loc_tree;
3583 off = ptob(anon_index);
3584 } else if (vp) {
3585 shm_locality = vp->v_locality;
3586 if (shm_locality == NULL)
3587 return (NULL);
3588 ASSERT(shm_locality->loc_count != 0);
3589 tree = shm_locality->loc_tree;
3590 off = vn_off;
3591 }
3592
3593 if (tree == NULL)
3594 return (NULL);
3595
3596 /*
3597 * Lookup policy segment for offset into shared object and return
3598 * policy info
3599 */
3600 rw_enter(&shm_locality->loc_lock, RW_READER);
3601 policy_info = NULL;
3602 policy_seg = avl_find(tree, &off, &where);
3603 if (policy_seg)
3604 policy_info = &policy_seg->shm_policy;
3605 rw_exit(&shm_locality->loc_lock);
3606
3607 return (policy_info);
3608 }
3609
3610 /*
3611 * Default memory allocation policy for kernel segmap pages
3612 */
3613 lgrp_mem_policy_t lgrp_segmap_default_policy = LGRP_MEM_POLICY_RANDOM;
3614
3615 /*
3616 * Return lgroup to use for allocating memory
3617 * given the segment and address
3618 *
3619 * There isn't any mutual exclusion that exists between calls
3620 * to this routine and DR, so this routine and whomever calls it
3621 * should be mindful of the possibility that the lgrp returned
3622 * may be deleted. If this happens, dereferences of the lgrp
3623 * pointer will still be safe, but the resources in the lgrp will
3624 * be gone, and LGRP_EXISTS() will no longer be true.
3625 */
3626 lgrp_t *
3627 lgrp_mem_choose(struct seg *seg, caddr_t vaddr, size_t pgsz)
3628 {
3629 int i;
3630 lgrp_t *lgrp;
3631 klgrpset_t lgrpset;
3632 int lgrps_spanned;
3633 unsigned long off;
3634 lgrp_mem_policy_t policy;
3635 lgrp_mem_policy_info_t *policy_info;
3636 ushort_t random;
3637 int stat = 0;
3638 extern struct seg *segkmap;
3639
3640 /*
3641 * Just return null if the lgrp framework hasn't finished
3642 * initializing or if this is a UMA machine.
3643 */
3644 if (nlgrps == 1 || !lgrp_initialized)
3645 return (lgrp_root);
3646
3647 /*
3648 * Get memory allocation policy for this segment
3649 */
3650 policy = lgrp_mem_default_policy;
3651 if (seg != NULL) {
3652 if (seg->s_as == &kas) {
3653 if (seg == segkmap)
3654 policy = lgrp_segmap_default_policy;
3655 if (policy == LGRP_MEM_POLICY_RANDOM_PROC ||
3656 policy == LGRP_MEM_POLICY_RANDOM_PSET)
3657 policy = LGRP_MEM_POLICY_RANDOM;
3658 } else {
3659 policy_info = lgrp_mem_policy_get(seg, vaddr);
3660 if (policy_info != NULL) {
3661 policy = policy_info->mem_policy;
3662 if (policy == LGRP_MEM_POLICY_NEXT_SEG) {
3663 lgrp_id_t id = policy_info->mem_lgrpid;
3664 ASSERT(id != LGRP_NONE);
3665 ASSERT(id < NLGRPS_MAX);
3666 lgrp = lgrp_table[id];
3667 if (!LGRP_EXISTS(lgrp)) {
3668 policy = LGRP_MEM_POLICY_NEXT;
3669 } else {
3670 lgrp_stat_add(id,
3671 LGRP_NUM_NEXT_SEG, 1);
3672 return (lgrp);
3673 }
3674 }
3675 }
3676 }
3677 }
3678 lgrpset = 0;
3679
3680 /*
3681 * Initialize lgroup to home by default
3682 */
3683 lgrp = lgrp_home_lgrp();
3684
3685 /*
3686 * When homing threads on root lgrp, override default memory
3687 * allocation policies with root lgroup memory allocation policy
3688 */
3689 if (lgrp == lgrp_root)
3690 policy = lgrp_mem_policy_root;
3691
3692 /*
3693 * Implement policy
3694 */
3695 switch (policy) {
3696 case LGRP_MEM_POLICY_NEXT_CPU:
3697
3698 /*
3699 * Return lgroup of current CPU which faulted on memory
3700 * If the CPU isn't currently in an lgrp, then opt to
3701 * allocate from the root.
3702 *
3703 * Kernel preemption needs to be disabled here to prevent
3704 * the current CPU from going away before lgrp is found.
3705 */
3706 if (LGRP_CPU_HAS_NO_LGRP(CPU)) {
3707 lgrp = lgrp_root;
3708 } else {
3709 kpreempt_disable();
3710 lgrp = lgrp_cpu_to_lgrp(CPU);
3711 kpreempt_enable();
3712 }
3713 break;
3714
3715 case LGRP_MEM_POLICY_NEXT:
3716 case LGRP_MEM_POLICY_DEFAULT:
3717 default:
3718
3719 /*
3720 * Just return current thread's home lgroup
3721 * for default policy (next touch)
3722 * If the thread is homed to the root,
3723 * then the default policy is random across lgroups.
3724 * Fallthrough to the random case.
3725 */
3726 if (lgrp != lgrp_root) {
3727 if (policy == LGRP_MEM_POLICY_NEXT)
3728 lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_NEXT, 1);
3729 else
3730 lgrp_stat_add(lgrp->lgrp_id,
3731 LGRP_NUM_DEFAULT, 1);
3732 break;
3733 }
3734 /* FALLTHROUGH */
3735 case LGRP_MEM_POLICY_RANDOM:
3736
3737 /*
3738 * Return a random leaf lgroup with memory
3739 */
3740 lgrpset = lgrp_root->lgrp_set[LGRP_RSRC_MEM];
3741 /*
3742 * Count how many lgroups are spanned
3743 */
3744 klgrpset_nlgrps(lgrpset, lgrps_spanned);
3745
3746 /*
3747 * There may be no memnodes in the root lgroup during DR copy
3748 * rename on a system with only two boards (memnodes)
3749 * configured. In this case just return the root lgrp.
3750 */
3751 if (lgrps_spanned == 0) {
3752 lgrp = lgrp_root;
3753 break;
3754 }
3755
3756 /*
3757 * Pick a random offset within lgroups spanned
3758 * and return lgroup at that offset
3759 */
3760 random = (ushort_t)gethrtime() >> 4;
3761 off = random % lgrps_spanned;
3762 ASSERT(off <= lgrp_alloc_max);
3763
3764 for (i = 0; i <= lgrp_alloc_max; i++) {
3765 if (!klgrpset_ismember(lgrpset, i))
3766 continue;
3767 if (off)
3768 off--;
3769 else {
3770 lgrp = lgrp_table[i];
3771 lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_RANDOM,
3772 1);
3773 break;
3774 }
3775 }
3776 break;
3777
3778 case LGRP_MEM_POLICY_RANDOM_PROC:
3779
3780 /*
3781 * Grab copy of bitmask of lgroups spanned by
3782 * this process
3783 */
3784 klgrpset_copy(lgrpset, curproc->p_lgrpset);
3785 stat = LGRP_NUM_RANDOM_PROC;
3786
3787 /* FALLTHROUGH */
3788 case LGRP_MEM_POLICY_RANDOM_PSET:
3789
3790 if (!stat)
3791 stat = LGRP_NUM_RANDOM_PSET;
3792
3793 if (klgrpset_isempty(lgrpset)) {
3794 /*
3795 * Grab copy of bitmask of lgroups spanned by
3796 * this processor set
3797 */
3798 kpreempt_disable();
3799 klgrpset_copy(lgrpset,
3800 curthread->t_cpupart->cp_lgrpset);
3801 kpreempt_enable();
3802 }
3803
3804 /*
3805 * Count how many lgroups are spanned
3806 */
3807 klgrpset_nlgrps(lgrpset, lgrps_spanned);
3808 ASSERT(lgrps_spanned <= nlgrps);
3809
3810 /*
3811 * Probably lgrps_spanned should be always non-zero, but to be
3812 * on the safe side we return lgrp_root if it is empty.
3813 */
3814 if (lgrps_spanned == 0) {
3815 lgrp = lgrp_root;
3816 break;
3817 }
3818
3819 /*
3820 * Pick a random offset within lgroups spanned
3821 * and return lgroup at that offset
3822 */
3823 random = (ushort_t)gethrtime() >> 4;
3824 off = random % lgrps_spanned;
3825 ASSERT(off <= lgrp_alloc_max);
3826
3827 for (i = 0; i <= lgrp_alloc_max; i++) {
3828 if (!klgrpset_ismember(lgrpset, i))
3829 continue;
3830 if (off)
3831 off--;
3832 else {
3833 lgrp = lgrp_table[i];
3834 lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_RANDOM,
3835 1);
3836 break;
3837 }
3838 }
3839 break;
3840
3841 case LGRP_MEM_POLICY_ROUNDROBIN:
3842
3843 /*
3844 * Use offset within segment to determine
3845 * offset from home lgroup to choose for
3846 * next lgroup to allocate memory from
3847 */
3848 off = ((unsigned long)(vaddr - seg->s_base) / pgsz) %
3849 (lgrp_alloc_max + 1);
3850
3851 kpreempt_disable();
3852 lgrpset = lgrp_root->lgrp_set[LGRP_RSRC_MEM];
3853 i = lgrp->lgrp_id;
3854 kpreempt_enable();
3855
3856 while (off > 0) {
3857 i = (i + 1) % (lgrp_alloc_max + 1);
3858 lgrp = lgrp_table[i];
3859 if (klgrpset_ismember(lgrpset, i))
3860 off--;
3861 }
3862 lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ROUNDROBIN, 1);
3863
3864 break;
3865 }
3866
3867 ASSERT(lgrp != NULL);
3868 return (lgrp);
3869 }
3870
3871 /*
3872 * Return the number of pages in an lgroup
3873 *
3874 * NOTE: NUMA test (numat) driver uses this, so changing arguments or semantics
3875 * could cause tests that rely on the numat driver to fail....
3876 */
3877 pgcnt_t
3878 lgrp_mem_size(lgrp_id_t lgrpid, lgrp_mem_query_t query)
3879 {
3880 lgrp_t *lgrp;
3881
3882 lgrp = lgrp_table[lgrpid];
3883 if (!LGRP_EXISTS(lgrp) ||
3884 klgrpset_isempty(lgrp->lgrp_set[LGRP_RSRC_MEM]) ||
3885 !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid))
3886 return (0);
3887
3888 return (lgrp_plat_mem_size(lgrp->lgrp_plathand, query));
3889 }
3890
3891 /*
3892 * Initialize lgroup shared memory allocation policy support
3893 */
3894 void
3895 lgrp_shm_policy_init(struct anon_map *amp, vnode_t *vp)
3896 {
3897 lgrp_shm_locality_t *shm_locality;
3898
3899 /*
3900 * Initialize locality field in anon_map
3901 * Don't need any locks because this is called when anon_map is
3902 * allocated, but not used anywhere yet.
3903 */
3904 if (amp) {
3905 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
3906 if (amp->locality == NULL) {
3907 /*
3908 * Allocate and initialize shared memory locality info
3909 * and set anon_map locality pointer to it
3910 * Drop lock across kmem_alloc(KM_SLEEP)
3911 */
3912 ANON_LOCK_EXIT(&->a_rwlock);
3913 shm_locality = kmem_alloc(sizeof (*shm_locality),
3914 KM_SLEEP);
3915 rw_init(&shm_locality->loc_lock, NULL, RW_DEFAULT,
3916 NULL);
3917 shm_locality->loc_count = 1; /* not used for amp */
3918 shm_locality->loc_tree = NULL;
3919
3920 /*
3921 * Reacquire lock and check to see whether anyone beat
3922 * us to initializing the locality info
3923 */
3924 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
3925 if (amp->locality != NULL) {
3926 rw_destroy(&shm_locality->loc_lock);
3927 kmem_free(shm_locality,
3928 sizeof (*shm_locality));
3929 } else
3930 amp->locality = shm_locality;
3931 }
3932 ANON_LOCK_EXIT(&->a_rwlock);
3933 return;
3934 }
3935
3936 /*
3937 * Allocate shared vnode policy info if vnode is not locality aware yet
3938 */
3939 mutex_enter(&vp->v_lock);
3940 if ((vp->v_flag & V_LOCALITY) == 0) {
3941 /*
3942 * Allocate and initialize shared memory locality info
3943 */
3944 mutex_exit(&vp->v_lock);
3945 shm_locality = kmem_alloc(sizeof (*shm_locality), KM_SLEEP);
3946 rw_init(&shm_locality->loc_lock, NULL, RW_DEFAULT, NULL);
3947 shm_locality->loc_count = 1;
3948 shm_locality->loc_tree = NULL;
3949
3950 /*
3951 * Point vnode locality field at shared vnode policy info
3952 * and set locality aware flag in vnode
3953 */
3954 mutex_enter(&vp->v_lock);
3955 if ((vp->v_flag & V_LOCALITY) == 0) {
3956 vp->v_locality = shm_locality;
3957 vp->v_flag |= V_LOCALITY;
3958 } else {
3959 /*
3960 * Lost race so free locality info and increment count.
3961 */
3962 rw_destroy(&shm_locality->loc_lock);
3963 kmem_free(shm_locality, sizeof (*shm_locality));
3964 shm_locality = vp->v_locality;
3965 shm_locality->loc_count++;
3966 }
3967 mutex_exit(&vp->v_lock);
3968
3969 return;
3970 }
3971
3972 /*
3973 * Increment reference count of number of segments mapping this vnode
3974 * shared
3975 */
3976 shm_locality = vp->v_locality;
3977 shm_locality->loc_count++;
3978 mutex_exit(&vp->v_lock);
3979 }
3980
3981 /*
3982 * Destroy the given shared memory policy segment tree
3983 */
3984 void
3985 lgrp_shm_policy_tree_destroy(avl_tree_t *tree)
3986 {
3987 lgrp_shm_policy_seg_t *cur;
3988 lgrp_shm_policy_seg_t *next;
3989
3990 if (tree == NULL)
3991 return;
3992
3993 cur = (lgrp_shm_policy_seg_t *)avl_first(tree);
3994 while (cur != NULL) {
3995 next = AVL_NEXT(tree, cur);
3996 avl_remove(tree, cur);
3997 kmem_free(cur, sizeof (*cur));
3998 cur = next;
3999 }
4000 kmem_free(tree, sizeof (avl_tree_t));
4001 }
4002
4003 /*
4004 * Uninitialize lgroup shared memory allocation policy support
4005 */
4006 void
4007 lgrp_shm_policy_fini(struct anon_map *amp, vnode_t *vp)
4008 {
4009 lgrp_shm_locality_t *shm_locality;
4010
4011 /*
4012 * For anon_map, deallocate shared memory policy tree and
4013 * zero locality field
4014 * Don't need any locks because anon_map is being freed
4015 */
4016 if (amp) {
4017 if (amp->locality == NULL)
4018 return;
4019 shm_locality = amp->locality;
4020 shm_locality->loc_count = 0; /* not really used for amp */
4021 rw_destroy(&shm_locality->loc_lock);
4022 lgrp_shm_policy_tree_destroy(shm_locality->loc_tree);
4023 kmem_free(shm_locality, sizeof (*shm_locality));
4024 amp->locality = 0;
4025 return;
4026 }
4027
4028 /*
4029 * For vnode, decrement reference count of segments mapping this vnode
4030 * shared and delete locality info if reference count drops to 0
4031 */
4032 mutex_enter(&vp->v_lock);
4033 shm_locality = vp->v_locality;
4034 shm_locality->loc_count--;
4035
4036 if (shm_locality->loc_count == 0) {
4037 rw_destroy(&shm_locality->loc_lock);
4038 lgrp_shm_policy_tree_destroy(shm_locality->loc_tree);
4039 kmem_free(shm_locality, sizeof (*shm_locality));
4040 vp->v_locality = 0;
4041 vp->v_flag &= ~V_LOCALITY;
4042 }
4043 mutex_exit(&vp->v_lock);
4044 }
4045
4046 /*
4047 * Compare two shared memory policy segments
4048 * Used by AVL tree code for searching
4049 */
4050 int
4051 lgrp_shm_policy_compar(const void *x, const void *y)
4052 {
4053 lgrp_shm_policy_seg_t *a = (lgrp_shm_policy_seg_t *)x;
4054 lgrp_shm_policy_seg_t *b = (lgrp_shm_policy_seg_t *)y;
4055
4056 if (a->shm_off < b->shm_off)
4057 return (-1);
4058 if (a->shm_off >= b->shm_off + b->shm_size)
4059 return (1);
4060 return (0);
4061 }
4062
4063 /*
4064 * Concatenate seg1 with seg2 and remove seg2
4065 */
4066 static int
4067 lgrp_shm_policy_concat(avl_tree_t *tree, lgrp_shm_policy_seg_t *seg1,
4068 lgrp_shm_policy_seg_t *seg2)
4069 {
4070 if (!seg1 || !seg2 ||
4071 seg1->shm_off + seg1->shm_size != seg2->shm_off ||
4072 seg1->shm_policy.mem_policy != seg2->shm_policy.mem_policy)
4073 return (-1);
4074
4075 seg1->shm_size += seg2->shm_size;
4076 avl_remove(tree, seg2);
4077 kmem_free(seg2, sizeof (*seg2));
4078 return (0);
4079 }
4080
4081 /*
4082 * Split segment at given offset and return rightmost (uppermost) segment
4083 * Assumes that there are no overlapping segments
4084 */
4085 static lgrp_shm_policy_seg_t *
4086 lgrp_shm_policy_split(avl_tree_t *tree, lgrp_shm_policy_seg_t *seg,
4087 u_offset_t off)
4088 {
4089 lgrp_shm_policy_seg_t *newseg;
4090 avl_index_t where;
4091
4092 ASSERT(seg != NULL);
4093 ASSERT(off >= seg->shm_off && off <= seg->shm_off + seg->shm_size);
4094
4095 if (!seg || off < seg->shm_off || off > seg->shm_off +
4096 seg->shm_size)
4097 return (NULL);
4098
4099 if (off == seg->shm_off || off == seg->shm_off + seg->shm_size)
4100 return (seg);
4101
4102 /*
4103 * Adjust size of left segment and allocate new (right) segment
4104 */
4105 newseg = kmem_alloc(sizeof (lgrp_shm_policy_seg_t), KM_SLEEP);
4106 newseg->shm_policy = seg->shm_policy;
4107 newseg->shm_off = off;
4108 newseg->shm_size = seg->shm_size - (off - seg->shm_off);
4109 seg->shm_size = off - seg->shm_off;
4110
4111 /*
4112 * Find where to insert new segment in AVL tree and insert it
4113 */
4114 (void) avl_find(tree, &off, &where);
4115 avl_insert(tree, newseg, where);
4116
4117 return (newseg);
4118 }
4119
4120 /*
4121 * Set shared memory allocation policy on specified shared object at given
4122 * offset and length
4123 *
4124 * Return 0 if policy wasn't set already, 1 if policy was set already, and
4125 * -1 if can't set policy.
4126 */
4127 int
4128 lgrp_shm_policy_set(lgrp_mem_policy_t policy, struct anon_map *amp,
4129 ulong_t anon_index, vnode_t *vp, u_offset_t vn_off, size_t len)
4130 {
4131 u_offset_t eoff;
4132 lgrp_shm_policy_seg_t *next;
4133 lgrp_shm_policy_seg_t *newseg;
4134 u_offset_t off;
4135 u_offset_t oldeoff;
4136 lgrp_shm_policy_seg_t *prev;
4137 int retval;
4138 lgrp_shm_policy_seg_t *seg;
4139 lgrp_shm_locality_t *shm_locality;
4140 avl_tree_t *tree;
4141 avl_index_t where;
4142
4143 ASSERT(amp || vp);
4144 ASSERT((len & PAGEOFFSET) == 0);
4145
4146 if (len == 0)
4147 return (-1);
4148
4149 retval = 0;
4150
4151 /*
4152 * Get locality info and starting offset into shared object
4153 * Try anon map first and then vnode
4154 * Assume that no locks need to be held on anon_map or vnode, since
4155 * it should be protected by its reference count which must be nonzero
4156 * for an existing segment.
4157 */
4158 if (amp) {
4159 /*
4160 * Get policy info from anon_map
4161 *
4162 */
4163 ASSERT(amp->refcnt != 0);
4164 if (amp->locality == NULL)
4165 lgrp_shm_policy_init(amp, NULL);
4166 shm_locality = amp->locality;
4167 off = ptob(anon_index);
4168 } else if (vp) {
4169 /*
4170 * Get policy info from vnode
4171 */
4172 if ((vp->v_flag & V_LOCALITY) == 0 || vp->v_locality == NULL)
4173 lgrp_shm_policy_init(NULL, vp);
4174 shm_locality = vp->v_locality;
4175 ASSERT(shm_locality->loc_count != 0);
4176 off = vn_off;
4177 } else
4178 return (-1);
4179
4180 ASSERT((off & PAGEOFFSET) == 0);
4181
4182 /*
4183 * Figure out default policy
4184 */
4185 if (policy == LGRP_MEM_POLICY_DEFAULT)
4186 policy = lgrp_mem_policy_default(len, MAP_SHARED);
4187
4188 /*
4189 * Create AVL tree if there isn't one yet
4190 * and set locality field to point at it
4191 */
4192 rw_enter(&shm_locality->loc_lock, RW_WRITER);
4193 tree = shm_locality->loc_tree;
4194 if (!tree) {
4195 rw_exit(&shm_locality->loc_lock);
4196
4197 tree = kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
4198
4199 rw_enter(&shm_locality->loc_lock, RW_WRITER);
4200 if (shm_locality->loc_tree == NULL) {
4201 avl_create(tree, lgrp_shm_policy_compar,
4202 sizeof (lgrp_shm_policy_seg_t),
4203 offsetof(lgrp_shm_policy_seg_t, shm_tree));
4204 shm_locality->loc_tree = tree;
4205 } else {
4206 /*
4207 * Another thread managed to set up the tree
4208 * before we could. Free the tree we allocated
4209 * and use the one that's already there.
4210 */
4211 kmem_free(tree, sizeof (*tree));
4212 tree = shm_locality->loc_tree;
4213 }
4214 }
4215
4216 /*
4217 * Set policy
4218 *
4219 * Need to maintain hold on writer's lock to keep tree from
4220 * changing out from under us
4221 */
4222 while (len != 0) {
4223 /*
4224 * Find policy segment for specified offset into shared object
4225 */
4226 seg = avl_find(tree, &off, &where);
4227
4228 /*
4229 * Didn't find any existing segment that contains specified
4230 * offset, so allocate new segment, insert it, and concatenate
4231 * with adjacent segments if possible
4232 */
4233 if (seg == NULL) {
4234 newseg = kmem_alloc(sizeof (lgrp_shm_policy_seg_t),
4235 KM_SLEEP);
4236 newseg->shm_policy.mem_policy = policy;
4237 newseg->shm_policy.mem_lgrpid = LGRP_NONE;
4238 newseg->shm_off = off;
4239 avl_insert(tree, newseg, where);
4240
4241 /*
4242 * Check to see whether new segment overlaps with next
4243 * one, set length of new segment accordingly, and
4244 * calculate remaining length and next offset
4245 */
4246 seg = AVL_NEXT(tree, newseg);
4247 if (seg == NULL || off + len <= seg->shm_off) {
4248 newseg->shm_size = len;
4249 len = 0;
4250 } else {
4251 newseg->shm_size = seg->shm_off - off;
4252 off = seg->shm_off;
4253 len -= newseg->shm_size;
4254 }
4255
4256 /*
4257 * Try to concatenate new segment with next and
4258 * previous ones, since they might have the same policy
4259 * now. Grab previous and next segments first because
4260 * they will change on concatenation.
4261 */
4262 prev = AVL_PREV(tree, newseg);
4263 next = AVL_NEXT(tree, newseg);
4264 (void) lgrp_shm_policy_concat(tree, newseg, next);
4265 (void) lgrp_shm_policy_concat(tree, prev, newseg);
4266
4267 continue;
4268 }
4269
4270 eoff = off + len;
4271 oldeoff = seg->shm_off + seg->shm_size;
4272
4273 /*
4274 * Policy set already?
4275 */
4276 if (policy == seg->shm_policy.mem_policy) {
4277 /*
4278 * Nothing left to do if offset and length
4279 * fall within this segment
4280 */
4281 if (eoff <= oldeoff) {
4282 retval = 1;
4283 break;
4284 } else {
4285 len = eoff - oldeoff;
4286 off = oldeoff;
4287 continue;
4288 }
4289 }
4290
4291 /*
4292 * Specified offset and length match existing segment exactly
4293 */
4294 if (off == seg->shm_off && len == seg->shm_size) {
4295 /*
4296 * Set policy and update current length
4297 */
4298 seg->shm_policy.mem_policy = policy;
4299 seg->shm_policy.mem_lgrpid = LGRP_NONE;
4300 len = 0;
4301
4302 /*
4303 * Try concatenating new segment with previous and next
4304 * segments, since they might have the same policy now.
4305 * Grab previous and next segments first because they
4306 * will change on concatenation.
4307 */
4308 prev = AVL_PREV(tree, seg);
4309 next = AVL_NEXT(tree, seg);
4310 (void) lgrp_shm_policy_concat(tree, seg, next);
4311 (void) lgrp_shm_policy_concat(tree, prev, seg);
4312 } else {
4313 /*
4314 * Specified offset and length only apply to part of
4315 * existing segment
4316 */
4317
4318 /*
4319 * New segment starts in middle of old one, so split
4320 * new one off near beginning of old one
4321 */
4322 newseg = NULL;
4323 if (off > seg->shm_off) {
4324 newseg = lgrp_shm_policy_split(tree, seg, off);
4325
4326 /*
4327 * New segment ends where old one did, so try
4328 * to concatenate with next segment
4329 */
4330 if (eoff == oldeoff) {
4331 newseg->shm_policy.mem_policy = policy;
4332 newseg->shm_policy.mem_lgrpid =
4333 LGRP_NONE;
4334 (void) lgrp_shm_policy_concat(tree,
4335 newseg, AVL_NEXT(tree, newseg));
4336 break;
4337 }
4338 }
4339
4340 /*
4341 * New segment ends before old one, so split off end of
4342 * old one
4343 */
4344 if (eoff < oldeoff) {
4345 if (newseg) {
4346 (void) lgrp_shm_policy_split(tree,
4347 newseg, eoff);
4348 newseg->shm_policy.mem_policy = policy;
4349 newseg->shm_policy.mem_lgrpid =
4350 LGRP_NONE;
4351 } else {
4352 (void) lgrp_shm_policy_split(tree, seg,
4353 eoff);
4354 seg->shm_policy.mem_policy = policy;
4355 seg->shm_policy.mem_lgrpid = LGRP_NONE;
4356 }
4357
4358 if (off == seg->shm_off)
4359 (void) lgrp_shm_policy_concat(tree,
4360 AVL_PREV(tree, seg), seg);
4361 break;
4362 }
4363
4364 /*
4365 * Calculate remaining length and next offset
4366 */
4367 len = eoff - oldeoff;
4368 off = oldeoff;
4369 }
4370 }
4371
4372 rw_exit(&shm_locality->loc_lock);
4373 return (retval);
4374 }
4375
4376 /*
4377 * Return the best memnode from which to allocate memory given
4378 * an lgroup.
4379 *
4380 * "c" is for cookie, which is good enough for me.
4381 * It references a cookie struct that should be zero'ed to initialize.
4382 * The cookie should live on the caller's stack.
4383 *
4384 * The routine returns -1 when:
4385 * - traverse is 0, and all the memnodes in "lgrp" have been returned.
4386 * - traverse is 1, and all the memnodes in the system have been
4387 * returned.
4388 */
4389 int
4390 lgrp_memnode_choose(lgrp_mnode_cookie_t *c)
4391 {
4392 lgrp_t *lp = c->lmc_lgrp;
4393 mnodeset_t nodes = c->lmc_nodes;
4394 int cnt = c->lmc_cnt;
4395 int offset, mnode;
4396
4397 extern int max_mem_nodes;
4398
4399 /*
4400 * If the set is empty, and the caller is willing, traverse
4401 * up the hierarchy until we find a non-empty set.
4402 */
4403 while (nodes == (mnodeset_t)0 || cnt <= 0) {
4404 if (c->lmc_scope == LGRP_SRCH_LOCAL ||
4405 ((lp = lp->lgrp_parent) == NULL))
4406 return (-1);
4407
4408 nodes = lp->lgrp_mnodes & ~(c->lmc_tried);
4409 cnt = lp->lgrp_nmnodes - c->lmc_ntried;
4410 }
4411
4412 /*
4413 * Select a memnode by picking one at a "random" offset.
4414 * Because of DR, memnodes can come and go at any time.
4415 * This code must be able to cope with the possibility
4416 * that the nodes count "cnt" is inconsistent with respect
4417 * to the number of elements actually in "nodes", and
4418 * therefore that the offset chosen could be greater than
4419 * the number of elements in the set (some memnodes may
4420 * have dissapeared just before cnt was read).
4421 * If this happens, the search simply wraps back to the
4422 * beginning of the set.
4423 */
4424 ASSERT(nodes != (mnodeset_t)0 && cnt > 0);
4425 offset = c->lmc_rand % cnt;
4426 do {
4427 for (mnode = 0; mnode < max_mem_nodes; mnode++)
4428 if (nodes & ((mnodeset_t)1 << mnode))
4429 if (!offset--)
4430 break;
4431 } while (mnode >= max_mem_nodes);
4432
4433 /* Found a node. Store state before returning. */
4434 c->lmc_lgrp = lp;
4435 c->lmc_nodes = (nodes & ~((mnodeset_t)1 << mnode));
4436 c->lmc_cnt = cnt - 1;
4437 c->lmc_tried = (c->lmc_tried | ((mnodeset_t)1 << mnode));
4438 c->lmc_ntried++;
4439
4440 return (mnode);
4441 }