Print this page
8115 parallel zfs mount
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/sys/lgrp.h
+++ new/usr/src/uts/common/sys/lgrp.h
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 + *
26 + * Copyright 2017 RackTop Systems.
25 27 */
26 28
27 29 #ifndef _LGRP_H
28 30 #define _LGRP_H
29 31
30 32 /*
31 33 * locality group definitions for kernel
32 34 */
33 35
34 36 #include <sys/types.h>
35 37
36 38 #ifdef __cplusplus
37 39 extern "C" {
38 40 #endif
39 41
40 42 #define LGRP_NONE (-1) /* non-existent lgroup ID */
41 43
42 -#if (!defined(_KERNEL) && !defined(_KMEMUSER))
44 +#if !defined(_KERNEL) && !defined(_FAKE_KERNEL) && !defined(_KMEMUSER)
43 45 typedef struct lgrp_mem_policy_info { int opaque[2]; } lgrp_mem_policy_info_t;
44 -#endif /* !_KERNEL && !_KMEMUSER */
46 +#endif /* !_KERNEL && !_FAKE_KERNEL && !_KMEMUSER */
45 47
46 -#if (defined(_KERNEL) || defined(_KMEMUSER))
48 +#if defined(_KERNEL) || defined(_FAKE_KERNEL) || defined(_KMEMUSER)
47 49 #include <sys/cpuvar.h>
48 50 #include <sys/bitmap.h>
49 51 #include <sys/vnode.h>
50 52 #include <vm/anon.h>
51 53 #include <vm/seg.h>
52 54 #include <sys/lgrp_user.h>
53 55 #include <sys/param.h>
54 56
55 57 typedef uint32_t lgrp_load_t; /* lgrp_loadavg type */
56 58 typedef uintptr_t lgrp_handle_t; /* lgrp handle */
57 59
58 60 #define LGRP_NONE_SUCH LGRP_NONE /* non-existent lgroup ID */
59 61 /* null platform handle */
60 62 #define LGRP_NULL_HANDLE ((lgrp_handle_t)0xbadbad)
61 63 #define LGRP_DEFAULT_HANDLE ((lgrp_handle_t)0xbabecafe) /* uma handle */
62 64 #define LGRP_ROOTID (0) /* root lgroup ID */
63 65
64 66 /*
65 67 * Maximum number of lgrps a platform may define.
66 68 */
67 69 #define NLGRPS_MAX 64
68 70 #define LGRP_LOADAVG_MAX UINT32_MAX
69 71
70 72 /*
71 73 * The load-average we expect for one cpu-bound thread's worth of load
72 74 */
73 75 #define LGRP_LOADAVG_THREAD_MAX 65516
74 76
75 77 /*
76 78 * The input to the load-average generating function for one cpu-bound thread's
77 79 * worth of load
78 80 */
79 81
80 82 #define LGRP_LOADAVG_IN_THREAD_MAX 128
81 83
82 84 /*
83 85 * LPL actions
84 86 */
85 87
86 88 typedef enum {
87 89 LPL_INCREMENT,
88 90 LPL_DECREMENT
89 91 } lpl_act_t;
90 92
91 93 /*
92 94 * lgroup statistics. Most of these are counters that are updated
93 95 * dynamically so they are hashed to CPU buckets to reduce cache
94 96 * interference. The remaining statistics are snapshots of kernel
95 97 * data, so they aren't stored in the array of counter stats.
96 98 *
97 99 * For the hashed stats to make sense, you have to sum all the buckets for
98 100 * that stat, hence macros are provided to read the stats.
99 101 */
100 102
101 103 #define LGRP_NUM_CPU_BUCKETS 8 /* must be power of 2 */
102 104 #define LGRP_CPU_BUCKET_MASK (LGRP_NUM_CPU_BUCKETS - 1)
103 105
104 106 /*
105 107 * Flags for what to do with lgroup memory policy
106 108 * Used for heap and stack where policy is extended to new segments added to
107 109 * the end
108 110 */
109 111 #define LGRP_MP_FLAG_EXTEND_UP 0x1 /* policy should extend up */
110 112 #define LGRP_MP_FLAG_EXTEND_DOWN 0x2 /* policy should extend down */
111 113
112 114 #define LGRP_STAT(stats, bucket, whichstat) \
113 115 ((stats)->ls_data[bucket][whichstat])
114 116
115 117 /* Return a pointer suitable for an atomic 64-bit op on the bucket */
116 118 #define LGRP_STAT_WRITE_PTR(stats, whichstat) \
117 119 (&LGRP_STAT(stats, (CPU->cpu_id) & LGRP_CPU_BUCKET_MASK, \
118 120 whichstat))
119 121
120 122 /* Sum up all the buckets and return the value in 'val' */
121 123 #define LGRP_STAT_READ(stats, whichstat, val) { \
122 124 int bkt; \
123 125 for (val = 0, bkt = 0; bkt < LGRP_NUM_CPU_BUCKETS; bkt++) \
124 126 val += LGRP_STAT(stats, bkt, whichstat); \
125 127 }
126 128
127 129 /* Reset all buckets for the stat to 0 */
128 130 #define LGRP_STAT_RESET(stats, stat) { \
129 131 int i; \
130 132 for (i = 0; i < LGRP_NUM_CPU_BUCKETS; i++) \
131 133 LGRP_STAT(stats, i, stat) = 0; \
132 134 }
133 135
134 136 /*
135 137 * Define all of the statistics that are kept for lgrp kstats,
136 138 * and their corresponding text names.
137 139 */
138 140
139 141 typedef enum lgrp_stat_types {
140 142 LGRP_NUM_MIGR, /* # migrations away from this lgrp */
141 143 LGRP_NUM_ALLOC_FAIL, /* # times alloc fails for chosen lgrp */
142 144 LGRP_PM_SRC_PGS, /* # pages migrated from this lgrp */
143 145 LGRP_PM_DEST_PGS, /* # pages migrated to this lgrp */
144 146 LGRP_PM_FAIL_ALLOC_PGS, /* # pages failed to migrate to this lgrp */
145 147 LGRP_PM_FAIL_LOCK_PGS, /* # pages failed to migrate from this lgrp */
146 148 LGRP_PMM_PGS, /* # pages marked to migrate from this lgrp */
147 149 LGRP_PMM_FAIL_PGS, /* # pages marked to migrate from this lgrp */
148 150 LGRP_NUM_DEFAULT, /* # of times default policy applied */
149 151 LGRP_NUM_NEXT, /* # of times next touch policy applied */
150 152 LGRP_NUM_RANDOM, /* # of times random policy applied */
151 153 LGRP_NUM_RANDOM_PROC, /* # of times random proc policy applied */
152 154 LGRP_NUM_RANDOM_PSET, /* # of times random pset policy applied */
153 155 LGRP_NUM_ROUNDROBIN, /* # of times round robin policy applied */
154 156 LGRP_NUM_NEXT_SEG, /* # of times next to seg policy applied */
155 157 LGRP_NUM_COUNTER_STATS, /* always last */
156 158 LGRP_CTR_STATS_ALLOC = 16 /* cache-align pad - multiple of 8 */
157 159 /* always keep >= LGRP_NUM_COUNTER_STATS */
158 160 } lgrp_stat_t;
159 161
160 162 typedef enum lgrp_snap_stat_types {
161 163 LGRP_NUM_CPUS, /* number of CPUs */
162 164 LGRP_NUM_PG_FREE, /* # of free pages */
163 165 LGRP_NUM_PG_AVAIL, /* # of allocatable physical pages */
164 166 LGRP_NUM_PG_INSTALL, /* # of installed physical pages */
165 167 LGRP_LOADAVG, /* unscaled load average of this lgrp */
166 168 LGRP_LOADAVG_SCALE, /* load unit of one CPU bound thread */
167 169 LGRP_NUM_SNAPSHOT_STATS /* always last */
168 170 } lgrp_snap_stat_t;
169 171
170 172 #define LGRP_KSTAT_NAMES \
171 173 static char *lgrp_kstat_names[] = { \
172 174 \
173 175 /* Counter stats */ \
174 176 "lwp migrations", \
175 177 "alloc fail", \
176 178 "pages migrated from", \
177 179 "pages migrated to", \
178 180 "pages failed to migrate to", \
179 181 "pages failed to migrate from", \
180 182 "pages marked for migration", \
181 183 "pages failed to mark", \
182 184 "default policy", \
183 185 "next-touch policy", \
184 186 "random policy", \
185 187 "span process policy", \
186 188 "span psrset policy", \
187 189 "round robin policy", \
188 190 "next-seg policy", \
189 191 \
190 192 /* Snapshot stats */ \
191 193 "cpus", \
192 194 "pages free", \
193 195 "pages avail", \
194 196 "pages installed", \
195 197 "load average", \
196 198 "loadscale" \
197 199 }
198 200
199 201 #define LGRP_NUM_STATS ((int)LGRP_NUM_COUNTER_STATS + \
200 202 (int)LGRP_NUM_SNAPSHOT_STATS)
201 203
202 204 /*
203 205 * The contents of this structure are opaque and should only be
204 206 * accessed through the LGRP_STAT macro.
205 207 */
206 208 struct lgrp_stats {
207 209 int64_t ls_data[LGRP_NUM_CPU_BUCKETS][LGRP_CTR_STATS_ALLOC];
208 210 };
209 211
210 212 /* The kernel's version of a bitmap of lgroups */
211 213 typedef uint64_t klgrpset_t;
212 214
213 215 /*
214 216 * This really belongs in memnode.h, but it must be defined here to avoid
215 217 * recursive inclusion problems. Note that memnode.h includes this header.
216 218 */
217 219 typedef uint64_t mnodeset_t;
218 220
219 221 /*
220 222 * lgroup structure
221 223 *
222 224 * Visible to generic code and contains the lgroup ID, CPUs in this lgroup,
223 225 * and a platform handle used to identify this lgroup to the lgroup platform
224 226 * support code
225 227 */
226 228 typedef struct lgrp {
227 229
228 230 lgrp_id_t lgrp_id; /* which lgroup */
229 231 int lgrp_latency;
230 232 lgrp_handle_t lgrp_plathand; /* handle for platform calls */
231 233 struct lgrp *lgrp_parent; /* parent lgroup */
232 234 uint_t lgrp_reserved1; /* filler */
233 235 uint_t lgrp_childcnt; /* number of children lgroups */
234 236 klgrpset_t lgrp_children; /* children lgroups */
235 237 klgrpset_t lgrp_leaves; /* (direct decendant) leaf lgroups */
236 238
237 239 /*
238 240 * set of lgroups containing a given type of resource
239 241 * at this level of locality
240 242 */
241 243 klgrpset_t lgrp_set[LGRP_RSRC_COUNT];
242 244
243 245 mnodeset_t lgrp_mnodes; /* set of memory nodes in this lgroup */
244 246 uint_t lgrp_nmnodes; /* number of memnodes */
245 247 uint_t lgrp_reserved2; /* filler */
246 248
247 249 struct cpu *lgrp_cpu; /* pointer to a cpu may be null */
248 250 uint_t lgrp_cpucnt; /* number of cpus in this lgrp */
249 251 kstat_t *lgrp_kstat; /* per-lgrp kstats */
250 252 } lgrp_t;
251 253
252 254 /*
253 255 * lgroup load average structure
254 256 */
255 257
256 258 typedef struct lgrp_ld {
257 259 lgrp_load_t lpl_loadavg; /* load average */
258 260 uint_t lpl_ncpu; /* how many cpus */
259 261 lgrp_id_t lpl_lgrpid; /* which group this lpl part of */
260 262 lgrp_t *lpl_lgrp; /* ptr to lpl's lgrp */
261 263 struct lgrp_ld *lpl_parent; /* lpl of parent lgrp */
262 264 struct cpu *lpl_cpus; /* list of cpus in lpl */
263 265 /* NULL for non-leaf lgrps */
264 266 uint_t lpl_nrset; /* no. of leaf lpls for lgrp */
265 267 hrtime_t lpl_homed_time; /* time of last homing to this lpl */
266 268 uint_t lpl_rset_sz; /* Resource set capacity */
267 269 struct lgrp_ld **lpl_rset; /* leaf lpls for lgrp */
268 270 /* contains ptr to self for leaf lgrp */
269 271 int *lpl_id2rset; /* mapping of lgrpid to rset index */
270 272 } lpl_t;
271 273
272 274 /*
273 275 * 1 << LGRP_MAX_EFFECT_SHFT == lgrp_loadavg_max_effect
274 276 */
275 277 #define LGRP_MAX_EFFECT_SHFT 16
276 278
277 279 /*
278 280 * Operations handled by lgrp_config()
279 281 */
280 282 typedef enum lgrp_config_flag {
281 283 LGRP_CONFIG_NOP,
282 284 LGRP_CONFIG_CPU_ADD,
283 285 LGRP_CONFIG_CPU_DEL,
284 286 LGRP_CONFIG_CPU_ONLINE,
285 287 LGRP_CONFIG_CPU_OFFLINE,
286 288 LGRP_CONFIG_CPUPART_ADD,
287 289 LGRP_CONFIG_CPUPART_DEL,
288 290 LGRP_CONFIG_MEM_ADD,
289 291 LGRP_CONFIG_MEM_DEL,
290 292 LGRP_CONFIG_MEM_RENAME,
291 293 LGRP_CONFIG_GEN_UPDATE,
292 294 LGRP_CONFIG_FLATTEN,
293 295 LGRP_CONFIG_LAT_CHANGE_ALL,
294 296 LGRP_CONFIG_LAT_CHANGE
295 297 } lgrp_config_flag_t;
296 298
297 299 /*
298 300 * Stages of lgroup framework initialization (done through lgrp_init()):
299 301 *
300 302 * 1) Initialize common and platform specific code (called in mlsetup())
301 303 *
302 304 * 2) Setup root lgroup and add CPU 0 to lgroup(s) (called near beginning of
303 305 * main() before startup())
304 306 *
305 307 * 3) Probe from CPU 0 and copy and release any BOP_ALLOC-ed memory temporarily
306 308 * allocated before kernel memory allocator is setup (called in main()
307 309 * after startup(), gethrtime() is setup, and before interrupts enabled)
308 310 *
309 311 * 4) Check for null proc LPA on Starcat, collapse lgroup topology (if
310 312 * necessary), setup lgroup kstats, etc. (called before start_other_cpus())
311 313 *
312 314 * 5) Finish any lgroup initialization needed including updating lgroup
313 315 * topology after all CPUs started (called after start_other_cpus())
314 316 */
315 317 typedef enum lgrp_init_stages {
316 318 LGRP_INIT_STAGE1,
317 319 LGRP_INIT_STAGE2,
318 320 LGRP_INIT_STAGE3,
319 321 LGRP_INIT_STAGE4,
320 322 LGRP_INIT_STAGE5
321 323 } lgrp_init_stages_t;
322 324
323 325 /*
324 326 * Memory allocation policies
325 327 */
326 328 typedef enum lgrp_mem_policy {
327 329 LGRP_MEM_POLICY_DEFAULT,
328 330 LGRP_MEM_POLICY_NEXT, /* near LWP to next touch */
329 331 LGRP_MEM_POLICY_RANDOM_PROC, /* randomly across process */
330 332 LGRP_MEM_POLICY_RANDOM_PSET, /* randomly across processor set */
331 333 LGRP_MEM_POLICY_RANDOM, /* randomly across all lgroups */
332 334 LGRP_MEM_POLICY_ROUNDROBIN, /* round robin across all lgroups */
333 335 LGRP_MEM_POLICY_NEXT_CPU, /* Near next CPU to touch memory */
334 336 LGRP_MEM_POLICY_NEXT_SEG, /* lgrp specified directly by seg */
335 337 LGRP_NUM_MEM_POLICIES
336 338 } lgrp_mem_policy_t;
337 339
338 340 /*
339 341 * Search scopes for finding resouces
340 342 */
341 343 typedef enum lgrp_res_ss {
342 344 LGRP_SRCH_LOCAL, /* Search local lgroup only */
343 345 LGRP_SRCH_HIER /* Search entire hierarchy */
344 346 } lgrp_res_ss_t;
345 347
346 348 /*
347 349 * Cookie used for lgrp mnode selection
348 350 */
349 351 typedef struct lgrp_mnode_cookie {
350 352 lgrp_t *lmc_lgrp; /* lgrp under consideration */
351 353 mnodeset_t lmc_nodes; /* nodes not yet tried in lgrp */
352 354 int lmc_cnt; /* how many nodes in untried set */
353 355 mnodeset_t lmc_tried; /* nodes already tried */
354 356 int lmc_ntried; /* how many nodes in tried set */
355 357 lgrp_res_ss_t lmc_scope; /* consider non-local nodes? */
356 358 ushort_t lmc_rand; /* a "random" number */
357 359 } lgrp_mnode_cookie_t;
358 360
359 361 /*
360 362 * Information needed to implement memory allocation policy
361 363 */
362 364 typedef struct lgrp_mem_policy_info {
363 365 int mem_policy; /* memory allocation policy */
364 366 lgrp_id_t mem_lgrpid; /* lgroup id */
365 367 } lgrp_mem_policy_info_t;
366 368
367 369 /*
368 370 * Shared memory policy segment
369 371 */
370 372 typedef struct lgrp_shm_policy_seg {
371 373 u_offset_t shm_off; /* offset into shared object */
372 374 size_t shm_size; /* size of segment */
373 375 lgrp_mem_policy_info_t shm_policy; /* memory allocation policy */
374 376 avl_node_t shm_tree; /* AVL tree */
375 377 } lgrp_shm_policy_seg_t;
376 378
377 379 /*
378 380 * Shared memory locality info
379 381 */
380 382 typedef struct lgrp_shm_locality {
381 383 size_t loc_count; /* reference count */
382 384 avl_tree_t *loc_tree; /* policy segment tree */
383 385 krwlock_t loc_lock; /* protects tree */
384 386 } lgrp_shm_locality_t;
385 387
386 388 /*
387 389 * Queries that may be made to determine lgroup memory size
388 390 */
389 391 typedef enum {
390 392 LGRP_MEM_SIZE_FREE, /* number of free pages */
391 393 LGRP_MEM_SIZE_AVAIL, /* number of pages in phys_avail */
392 394 LGRP_MEM_SIZE_INSTALL /* number of pages in phys_install */
393 395 } lgrp_mem_query_t;
394 396
395 397 /*
396 398 * Argument for the memory copy-rename operation, contains the source and the
397 399 * destination platform handles.
398 400 */
399 401 typedef struct lgrp_config_mem_rename {
400 402 lgrp_handle_t lmem_rename_from;
401 403 lgrp_handle_t lmem_rename_to;
402 404 } lgrp_config_mem_rename_t;
403 405
404 406 /* Macro to clear an lgroup bitmap */
405 407 #define klgrpset_clear(klgrpset) \
406 408 (klgrpset) = (klgrpset_t)0
407 409
408 410 /* Macro to fill an lgroup bitmap */
409 411 #define klgrpset_fill(klgrpset) \
410 412 (klgrpset) = (klgrpset_t)(-1)
411 413
412 414 /* Macro to add an lgroup to an lgroup bitmap */
413 415 #define klgrpset_add(klgrpset, lgrpid) \
414 416 (klgrpset) |= ((klgrpset_t)1 << (lgrpid))
415 417
416 418 /* Macro to delete an lgroup from an lgroup bitmap */
417 419 #define klgrpset_del(klgrpset, lgrpid) \
418 420 (klgrpset) &= ~((klgrpset_t)1 << (lgrpid))
419 421
420 422 /* Macro to copy a klgrpset into another klgrpset */
421 423 #define klgrpset_copy(klgrpset_to, klgrpset_from) \
422 424 (klgrpset_to) = (klgrpset_from)
423 425
424 426 /* Macro to perform an 'and' operation on a pair of lgroup bitmaps */
425 427 #define klgrpset_and(klgrpset_rslt, klgrpset_arg) \
426 428 (klgrpset_rslt) &= (klgrpset_arg)
427 429
428 430 /* Macro to perform an 'or' operation on a pair of lgroup bitmaps */
429 431 #define klgrpset_or(klgrpset_rslt, klgrpset_arg) \
430 432 (klgrpset_rslt) |= (klgrpset_arg)
431 433
432 434 /* Macro to perform a 'diff' operation on a pair of lgroup bitmaps */
433 435 #define klgrpset_diff(klgrpset_rslt, klgrpset_arg) \
434 436 (klgrpset_rslt) &= ~(klgrpset_arg)
435 437
436 438 /* Macro to check if an lgroup is a member of an lgrpset */
437 439 #define klgrpset_ismember(klgrpset, lgrpid) \
438 440 ((klgrpset) & ((klgrpset_t)1 << (lgrpid)))
439 441
440 442 /* Macro to check if an lgroup bitmap is empty */
441 443 #define klgrpset_isempty(klgrpset) \
442 444 ((klgrpset) == (klgrpset_t)0)
443 445
444 446 /* Macro to check if two lgrpsets intersect */
445 447 #define klgrpset_intersects(klgrpset1, klgrpset2) \
446 448 ((klgrpset1) & (klgrpset2))
447 449
448 450 /* Macro to count the number of members in an lgrpset */
449 451 #define klgrpset_nlgrps(klgrpset, count) \
450 452 { \
451 453 lgrp_id_t lgrpid; \
452 454 for (lgrpid = 0, count = 0; lgrpid <= lgrp_alloc_max; lgrpid++) {\
453 455 if (klgrpset_ismember(klgrpset, lgrpid)) \
454 456 count++; \
455 457 } \
456 458 }
457 459
458 460 /* Macro to get total memory size (in bytes) of a given set of lgroups */
459 461 #define klgrpset_totalsize(klgrpset, size) \
460 462 { \
461 463 lgrp_handle_t hand; \
462 464 lgrp_id_t lgrpid; \
463 465 \
464 466 for (lgrpid = 0, size = 0; lgrpid <= lgrp_alloc_max; lgrpid++) {\
465 467 if (klgrpset_ismember(klgrpset, lgrpid) && \
466 468 lgrp_table[lgrpid]) { \
467 469 hand = lgrp_table[lgrpid]->lgrp_plathand; \
468 470 size += lgrp_plat_mem_size(hand, \
469 471 LGRP_MEM_SIZE_AVAIL) * PAGESIZE; \
470 472 } \
471 473 } \
472 474 }
473 475
474 476 /*
475 477 * Does this lgroup exist?
476 478 */
477 479 #define LGRP_EXISTS(lgrp) \
478 480 (lgrp != NULL && lgrp->lgrp_id != LGRP_NONE)
479 481
480 482 /*
481 483 * Macro for testing if a CPU is contained in an lgrp.
482 484 */
483 485 #define LGRP_CONTAINS_CPU(lgrp, cpu) \
484 486 (klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_CPU], \
485 487 cpu->cpu_lpl->lpl_lgrpid))
486 488
487 489 /*
488 490 * Initialize an lgrp_mnode_cookie
489 491 */
490 492 #define LGRP_MNODE_COOKIE_INIT(c, lgrp, scope) \
491 493 { \
492 494 bzero(&(c), sizeof (lgrp_mnode_cookie_t)); \
493 495 (&(c))->lmc_lgrp = lgrp; \
494 496 (&(c))->lmc_nodes = lgrp->lgrp_mnodes; \
495 497 (&(c))->lmc_cnt = lgrp->lgrp_nmnodes; \
496 498 (&(c))->lmc_scope = scope; \
497 499 (&(c))->lmc_rand = (ushort_t)gethrtime_unscaled() >> 4; \
498 500 }
499 501
500 502 /*
501 503 * Upgrade cookie scope from LGRP_SRCH_LOCAL to LGRP_SRCH_HIER.
502 504 */
503 505 #define LGRP_MNODE_COOKIE_UPGRADE(c) \
504 506 { \
505 507 ASSERT((&(c))->lmc_scope == LGRP_SRCH_LOCAL); \
506 508 (&(c))->lmc_scope = LGRP_SRCH_HIER; \
507 509 }
508 510
509 511 /*
510 512 * Macro to see whether memory allocation policy can be reapplied
511 513 */
512 514 #define LGRP_MEM_POLICY_REAPPLICABLE(p) \
513 515 (p == LGRP_MEM_POLICY_NEXT)
514 516
515 517 /*
516 518 * Return true if lgrp has CPU resources in the cpupart
517 519 */
518 520 #define LGRP_CPUS_IN_PART(lgrpid, cpupart) \
519 521 (cpupart->cp_lgrploads[lgrpid].lpl_ncpu > 0)
520 522
521 523 extern int lgrp_alloc_max;
522 524 extern lgrp_t *lgrp_table[NLGRPS_MAX]; /* indexed by lgrp_id */
523 525 extern int nlgrps; /* number of lgroups in machine */
524 526 extern int nlgrpsmax; /* max number of lgroups on platform */
525 527 extern lgrp_gen_t lgrp_gen; /* generation of lgroup hierarchy */
526 528 extern int lgrp_initialized; /* single-CPU initialization done */
527 529 extern int lgrp_topo_initialized; /* lgrp topology constructed */
528 530 extern lgrp_t *lgrp_root; /* root lgroup */
529 531 extern unsigned int lgrp_topo_levels;
530 532 extern lpl_t *lpl_bootstrap; /* bootstrap lpl for non-active CPUs */
531 533
532 534
533 535 /* generic interfaces */
534 536
535 537 /*
536 538 * lgroup management
537 539 */
538 540 int lgrp_optimizations(void);
539 541 void lgrp_init(lgrp_init_stages_t);
540 542 lgrp_t *lgrp_create(void);
541 543 void lgrp_destroy(lgrp_t *);
542 544 void lgrp_config(lgrp_config_flag_t, uintptr_t, uintptr_t);
543 545 lgrp_t *lgrp_hand_to_lgrp(lgrp_handle_t);
544 546
545 547 /*
546 548 * lgroup stats
547 549 */
548 550 void lgrp_kstat_create(struct cpu *);
549 551 void lgrp_kstat_destroy(struct cpu *);
550 552 void lgrp_stat_add(lgrp_id_t, lgrp_stat_t, int64_t);
551 553 int64_t lgrp_stat_read(lgrp_id_t, lgrp_stat_t);
552 554
553 555 /*
554 556 * lgroup memory
555 557 */
556 558 lgrp_mem_policy_t lgrp_madv_to_policy(uchar_t, size_t, int);
557 559 pgcnt_t lgrp_mem_size(lgrp_id_t, lgrp_mem_query_t);
558 560 lgrp_t *lgrp_mem_choose(struct seg *, caddr_t, size_t);
559 561 int lgrp_memnode_choose(lgrp_mnode_cookie_t *);
560 562 lgrp_mem_policy_t lgrp_mem_policy_default(size_t, int);
561 563 int lgrp_mnode_update(klgrpset_t, klgrpset_t *);
562 564 lgrp_t *lgrp_pfn_to_lgrp(pfn_t);
563 565 lgrp_t *lgrp_phys_to_lgrp(u_longlong_t); /* used by numat driver */
564 566 int lgrp_privm_policy_set(lgrp_mem_policy_t, lgrp_mem_policy_info_t *,
565 567 size_t);
566 568 void lgrp_shm_policy_init(struct anon_map *, vnode_t *);
567 569 void lgrp_shm_policy_fini(struct anon_map *, vnode_t *);
568 570 lgrp_mem_policy_info_t *lgrp_shm_policy_get(struct anon_map *, ulong_t,
569 571 vnode_t *, u_offset_t);
570 572 int lgrp_shm_policy_set(lgrp_mem_policy_t, struct anon_map *, ulong_t,
571 573 vnode_t *, u_offset_t, size_t);
572 574
573 575 /*
574 576 * Used by numat driver
575 577 */
576 578 int lgrp_query_cpu(processorid_t, lgrp_id_t *);
577 579 int lgrp_query_load(processorid_t, lgrp_load_t *);
578 580
579 581 /*
580 582 * lgroup thread placement
581 583 */
582 584 lpl_t *lgrp_affinity_best(kthread_t *, struct cpupart *, lgrp_id_t,
583 585 boolean_t);
584 586 void lgrp_affinity_init(lgrp_affinity_t **);
585 587 void lgrp_affinity_free(lgrp_affinity_t **);
586 588 lpl_t *lgrp_choose(kthread_t *t, struct cpupart *);
587 589 lgrp_t *lgrp_home_lgrp(void);
588 590 lgrp_id_t lgrp_home_id(kthread_t *);
589 591 void lgrp_loadavg(lpl_t *, uint_t, int);
590 592 void lgrp_move_thread(kthread_t *, lpl_t *, int);
591 593 uint64_t lgrp_get_trthr_migrations(void);
592 594 void lgrp_update_trthr_migrations(uint64_t);
593 595
594 596 /*
595 597 * lgroup topology
596 598 */
597 599 int lgrp_leaf_add(lgrp_t *, lgrp_t **, int, klgrpset_t *);
598 600 int lgrp_leaf_delete(lgrp_t *, lgrp_t **, int, klgrpset_t *);
599 601 int lgrp_rsets_empty(klgrpset_t *);
600 602 int lgrp_rsets_member(klgrpset_t *, lgrp_id_t);
601 603 int lgrp_topo_flatten(int, lgrp_t **, int, klgrpset_t *);
602 604 int lgrp_topo_ht_limit(void);
603 605 int lgrp_topo_ht_limit_default(void);
604 606 int lgrp_topo_ht_limit_set(int);
605 607 int lgrp_topo_update(lgrp_t **, int, klgrpset_t *);
606 608
607 609 /*
608 610 * lpl topology
609 611 */
610 612 void lpl_topo_bootstrap(lpl_t *, int);
611 613 int lpl_topo_flatten(int);
612 614 int lpl_topo_verify(struct cpupart *);
613 615
614 616
615 617 /* platform interfaces */
616 618 void lgrp_plat_init(lgrp_init_stages_t);
617 619 lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid);
618 620 void lgrp_plat_config(lgrp_config_flag_t, uintptr_t);
619 621 lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t);
620 622 lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t);
621 623 int lgrp_plat_max_lgrps(void);
622 624 pgcnt_t lgrp_plat_mem_size(lgrp_handle_t, lgrp_mem_query_t);
623 625 int lgrp_plat_latency(lgrp_handle_t, lgrp_handle_t);
|
↓ open down ↓ |
567 lines elided |
↑ open up ↑ |
624 626 lgrp_handle_t lgrp_plat_root_hand(void);
625 627
626 628 extern uint32_t lgrp_expand_proc_thresh;
627 629 extern uint32_t lgrp_expand_proc_diff;
628 630 extern pgcnt_t lgrp_mem_free_thresh;
629 631 extern uint32_t lgrp_loadavg_tolerance;
630 632 extern uint32_t lgrp_loadavg_max_effect;
631 633 extern uint32_t lgrp_load_thresh;
632 634 extern lgrp_mem_policy_t lgrp_mem_policy_root;
633 635
634 -#endif /* _KERNEL && _KMEMUSER */
636 +#endif /* _KERNEL || _FAKE_KERNEL || _KMEMUSER */
635 637
636 638 #ifdef __cplusplus
637 639 }
638 640 #endif
639 641
640 642 #endif /* _LGRP_H */
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX