Print this page
8115 parallel zfs mount
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/sys/cpuvar.h
+++ new/usr/src/uts/common/sys/cpuvar.h
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright (c) 2012 by Delphix. All rights reserved.
25 25 * Copyright 2014 Igor Kozhukhov <ikozhukhov@gmail.com>.
26 + * Copyright 2017 RackTop Systems.
26 27 */
27 28
28 29 #ifndef _SYS_CPUVAR_H
29 30 #define _SYS_CPUVAR_H
30 31
31 32 #include <sys/thread.h>
32 33 #include <sys/sysinfo.h> /* has cpu_stat_t definition */
33 34 #include <sys/disp.h>
34 35 #include <sys/processor.h>
36 +#include <sys/kcpc.h> /* has kcpc_ctx_t definition */
35 37
36 38 #include <sys/loadavg.h>
37 39 #if (defined(_KERNEL) || defined(_KMEMUSER)) && defined(_MACHDEP)
38 40 #include <sys/machcpuvar.h>
39 41 #endif
40 42
41 43 #include <sys/types.h>
42 44 #include <sys/file.h>
43 45 #include <sys/bitmap.h>
44 46 #include <sys/rwlock.h>
45 47 #include <sys/msacct.h>
46 48 #if defined(__GNUC__) && defined(_ASM_INLINES) && defined(_KERNEL) && \
47 49 (defined(__i386) || defined(__amd64))
48 50 #include <asm/cpuvar.h>
49 51 #endif
50 52
51 53 #ifdef __cplusplus
52 54 extern "C" {
53 55 #endif
54 56
55 57 struct squeue_set_s;
56 58
57 59 #define CPU_CACHE_COHERENCE_SIZE 64
58 60
59 61 /*
60 62 * For fast event tracing.
61 63 */
62 64 struct ftrace_record;
63 65 typedef struct ftrace_data {
64 66 int ftd_state; /* ftrace flags */
65 67 kmutex_t ftd_unused; /* ftrace buffer lock, unused */
66 68 struct ftrace_record *ftd_cur; /* current record */
67 69 struct ftrace_record *ftd_first; /* first record */
68 70 struct ftrace_record *ftd_last; /* last record */
69 71 } ftrace_data_t;
70 72
71 73 struct cyc_cpu;
72 74 struct nvlist;
73 75
74 76 /*
75 77 * Per-CPU data.
76 78 *
77 79 * Be careful adding new members: if they are not the same in all modules (e.g.
78 80 * change size depending on a #define), CTF uniquification can fail to work
79 81 * properly. Furthermore, this is transitive in that it applies recursively to
80 82 * all types pointed to by cpu_t.
81 83 */
82 84 typedef struct cpu {
83 85 processorid_t cpu_id; /* CPU number */
84 86 processorid_t cpu_seqid; /* sequential CPU id (0..ncpus-1) */
85 87 volatile cpu_flag_t cpu_flags; /* flags indicating CPU state */
86 88 struct cpu *cpu_self; /* pointer to itself */
87 89 kthread_t *cpu_thread; /* current thread */
88 90 kthread_t *cpu_idle_thread; /* idle thread for this CPU */
89 91 kthread_t *cpu_pause_thread; /* pause thread for this CPU */
90 92 klwp_id_t cpu_lwp; /* current lwp (if any) */
91 93 klwp_id_t cpu_fpowner; /* currently loaded fpu owner */
92 94 struct cpupart *cpu_part; /* partition with this CPU */
93 95 struct lgrp_ld *cpu_lpl; /* pointer to this cpu's load */
94 96 int cpu_cache_offset; /* see kmem.c for details */
95 97
96 98 /*
97 99 * Links to other CPUs. It is safe to walk these lists if
98 100 * one of the following is true:
99 101 * - cpu_lock held
100 102 * - preemption disabled via kpreempt_disable
101 103 * - PIL >= DISP_LEVEL
102 104 * - acting thread is an interrupt thread
103 105 * - all other CPUs are paused
104 106 */
105 107 struct cpu *cpu_next; /* next existing CPU */
106 108 struct cpu *cpu_prev; /* prev existing CPU */
107 109 struct cpu *cpu_next_onln; /* next online (enabled) CPU */
108 110 struct cpu *cpu_prev_onln; /* prev online (enabled) CPU */
109 111 struct cpu *cpu_next_part; /* next CPU in partition */
110 112 struct cpu *cpu_prev_part; /* prev CPU in partition */
111 113 struct cpu *cpu_next_lgrp; /* next CPU in latency group */
112 114 struct cpu *cpu_prev_lgrp; /* prev CPU in latency group */
113 115 struct cpu *cpu_next_lpl; /* next CPU in lgrp partition */
114 116 struct cpu *cpu_prev_lpl;
115 117
116 118 struct cpu_pg *cpu_pg; /* cpu's processor groups */
117 119
118 120 void *cpu_reserved[4]; /* reserved for future use */
119 121
120 122 /*
121 123 * Scheduling variables.
122 124 */
123 125 disp_t *cpu_disp; /* dispatch queue data */
124 126 /*
125 127 * Note that cpu_disp is set before the CPU is added to the system
126 128 * and is never modified. Hence, no additional locking is needed
127 129 * beyond what's necessary to access the cpu_t structure.
128 130 */
129 131 char cpu_runrun; /* scheduling flag - set to preempt */
130 132 char cpu_kprunrun; /* force kernel preemption */
131 133 pri_t cpu_chosen_level; /* priority at which cpu */
132 134 /* was chosen for scheduling */
133 135 kthread_t *cpu_dispthread; /* thread selected for dispatch */
134 136 disp_lock_t cpu_thread_lock; /* dispatcher lock on current thread */
135 137 uint8_t cpu_disp_flags; /* flags used by dispatcher */
136 138 /*
137 139 * The following field is updated when ever the cpu_dispthread
138 140 * changes. Also in places, where the current thread(cpu_dispthread)
139 141 * priority changes. This is used in disp_lowpri_cpu()
140 142 */
141 143 pri_t cpu_dispatch_pri; /* priority of cpu_dispthread */
142 144 clock_t cpu_last_swtch; /* last time switched to new thread */
143 145
144 146 /*
145 147 * Interrupt data.
146 148 */
147 149 caddr_t cpu_intr_stack; /* interrupt stack */
148 150 kthread_t *cpu_intr_thread; /* interrupt thread list */
149 151 uint_t cpu_intr_actv; /* interrupt levels active (bitmask) */
150 152 int cpu_base_spl; /* priority for highest rupt active */
151 153
152 154 /*
153 155 * Statistics.
154 156 */
155 157 cpu_stats_t cpu_stats; /* per-CPU statistics */
156 158 struct kstat *cpu_info_kstat; /* kstat for cpu info */
157 159
158 160 uintptr_t cpu_profile_pc; /* kernel PC in profile interrupt */
159 161 uintptr_t cpu_profile_upc; /* user PC in profile interrupt */
160 162 uintptr_t cpu_profile_pil; /* PIL when profile interrupted */
161 163
162 164 ftrace_data_t cpu_ftrace; /* per cpu ftrace data */
163 165
164 166 clock_t cpu_deadman_counter; /* used by deadman() */
165 167 uint_t cpu_deadman_countdown; /* used by deadman() */
166 168
167 169 kmutex_t cpu_cpc_ctxlock; /* protects context for idle thread */
168 170 kcpc_ctx_t *cpu_cpc_ctx; /* performance counter context */
169 171
170 172 /*
171 173 * Configuration information for the processor_info system call.
172 174 */
173 175 processor_info_t cpu_type_info; /* config info */
174 176 time_t cpu_state_begin; /* when CPU entered current state */
175 177 char cpu_cpr_flags; /* CPR related info */
176 178 struct cyc_cpu *cpu_cyclic; /* per cpu cyclic subsystem data */
177 179 struct squeue_set_s *cpu_squeue_set; /* per cpu squeue set */
178 180 struct nvlist *cpu_props; /* pool-related properties */
179 181
180 182 krwlock_t cpu_ft_lock; /* DTrace: fasttrap lock */
181 183 uintptr_t cpu_dtrace_caller; /* DTrace: caller, if any */
182 184 hrtime_t cpu_dtrace_chillmark; /* DTrace: chill mark time */
183 185 hrtime_t cpu_dtrace_chilled; /* DTrace: total chill time */
184 186 uint64_t cpu_dtrace_probes; /* DTrace: total probes fired */
185 187 hrtime_t cpu_dtrace_nsec; /* DTrace: ns in dtrace_probe */
186 188
187 189 volatile uint16_t cpu_mstate; /* cpu microstate */
188 190 volatile uint16_t cpu_mstate_gen; /* generation counter */
189 191 volatile hrtime_t cpu_mstate_start; /* cpu microstate start time */
190 192 volatile hrtime_t cpu_acct[NCMSTATES]; /* cpu microstate data */
191 193 hrtime_t cpu_intracct[NCMSTATES]; /* interrupt mstate data */
192 194 hrtime_t cpu_waitrq; /* cpu run-queue wait time */
193 195 struct loadavg_s cpu_loadavg; /* loadavg info for this cpu */
194 196
195 197 char *cpu_idstr; /* for printing and debugging */
196 198 char *cpu_brandstr; /* for printing */
197 199
198 200 /*
199 201 * Sum of all device interrupt weights that are currently directed at
200 202 * this cpu. Cleared at start of interrupt redistribution.
201 203 */
202 204 int32_t cpu_intr_weight;
203 205 void *cpu_vm_data;
204 206
205 207 struct cpu_physid *cpu_physid; /* physical associations */
206 208
207 209 uint64_t cpu_curr_clock; /* current clock freq in Hz */
208 210 char *cpu_supp_freqs; /* supported freqs in Hz */
209 211
210 212 uintptr_t cpu_cpcprofile_pc; /* kernel PC in cpc interrupt */
211 213 uintptr_t cpu_cpcprofile_upc; /* user PC in cpc interrupt */
212 214
213 215 /*
214 216 * Interrupt load factor used by dispatcher & softcall
215 217 */
216 218 hrtime_t cpu_intrlast; /* total interrupt time (nsec) */
217 219 int cpu_intrload; /* interrupt load factor (0-99%) */
218 220
219 221 uint_t cpu_rotor; /* for cheap pseudo-random numbers */
220 222
221 223 struct cu_cpu_info *cpu_cu_info; /* capacity & util. info */
222 224
223 225 /*
224 226 * cpu_generation is updated whenever CPU goes on-line or off-line.
225 227 * Updates to cpu_generation are protected by cpu_lock.
226 228 *
227 229 * See CPU_NEW_GENERATION() macro below.
228 230 */
229 231 volatile uint_t cpu_generation; /* tracking on/off-line */
230 232
231 233 /*
232 234 * New members must be added /before/ this member, as the CTF tools
233 235 * rely on this being the last field before cpu_m, so they can
234 236 * correctly calculate the offset when synthetically adding the cpu_m
235 237 * member in objects that do not have it. This fixup is required for
236 238 * uniquification to work correctly.
237 239 */
238 240 uintptr_t cpu_m_pad;
239 241
240 242 #if (defined(_KERNEL) || defined(_KMEMUSER)) && defined(_MACHDEP)
241 243 struct machcpu cpu_m; /* per architecture info */
242 244 #endif
243 245 } cpu_t;
244 246
245 247 /*
246 248 * The cpu_core structure consists of per-CPU state available in any context.
247 249 * On some architectures, this may mean that the page(s) containing the
248 250 * NCPU-sized array of cpu_core structures must be locked in the TLB -- it
249 251 * is up to the platform to assure that this is performed properly. Note that
250 252 * the structure is sized to avoid false sharing.
251 253 */
252 254 #define CPUC_SIZE (sizeof (uint16_t) + sizeof (uint8_t) + \
253 255 sizeof (uintptr_t) + sizeof (kmutex_t))
254 256 #define CPUC_PADSIZE CPU_CACHE_COHERENCE_SIZE - CPUC_SIZE
255 257
256 258 typedef struct cpu_core {
257 259 uint16_t cpuc_dtrace_flags; /* DTrace flags */
258 260 uint8_t cpuc_dcpc_intr_state; /* DCPC provider intr state */
259 261 uint8_t cpuc_pad[CPUC_PADSIZE]; /* padding */
260 262 uintptr_t cpuc_dtrace_illval; /* DTrace illegal value */
261 263 kmutex_t cpuc_pid_lock; /* DTrace pid provider lock */
262 264 } cpu_core_t;
263 265
264 266 #ifdef _KERNEL
265 267 extern cpu_core_t cpu_core[];
266 268 #endif /* _KERNEL */
267 269
268 270 /*
269 271 * CPU_ON_INTR() macro. Returns non-zero if currently on interrupt stack.
270 272 * Note that this isn't a test for a high PIL. For example, cpu_intr_actv
271 273 * does not get updated when we go through sys_trap from TL>0 at high PIL.
272 274 * getpil() should be used instead to check for PIL levels.
273 275 */
274 276 #define CPU_ON_INTR(cpup) ((cpup)->cpu_intr_actv >> (LOCK_LEVEL + 1))
275 277
276 278 /*
277 279 * Check to see if an interrupt thread might be active at a given ipl.
278 280 * If so return true.
279 281 * We must be conservative--it is ok to give a false yes, but a false no
280 282 * will cause disaster. (But if the situation changes after we check it is
281 283 * ok--the caller is trying to ensure that an interrupt routine has been
282 284 * exited).
283 285 * This is used when trying to remove an interrupt handler from an autovector
284 286 * list in avintr.c.
285 287 */
286 288 #define INTR_ACTIVE(cpup, level) \
287 289 ((level) <= LOCK_LEVEL ? \
288 290 ((cpup)->cpu_intr_actv & (1 << (level))) : (CPU_ON_INTR(cpup)))
289 291
290 292 /*
291 293 * CPU_PSEUDO_RANDOM() returns a per CPU value that changes each time one
292 294 * looks at it. It's meant as a cheap mechanism to be incorporated in routines
293 295 * wanting to avoid biasing, but where true randomness isn't needed (just
294 296 * something that changes).
295 297 */
296 298 #define CPU_PSEUDO_RANDOM() (CPU->cpu_rotor++)
297 299
298 300 #if defined(_KERNEL) || defined(_KMEMUSER) || defined(_BOOT)
299 301
300 302 #define INTR_STACK_SIZE MAX(DEFAULTSTKSZ, PAGESIZE)
301 303
302 304 /* MEMBERS PROTECTED BY "atomicity": cpu_flags */
303 305
304 306 /*
305 307 * Flags in the CPU structure.
306 308 *
307 309 * These are protected by cpu_lock (except during creation).
308 310 *
309 311 * Offlined-CPUs have three stages of being offline:
310 312 *
311 313 * CPU_ENABLE indicates that the CPU is participating in I/O interrupts
312 314 * that can be directed at a number of different CPUs. If CPU_ENABLE
313 315 * is off, the CPU will not be given interrupts that can be sent elsewhere,
314 316 * but will still get interrupts from devices associated with that CPU only,
315 317 * and from other CPUs.
316 318 *
317 319 * CPU_OFFLINE indicates that the dispatcher should not allow any threads
318 320 * other than interrupt threads to run on that CPU. A CPU will not have
319 321 * CPU_OFFLINE set if there are any bound threads (besides interrupts).
320 322 *
321 323 * CPU_QUIESCED is set if p_offline was able to completely turn idle the
322 324 * CPU and it will not have to run interrupt threads. In this case it'll
323 325 * stay in the idle loop until CPU_QUIESCED is turned off.
324 326 *
325 327 * CPU_FROZEN is used only by CPR to mark CPUs that have been successfully
326 328 * suspended (in the suspend path), or have yet to be resumed (in the resume
327 329 * case).
328 330 *
329 331 * On some platforms CPUs can be individually powered off.
330 332 * The following flags are set for powered off CPUs: CPU_QUIESCED,
331 333 * CPU_OFFLINE, and CPU_POWEROFF. The following flags are cleared:
332 334 * CPU_RUNNING, CPU_READY, CPU_EXISTS, and CPU_ENABLE.
333 335 */
334 336 #define CPU_RUNNING 0x001 /* CPU running */
335 337 #define CPU_READY 0x002 /* CPU ready for cross-calls */
336 338 #define CPU_QUIESCED 0x004 /* CPU will stay in idle */
337 339 #define CPU_EXISTS 0x008 /* CPU is configured */
338 340 #define CPU_ENABLE 0x010 /* CPU enabled for interrupts */
339 341 #define CPU_OFFLINE 0x020 /* CPU offline via p_online */
340 342 #define CPU_POWEROFF 0x040 /* CPU is powered off */
341 343 #define CPU_FROZEN 0x080 /* CPU is frozen via CPR suspend */
342 344 #define CPU_SPARE 0x100 /* CPU offline available for use */
343 345 #define CPU_FAULTED 0x200 /* CPU offline diagnosed faulty */
344 346
345 347 #define FMT_CPU_FLAGS \
346 348 "\20\12fault\11spare\10frozen" \
347 349 "\7poweroff\6offline\5enable\4exist\3quiesced\2ready\1run"
348 350
349 351 #define CPU_ACTIVE(cpu) (((cpu)->cpu_flags & CPU_OFFLINE) == 0)
350 352
351 353 /*
352 354 * Flags for cpu_offline(), cpu_faulted(), and cpu_spare().
353 355 */
354 356 #define CPU_FORCED 0x0001 /* Force CPU offline */
355 357
356 358 /*
357 359 * DTrace flags.
358 360 */
359 361 #define CPU_DTRACE_NOFAULT 0x0001 /* Don't fault */
360 362 #define CPU_DTRACE_DROP 0x0002 /* Drop this ECB */
361 363 #define CPU_DTRACE_BADADDR 0x0004 /* DTrace fault: bad address */
362 364 #define CPU_DTRACE_BADALIGN 0x0008 /* DTrace fault: bad alignment */
363 365 #define CPU_DTRACE_DIVZERO 0x0010 /* DTrace fault: divide by zero */
364 366 #define CPU_DTRACE_ILLOP 0x0020 /* DTrace fault: illegal operation */
365 367 #define CPU_DTRACE_NOSCRATCH 0x0040 /* DTrace fault: out of scratch */
366 368 #define CPU_DTRACE_KPRIV 0x0080 /* DTrace fault: bad kernel access */
367 369 #define CPU_DTRACE_UPRIV 0x0100 /* DTrace fault: bad user access */
368 370 #define CPU_DTRACE_TUPOFLOW 0x0200 /* DTrace fault: tuple stack overflow */
369 371 #if defined(__sparc)
370 372 #define CPU_DTRACE_FAKERESTORE 0x0400 /* pid provider hint to getreg */
371 373 #endif
372 374 #define CPU_DTRACE_ENTRY 0x0800 /* pid provider hint to ustack() */
373 375 #define CPU_DTRACE_BADSTACK 0x1000 /* DTrace fault: bad stack */
374 376
375 377 #define CPU_DTRACE_FAULT (CPU_DTRACE_BADADDR | CPU_DTRACE_BADALIGN | \
376 378 CPU_DTRACE_DIVZERO | CPU_DTRACE_ILLOP | \
377 379 CPU_DTRACE_NOSCRATCH | CPU_DTRACE_KPRIV | \
378 380 CPU_DTRACE_UPRIV | CPU_DTRACE_TUPOFLOW | \
379 381 CPU_DTRACE_BADSTACK)
380 382 #define CPU_DTRACE_ERROR (CPU_DTRACE_FAULT | CPU_DTRACE_DROP)
381 383
382 384 /*
383 385 * Dispatcher flags
384 386 * These flags must be changed only by the current CPU.
385 387 */
386 388 #define CPU_DISP_DONTSTEAL 0x01 /* CPU undergoing context swtch */
387 389 #define CPU_DISP_HALTED 0x02 /* CPU halted waiting for interrupt */
388 390
389 391 /* Note: inside ifdef: _KERNEL || _KMEMUSER || _BOOT */
390 392 #if defined(_MACHDEP)
391 393
392 394 /*
393 395 * Macros for manipulating sets of CPUs as a bitmap. Note that this
394 396 * bitmap may vary in size depending on the maximum CPU id a specific
395 397 * platform supports. This may be different than the number of CPUs
396 398 * the platform supports, since CPU ids can be sparse. We define two
397 399 * sets of macros; one for platforms where the maximum CPU id is less
398 400 * than the number of bits in a single word (32 in a 32-bit kernel,
399 401 * 64 in a 64-bit kernel), and one for platforms that require bitmaps
400 402 * of more than one word.
401 403 */
402 404
403 405 #define CPUSET_WORDS BT_BITOUL(NCPU)
404 406 #define CPUSET_NOTINSET ((uint_t)-1)
405 407
406 408 #if CPUSET_WORDS > 1
407 409
408 410 typedef struct cpuset {
409 411 ulong_t cpub[CPUSET_WORDS];
410 412 } cpuset_t;
411 413
412 414 /*
413 415 * Private functions for manipulating cpusets that do not fit in a
414 416 * single word. These should not be used directly; instead the
415 417 * CPUSET_* macros should be used so the code will be portable
416 418 * across different definitions of NCPU.
417 419 */
418 420 extern void cpuset_all(cpuset_t *);
419 421 extern void cpuset_all_but(cpuset_t *, uint_t);
420 422 extern int cpuset_isnull(cpuset_t *);
421 423 extern int cpuset_cmp(cpuset_t *, cpuset_t *);
422 424 extern void cpuset_only(cpuset_t *, uint_t);
423 425 extern uint_t cpuset_find(cpuset_t *);
424 426 extern void cpuset_bounds(cpuset_t *, uint_t *, uint_t *);
425 427
426 428 #define CPUSET_ALL(set) cpuset_all(&(set))
427 429 #define CPUSET_ALL_BUT(set, cpu) cpuset_all_but(&(set), cpu)
428 430 #define CPUSET_ONLY(set, cpu) cpuset_only(&(set), cpu)
429 431 #define CPU_IN_SET(set, cpu) BT_TEST((set).cpub, cpu)
430 432 #define CPUSET_ADD(set, cpu) BT_SET((set).cpub, cpu)
431 433 #define CPUSET_DEL(set, cpu) BT_CLEAR((set).cpub, cpu)
432 434 #define CPUSET_ISNULL(set) cpuset_isnull(&(set))
433 435 #define CPUSET_ISEQUAL(set1, set2) cpuset_cmp(&(set1), &(set2))
434 436
435 437 /*
436 438 * Find one CPU in the cpuset.
437 439 * Sets "cpu" to the id of the found CPU, or CPUSET_NOTINSET if no cpu
438 440 * could be found. (i.e. empty set)
439 441 */
440 442 #define CPUSET_FIND(set, cpu) { \
441 443 cpu = cpuset_find(&(set)); \
442 444 }
443 445
444 446 /*
445 447 * Determine the smallest and largest CPU id in the set. Returns
446 448 * CPUSET_NOTINSET in smallest and largest when set is empty.
447 449 */
448 450 #define CPUSET_BOUNDS(set, smallest, largest) { \
449 451 cpuset_bounds(&(set), &(smallest), &(largest)); \
450 452 }
451 453
452 454 /*
453 455 * Atomic cpuset operations
454 456 * These are safe to use for concurrent cpuset manipulations.
455 457 * "xdel" and "xadd" are exclusive operations, that set "result" to "0"
456 458 * if the add or del was successful, or "-1" if not successful.
457 459 * (e.g. attempting to add a cpu to a cpuset that's already there, or
458 460 * deleting a cpu that's not in the cpuset)
459 461 */
460 462
461 463 #define CPUSET_ATOMIC_DEL(set, cpu) BT_ATOMIC_CLEAR((set).cpub, (cpu))
462 464 #define CPUSET_ATOMIC_ADD(set, cpu) BT_ATOMIC_SET((set).cpub, (cpu))
463 465
464 466 #define CPUSET_ATOMIC_XADD(set, cpu, result) \
465 467 BT_ATOMIC_SET_EXCL((set).cpub, cpu, result)
466 468
467 469 #define CPUSET_ATOMIC_XDEL(set, cpu, result) \
468 470 BT_ATOMIC_CLEAR_EXCL((set).cpub, cpu, result)
469 471
470 472
471 473 #define CPUSET_OR(set1, set2) { \
472 474 int _i; \
473 475 for (_i = 0; _i < CPUSET_WORDS; _i++) \
474 476 (set1).cpub[_i] |= (set2).cpub[_i]; \
475 477 }
476 478
477 479 #define CPUSET_XOR(set1, set2) { \
478 480 int _i; \
479 481 for (_i = 0; _i < CPUSET_WORDS; _i++) \
480 482 (set1).cpub[_i] ^= (set2).cpub[_i]; \
481 483 }
482 484
483 485 #define CPUSET_AND(set1, set2) { \
484 486 int _i; \
485 487 for (_i = 0; _i < CPUSET_WORDS; _i++) \
486 488 (set1).cpub[_i] &= (set2).cpub[_i]; \
487 489 }
488 490
489 491 #define CPUSET_ZERO(set) { \
490 492 int _i; \
491 493 for (_i = 0; _i < CPUSET_WORDS; _i++) \
492 494 (set).cpub[_i] = 0; \
493 495 }
494 496
495 497 #elif CPUSET_WORDS == 1
496 498
497 499 typedef ulong_t cpuset_t; /* a set of CPUs */
498 500
499 501 #define CPUSET(cpu) (1UL << (cpu))
500 502
501 503 #define CPUSET_ALL(set) ((void)((set) = ~0UL))
502 504 #define CPUSET_ALL_BUT(set, cpu) ((void)((set) = ~CPUSET(cpu)))
503 505 #define CPUSET_ONLY(set, cpu) ((void)((set) = CPUSET(cpu)))
504 506 #define CPU_IN_SET(set, cpu) ((set) & CPUSET(cpu))
505 507 #define CPUSET_ADD(set, cpu) ((void)((set) |= CPUSET(cpu)))
506 508 #define CPUSET_DEL(set, cpu) ((void)((set) &= ~CPUSET(cpu)))
507 509 #define CPUSET_ISNULL(set) ((set) == 0)
508 510 #define CPUSET_ISEQUAL(set1, set2) ((set1) == (set2))
509 511 #define CPUSET_OR(set1, set2) ((void)((set1) |= (set2)))
510 512 #define CPUSET_XOR(set1, set2) ((void)((set1) ^= (set2)))
511 513 #define CPUSET_AND(set1, set2) ((void)((set1) &= (set2)))
512 514 #define CPUSET_ZERO(set) ((void)((set) = 0))
513 515
514 516 #define CPUSET_FIND(set, cpu) { \
515 517 cpu = (uint_t)(lowbit(set) - 1); \
516 518 }
517 519
518 520 #define CPUSET_BOUNDS(set, smallest, largest) { \
519 521 smallest = (uint_t)(lowbit(set) - 1); \
520 522 largest = (uint_t)(highbit(set) - 1); \
521 523 }
522 524
523 525 #define CPUSET_ATOMIC_DEL(set, cpu) atomic_and_ulong(&(set), ~CPUSET(cpu))
524 526 #define CPUSET_ATOMIC_ADD(set, cpu) atomic_or_ulong(&(set), CPUSET(cpu))
525 527
526 528 #define CPUSET_ATOMIC_XADD(set, cpu, result) \
527 529 { result = atomic_set_long_excl(&(set), (cpu)); }
528 530
529 531 #define CPUSET_ATOMIC_XDEL(set, cpu, result) \
530 532 { result = atomic_clear_long_excl(&(set), (cpu)); }
531 533
532 534 #else /* CPUSET_WORDS <= 0 */
533 535
534 536 #error NCPU is undefined or invalid
535 537
536 538 #endif /* CPUSET_WORDS */
537 539
538 540 extern cpuset_t cpu_seqid_inuse;
539 541
540 542 #endif /* _MACHDEP */
541 543 #endif /* _KERNEL || _KMEMUSER || _BOOT */
542 544
543 545 #define CPU_CPR_OFFLINE 0x0
544 546 #define CPU_CPR_ONLINE 0x1
545 547 #define CPU_CPR_IS_OFFLINE(cpu) (((cpu)->cpu_cpr_flags & CPU_CPR_ONLINE) == 0)
546 548 #define CPU_CPR_IS_ONLINE(cpu) ((cpu)->cpu_cpr_flags & CPU_CPR_ONLINE)
547 549 #define CPU_SET_CPR_FLAGS(cpu, flag) ((cpu)->cpu_cpr_flags |= flag)
548 550
549 551 #if defined(_KERNEL) || defined(_KMEMUSER)
550 552
551 553 extern struct cpu *cpu[]; /* indexed by CPU number */
552 554 extern struct cpu **cpu_seq; /* indexed by sequential CPU id */
553 555 extern cpu_t *cpu_list; /* list of CPUs */
554 556 extern cpu_t *cpu_active; /* list of active CPUs */
555 557 extern int ncpus; /* number of CPUs present */
556 558 extern int ncpus_online; /* number of CPUs not quiesced */
557 559 extern int max_ncpus; /* max present before ncpus is known */
558 560 extern int boot_max_ncpus; /* like max_ncpus but for real */
559 561 extern int boot_ncpus; /* # cpus present @ boot */
560 562 extern processorid_t max_cpuid; /* maximum CPU number */
561 563 extern struct cpu *cpu_inmotion; /* offline or partition move target */
562 564 extern cpu_t *clock_cpu_list;
563 565 extern processorid_t max_cpu_seqid_ever; /* maximum seqid ever given */
564 566
565 567 #if defined(__i386) || defined(__amd64)
566 568 extern struct cpu *curcpup(void);
567 569 #define CPU (curcpup()) /* Pointer to current CPU */
568 570 #else
569 571 #define CPU (curthread->t_cpu) /* Pointer to current CPU */
570 572 #endif
571 573
572 574 /*
573 575 * CPU_CURRENT indicates to thread_affinity_set to use CPU->cpu_id
574 576 * as the target and to grab cpu_lock instead of requiring the caller
575 577 * to grab it.
576 578 */
577 579 #define CPU_CURRENT -3
578 580
579 581 /*
580 582 * Per-CPU statistics
581 583 *
582 584 * cpu_stats_t contains numerous system and VM-related statistics, in the form
583 585 * of gauges or monotonically-increasing event occurrence counts.
584 586 */
585 587
586 588 #define CPU_STATS_ENTER_K() kpreempt_disable()
587 589 #define CPU_STATS_EXIT_K() kpreempt_enable()
588 590
589 591 #define CPU_STATS_ADD_K(class, stat, amount) \
590 592 { kpreempt_disable(); /* keep from switching CPUs */\
591 593 CPU_STATS_ADDQ(CPU, class, stat, amount); \
592 594 kpreempt_enable(); \
593 595 }
594 596
595 597 #define CPU_STATS_ADDQ(cp, class, stat, amount) { \
596 598 extern void __dtrace_probe___cpu_##class##info_##stat(uint_t, \
597 599 uint64_t *, cpu_t *); \
598 600 uint64_t *stataddr = &((cp)->cpu_stats.class.stat); \
599 601 __dtrace_probe___cpu_##class##info_##stat((amount), \
600 602 stataddr, cp); \
601 603 *(stataddr) += (amount); \
602 604 }
603 605
604 606 #define CPU_STATS(cp, stat) \
605 607 ((cp)->cpu_stats.stat)
606 608
|
↓ open down ↓ |
562 lines elided |
↑ open up ↑ |
607 609 /*
608 610 * Increment CPU generation value.
609 611 * This macro should be called whenever CPU goes on-line or off-line.
610 612 * Updates to cpu_generation should be protected by cpu_lock.
611 613 */
612 614 #define CPU_NEW_GENERATION(cp) ((cp)->cpu_generation++)
613 615
614 616 #endif /* _KERNEL || _KMEMUSER */
615 617
616 618 /*
617 - * CPU support routines.
619 + * CPU support routines (not for genassym.c)
618 620 */
619 -#if defined(_KERNEL) && defined(__STDC__) /* not for genassym.c */
621 +#if defined(_KERNEL) || defined(_FAKE_KERNEL) && defined(__STDC__)
620 622
621 623 struct zone;
622 624
623 625 void cpu_list_init(cpu_t *);
624 626 void cpu_add_unit(cpu_t *);
625 627 void cpu_del_unit(int cpuid);
626 628 void cpu_add_active(cpu_t *);
627 629 void cpu_kstat_init(cpu_t *);
628 630 void cpu_visibility_add(cpu_t *, struct zone *);
629 631 void cpu_visibility_remove(cpu_t *, struct zone *);
630 632 void cpu_visibility_configure(cpu_t *, struct zone *);
631 633 void cpu_visibility_unconfigure(cpu_t *, struct zone *);
632 634 void cpu_visibility_online(cpu_t *, struct zone *);
633 635 void cpu_visibility_offline(cpu_t *, struct zone *);
634 636 void cpu_create_intrstat(cpu_t *);
635 637 void cpu_delete_intrstat(cpu_t *);
636 638 int cpu_kstat_intrstat_update(kstat_t *, int);
637 639 void cpu_intr_swtch_enter(kthread_t *);
638 640 void cpu_intr_swtch_exit(kthread_t *);
639 641
640 642 void mbox_lock_init(void); /* initialize cross-call locks */
641 643 void mbox_init(int cpun); /* initialize cross-calls */
642 644 void poke_cpu(int cpun); /* interrupt another CPU (to preempt) */
643 645
644 646 /*
645 647 * values for safe_list. Pause state that CPUs are in.
646 648 */
647 649 #define PAUSE_IDLE 0 /* normal state */
648 650 #define PAUSE_READY 1 /* paused thread ready to spl */
649 651 #define PAUSE_WAIT 2 /* paused thread is spl-ed high */
650 652 #define PAUSE_DIE 3 /* tell pause thread to leave */
651 653 #define PAUSE_DEAD 4 /* pause thread has left */
652 654
653 655 void mach_cpu_pause(volatile char *);
654 656
655 657 void pause_cpus(cpu_t *off_cp, void *(*func)(void *));
656 658 void start_cpus(void);
657 659 int cpus_paused(void);
658 660
659 661 void cpu_pause_init(void);
660 662 cpu_t *cpu_get(processorid_t cpun); /* get the CPU struct associated */
661 663
662 664 int cpu_online(cpu_t *cp); /* take cpu online */
663 665 int cpu_offline(cpu_t *cp, int flags); /* take cpu offline */
664 666 int cpu_spare(cpu_t *cp, int flags); /* take cpu to spare */
665 667 int cpu_faulted(cpu_t *cp, int flags); /* take cpu to faulted */
666 668 int cpu_poweron(cpu_t *cp); /* take powered-off cpu to offline */
667 669 int cpu_poweroff(cpu_t *cp); /* take offline cpu to powered-off */
668 670
669 671 cpu_t *cpu_intr_next(cpu_t *cp); /* get next online CPU taking intrs */
670 672 int cpu_intr_count(cpu_t *cp); /* count # of CPUs handling intrs */
671 673 int cpu_intr_on(cpu_t *cp); /* CPU taking I/O interrupts? */
672 674 void cpu_intr_enable(cpu_t *cp); /* enable I/O interrupts */
673 675 int cpu_intr_disable(cpu_t *cp); /* disable I/O interrupts */
674 676 void cpu_intr_alloc(cpu_t *cp, int n); /* allocate interrupt threads */
675 677
676 678 /*
677 679 * Routines for checking CPU states.
678 680 */
679 681 int cpu_is_online(cpu_t *); /* check if CPU is online */
680 682 int cpu_is_nointr(cpu_t *); /* check if CPU can service intrs */
681 683 int cpu_is_active(cpu_t *); /* check if CPU can run threads */
682 684 int cpu_is_offline(cpu_t *); /* check if CPU is offline */
683 685 int cpu_is_poweredoff(cpu_t *); /* check if CPU is powered off */
684 686
685 687 int cpu_flagged_online(cpu_flag_t); /* flags show CPU is online */
686 688 int cpu_flagged_nointr(cpu_flag_t); /* flags show CPU not handling intrs */
687 689 int cpu_flagged_active(cpu_flag_t); /* flags show CPU scheduling threads */
688 690 int cpu_flagged_offline(cpu_flag_t); /* flags show CPU is offline */
689 691 int cpu_flagged_poweredoff(cpu_flag_t); /* flags show CPU is powered off */
690 692
691 693 /*
692 694 * The processor_info(2) state of a CPU is a simplified representation suitable
693 695 * for use by an application program. Kernel subsystems should utilize the
694 696 * internal per-CPU state as given by the cpu_flags member of the cpu structure,
695 697 * as this information may include platform- or architecture-specific state
696 698 * critical to a subsystem's disposition of a particular CPU.
697 699 */
698 700 void cpu_set_state(cpu_t *); /* record/timestamp current state */
699 701 int cpu_get_state(cpu_t *); /* get current cpu state */
700 702 const char *cpu_get_state_str(cpu_t *); /* get current cpu state as string */
701 703
702 704
703 705 void cpu_set_curr_clock(uint64_t); /* indicate the current CPU's freq */
704 706 void cpu_set_supp_freqs(cpu_t *, const char *); /* set the CPU supported */
705 707 /* frequencies */
706 708
707 709 int cpu_configure(int);
708 710 int cpu_unconfigure(int);
709 711 void cpu_destroy_bound_threads(cpu_t *cp);
710 712
711 713 extern int cpu_bind_thread(kthread_t *tp, processorid_t bind,
712 714 processorid_t *obind, int *error);
713 715 extern int cpu_unbind(processorid_t cpu_id, boolean_t force);
714 716 extern void thread_affinity_set(kthread_t *t, int cpu_id);
715 717 extern void thread_affinity_clear(kthread_t *t);
716 718 extern void affinity_set(int cpu_id);
717 719 extern void affinity_clear(void);
718 720 extern void init_cpu_mstate(struct cpu *, int);
719 721 extern void term_cpu_mstate(struct cpu *);
720 722 extern void new_cpu_mstate(int, hrtime_t);
721 723 extern void get_cpu_mstate(struct cpu *, hrtime_t *);
722 724 extern void thread_nomigrate(void);
723 725 extern void thread_allowmigrate(void);
724 726 extern void weakbinding_stop(void);
725 727 extern void weakbinding_start(void);
726 728
727 729 /*
728 730 * The following routines affect the CPUs participation in interrupt processing,
729 731 * if that is applicable on the architecture. This only affects interrupts
730 732 * which aren't directed at the processor (not cross calls).
731 733 *
732 734 * cpu_disable_intr returns non-zero if interrupts were previously enabled.
733 735 */
734 736 int cpu_disable_intr(struct cpu *cp); /* stop issuing interrupts to cpu */
735 737 void cpu_enable_intr(struct cpu *cp); /* start issuing interrupts to cpu */
736 738
737 739 /*
738 740 * The mutex cpu_lock protects cpu_flags for all CPUs, as well as the ncpus
739 741 * and ncpus_online counts.
740 742 */
741 743 extern kmutex_t cpu_lock; /* lock protecting CPU data */
742 744
743 745 /*
744 746 * CPU state change events
745 747 *
746 748 * Various subsystems need to know when CPUs change their state. They get this
747 749 * information by registering CPU state change callbacks using
748 750 * register_cpu_setup_func(). Whenever any CPU changes its state, the callback
749 751 * function is called. The callback function is passed three arguments:
750 752 *
751 753 * Event, described by cpu_setup_t
752 754 * CPU ID
753 755 * Transparent pointer passed when registering the callback
754 756 *
755 757 * The callback function is called with cpu_lock held. The return value from the
756 758 * callback function is usually ignored, except for CPU_CONFIG and CPU_UNCONFIG
757 759 * events. For these two events, non-zero return value indicates a failure and
758 760 * prevents successful completion of the operation.
759 761 *
760 762 * New events may be added in the future. Callback functions should ignore any
761 763 * events that they do not understand.
762 764 *
763 765 * The following events provide notification callbacks:
764 766 *
765 767 * CPU_INIT A new CPU is started and added to the list of active CPUs
766 768 * This event is only used during boot
767 769 *
768 770 * CPU_CONFIG A newly inserted CPU is prepared for starting running code
769 771 * This event is called by DR code
770 772 *
771 773 * CPU_UNCONFIG CPU has been powered off and needs cleanup
772 774 * This event is called by DR code
773 775 *
774 776 * CPU_ON CPU is enabled but does not run anything yet
775 777 *
776 778 * CPU_INTR_ON CPU is enabled and has interrupts enabled
777 779 *
778 780 * CPU_OFF CPU is going offline but can still run threads
779 781 *
780 782 * CPU_CPUPART_OUT CPU is going to move out of its partition
781 783 *
782 784 * CPU_CPUPART_IN CPU is going to move to a new partition
783 785 *
784 786 * CPU_SETUP CPU is set up during boot and can run threads
785 787 */
786 788 typedef enum {
787 789 CPU_INIT,
788 790 CPU_CONFIG,
789 791 CPU_UNCONFIG,
790 792 CPU_ON,
791 793 CPU_OFF,
792 794 CPU_CPUPART_IN,
793 795 CPU_CPUPART_OUT,
794 796 CPU_SETUP,
795 797 CPU_INTR_ON
796 798 } cpu_setup_t;
797 799
798 800 typedef int cpu_setup_func_t(cpu_setup_t, int, void *);
799 801
800 802 /*
801 803 * Routines used to register interest in cpu's being added to or removed
802 804 * from the system.
803 805 */
804 806 extern void register_cpu_setup_func(cpu_setup_func_t *, void *);
805 807 extern void unregister_cpu_setup_func(cpu_setup_func_t *, void *);
806 808 extern void cpu_state_change_notify(int, cpu_setup_t);
807 809
808 810 /*
809 811 * Call specified function on the given CPU
810 812 */
811 813 typedef void (*cpu_call_func_t)(uintptr_t, uintptr_t);
812 814 extern void cpu_call(cpu_t *, cpu_call_func_t, uintptr_t, uintptr_t);
813 815
814 816
815 817 /*
|
↓ open down ↓ |
186 lines elided |
↑ open up ↑ |
816 818 * Create various strings that describe the given CPU for the
817 819 * processor_info system call and configuration-related kstats.
818 820 */
819 821 #define CPU_IDSTRLEN 100
820 822
821 823 extern void init_cpu_info(struct cpu *);
822 824 extern void populate_idstr(struct cpu *);
823 825 extern void cpu_vm_data_init(struct cpu *);
824 826 extern void cpu_vm_data_destroy(struct cpu *);
825 827
826 -#endif /* _KERNEL */
828 +#endif /* _KERNEL || _FAKE_KERNEL */
827 829
828 830 #ifdef __cplusplus
829 831 }
830 832 #endif
831 833
832 834 #endif /* _SYS_CPUVAR_H */
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX