Print this page
11630 remove checks for 64-bit capable hardware
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/sun4/os/startup.c
+++ new/usr/src/uts/sun4/os/startup.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 25 * Copyright 2019 Peter Tribble.
26 26 */
27 27
28 28 #include <sys/machsystm.h>
29 29 #include <sys/archsystm.h>
30 30 #include <sys/vm.h>
31 31 #include <sys/cpu.h>
32 32 #include <sys/atomic.h>
33 33 #include <sys/reboot.h>
34 34 #include <sys/kdi.h>
35 35 #include <sys/bootconf.h>
36 36 #include <sys/memlist_plat.h>
37 37 #include <sys/memlist_impl.h>
38 38 #include <sys/prom_plat.h>
39 39 #include <sys/prom_isa.h>
40 40 #include <sys/autoconf.h>
41 41 #include <sys/ivintr.h>
42 42 #include <sys/fpu/fpusystm.h>
43 43 #include <sys/iommutsb.h>
44 44 #include <vm/vm_dep.h>
45 45 #include <vm/seg_dev.h>
46 46 #include <vm/seg_kmem.h>
47 47 #include <vm/seg_kpm.h>
48 48 #include <vm/seg_map.h>
49 49 #include <vm/seg_kp.h>
50 50 #include <sys/sysconf.h>
51 51 #include <vm/hat_sfmmu.h>
52 52 #include <sys/kobj.h>
53 53 #include <sys/sun4asi.h>
54 54 #include <sys/clconf.h>
55 55 #include <sys/platform_module.h>
56 56 #include <sys/panic.h>
57 57 #include <sys/cpu_sgnblk_defs.h>
58 58 #include <sys/clock.h>
59 59 #include <sys/cmn_err.h>
60 60 #include <sys/dumphdr.h>
61 61 #include <sys/promif.h>
62 62 #include <sys/prom_debug.h>
63 63 #include <sys/traptrace.h>
64 64 #include <sys/memnode.h>
65 65 #include <sys/mem_cage.h>
66 66 #include <sys/mmu.h>
67 67 #include <sys/swap.h>
68 68
69 69 extern void setup_trap_table(void);
70 70 extern int cpu_intrq_setup(struct cpu *);
71 71 extern void cpu_intrq_register(struct cpu *);
72 72 extern void contig_mem_init(void);
73 73 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
74 74 extern void mach_dump_buffer_init(void);
75 75 extern void mach_descrip_init(void);
76 76 extern void mach_descrip_startup_fini(void);
77 77 extern void mach_memscrub(void);
78 78 extern void mach_fpras(void);
79 79 extern void mach_cpu_halt_idle(void);
80 80 extern void mach_hw_copy_limit(void);
81 81 extern void load_mach_drivers(void);
82 82 extern void load_tod_module(void);
83 83 #pragma weak load_tod_module
84 84
85 85 extern int ndata_alloc_mmfsa(struct memlist *ndata);
86 86 #pragma weak ndata_alloc_mmfsa
87 87
88 88 extern void cif_init(void);
89 89 #pragma weak cif_init
90 90
91 91 extern void parse_idprom(void);
92 92 extern void add_vx_handler(char *, int, void (*)(cell_t *));
93 93 extern void mem_config_init(void);
94 94 extern void memseg_remap_init(void);
95 95
96 96 extern void mach_kpm_init(void);
97 97 extern void pcf_init();
98 98 extern int size_pse_array(pgcnt_t, int);
99 99 extern void pg_init();
100 100
101 101 /*
102 102 * External Data:
103 103 */
104 104 extern int vac_size; /* cache size in bytes */
105 105 extern uint_t vac_mask; /* VAC alignment consistency mask */
106 106 extern uint_t vac_colors;
107 107
108 108 /*
109 109 * Global Data Definitions:
110 110 */
111 111
112 112 /*
113 113 * XXX - Don't port this to new architectures
114 114 * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
115 115 * 'romp' has no use with a prom with an IEEE 1275 client interface.
116 116 * The driver doesn't use the value, but it depends on the symbol.
117 117 */
118 118 void *romp; /* veritas driver won't load without romp 4154976 */
119 119 /*
120 120 * Declare these as initialized data so we can patch them.
121 121 */
122 122 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */
123 123 pgcnt_t segkpsize =
124 124 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */
125 125 uint_t segmap_percent = 6; /* Size of segmap segment */
126 126
127 127 int use_cache = 1; /* cache not reliable (605 bugs) with MP */
128 128 int vac_copyback = 1;
129 129 char *cache_mode = NULL;
130 130 int use_mix = 1;
131 131 int prom_debug = 0;
132 132
133 133 caddr_t boot_tba; /* %tba at boot - used by kmdb */
134 134 uint_t tba_taken_over = 0;
135 135
136 136 caddr_t s_text; /* start of kernel text segment */
137 137 caddr_t e_text; /* end of kernel text segment */
138 138 caddr_t s_data; /* start of kernel data segment */
139 139 caddr_t e_data; /* end of kernel data segment */
140 140
141 141 caddr_t modtext; /* beginning of module text */
142 142 size_t modtext_sz; /* size of module text */
143 143 caddr_t moddata; /* beginning of module data reserve */
144 144 caddr_t e_moddata; /* end of module data reserve */
145 145
146 146 /*
147 147 * End of first block of contiguous kernel in 32-bit virtual address space
148 148 */
149 149 caddr_t econtig32; /* end of first blk of contiguous kernel */
150 150
151 151 caddr_t ncbase; /* beginning of non-cached segment */
152 152 caddr_t ncend; /* end of non-cached segment */
153 153
154 154 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */
155 155 caddr_t nalloc_base; /* beginning of nucleus allocation */
156 156 caddr_t nalloc_end; /* end of nucleus allocatable memory */
157 157 caddr_t valloc_base; /* beginning of kvalloc segment */
158 158
159 159 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */
160 160 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */
161 161 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */
162 162 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */
163 163 int kmem64_szc; /* page size code */
164 164 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */
165 165
166 166 uintptr_t shm_alignment; /* VAC address consistency modulus */
167 167 struct memlist *phys_install; /* Total installed physical memory */
168 168 struct memlist *phys_avail; /* Available (unreserved) physical memory */
169 169 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */
170 170 struct memlist *nopp_list; /* pages with no backing page structs */
171 171 struct memlist ndata; /* memlist of nucleus allocatable memory */
172 172 int memexp_flag; /* memory expansion card flag */
173 173 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */
174 174 pgcnt_t obp_pages; /* Physical pages used by OBP */
175 175
176 176 /*
177 177 * VM data structures
178 178 */
179 179 long page_hashsz; /* Size of page hash table (power of two) */
180 180 unsigned int page_hashsz_shift; /* log2(page_hashsz) */
181 181 struct page *pp_base; /* Base of system page struct array */
182 182 size_t pp_sz; /* Size in bytes of page struct array */
183 183 struct page **page_hash; /* Page hash table */
184 184 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */
185 185 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */
186 186 int pse_shift; /* log2(pse_table_size) */
187 187 struct seg ktextseg; /* Segment used for kernel executable image */
188 188 struct seg kvalloc; /* Segment used for "valloc" mapping */
189 189 struct seg kpseg; /* Segment used for pageable kernel virt mem */
190 190 struct seg ktexthole; /* Segment used for nucleus text hole */
191 191 struct seg kmapseg; /* Segment used for generic kernel mappings */
192 192 struct seg kpmseg; /* Segment used for physical mapping */
193 193 struct seg kdebugseg; /* Segment used for the kernel debugger */
194 194
195 195 void *kpm_pp_base; /* Base of system kpm_page array */
196 196 size_t kpm_pp_sz; /* Size of system kpm_page array */
197 197 pgcnt_t kpm_npages; /* How many kpm pages are managed */
198 198
199 199 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */
200 200 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */
201 201 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */
202 202
203 203 int segzio_fromheap = 0; /* zio allocations occur from heap */
204 204 caddr_t segzio_base; /* Base address of segzio */
205 205 pgcnt_t segziosize = 0; /* size of zio segment in pages */
206 206
207 207 /*
208 208 * A static DR page_t VA map is reserved that can map the page structures
209 209 * for a domain's entire RA space. The pages that backs this space are
210 210 * dynamically allocated and need not be physically contiguous. The DR
211 211 * map size is derived from KPM size.
212 212 */
213 213 int ppvm_enable = 0; /* Static virtual map for page structs */
214 214 page_t *ppvm_base; /* Base of page struct map */
215 215 pgcnt_t ppvm_size = 0; /* Size of page struct map */
216 216
217 217 /*
218 218 * debugger pages (if allocated)
219 219 */
220 220 struct vnode kdebugvp;
221 221
222 222 /*
223 223 * VA range available to the debugger
224 224 */
225 225 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
226 226 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
227 227
228 228 /*
229 229 * Segment for relocated kernel structures in 64-bit large RAM kernels
230 230 */
231 231 struct seg kmem64;
232 232
233 233 struct memseg *memseg_free;
234 234
235 235 struct vnode unused_pages_vp;
236 236
237 237 /*
238 238 * VM data structures allocated early during boot.
239 239 */
240 240 size_t pagehash_sz;
241 241 uint64_t memlist_sz;
242 242
243 243 char tbr_wr_addr_inited = 0;
244 244
245 245 caddr_t mpo_heap32_buf = NULL;
246 246 size_t mpo_heap32_bufsz = 0;
247 247
248 248 /*
249 249 * Static Routines:
250 250 */
251 251 static int ndata_alloc_memseg(struct memlist *, size_t);
252 252 static void memlist_new(uint64_t, uint64_t, struct memlist **);
253 253 static void memlist_add(uint64_t, uint64_t,
254 254 struct memlist **, struct memlist **);
255 255 static void kphysm_init(void);
256 256 static void kvm_init(void);
257 257 static void install_kmem64_tte(void);
258 258
259 259 static void startup_init(void);
260 260 static void startup_memlist(void);
261 261 static void startup_modules(void);
262 262 static void startup_bop_gone(void);
263 263 static void startup_vm(void);
264 264 static void startup_end(void);
265 265 static void setup_cage_params(void);
266 266 static void startup_create_io_node(void);
267 267
268 268 static pgcnt_t npages;
|
↓ open down ↓ |
268 lines elided |
↑ open up ↑ |
269 269 static struct memlist *memlist;
270 270 void *memlist_end;
271 271
272 272 static pgcnt_t bop_alloc_pages;
273 273 static caddr_t hblk_base;
274 274 uint_t hblk_alloc_dynamic = 0;
275 275 uint_t hblk1_min = H1MIN;
276 276
277 277
278 278 /*
279 - * Hook for down-rev firmware
280 - */
281 -static void do_prom_version_check(void);
282 -
283 -/*
284 279 * After receiving a thermal interrupt, this is the number of seconds
285 280 * to delay before shutting off the system, assuming
286 281 * shutdown fails. Use /etc/system to change the delay if this isn't
287 282 * large enough.
288 283 */
289 284 int thermal_powerdown_delay = 1200;
290 285
291 286 /*
292 287 * Used to hold off page relocations into the cage until OBP has completed
293 288 * its boot-time handoff of its resources to the kernel.
294 289 */
295 290 int page_relocate_ready = 0;
296 291
297 292 /*
298 293 * Indicate if kmem64 allocation was done in small chunks
299 294 */
300 295 int kmem64_smchunks = 0;
301 296
302 297 /*
303 298 * Enable some debugging messages concerning memory usage...
304 299 */
305 300 #ifdef DEBUGGING_MEM
306 301 static int debugging_mem;
307 302 static void
308 303 printmemlist(char *title, struct memlist *list)
309 304 {
310 305 if (!debugging_mem)
311 306 return;
312 307
313 308 printf("%s\n", title);
314 309
315 310 while (list) {
316 311 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
317 312 (uint32_t)(list->ml_address >> 32),
318 313 (uint32_t)list->ml_address,
319 314 (uint32_t)(list->ml_size >> 32),
320 315 (uint32_t)(list->ml_size));
321 316 list = list->ml_next;
322 317 }
323 318 }
324 319
325 320 void
326 321 printmemseg(struct memseg *memseg)
327 322 {
328 323 if (!debugging_mem)
329 324 return;
330 325
331 326 printf("memseg\n");
332 327
333 328 while (memseg) {
334 329 prom_printf("\tpage = 0x%p, epage = 0x%p, "
335 330 "pfn = 0x%x, epfn = 0x%x\n",
336 331 memseg->pages, memseg->epages,
337 332 memseg->pages_base, memseg->pages_end);
338 333 memseg = memseg->next;
339 334 }
340 335 }
341 336
342 337 #define debug_pause(str) halt((str))
343 338 #define MPRINTF(str) if (debugging_mem) prom_printf((str))
344 339 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a))
345 340 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b))
346 341 #define MPRINTF3(str, a, b, c) \
347 342 if (debugging_mem) prom_printf((str), (a), (b), (c))
348 343 #else /* DEBUGGING_MEM */
349 344 #define MPRINTF(str)
350 345 #define MPRINTF1(str, a)
351 346 #define MPRINTF2(str, a, b)
352 347 #define MPRINTF3(str, a, b, c)
353 348 #endif /* DEBUGGING_MEM */
354 349
355 350
356 351 /*
357 352 *
358 353 * Kernel's Virtual Memory Layout.
359 354 * /-----------------------\
360 355 * 0xFFFFFFFF.FFFFFFFF -| |-
361 356 * | OBP's virtual page |
362 357 * | tables |
363 358 * 0xFFFFFFFC.00000000 -|-----------------------|-
364 359 * : :
365 360 * : :
366 361 * -|-----------------------|-
367 362 * | segzio | (base and size vary)
368 363 * 0xFFFFFE00.00000000 -|-----------------------|-
369 364 * | | Ultrasparc I/II support
370 365 * | segkpm segment | up to 2TB of physical
371 366 * | (64-bit kernel ONLY) | memory, VAC has 2 colors
372 367 * | |
373 368 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment
374 369 * : :
375 370 * : :
376 371 * 0xFFFFF810.00000000 -|-----------------------|- hole_end
377 372 * | | ^
378 373 * | UltraSPARC I/II call | |
379 374 * | bug requires an extra | |
380 375 * | 4 GB of space between | |
381 376 * | hole and used RAM | |
382 377 * | | |
383 378 * 0xFFFFF800.00000000 -|-----------------------|- |
384 379 * | | |
385 380 * | Virtual Address Hole | UltraSPARC
386 381 * | on UltraSPARC I/II | I/II * ONLY *
387 382 * | | |
388 383 * 0x00000800.00000000 -|-----------------------|- |
389 384 * | | |
390 385 * | UltraSPARC I/II call | |
391 386 * | bug requires an extra | |
392 387 * | 4 GB of space between | |
393 388 * | hole and used RAM | |
394 389 * | | v
395 390 * 0x000007FF.00000000 -|-----------------------|- hole_start -----
396 391 * : : ^
397 392 * : : |
398 393 * |-----------------------| |
399 394 * | | |
400 395 * | ecache flush area | |
401 396 * | (twice largest e$) | |
402 397 * | | |
403 398 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ |
404 399 * | overmapped area | alignend_end |
405 400 * | (kmem64_alignsize | |
406 401 * | boundary) | |
407 402 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end |
408 403 * | | |
409 404 * | 64-bit kernel ONLY | |
410 405 * | | |
411 406 * | kmem64 segment | |
412 407 * | | |
413 408 * | (Relocated extra HME | Approximately
414 409 * | block allocations, | 1 TB of virtual
415 410 * | memnode freelists, | address space
416 411 * | HME hash buckets, | |
417 412 * | mml_table, kpmp_table,| |
418 413 * | page_t array and | |
419 414 * | hashblock pool to | |
420 415 * | avoid hard-coded | |
421 416 * | 32-bit vaddr | |
422 417 * | limitations) | |
423 418 * | | v
424 419 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base)
425 420 * | |
426 421 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB)
427 422 * | |
428 423 * 0x00000300.00000000 -|-----------------------|- SYSBASE
429 424 * : :
430 425 * : :
431 426 * -|-----------------------|-
432 427 * | |
433 428 * | segmap segment | SEGMAPSIZE (1/8th physmem,
434 429 * | | 256G MAX)
435 430 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE
436 431 * : :
437 432 * : :
438 433 * -|-----------------------|-
439 434 * | |
440 435 * | segkp | SEGKPSIZE (2GB)
441 436 * | |
442 437 * | |
443 438 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE
444 439 * | |
445 440 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE
446 441 * | | (SEGKPBASE - 0x400000)
447 442 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE
448 443 * | | (MEMSCRUBBASE - NCARGS)
449 444 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE
450 445 * | | (ARGSBASE - PPMAPSIZE)
451 446 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE
452 447 * | |
453 448 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE
454 449 * | |
455 450 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE
456 451 * : :
457 452 * : :
458 453 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR
459 454 * | |
460 455 * | OBP |
461 456 * | |
462 457 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR
463 458 * | kmdb |
464 459 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE
465 460 * : :
466 461 * : :
467 462 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32
468 463 * | |
469 464 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 =
470 465 * | | ~64MB)
471 466 * -|-----------------------|
472 467 * | IVSIZE |
473 468 * 0x00000000.70004000 -|-----------------------|
474 469 * | panicbuf |
475 470 * 0x00000000.70002000 -|-----------------------|
476 471 * | PAGESIZE |
477 472 * 0x00000000.70000000 -|-----------------------|- SYSBASE32
478 473 * | boot-time |
479 474 * | temporary space |
480 475 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE
481 476 * : :
482 477 * : :
483 478 * | |
484 479 * |-----------------------|- econtig32
485 480 * | vm structures |
486 481 * 0x00000000.01C00000 |-----------------------|- nalloc_end
487 482 * | TSBs |
488 483 * |-----------------------|- end/nalloc_base
489 484 * | kernel data & bss |
490 485 * 0x00000000.01800000 -|-----------------------|
491 486 * : nucleus text hole :
492 487 * 0x00000000.01400000 -|-----------------------|
493 488 * : :
494 489 * |-----------------------|
495 490 * | module text |
496 491 * |-----------------------|- e_text/modtext
497 492 * | kernel text |
498 493 * |-----------------------|
499 494 * | trap table (48k) |
500 495 * 0x00000000.01000000 -|-----------------------|- KERNELBASE
501 496 * | reserved for trapstat |} TSTAT_TOTAL_SIZE
502 497 * |-----------------------|
503 498 * | |
504 499 * | invalid |
505 500 * | |
506 501 * 0x00000000.00000000 _|_______________________|
507 502 *
508 503 *
509 504 *
510 505 * 32-bit User Virtual Memory Layout.
511 506 * /-----------------------\
512 507 * | |
513 508 * | invalid |
514 509 * | |
515 510 * 0xFFC00000 -|-----------------------|- USERLIMIT
516 511 * | user stack |
517 512 * : :
518 513 * : :
519 514 * : :
520 515 * | user data |
521 516 * -|-----------------------|-
522 517 * | user text |
523 518 * 0x00002000 -|-----------------------|-
524 519 * | invalid |
525 520 * 0x00000000 _|_______________________|
526 521 *
527 522 *
528 523 *
529 524 * 64-bit User Virtual Memory Layout.
530 525 * /-----------------------\
531 526 * | |
532 527 * | invalid |
533 528 * | |
534 529 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
535 530 * | user stack |
536 531 * : :
537 532 * : :
538 533 * : :
539 534 * | user data |
540 535 * -|-----------------------|-
541 536 * | user text |
542 537 * 0x00000000.01000000 -|-----------------------|-
543 538 * | invalid |
544 539 * 0x00000000.00000000 _|_______________________|
545 540 */
546 541
547 542 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
548 543 extern uint64_t ecache_flush_address(void);
549 544
550 545 #pragma weak load_platform_modules
551 546 #pragma weak plat_startup_memlist
552 547 #pragma weak ecache_init_scrub_flush_area
553 548 #pragma weak ecache_flush_address
554 549
555 550
556 551 /*
557 552 * By default the DR Cage is enabled for maximum OS
558 553 * MPSS performance. Users needing to disable the cage mechanism
559 554 * can set this variable to zero via /etc/system.
560 555 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
561 556 * will result in loss of DR functionality.
562 557 * Platforms wishing to disable kernel Cage by default
563 558 * should do so in their set_platform_defaults() routine.
564 559 */
565 560 int kernel_cage_enable = 1;
566 561
567 562 static void
568 563 setup_cage_params(void)
569 564 {
570 565 void (*func)(void);
571 566
572 567 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
573 568 if (func != NULL) {
574 569 (*func)();
575 570 return;
576 571 }
577 572
578 573 if (kernel_cage_enable == 0) {
579 574 return;
580 575 }
581 576 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
582 577
583 578 if (kcage_on) {
584 579 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
585 580 } else {
586 581 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
587 582 }
588 583
589 584 }
590 585
591 586 /*
592 587 * Machine-dependent startup code
593 588 */
594 589 void
595 590 startup(void)
596 591 {
597 592 startup_init();
598 593 if (&startup_platform)
599 594 startup_platform();
600 595 startup_memlist();
601 596 startup_modules();
602 597 setup_cage_params();
603 598 startup_bop_gone();
604 599 startup_vm();
605 600 startup_end();
606 601 }
607 602
608 603 struct regs sync_reg_buf;
609 604 uint64_t sync_tt;
610 605
611 606 void
612 607 sync_handler(void)
613 608 {
614 609 struct panic_trap_info ti;
615 610 int i;
616 611
617 612 /*
618 613 * Prevent trying to talk to the other CPUs since they are
619 614 * sitting in the prom and won't reply.
620 615 */
621 616 for (i = 0; i < NCPU; i++) {
622 617 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
623 618 cpu[i]->cpu_flags &= ~CPU_READY;
624 619 cpu[i]->cpu_flags |= CPU_QUIESCED;
625 620 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
626 621 }
627 622 }
628 623
629 624 /*
630 625 * Force a serial dump, since there are no CPUs to help.
631 626 */
632 627 dump_plat_mincpu = 0;
633 628
634 629 /*
635 630 * We've managed to get here without going through the
636 631 * normal panic code path. Try and save some useful
637 632 * information.
638 633 */
639 634 if (!panicstr && (curthread->t_panic_trap == NULL)) {
640 635 ti.trap_type = sync_tt;
641 636 ti.trap_regs = &sync_reg_buf;
642 637 ti.trap_addr = NULL;
643 638 ti.trap_mmu_fsr = 0x0;
644 639
645 640 curthread->t_panic_trap = &ti;
646 641 }
647 642
648 643 /*
649 644 * If we're re-entering the panic path, update the signature
650 645 * block so that the SC knows we're in the second part of panic.
651 646 */
652 647 if (panicstr)
653 648 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
654 649
655 650 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
656 651 panic("sync initiated");
657 652 }
658 653
659 654
660 655 static void
661 656 startup_init(void)
662 657 {
663 658 /*
664 659 * We want to save the registers while we're still in OBP
665 660 * so that we know they haven't been fiddled with since.
666 661 * (In principle, OBP can't change them just because it
667 662 * makes a callback, but we'd rather not depend on that
668 663 * behavior.)
669 664 */
670 665 char sync_str[] =
671 666 "warning @ warning off : sync "
672 667 "%%tl-c %%tstate h# %p x! "
673 668 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
674 669 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
675 670 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
676 671 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
677 672 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
678 673 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
679 674 "%%y h# %p l! %%tl-c %%tt h# %p x! "
680 675 "sync ; warning !";
681 676
682 677 /*
683 678 * 20 == num of %p substrings
684 679 * 16 == max num of chars %p will expand to.
685 680 */
686 681 char bp[sizeof (sync_str) + 16 * 20];
687 682
688 683 /*
689 684 * Initialize ptl1 stack for the 1st CPU.
690 685 */
691 686 ptl1_init_cpu(&cpu0);
692 687
693 688 /*
694 689 * Initialize the address map for cache consistent mappings
695 690 * to random pages; must be done after vac_size is set.
696 691 */
697 692 ppmapinit();
698 693
699 694 /*
700 695 * Initialize the PROM callback handler.
701 696 */
702 697 init_vx_handler();
703 698
704 699 /*
705 700 * have prom call sync_callback() to handle the sync and
706 701 * save some useful information which will be stored in the
707 702 * core file later.
708 703 */
709 704 (void) sprintf((char *)bp, sync_str,
710 705 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
711 706 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
712 707 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
713 708 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
714 709 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
715 710 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
716 711 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
717 712 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
718 713 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
719 714 (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
720 715 prom_interpret(bp, 0, 0, 0, 0, 0);
721 716 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
722 717 }
723 718
724 719
725 720 size_t
726 721 calc_pp_sz(pgcnt_t npages)
727 722 {
728 723
729 724 return (npages * sizeof (struct page));
730 725 }
731 726
732 727 size_t
733 728 calc_kpmpp_sz(pgcnt_t npages)
734 729 {
735 730
736 731 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
737 732 kpm_pgsz = 1ull << kpm_pgshft;
738 733 kpm_pgoff = kpm_pgsz - 1;
739 734 kpmp2pshft = kpm_pgshft - PAGESHIFT;
740 735 kpmpnpgs = 1 << kpmp2pshft;
741 736
742 737 if (kpm_smallpages == 0) {
743 738 /*
744 739 * Avoid fragmentation problems in kphysm_init()
745 740 * by allocating for all of physical memory
746 741 */
747 742 kpm_npages = ptokpmpr(physinstalled);
748 743 return (kpm_npages * sizeof (kpm_page_t));
749 744 } else {
750 745 kpm_npages = npages;
751 746 return (kpm_npages * sizeof (kpm_spage_t));
752 747 }
753 748 }
754 749
755 750 size_t
756 751 calc_pagehash_sz(pgcnt_t npages)
757 752 {
758 753 /* LINTED */
759 754 ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page))));
760 755 /*
761 756 * The page structure hash table size is a power of 2
762 757 * such that the average hash chain length is PAGE_HASHAVELEN.
763 758 */
764 759 page_hashsz = npages / PAGE_HASHAVELEN;
765 760 page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1),
766 761 highbit(page_hashsz));
767 762 page_hashsz = 1 << page_hashsz_shift;
768 763 return (page_hashsz * sizeof (struct page *));
769 764 }
770 765
771 766 int testkmem64_smchunks = 0;
772 767
773 768 int
774 769 alloc_kmem64(caddr_t base, caddr_t end)
775 770 {
776 771 int i;
777 772 caddr_t aligned_end = NULL;
778 773
779 774 if (testkmem64_smchunks)
780 775 return (1);
781 776
782 777 /*
783 778 * Make one large memory alloc after figuring out the 64-bit size. This
784 779 * will enable use of the largest page size appropriate for the system
785 780 * architecture.
786 781 */
787 782 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
788 783 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
789 784 for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
790 785 size_t alloc_size, alignsize;
791 786 #if !defined(C_OBP)
792 787 unsigned long long pa;
793 788 #endif /* !C_OBP */
794 789
795 790 if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
796 791 continue;
797 792 alignsize = TTEBYTES(i);
798 793 kmem64_szc = i;
799 794
800 795 /* limit page size for small memory */
801 796 if (mmu_btop(alignsize) > (npages >> 2))
802 797 continue;
803 798
804 799 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
805 800 alloc_size = aligned_end - base;
806 801 #if !defined(C_OBP)
807 802 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
808 803 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
809 804 kmem64_pabase = pa;
810 805 kmem64_aligned_end = aligned_end;
811 806 install_kmem64_tte();
812 807 break;
813 808 } else {
814 809 prom_free_phys(alloc_size, pa);
815 810 }
816 811 }
817 812 #else /* !C_OBP */
818 813 if (prom_alloc(base, alloc_size, alignsize) == base) {
819 814 kmem64_pabase = va_to_pa(kmem64_base);
820 815 kmem64_aligned_end = aligned_end;
821 816 break;
822 817 }
823 818 #endif /* !C_OBP */
824 819 if (i == TTE8K) {
825 820 #ifdef sun4v
826 821 /* return failure to try small allocations */
827 822 return (1);
828 823 #else
829 824 prom_panic("kmem64 allocation failure");
830 825 #endif
831 826 }
832 827 }
833 828 ASSERT(aligned_end != NULL);
834 829 return (0);
835 830 }
836 831
837 832 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
838 833 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
839 834
840 835 #if !defined(C_OBP)
841 836 /*
842 837 * Install a temporary tte handler in OBP for kmem64 area.
843 838 *
844 839 * We map kmem64 area with large pages before the trap table is taken
845 840 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
846 841 * the same area. Duplicate tlb entries with different page sizes
847 842 * cause unpredicatble behavior. To avoid this, we don't create
848 843 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
849 844 * OBP). Instead, we manage translations with a temporary va>tte-data
850 845 * handler (kmem64-tte). This handler is replaced by unix-tte when
851 846 * the trap table is taken over.
852 847 *
853 848 * The temporary handler knows the physical address of the kmem64
854 849 * area. It uses the prom's pgmap@ Forth word for other addresses.
855 850 *
856 851 * We have to use BOP_ALLOC() method for C-OBP platforms because
857 852 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
858 853 * sun4u platforms. On sun4u we flush tlb after trap table is taken
859 854 * over if we use large pages for kernel heap and kmem64. Since sun4u
860 855 * prom (unlike sun4v) calls va>tte-data first for client address
861 856 * translation prom's ttes for kmem64 can't get into TLB even if we
862 857 * later switch to prom's trap table again. C-OBP uses 4M pages for
863 858 * client mappings when possible so on all platforms we get the
864 859 * benefit from large mappings for kmem64 area immediately during
865 860 * boot.
866 861 *
867 862 * pseudo code:
868 863 * if (context != 0) {
869 864 * return false
870 865 * } else if (miss_va in range[kmem64_base, kmem64_end)) {
871 866 * tte = tte_template +
872 867 * (((miss_va & pagemask) - kmem64_base));
873 868 * return tte, true
874 869 * } else {
875 870 * return pgmap@ result
876 871 * }
877 872 */
878 873 char kmem64_obp_str[] =
879 874 "h# %lx constant kmem64-base "
880 875 "h# %lx constant kmem64-end "
881 876 "h# %lx constant kmem64-pagemask "
882 877 "h# %lx constant kmem64-template "
883 878
884 879 ": kmem64-tte ( addr cnum -- false | tte-data true ) "
885 880 " if ( addr ) "
886 881 " drop false exit then ( false ) "
887 882 " dup kmem64-base kmem64-end within if ( addr ) "
888 883 " kmem64-pagemask and ( addr' ) "
889 884 " kmem64-base - ( addr' ) "
890 885 " kmem64-template + ( tte ) "
891 886 " true ( tte true ) "
892 887 " else ( addr ) "
893 888 " pgmap@ ( tte ) "
894 889 " dup 0< if true else drop false then ( tte true | false ) "
895 890 " then ( tte true | false ) "
896 891 "; "
897 892
898 893 "' kmem64-tte is va>tte-data "
899 894 ;
900 895
901 896 static void
902 897 install_kmem64_tte()
903 898 {
904 899 char b[sizeof (kmem64_obp_str) + (4 * 16)];
905 900 tte_t tte;
906 901
907 902 PRM_DEBUG(kmem64_pabase);
908 903 PRM_DEBUG(kmem64_szc);
909 904 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
910 905 PROC_DATA | HAT_NOSYNC, kmem64_szc);
911 906 PRM_DEBUG(tte.ll);
912 907 (void) sprintf(b, kmem64_obp_str,
913 908 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
914 909 ASSERT(strlen(b) < sizeof (b));
915 910 prom_interpret(b, 0, 0, 0, 0, 0);
916 911 }
917 912 #endif /* !C_OBP */
918 913
919 914 /*
920 915 * As OBP takes up some RAM when the system boots, pages will already be "lost"
921 916 * to the system and reflected in npages by the time we see it.
922 917 *
923 918 * We only want to allocate kernel structures in the 64-bit virtual address
924 919 * space on systems with enough RAM to make the overhead of keeping track of
925 920 * an extra kernel memory segment worthwhile.
926 921 *
927 922 * Since OBP has already performed its memory allocations by this point, if we
928 923 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
929 924 * memory in the 64-bit virtual address space; otherwise keep allocations
930 925 * contiguous with we've mapped so far in the 32-bit virtual address space.
931 926 */
932 927 #define MINMOVE_RAM_MB ((size_t)1900)
933 928 #define MB_TO_BYTES(mb) ((mb) * 1048576ul)
934 929 #define BYTES_TO_MB(b) ((b) / 1048576ul)
935 930
936 931 pgcnt_t tune_npages = (pgcnt_t)
937 932 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
938 933
939 934 #pragma weak page_set_colorequiv_arr_cpu
940 935 extern void page_set_colorequiv_arr_cpu(void);
941 936 extern void page_set_colorequiv_arr(void);
942 937
943 938 static pgcnt_t ramdisk_npages;
944 939 static struct memlist *old_phys_avail;
945 940
946 941 kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
947 942
948 943 static void
949 944 startup_memlist(void)
950 945 {
951 946 size_t hmehash_sz, pagelist_sz, tt_sz;
952 947 size_t psetable_sz;
953 948 caddr_t alloc_base;
954 949 caddr_t memspace;
955 950 struct memlist *cur;
956 951 size_t syslimit = (size_t)SYSLIMIT;
957 952 size_t sysbase = (size_t)SYSBASE;
958 953
959 954 /*
960 955 * Initialize enough of the system to allow kmem_alloc to work by
961 956 * calling boot to allocate its memory until the time that
962 957 * kvm_init is completed. The page structs are allocated after
963 958 * rounding up end to the nearest page boundary; the memsegs are
964 959 * initialized and the space they use comes from the kernel heap.
965 960 * With appropriate initialization, they can be reallocated later
966 961 * to a size appropriate for the machine's configuration.
967 962 *
968 963 * At this point, memory is allocated for things that will never
969 964 * need to be freed, this used to be "valloced". This allows a
970 965 * savings as the pages don't need page structures to describe
971 966 * them because them will not be managed by the vm system.
972 967 */
973 968
974 969 /*
975 970 * We're loaded by boot with the following configuration (as
976 971 * specified in the sun4u/conf/Mapfile):
977 972 *
978 973 * text: 4 MB chunk aligned on a 4MB boundary
979 974 * data & bss: 4 MB chunk aligned on a 4MB boundary
980 975 *
981 976 * These two chunks will eventually be mapped by 2 locked 4MB
982 977 * ttes and will represent the nucleus of the kernel. This gives
983 978 * us some free space that is already allocated, some or all of
984 979 * which is made available to kernel module text.
985 980 *
986 981 * The free space in the data-bss chunk is used for nucleus
987 982 * allocatable data structures and we reserve it using the
988 983 * nalloc_base and nalloc_end variables. This space is currently
989 984 * being used for hat data structures required for tlb miss
990 985 * handling operations. We align nalloc_base to a l2 cache
991 986 * linesize because this is the line size the hardware uses to
992 987 * maintain cache coherency.
993 988 * 512K is carved out for module data.
994 989 */
995 990
996 991 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
997 992 e_moddata = moddata + MODDATA;
998 993 nalloc_base = e_moddata;
999 994
1000 995 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
1001 996 valloc_base = nalloc_base;
1002 997
1003 998 /*
1004 999 * Calculate the start of the data segment.
1005 1000 */
1006 1001 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
1007 1002 prom_panic("nucleus data overflow");
1008 1003
1009 1004 PRM_DEBUG(moddata);
1010 1005 PRM_DEBUG(nalloc_base);
1011 1006 PRM_DEBUG(nalloc_end);
1012 1007
1013 1008 /*
1014 1009 * Remember any slop after e_text so we can give it to the modules.
1015 1010 */
1016 1011 PRM_DEBUG(e_text);
1017 1012 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
1018 1013 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
1019 1014 prom_panic("nucleus text overflow");
1020 1015 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
1021 1016 modtext;
1022 1017 PRM_DEBUG(modtext);
1023 1018 PRM_DEBUG(modtext_sz);
1024 1019
1025 1020 init_boot_memlists();
1026 1021 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1027 1022 &boot_physavail, &boot_physavail_len,
1028 1023 &boot_virtavail, &boot_virtavail_len);
1029 1024
1030 1025 /*
1031 1026 * Remember what the physically available highest page is
1032 1027 * so that dumpsys works properly, and find out how much
1033 1028 * memory is installed.
1034 1029 */
1035 1030 installed_top_size_memlist_array(boot_physinstalled,
1036 1031 boot_physinstalled_len, &physmax, &physinstalled);
1037 1032 PRM_DEBUG(physinstalled);
1038 1033 PRM_DEBUG(physmax);
1039 1034
1040 1035 /* Fill out memory nodes config structure */
1041 1036 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
1042 1037
1043 1038 /*
1044 1039 * npages is the maximum of available physical memory possible.
1045 1040 * (ie. it will never be more than this)
1046 1041 *
1047 1042 * When we boot from a ramdisk, the ramdisk memory isn't free, so
1048 1043 * using phys_avail will underestimate what will end up being freed.
1049 1044 * A better initial guess is just total memory minus the kernel text
1050 1045 */
1051 1046 npages = physinstalled - btop(MMU_PAGESIZE4M);
1052 1047
1053 1048 /*
1054 1049 * First allocate things that can go in the nucleus data page
1055 1050 * (fault status, TSBs, dmv, CPUs)
1056 1051 */
1057 1052 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
1058 1053
1059 1054 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
1060 1055 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
1061 1056
1062 1057 if (ndata_alloc_tsbs(&ndata, npages) != 0)
1063 1058 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
1064 1059
1065 1060 if (ndata_alloc_dmv(&ndata) != 0)
1066 1061 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
1067 1062
1068 1063 if (ndata_alloc_page_mutexs(&ndata) != 0)
1069 1064 cmn_err(CE_PANIC,
1070 1065 "no more nucleus memory after page free lists alloc");
1071 1066
1072 1067 if (ndata_alloc_hat(&ndata) != 0)
1073 1068 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
1074 1069
1075 1070 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
1076 1071 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
1077 1072
1078 1073 /*
1079 1074 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1080 1075 *
1081 1076 * There are comments all over the SFMMU code warning of dire
1082 1077 * consequences if the TSBs are moved out of 32-bit space. This
1083 1078 * is largely because the asm code uses "sethi %hi(addr)"-type
1084 1079 * instructions which will not provide the expected result if the
1085 1080 * address is a 64-bit one.
1086 1081 *
1087 1082 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1088 1083 */
1089 1084 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
1090 1085 PRM_DEBUG(alloc_base);
1091 1086
1092 1087 alloc_base = sfmmu_ktsb_alloc(alloc_base);
1093 1088 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1094 1089 PRM_DEBUG(alloc_base);
1095 1090
1096 1091 /*
1097 1092 * Allocate IOMMU TSB array. We do this here so that the physical
1098 1093 * memory gets deducted from the PROM's physical memory list.
1099 1094 */
1100 1095 alloc_base = iommu_tsb_init(alloc_base);
1101 1096 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1102 1097 PRM_DEBUG(alloc_base);
1103 1098
1104 1099 /*
1105 1100 * Allow for an early allocation of physically contiguous memory.
1106 1101 */
1107 1102 alloc_base = contig_mem_prealloc(alloc_base, npages);
1108 1103
1109 1104 /*
1110 1105 * Platforms like Starcat and OPL need special structures assigned in
1111 1106 * 32-bit virtual address space because their probing routines execute
1112 1107 * FCode, and FCode can't handle 64-bit virtual addresses...
1113 1108 */
1114 1109 if (&plat_startup_memlist) {
1115 1110 alloc_base = plat_startup_memlist(alloc_base);
1116 1111 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
1117 1112 ecache_alignsize);
1118 1113 PRM_DEBUG(alloc_base);
1119 1114 }
1120 1115
1121 1116 /*
1122 1117 * Save off where the contiguous allocations to date have ended
1123 1118 * in econtig32.
1124 1119 */
1125 1120 econtig32 = alloc_base;
1126 1121 PRM_DEBUG(econtig32);
1127 1122 if (econtig32 > (caddr_t)KERNEL_LIMIT32)
1128 1123 cmn_err(CE_PANIC, "econtig32 too big");
1129 1124
1130 1125 pp_sz = calc_pp_sz(npages);
1131 1126 PRM_DEBUG(pp_sz);
1132 1127 if (kpm_enable) {
1133 1128 kpm_pp_sz = calc_kpmpp_sz(npages);
1134 1129 PRM_DEBUG(kpm_pp_sz);
1135 1130 }
1136 1131
1137 1132 hmehash_sz = calc_hmehash_sz(npages);
1138 1133 PRM_DEBUG(hmehash_sz);
1139 1134
1140 1135 pagehash_sz = calc_pagehash_sz(npages);
1141 1136 PRM_DEBUG(pagehash_sz);
1142 1137
1143 1138 pagelist_sz = calc_free_pagelist_sz();
1144 1139 PRM_DEBUG(pagelist_sz);
1145 1140
1146 1141 #ifdef TRAPTRACE
1147 1142 tt_sz = calc_traptrace_sz();
1148 1143 PRM_DEBUG(tt_sz);
1149 1144 #else
1150 1145 tt_sz = 0;
1151 1146 #endif /* TRAPTRACE */
1152 1147
1153 1148 /*
1154 1149 * Place the array that protects pp->p_selock in the kmem64 wad.
1155 1150 */
1156 1151 pse_shift = size_pse_array(npages, max_ncpus);
1157 1152 PRM_DEBUG(pse_shift);
1158 1153 pse_table_size = 1 << pse_shift;
1159 1154 PRM_DEBUG(pse_table_size);
1160 1155 psetable_sz = roundup(
1161 1156 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
1162 1157 PRM_DEBUG(psetable_sz);
1163 1158
1164 1159 /*
1165 1160 * Now allocate the whole wad
1166 1161 */
1167 1162 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
1168 1163 pagelist_sz + tt_sz + psetable_sz;
1169 1164 kmem64_sz = roundup(kmem64_sz, PAGESIZE);
1170 1165 kmem64_base = (caddr_t)syslimit;
1171 1166 kmem64_end = kmem64_base + kmem64_sz;
1172 1167 if (alloc_kmem64(kmem64_base, kmem64_end)) {
1173 1168 /*
1174 1169 * Attempt for kmem64 to allocate one big
1175 1170 * contiguous chunk of memory failed.
1176 1171 * We get here because we are sun4v.
1177 1172 * We will proceed by breaking up
1178 1173 * the allocation into two attempts.
1179 1174 * First, we allocate kpm_pp_sz, hmehash_sz,
1180 1175 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as
1181 1176 * one contiguous chunk. This is a much smaller
1182 1177 * chunk and we should get it, if not we panic.
1183 1178 * Note that hmehash and tt need to be physically
1184 1179 * (in the real address sense) contiguous.
1185 1180 * Next, we use bop_alloc_chunk() to
1186 1181 * to allocate the page_t structures.
1187 1182 * This will allow the page_t to be allocated
1188 1183 * in multiple smaller chunks.
1189 1184 * In doing so, the assumption that page_t is
1190 1185 * physically contiguous no longer hold, this is ok
1191 1186 * for sun4v but not for sun4u.
1192 1187 */
1193 1188 size_t tmp_size;
1194 1189 caddr_t tmp_base;
1195 1190
1196 1191 pp_sz = roundup(pp_sz, PAGESIZE);
1197 1192
1198 1193 /*
1199 1194 * Allocate kpm_pp_sz, hmehash_sz,
1200 1195 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz
1201 1196 */
1202 1197 tmp_base = kmem64_base + pp_sz;
1203 1198 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
1204 1199 pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
1205 1200 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
1206 1201 prom_panic("kmem64 prom_alloc contig failed");
1207 1202 PRM_DEBUG(tmp_base);
1208 1203 PRM_DEBUG(tmp_size);
1209 1204
1210 1205 /*
1211 1206 * Allocate the page_ts
1212 1207 */
1213 1208 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
1214 1209 prom_panic("kmem64 bop_alloc_chunk page_t failed");
1215 1210 PRM_DEBUG(kmem64_base);
1216 1211 PRM_DEBUG(pp_sz);
1217 1212
1218 1213 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
1219 1214 ASSERT(kmem64_aligned_end >= kmem64_end);
1220 1215
1221 1216 kmem64_smchunks = 1;
1222 1217 } else {
1223 1218
1224 1219 /*
1225 1220 * We need to adjust pp_sz for the normal
1226 1221 * case where kmem64 can allocate one large chunk
1227 1222 */
1228 1223 if (kpm_smallpages == 0) {
1229 1224 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
1230 1225 } else {
1231 1226 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
1232 1227 sizeof (kpm_spage_t));
1233 1228 }
1234 1229 pp_sz = npages * sizeof (struct page);
1235 1230 }
1236 1231
1237 1232 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
1238 1233 cmn_err(CE_PANIC, "not enough kmem64 space");
1239 1234 PRM_DEBUG(kmem64_base);
1240 1235 PRM_DEBUG(kmem64_end);
1241 1236 PRM_DEBUG(kmem64_aligned_end);
1242 1237
1243 1238 /*
1244 1239 * ... and divy it up
1245 1240 */
1246 1241 alloc_base = kmem64_base;
1247 1242
1248 1243 pp_base = (page_t *)alloc_base;
1249 1244 alloc_base += pp_sz;
1250 1245 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1251 1246 PRM_DEBUG(pp_base);
1252 1247 PRM_DEBUG(npages);
1253 1248
1254 1249 if (kpm_enable) {
1255 1250 kpm_pp_base = alloc_base;
1256 1251 if (kpm_smallpages == 0) {
1257 1252 /* kpm_npages based on physinstalled, don't reset */
1258 1253 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
1259 1254 } else {
1260 1255 kpm_npages = ptokpmpr(npages);
1261 1256 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
1262 1257 }
1263 1258 alloc_base += kpm_pp_sz;
1264 1259 alloc_base =
1265 1260 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1266 1261 PRM_DEBUG(kpm_pp_base);
1267 1262 }
1268 1263
1269 1264 alloc_base = alloc_hmehash(alloc_base);
1270 1265 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1271 1266 PRM_DEBUG(alloc_base);
1272 1267
1273 1268 page_hash = (page_t **)alloc_base;
1274 1269 alloc_base += pagehash_sz;
1275 1270 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1276 1271 PRM_DEBUG(page_hash);
1277 1272
1278 1273 alloc_base = alloc_page_freelists(alloc_base);
1279 1274 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1280 1275 PRM_DEBUG(alloc_base);
1281 1276
1282 1277 #ifdef TRAPTRACE
1283 1278 ttrace_buf = alloc_base;
1284 1279 alloc_base += tt_sz;
1285 1280 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1286 1281 PRM_DEBUG(alloc_base);
1287 1282 #endif /* TRAPTRACE */
1288 1283
1289 1284 pse_mutex = (pad_mutex_t *)alloc_base;
1290 1285 alloc_base += psetable_sz;
1291 1286 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1292 1287 PRM_DEBUG(alloc_base);
1293 1288
1294 1289 /*
1295 1290 * Note that if we use small chunk allocations for
1296 1291 * kmem64, we need to ensure kmem64_end is the same as
1297 1292 * kmem64_aligned_end to prevent subsequent logic from
1298 1293 * trying to reuse the overmapping.
1299 1294 * Otherwise we adjust kmem64_end to what we really allocated.
1300 1295 */
1301 1296 if (kmem64_smchunks) {
1302 1297 kmem64_end = kmem64_aligned_end;
1303 1298 } else {
1304 1299 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
1305 1300 }
1306 1301 kmem64_sz = kmem64_end - kmem64_base;
1307 1302
1308 1303 if (&ecache_init_scrub_flush_area) {
1309 1304 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
1310 1305 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
1311 1306 }
1312 1307
1313 1308 /*
1314 1309 * If physmem is patched to be non-zero, use it instead of
1315 1310 * the monitor value unless physmem is larger than the total
1316 1311 * amount of memory on hand.
1317 1312 */
1318 1313 if (physmem == 0 || physmem > npages)
1319 1314 physmem = npages;
1320 1315
1321 1316 /*
1322 1317 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
1323 1318 * is mounted as root. This memory is held down by OBP and unlike
1324 1319 * the stub boot_archive is never released.
1325 1320 *
1326 1321 * In order to get things sized correctly on lower memory
1327 1322 * machines (where the memory used by the ramdisk represents
1328 1323 * a significant portion of memory), physmem is adjusted.
1329 1324 *
1330 1325 * This is done by subtracting the ramdisk_size which is set
1331 1326 * to the size of the ramdisk (in Kb) in /etc/system at the
1332 1327 * time the miniroot archive is constructed.
1333 1328 */
1334 1329 if (root_is_ramdisk == B_TRUE) {
1335 1330 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
1336 1331 physmem -= ramdisk_npages;
1337 1332 }
1338 1333
1339 1334 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
1340 1335 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
1341 1336
1342 1337 /*
1343 1338 * Allocate space for the interrupt vector table.
1344 1339 */
1345 1340 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
1346 1341 if (memspace != (caddr_t)intr_vec_table)
1347 1342 prom_panic("interrupt vector table allocation failure");
1348 1343
1349 1344 /*
1350 1345 * Between now and when we finish copying in the memory lists,
1351 1346 * allocations happen so the space gets fragmented and the
1352 1347 * lists longer. Leave enough space for lists twice as
1353 1348 * long as we have now; then roundup to a pagesize.
1354 1349 */
1355 1350 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
1356 1351 prom_phys_avail_len() + prom_virt_avail_len());
1357 1352 memlist_sz *= 2;
1358 1353 memlist_sz = roundup(memlist_sz, PAGESIZE);
1359 1354 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
1360 1355 if (memspace == NULL)
1361 1356 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
1362 1357
1363 1358 memlist = (struct memlist *)memspace;
1364 1359 memlist_end = (char *)memspace + memlist_sz;
1365 1360 PRM_DEBUG(memlist);
1366 1361 PRM_DEBUG(memlist_end);
1367 1362
1368 1363 PRM_DEBUG(sysbase);
1369 1364 PRM_DEBUG(syslimit);
1370 1365 kernelheap_init((void *)sysbase, (void *)syslimit,
1371 1366 (caddr_t)sysbase + PAGESIZE, NULL, NULL);
1372 1367
1373 1368 /*
1374 1369 * Take the most current snapshot we can by calling mem-update.
1375 1370 */
1376 1371 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1377 1372 &boot_physavail, &boot_physavail_len,
1378 1373 &boot_virtavail, &boot_virtavail_len);
1379 1374
1380 1375 /*
1381 1376 * Remove the space used by prom_alloc from the kernel heap
1382 1377 * plus the area actually used by the OBP (if any)
1383 1378 * ignoring virtual addresses in virt_avail, above syslimit.
1384 1379 */
1385 1380 virt_avail = memlist;
1386 1381 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1387 1382
1388 1383 for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) {
1389 1384 uint64_t range_base, range_size;
1390 1385
1391 1386 if ((range_base = cur->ml_address + cur->ml_size) <
1392 1387 (uint64_t)sysbase)
1393 1388 continue;
1394 1389 if (range_base >= (uint64_t)syslimit)
1395 1390 break;
1396 1391 /*
1397 1392 * Limit the range to end at syslimit.
1398 1393 */
1399 1394 range_size = MIN(cur->ml_next->ml_address,
1400 1395 (uint64_t)syslimit) - range_base;
1401 1396 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
1402 1397 0, 0, (void *)range_base, (void *)(range_base + range_size),
1403 1398 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1404 1399 }
1405 1400
1406 1401 phys_avail = memlist;
1407 1402 copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1408 1403
1409 1404 /*
1410 1405 * Add any extra memory at the end of the ndata region if there's at
1411 1406 * least a page to add. There might be a few more pages available in
1412 1407 * the middle of the ndata region, but for now they are ignored.
1413 1408 */
1414 1409 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
1415 1410 if (nalloc_base == NULL)
1416 1411 nalloc_base = nalloc_end;
1417 1412 ndata_remain_sz = nalloc_end - nalloc_base;
1418 1413
1419 1414 /*
1420 1415 * Copy physinstalled list into kernel space.
1421 1416 */
1422 1417 phys_install = memlist;
1423 1418 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
1424 1419
1425 1420 /*
1426 1421 * Create list of physical addrs we don't need pp's for:
1427 1422 * kernel text 4M page
1428 1423 * kernel data 4M page - ndata_remain_sz
1429 1424 * kmem64 pages
1430 1425 *
1431 1426 * NB if adding any pages here, make sure no kpm page
1432 1427 * overlaps can occur (see ASSERTs in kphysm_memsegs)
1433 1428 */
1434 1429 nopp_list = memlist;
1435 1430 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
1436 1431 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
1437 1432 &memlist, &nopp_list);
1438 1433
1439 1434 /* Don't add to nopp_list if kmem64 was allocated in smchunks */
1440 1435 if (!kmem64_smchunks)
1441 1436 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
1442 1437
1443 1438 if ((caddr_t)memlist > (memspace + memlist_sz))
1444 1439 prom_panic("memlist overflow");
1445 1440
1446 1441 /*
1447 1442 * Size the pcf array based on the number of cpus in the box at
1448 1443 * boot time.
1449 1444 */
1450 1445 pcf_init();
1451 1446
1452 1447 /*
1453 1448 * Initialize the page structures from the memory lists.
1454 1449 */
1455 1450 kphysm_init();
1456 1451
1457 1452 availrmem_initial = availrmem = freemem;
1458 1453 PRM_DEBUG(availrmem);
1459 1454
1460 1455 /*
1461 1456 * Some of the locks depend on page_hashsz being set!
1462 1457 * kmem_init() depends on this; so, keep it here.
1463 1458 */
1464 1459 page_lock_init();
1465 1460
1466 1461 /*
1467 1462 * Initialize kernel memory allocator.
1468 1463 */
1469 1464 kmem_init();
1470 1465
1471 1466 /*
1472 1467 * Factor in colorequiv to check additional 'equivalent' bins
1473 1468 */
1474 1469 if (&page_set_colorequiv_arr_cpu != NULL)
1475 1470 page_set_colorequiv_arr_cpu();
1476 1471 else
1477 1472 page_set_colorequiv_arr();
1478 1473
1479 1474 /*
1480 1475 * Initialize bp_mapin().
1481 1476 */
1482 1477 bp_init(shm_alignment, HAT_STRICTORDER);
1483 1478
1484 1479 /*
1485 1480 * Reserve space for MPO mblock structs from the 32-bit heap.
1486 1481 */
1487 1482
1488 1483 if (mpo_heap32_bufsz > (size_t)0) {
1489 1484 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
1490 1485 PAGESIZE, 0, 0, mpo_heap32_buf,
1491 1486 mpo_heap32_buf + mpo_heap32_bufsz,
1492 1487 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1493 1488 }
1494 1489 mem_config_init();
1495 1490 }
1496 1491
1497 1492 static void
1498 1493 startup_modules(void)
1499 1494 {
1500 1495 int nhblk1, nhblk8;
1501 1496 size_t nhblksz;
1502 1497 pgcnt_t pages_per_hblk;
1503 1498 size_t hme8blk_sz, hme1blk_sz;
1504 1499
1505 1500 /*
1506 1501 * The system file /etc/system was read already under startup_memlist.
1507 1502 */
1508 1503 if (&set_platform_defaults)
1509 1504 set_platform_defaults();
|
↓ open down ↓ |
1216 lines elided |
↑ open up ↑ |
1510 1505
1511 1506 /*
1512 1507 * Calculate default settings of system parameters based upon
1513 1508 * maxusers, yet allow to be overridden via the /etc/system file.
1514 1509 */
1515 1510 param_calc(0);
1516 1511
1517 1512 mod_setup();
1518 1513
1519 1514 /*
1520 - * If we are running firmware that isn't 64-bit ready
1521 - * then complain and halt.
1522 - */
1523 - do_prom_version_check();
1524 -
1525 - /*
1526 1515 * Initialize system parameters
1527 1516 */
1528 1517 param_init();
1529 1518
1530 1519 /*
1531 1520 * maxmem is the amount of physical memory we're playing with.
1532 1521 */
1533 1522 maxmem = physmem;
1534 1523
1535 1524 /* Set segkp limits. */
1536 1525 ncbase = kdi_segdebugbase;
1537 1526 ncend = kdi_segdebugbase;
1538 1527
1539 1528 /*
1540 1529 * Initialize the hat layer.
1541 1530 */
1542 1531 hat_init();
1543 1532
1544 1533 /*
1545 1534 * Initialize segment management stuff.
1546 1535 */
1547 1536 seg_init();
1548 1537
1549 1538 /*
1550 1539 * Create the va>tte handler, so the prom can understand
1551 1540 * kernel translations. The handler is installed later, just
1552 1541 * as we are about to take over the trap table from the prom.
1553 1542 */
1554 1543 create_va_to_tte();
1555 1544
1556 1545 /*
1557 1546 * Load the forthdebugger (optional)
1558 1547 */
1559 1548 forthdebug_init();
1560 1549
1561 1550 /*
1562 1551 * Create OBP node for console input callbacks
1563 1552 * if it is needed.
1564 1553 */
1565 1554 startup_create_io_node();
1566 1555
1567 1556 if (modloadonly("fs", "specfs") == -1)
1568 1557 halt("Can't load specfs");
1569 1558
1570 1559 if (modloadonly("fs", "devfs") == -1)
1571 1560 halt("Can't load devfs");
1572 1561
1573 1562 if (modloadonly("fs", "procfs") == -1)
1574 1563 halt("Can't load procfs");
1575 1564
1576 1565 if (modloadonly("misc", "swapgeneric") == -1)
1577 1566 halt("Can't load swapgeneric");
1578 1567
1579 1568 (void) modloadonly("sys", "lbl_edition");
1580 1569
1581 1570 dispinit();
1582 1571
1583 1572 /*
1584 1573 * Infer meanings to the members of the idprom buffer.
1585 1574 */
1586 1575 parse_idprom();
1587 1576
1588 1577 /* Read cluster configuration data. */
1589 1578 clconf_init();
1590 1579
1591 1580 setup_ddi();
1592 1581
1593 1582 /*
1594 1583 * Lets take this opportunity to load the root device.
1595 1584 */
1596 1585 if (loadrootmodules() != 0)
1597 1586 debug_enter("Can't load the root filesystem");
1598 1587
1599 1588 /*
1600 1589 * Load tod driver module for the tod part found on this system.
1601 1590 * Recompute the cpu frequency/delays based on tod as tod part
1602 1591 * tends to keep time more accurately.
1603 1592 */
1604 1593 if (&load_tod_module)
1605 1594 load_tod_module();
1606 1595
1607 1596 /*
1608 1597 * Allow platforms to load modules which might
1609 1598 * be needed after bootops are gone.
1610 1599 */
1611 1600 if (&load_platform_modules)
1612 1601 load_platform_modules();
1613 1602
1614 1603 setcpudelay();
1615 1604
1616 1605 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1617 1606 &boot_physavail, &boot_physavail_len,
1618 1607 &boot_virtavail, &boot_virtavail_len);
1619 1608
1620 1609 /*
1621 1610 * Calculation and allocation of hmeblks needed to remap
1622 1611 * the memory allocated by PROM till now.
1623 1612 * Overestimate the number of hblk1 elements by assuming
1624 1613 * worst case of TTE64K mappings.
1625 1614 * sfmmu_hblk_alloc will panic if this calculation is wrong.
1626 1615 */
1627 1616 bop_alloc_pages = btopr(kmem64_end - kmem64_base);
1628 1617 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
1629 1618 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1630 1619 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
1631 1620
1632 1621 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
1633 1622
1634 1623 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */
1635 1624 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
1636 1625 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
1637 1626
1638 1627 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
1639 1628
1640 1629 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
1641 1630 nhblk8 = 0;
1642 1631 while (bop_alloc_pages > 1) {
1643 1632 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1644 1633 nhblk8 += bop_alloc_pages /= pages_per_hblk;
1645 1634 bop_alloc_pages *= hme8blk_sz;
1646 1635 bop_alloc_pages = btopr(bop_alloc_pages);
1647 1636 }
1648 1637 nhblk8 += 2;
1649 1638
1650 1639 /*
1651 1640 * Since hblk8's can hold up to 64k of mappings aligned on a 64k
1652 1641 * boundary, the number of hblk8's needed to map the entries in the
1653 1642 * boot_virtavail list needs to be adjusted to take this into
1654 1643 * consideration. Thus, we need to add additional hblk8's since it
1655 1644 * is possible that an hblk8 will not have all 8 slots used due to
1656 1645 * alignment constraints. Since there were boot_virtavail_len entries
1657 1646 * in that list, we need to add that many hblk8's to the number
1658 1647 * already calculated to make sure we don't underestimate.
1659 1648 */
1660 1649 nhblk8 += boot_virtavail_len;
1661 1650 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
1662 1651
1663 1652 /* Allocate in pagesize chunks */
1664 1653 nhblksz = roundup(nhblksz, MMU_PAGESIZE);
1665 1654 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
1666 1655 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
1667 1656 }
1668 1657
1669 1658 static void
1670 1659 startup_bop_gone(void)
1671 1660 {
1672 1661
1673 1662 /*
1674 1663 * Destroy the MD initialized at startup
1675 1664 * The startup initializes the MD framework
1676 1665 * using prom and BOP alloc free it now.
1677 1666 */
1678 1667 mach_descrip_startup_fini();
1679 1668
1680 1669 /*
1681 1670 * We're done with prom allocations.
1682 1671 */
1683 1672 bop_fini();
1684 1673
1685 1674 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1686 1675 &boot_physavail, &boot_physavail_len,
1687 1676 &boot_virtavail, &boot_virtavail_len);
1688 1677
1689 1678 /*
1690 1679 * setup physically contiguous area twice as large as the ecache.
1691 1680 * this is used while doing displacement flush of ecaches
1692 1681 */
1693 1682 if (&ecache_flush_address) {
1694 1683 ecache_flushaddr = ecache_flush_address();
1695 1684 if (ecache_flushaddr == (uint64_t)-1) {
1696 1685 cmn_err(CE_PANIC,
1697 1686 "startup: no memory to set ecache_flushaddr");
1698 1687 }
1699 1688 }
1700 1689
1701 1690 /*
1702 1691 * Virtual available next.
1703 1692 */
1704 1693 ASSERT(virt_avail != NULL);
1705 1694 memlist_free_list(virt_avail);
1706 1695 virt_avail = memlist;
1707 1696 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1708 1697
1709 1698 }
1710 1699
1711 1700
1712 1701 /*
1713 1702 * startup_fixup_physavail - called from mach_sfmmu.c after the final
1714 1703 * allocations have been performed. We can't call it in startup_bop_gone
1715 1704 * since later operations can cause obp to allocate more memory.
1716 1705 */
1717 1706 void
1718 1707 startup_fixup_physavail(void)
1719 1708 {
1720 1709 struct memlist *cur;
1721 1710 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
1722 1711
1723 1712 PRM_DEBUG(kmem64_overmap_size);
1724 1713
1725 1714 /*
1726 1715 * take the most current snapshot we can by calling mem-update
1727 1716 */
1728 1717 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1729 1718 &boot_physavail, &boot_physavail_len,
1730 1719 &boot_virtavail, &boot_virtavail_len);
1731 1720
1732 1721 /*
1733 1722 * Copy phys_avail list, again.
1734 1723 * Both the kernel/boot and the prom have been allocating
1735 1724 * from the original list we copied earlier.
1736 1725 */
1737 1726 cur = memlist;
1738 1727 copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1739 1728
1740 1729 /*
1741 1730 * Add any unused kmem64 memory from overmapped page
1742 1731 * (Note: va_to_pa does not work for kmem64_end)
1743 1732 */
1744 1733 if (kmem64_overmap_size) {
1745 1734 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
1746 1735 kmem64_overmap_size, &memlist, &cur);
1747 1736 }
1748 1737
1749 1738 /*
1750 1739 * Add any extra memory after e_data we added to the phys_avail list
1751 1740 * back to the old list.
1752 1741 */
1753 1742 if (ndata_remain_sz >= MMU_PAGESIZE)
1754 1743 memlist_add(va_to_pa(nalloc_base),
1755 1744 (uint64_t)ndata_remain_sz, &memlist, &cur);
1756 1745
1757 1746 /*
1758 1747 * There isn't any bounds checking on the memlist area
1759 1748 * so ensure it hasn't overgrown.
1760 1749 */
1761 1750 if ((caddr_t)memlist > (caddr_t)memlist_end)
1762 1751 cmn_err(CE_PANIC, "startup: memlist size exceeded");
1763 1752
1764 1753 /*
1765 1754 * The kernel removes the pages that were allocated for it from
1766 1755 * the freelist, but we now have to find any -extra- pages that
1767 1756 * the prom has allocated for it's own book-keeping, and remove
1768 1757 * them from the freelist too. sigh.
1769 1758 */
1770 1759 sync_memlists(phys_avail, cur);
1771 1760
1772 1761 ASSERT(phys_avail != NULL);
1773 1762
1774 1763 old_phys_avail = phys_avail;
1775 1764 phys_avail = cur;
1776 1765 }
1777 1766
1778 1767 void
1779 1768 update_kcage_ranges(uint64_t addr, uint64_t len)
1780 1769 {
1781 1770 pfn_t base = btop(addr);
1782 1771 pgcnt_t num = btop(len);
1783 1772 int rv;
1784 1773
1785 1774 rv = kcage_range_add(base, num, kcage_startup_dir);
1786 1775
1787 1776 if (rv == ENOMEM) {
1788 1777 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
1789 1778 (len == 0 ? 0 : BYTES_TO_MB(len)));
1790 1779 } else if (rv != 0) {
1791 1780 /* catch this in debug kernels */
1792 1781 ASSERT(0);
1793 1782
1794 1783 cmn_err(CE_WARN, "unexpected kcage_range_add"
1795 1784 " return value %d", rv);
1796 1785 }
1797 1786 }
1798 1787
1799 1788 static void
1800 1789 startup_vm(void)
1801 1790 {
1802 1791 size_t i;
1803 1792 struct segmap_crargs a;
1804 1793 struct segkpm_crargs b;
1805 1794
1806 1795 uint64_t avmem;
1807 1796 caddr_t va;
1808 1797 pgcnt_t max_phys_segkp;
1809 1798 int mnode;
1810 1799
1811 1800 extern int use_brk_lpg, use_stk_lpg;
1812 1801
1813 1802 /*
1814 1803 * get prom's mappings, create hments for them and switch
1815 1804 * to the kernel context.
1816 1805 */
1817 1806 hat_kern_setup();
1818 1807
1819 1808 /*
1820 1809 * Take over trap table
1821 1810 */
1822 1811 setup_trap_table();
1823 1812
1824 1813 /*
1825 1814 * Install the va>tte handler, so that the prom can handle
1826 1815 * misses and understand the kernel table layout in case
1827 1816 * we need call into the prom.
1828 1817 */
1829 1818 install_va_to_tte();
1830 1819
1831 1820 /*
1832 1821 * Set a flag to indicate that the tba has been taken over.
1833 1822 */
1834 1823 tba_taken_over = 1;
1835 1824
1836 1825 /* initialize MMU primary context register */
1837 1826 mmu_init_kcontext();
1838 1827
1839 1828 /*
1840 1829 * The boot cpu can now take interrupts, x-calls, x-traps
1841 1830 */
1842 1831 CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
1843 1832 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
1844 1833
1845 1834 /*
1846 1835 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
1847 1836 */
1848 1837 tbr_wr_addr_inited = 1;
1849 1838
1850 1839 /*
1851 1840 * Initialize VM system, and map kernel address space.
1852 1841 */
1853 1842 kvm_init();
1854 1843
1855 1844 ASSERT(old_phys_avail != NULL && phys_avail != NULL);
1856 1845 if (kernel_cage_enable) {
1857 1846 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
1858 1847 }
1859 1848 memlist_free_list(old_phys_avail);
1860 1849
1861 1850 /*
1862 1851 * If the following is true, someone has patched
1863 1852 * phsymem to be less than the number of pages that
1864 1853 * the system actually has. Remove pages until system
1865 1854 * memory is limited to the requested amount. Since we
1866 1855 * have allocated page structures for all pages, we
1867 1856 * correct the amount of memory we want to remove
1868 1857 * by the size of the memory used to hold page structures
1869 1858 * for the non-used pages.
1870 1859 */
1871 1860 if (physmem + ramdisk_npages < npages) {
1872 1861 pgcnt_t diff, off;
1873 1862 struct page *pp;
1874 1863 struct seg kseg;
1875 1864
1876 1865 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
1877 1866
1878 1867 off = 0;
1879 1868 diff = npages - (physmem + ramdisk_npages);
1880 1869 diff -= mmu_btopr(diff * sizeof (struct page));
1881 1870 kseg.s_as = &kas;
1882 1871 while (diff--) {
1883 1872 pp = page_create_va(&unused_pages_vp, (offset_t)off,
1884 1873 MMU_PAGESIZE, PG_WAIT | PG_EXCL,
1885 1874 &kseg, (caddr_t)off);
1886 1875 if (pp == NULL)
1887 1876 cmn_err(CE_PANIC, "limited physmem too much!");
1888 1877 page_io_unlock(pp);
1889 1878 page_downgrade(pp);
1890 1879 availrmem--;
1891 1880 off += MMU_PAGESIZE;
1892 1881 }
1893 1882 }
1894 1883
1895 1884 /*
1896 1885 * When printing memory, show the total as physmem less
1897 1886 * that stolen by a debugger.
1898 1887 */
1899 1888 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
1900 1889 (ulong_t)(physinstalled) << (PAGESHIFT - 10),
1901 1890 (ulong_t)(physinstalled) << (PAGESHIFT - 12));
1902 1891
1903 1892 avmem = (uint64_t)freemem << PAGESHIFT;
1904 1893 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
1905 1894
1906 1895 /*
1907 1896 * For small memory systems disable automatic large pages.
1908 1897 */
1909 1898 if (physmem < privm_lpg_min_physmem) {
1910 1899 use_brk_lpg = 0;
1911 1900 use_stk_lpg = 0;
1912 1901 }
1913 1902
1914 1903 /*
1915 1904 * Perform platform specific freelist processing
1916 1905 */
1917 1906 if (&plat_freelist_process) {
1918 1907 for (mnode = 0; mnode < max_mem_nodes; mnode++)
1919 1908 if (mem_node_config[mnode].exists)
1920 1909 plat_freelist_process(mnode);
1921 1910 }
1922 1911
1923 1912 /*
1924 1913 * Initialize the segkp segment type. We position it
1925 1914 * after the configured tables and buffers (whose end
1926 1915 * is given by econtig) and before V_WKBASE_ADDR.
1927 1916 * Also in this area is segkmap (size SEGMAPSIZE).
1928 1917 */
1929 1918
1930 1919 /* XXX - cache alignment? */
1931 1920 va = (caddr_t)SEGKPBASE;
1932 1921 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
1933 1922
1934 1923 max_phys_segkp = (physmem * 2);
1935 1924
1936 1925 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
1937 1926 segkpsize = btop(SEGKPDEFSIZE);
1938 1927 cmn_err(CE_WARN, "Illegal value for segkpsize. "
1939 1928 "segkpsize has been reset to %ld pages", segkpsize);
1940 1929 }
1941 1930
1942 1931 i = ptob(MIN(segkpsize, max_phys_segkp));
1943 1932
1944 1933 rw_enter(&kas.a_lock, RW_WRITER);
1945 1934 if (seg_attach(&kas, va, i, segkp) < 0)
1946 1935 cmn_err(CE_PANIC, "startup: cannot attach segkp");
1947 1936 if (segkp_create(segkp) != 0)
1948 1937 cmn_err(CE_PANIC, "startup: segkp_create failed");
1949 1938 rw_exit(&kas.a_lock);
1950 1939
1951 1940 /*
1952 1941 * kpm segment
1953 1942 */
1954 1943 segmap_kpm = kpm_enable &&
1955 1944 segmap_kpm && PAGESIZE == MAXBSIZE;
1956 1945
1957 1946 if (kpm_enable) {
1958 1947 rw_enter(&kas.a_lock, RW_WRITER);
1959 1948
1960 1949 /*
1961 1950 * The segkpm virtual range range is larger than the
1962 1951 * actual physical memory size and also covers gaps in
1963 1952 * the physical address range for the following reasons:
1964 1953 * . keep conversion between segkpm and physical addresses
1965 1954 * simple, cheap and unambiguous.
1966 1955 * . avoid extension/shrink of the the segkpm in case of DR.
1967 1956 * . avoid complexity for handling of virtual addressed
1968 1957 * caches, segkpm and the regular mapping scheme must be
1969 1958 * kept in sync wrt. the virtual color of mapped pages.
1970 1959 * Any accesses to virtual segkpm ranges not backed by
1971 1960 * physical memory will fall through the memseg pfn hash
1972 1961 * and will be handled in segkpm_fault.
1973 1962 * Additional kpm_size spaces needed for vac alias prevention.
1974 1963 */
1975 1964 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
1976 1965 segkpm) < 0)
1977 1966 cmn_err(CE_PANIC, "cannot attach segkpm");
1978 1967
1979 1968 b.prot = PROT_READ | PROT_WRITE;
1980 1969 b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
1981 1970
1982 1971 if (segkpm_create(segkpm, (caddr_t)&b) != 0)
1983 1972 panic("segkpm_create segkpm");
1984 1973
1985 1974 rw_exit(&kas.a_lock);
1986 1975
1987 1976 mach_kpm_init();
1988 1977 }
1989 1978
1990 1979 va = kpm_vbase + (kpm_size * vac_colors);
1991 1980
1992 1981 if (!segzio_fromheap) {
1993 1982 size_t size;
1994 1983 size_t physmem_b = mmu_ptob(physmem);
1995 1984
1996 1985 /* size is in bytes, segziosize is in pages */
1997 1986 if (segziosize == 0) {
1998 1987 size = physmem_b;
1999 1988 } else {
2000 1989 size = mmu_ptob(segziosize);
2001 1990 }
2002 1991
2003 1992 if (size < SEGZIOMINSIZE) {
2004 1993 size = SEGZIOMINSIZE;
2005 1994 } else if (size > SEGZIOMAXSIZE) {
2006 1995 size = SEGZIOMAXSIZE;
2007 1996 /*
2008 1997 * On 64-bit x86, we only have 2TB of KVA. This exists
2009 1998 * for parity with x86.
2010 1999 *
2011 2000 * SEGZIOMAXSIZE is capped at 512gb so that segzio
2012 2001 * doesn't consume all of KVA. However, if we have a
2013 2002 * system that has more thant 512gb of physical memory,
2014 2003 * we can actually consume about half of the difference
2015 2004 * between 512gb and the rest of the available physical
2016 2005 * memory.
2017 2006 */
2018 2007 if (physmem_b > SEGZIOMAXSIZE) {
2019 2008 size += (physmem_b - SEGZIOMAXSIZE) / 2;
2020 2009 }
2021 2010 }
2022 2011 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
2023 2012 /* put the base of the ZIO segment after the kpm segment */
2024 2013 segzio_base = va;
2025 2014 va += mmu_ptob(segziosize);
2026 2015 PRM_DEBUG(segziosize);
2027 2016 PRM_DEBUG(segzio_base);
2028 2017
2029 2018 /*
2030 2019 * On some platforms, kvm_init is called after the kpm
2031 2020 * sizes have been determined. On SPARC, kvm_init is called
2032 2021 * before, so we have to attach the kzioseg after kvm is
2033 2022 * initialized, otherwise we'll try to allocate from the boot
2034 2023 * area since the kernel heap hasn't yet been configured.
2035 2024 */
2036 2025 rw_enter(&kas.a_lock, RW_WRITER);
2037 2026
2038 2027 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2039 2028 &kzioseg);
2040 2029 (void) segkmem_zio_create(&kzioseg);
2041 2030
2042 2031 /* create zio area covering new segment */
2043 2032 segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2044 2033
2045 2034 rw_exit(&kas.a_lock);
2046 2035 }
2047 2036
2048 2037 if (ppvm_enable) {
2049 2038 caddr_t ppvm_max;
2050 2039
2051 2040 /*
2052 2041 * ppvm refers to the static VA space used to map
2053 2042 * the page_t's for dynamically added memory.
2054 2043 *
2055 2044 * ppvm_base should not cross a potential VA hole.
2056 2045 *
2057 2046 * ppvm_size should be large enough to map the
2058 2047 * page_t's needed to manage all of KPM range.
2059 2048 */
2060 2049 ppvm_size =
2061 2050 roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t),
2062 2051 MMU_PAGESIZE);
2063 2052 ppvm_max = (caddr_t)(0ull - ppvm_size);
2064 2053 ppvm_base = (page_t *)va;
2065 2054
2066 2055 if ((caddr_t)ppvm_base <= hole_end) {
2067 2056 cmn_err(CE_WARN,
2068 2057 "Memory DR disabled: invalid DR map base: 0x%p\n",
2069 2058 (void *)ppvm_base);
2070 2059 ppvm_enable = 0;
2071 2060 } else if ((caddr_t)ppvm_base > ppvm_max) {
2072 2061 uint64_t diff = (caddr_t)ppvm_base - ppvm_max;
2073 2062
2074 2063 cmn_err(CE_WARN,
2075 2064 "Memory DR disabled: insufficient DR map size:"
2076 2065 " 0x%lx (needed 0x%lx)\n",
2077 2066 ppvm_size - diff, ppvm_size);
2078 2067 ppvm_enable = 0;
2079 2068 }
2080 2069 PRM_DEBUG(ppvm_size);
2081 2070 PRM_DEBUG(ppvm_base);
2082 2071 }
2083 2072
2084 2073 /*
2085 2074 * Now create generic mapping segment. This mapping
2086 2075 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total
2087 2076 * virtual address is greater than the amount of free
2088 2077 * memory that is available, then we trim back the
2089 2078 * segment size to that amount
2090 2079 */
2091 2080 va = (caddr_t)SEGMAPBASE;
2092 2081
2093 2082 /*
2094 2083 * 1201049: segkmap base address must be MAXBSIZE aligned
2095 2084 */
2096 2085 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
2097 2086
2098 2087 /*
2099 2088 * Set size of segmap to percentage of freemem at boot,
2100 2089 * but stay within the allowable range
2101 2090 * Note we take percentage before converting from pages
2102 2091 * to bytes to avoid an overflow on 32-bit kernels.
2103 2092 */
2104 2093 i = mmu_ptob((freemem * segmap_percent) / 100);
2105 2094
2106 2095 if (i < MINMAPSIZE)
2107 2096 i = MINMAPSIZE;
2108 2097
2109 2098 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
2110 2099 i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
2111 2100
2112 2101 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */
2113 2102
2114 2103 rw_enter(&kas.a_lock, RW_WRITER);
2115 2104 if (seg_attach(&kas, va, i, segkmap) < 0)
2116 2105 cmn_err(CE_PANIC, "cannot attach segkmap");
2117 2106
2118 2107 a.prot = PROT_READ | PROT_WRITE;
2119 2108 a.shmsize = shm_alignment;
2120 2109 a.nfreelist = 0; /* use segmap driver defaults */
2121 2110
2122 2111 if (segmap_create(segkmap, (caddr_t)&a) != 0)
2123 2112 panic("segmap_create segkmap");
2124 2113 rw_exit(&kas.a_lock);
2125 2114
2126 2115 segdev_init();
2127 2116 }
2128 2117
2129 2118 static void
2130 2119 startup_end(void)
2131 2120 {
2132 2121 if ((caddr_t)memlist > (caddr_t)memlist_end)
2133 2122 panic("memlist overflow 2");
2134 2123 memlist_free_block((caddr_t)memlist,
2135 2124 ((caddr_t)memlist_end - (caddr_t)memlist));
2136 2125 memlist = NULL;
2137 2126
2138 2127 /* enable page_relocation since OBP is now done */
2139 2128 page_relocate_ready = 1;
2140 2129
2141 2130 /*
2142 2131 * Perform tasks that get done after most of the VM
2143 2132 * initialization has been done but before the clock
2144 2133 * and other devices get started.
2145 2134 */
2146 2135 kern_setup1();
2147 2136
2148 2137 /*
2149 2138 * Perform CPC initialization for this CPU.
2150 2139 */
2151 2140 kcpc_hw_init();
2152 2141
2153 2142 /*
2154 2143 * Intialize the VM arenas for allocating physically
2155 2144 * contiguus memory chunk for interrupt queues snd
2156 2145 * allocate/register boot cpu's queues, if any and
2157 2146 * allocate dump buffer for sun4v systems to store
2158 2147 * extra crash information during crash dump
2159 2148 */
2160 2149 contig_mem_init();
2161 2150 mach_descrip_init();
2162 2151
2163 2152 if (cpu_intrq_setup(CPU)) {
2164 2153 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
2165 2154 }
2166 2155 cpu_intrq_register(CPU);
2167 2156 mach_htraptrace_setup(CPU->cpu_id);
2168 2157 mach_htraptrace_configure(CPU->cpu_id);
2169 2158 mach_dump_buffer_init();
2170 2159
2171 2160 /*
2172 2161 * Initialize interrupt related stuff
2173 2162 */
2174 2163 cpu_intr_alloc(CPU, NINTR_THREADS);
2175 2164
2176 2165 (void) splzs(); /* allow hi clock ints but not zs */
2177 2166
2178 2167 /*
2179 2168 * Initialize errors.
2180 2169 */
2181 2170 error_init();
2182 2171
2183 2172 /*
2184 2173 * Note that we may have already used kernel bcopy before this
2185 2174 * point - but if you really care about this, adb the use_hw_*
2186 2175 * variables to 0 before rebooting.
2187 2176 */
2188 2177 mach_hw_copy_limit();
2189 2178
2190 2179 /*
2191 2180 * Install the "real" preemption guards before DDI services
2192 2181 * are available.
2193 2182 */
2194 2183 (void) prom_set_preprom(kern_preprom);
2195 2184 (void) prom_set_postprom(kern_postprom);
2196 2185 CPU->cpu_m.mutex_ready = 1;
2197 2186
2198 2187 /*
2199 2188 * Initialize segnf (kernel support for non-faulting loads).
2200 2189 */
2201 2190 segnf_init();
2202 2191
2203 2192 /*
2204 2193 * Configure the root devinfo node.
2205 2194 */
2206 2195 configure(); /* set up devices */
2207 2196 mach_cpu_halt_idle();
2208 2197 }
2209 2198
2210 2199
2211 2200 void
2212 2201 post_startup(void)
2213 2202 {
2214 2203 #ifdef PTL1_PANIC_DEBUG
2215 2204 extern void init_ptl1_thread(void);
2216 2205 #endif /* PTL1_PANIC_DEBUG */
2217 2206 extern void abort_sequence_init(void);
2218 2207
2219 2208 /*
2220 2209 * Set the system wide, processor-specific flags to be passed
2221 2210 * to userland via the aux vector for performance hints and
2222 2211 * instruction set extensions.
2223 2212 */
2224 2213 bind_hwcap();
2225 2214
2226 2215 /*
2227 2216 * Startup memory scrubber (if any)
2228 2217 */
2229 2218 mach_memscrub();
2230 2219
2231 2220 /*
2232 2221 * Allocate soft interrupt to handle abort sequence.
2233 2222 */
2234 2223 abort_sequence_init();
2235 2224
2236 2225 /*
2237 2226 * Configure the rest of the system.
2238 2227 * Perform forceloading tasks for /etc/system.
2239 2228 */
2240 2229 (void) mod_sysctl(SYS_FORCELOAD, NULL);
2241 2230 /*
2242 2231 * ON4.0: Force /proc module in until clock interrupt handle fixed
2243 2232 * ON4.0: This must be fixed or restated in /etc/systems.
2244 2233 */
2245 2234 (void) modload("fs", "procfs");
2246 2235
2247 2236 /* load machine class specific drivers */
2248 2237 load_mach_drivers();
2249 2238
2250 2239 /* load platform specific drivers */
2251 2240 if (&load_platform_drivers)
2252 2241 load_platform_drivers();
2253 2242
2254 2243 /* load vis simulation module, if we are running w/fpu off */
2255 2244 if (!fpu_exists) {
2256 2245 if (modload("misc", "vis") == -1)
2257 2246 halt("Can't load vis");
2258 2247 }
2259 2248
2260 2249 mach_fpras();
2261 2250
2262 2251 maxmem = freemem;
2263 2252
2264 2253 pg_init();
2265 2254
2266 2255 #ifdef PTL1_PANIC_DEBUG
2267 2256 init_ptl1_thread();
2268 2257 #endif /* PTL1_PANIC_DEBUG */
2269 2258 }
2270 2259
2271 2260 #ifdef PTL1_PANIC_DEBUG
2272 2261 int ptl1_panic_test = 0;
2273 2262 int ptl1_panic_xc_one_test = 0;
2274 2263 int ptl1_panic_xc_all_test = 0;
2275 2264 int ptl1_panic_xt_one_test = 0;
2276 2265 int ptl1_panic_xt_all_test = 0;
2277 2266 kthread_id_t ptl1_thread_p = NULL;
2278 2267 kcondvar_t ptl1_cv;
2279 2268 kmutex_t ptl1_mutex;
2280 2269 int ptl1_recurse_count_threshold = 0x40;
2281 2270 int ptl1_recurse_trap_threshold = 0x3d;
2282 2271 extern void ptl1_recurse(int, int);
2283 2272 extern void ptl1_panic_xt(int, int);
2284 2273
2285 2274 /*
2286 2275 * Called once per second by timeout() to wake up
2287 2276 * the ptl1_panic thread to see if it should cause
2288 2277 * a trap to the ptl1_panic() code.
2289 2278 */
2290 2279 /* ARGSUSED */
2291 2280 static void
2292 2281 ptl1_wakeup(void *arg)
2293 2282 {
2294 2283 mutex_enter(&ptl1_mutex);
2295 2284 cv_signal(&ptl1_cv);
2296 2285 mutex_exit(&ptl1_mutex);
2297 2286 }
2298 2287
2299 2288 /*
2300 2289 * ptl1_panic cross call function:
2301 2290 * Needed because xc_one() and xc_some() can pass
2302 2291 * 64 bit args but ptl1_recurse() expects ints.
2303 2292 */
2304 2293 static void
2305 2294 ptl1_panic_xc(void)
2306 2295 {
2307 2296 ptl1_recurse(ptl1_recurse_count_threshold,
2308 2297 ptl1_recurse_trap_threshold);
2309 2298 }
2310 2299
2311 2300 /*
2312 2301 * The ptl1 thread waits for a global flag to be set
2313 2302 * and uses the recurse thresholds to set the stack depth
2314 2303 * to cause a ptl1_panic() directly via a call to ptl1_recurse
2315 2304 * or indirectly via the cross call and cross trap functions.
2316 2305 *
2317 2306 * This is useful testing stack overflows and normal
2318 2307 * ptl1_panic() states with a know stack frame.
2319 2308 *
2320 2309 * ptl1_recurse() is an asm function in ptl1_panic.s that
2321 2310 * sets the {In, Local, Out, and Global} registers to a
2322 2311 * know state on the stack and just prior to causing a
2323 2312 * test ptl1_panic trap.
2324 2313 */
2325 2314 static void
2326 2315 ptl1_thread(void)
2327 2316 {
2328 2317 mutex_enter(&ptl1_mutex);
2329 2318 while (ptl1_thread_p) {
2330 2319 cpuset_t other_cpus;
2331 2320 int cpu_id;
2332 2321 int my_cpu_id;
2333 2322 int target_cpu_id;
2334 2323 int target_found;
2335 2324
2336 2325 if (ptl1_panic_test) {
2337 2326 ptl1_recurse(ptl1_recurse_count_threshold,
2338 2327 ptl1_recurse_trap_threshold);
2339 2328 }
2340 2329
2341 2330 /*
2342 2331 * Find potential targets for x-call and x-trap,
2343 2332 * if any exist while preempt is disabled we
2344 2333 * start a ptl1_panic if requested via a
2345 2334 * globals.
2346 2335 */
2347 2336 kpreempt_disable();
2348 2337 my_cpu_id = CPU->cpu_id;
2349 2338 other_cpus = cpu_ready_set;
2350 2339 CPUSET_DEL(other_cpus, CPU->cpu_id);
2351 2340 target_found = 0;
2352 2341 if (!CPUSET_ISNULL(other_cpus)) {
2353 2342 /*
2354 2343 * Pick the first one
2355 2344 */
2356 2345 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
2357 2346 if (cpu_id == my_cpu_id)
2358 2347 continue;
2359 2348
2360 2349 if (CPU_XCALL_READY(cpu_id)) {
2361 2350 target_cpu_id = cpu_id;
2362 2351 target_found = 1;
2363 2352 break;
2364 2353 }
2365 2354 }
2366 2355 ASSERT(target_found);
2367 2356
2368 2357 if (ptl1_panic_xc_one_test) {
2369 2358 xc_one(target_cpu_id,
2370 2359 (xcfunc_t *)ptl1_panic_xc, 0, 0);
2371 2360 }
2372 2361 if (ptl1_panic_xc_all_test) {
2373 2362 xc_some(other_cpus,
2374 2363 (xcfunc_t *)ptl1_panic_xc, 0, 0);
2375 2364 }
2376 2365 if (ptl1_panic_xt_one_test) {
2377 2366 xt_one(target_cpu_id,
2378 2367 (xcfunc_t *)ptl1_panic_xt, 0, 0);
2379 2368 }
2380 2369 if (ptl1_panic_xt_all_test) {
2381 2370 xt_some(other_cpus,
2382 2371 (xcfunc_t *)ptl1_panic_xt, 0, 0);
2383 2372 }
2384 2373 }
2385 2374 kpreempt_enable();
2386 2375 (void) timeout(ptl1_wakeup, NULL, hz);
2387 2376 (void) cv_wait(&ptl1_cv, &ptl1_mutex);
2388 2377 }
2389 2378 mutex_exit(&ptl1_mutex);
2390 2379 }
2391 2380
2392 2381 /*
2393 2382 * Called during early startup to create the ptl1_thread
2394 2383 */
2395 2384 void
2396 2385 init_ptl1_thread(void)
2397 2386 {
2398 2387 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
2399 2388 &p0, TS_RUN, 0);
2400 2389 }
2401 2390 #endif /* PTL1_PANIC_DEBUG */
2402 2391
2403 2392
2404 2393 static void
2405 2394 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
2406 2395 {
2407 2396 struct memlist *new;
2408 2397
2409 2398 new = *memlistp;
2410 2399 new->ml_address = start;
2411 2400 new->ml_size = len;
2412 2401 *memlistp = new + 1;
2413 2402 }
2414 2403
2415 2404 /*
2416 2405 * Add to a memory list.
2417 2406 * start = start of new memory segment
2418 2407 * len = length of new memory segment in bytes
2419 2408 * memlistp = pointer to array of available memory segment structures
2420 2409 * curmemlistp = memory list to which to add segment.
2421 2410 */
2422 2411 static void
2423 2412 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
2424 2413 struct memlist **curmemlistp)
2425 2414 {
2426 2415 struct memlist *new = *memlistp;
2427 2416
2428 2417 memlist_new(start, len, memlistp);
2429 2418 memlist_insert(new, curmemlistp);
2430 2419 }
2431 2420
2432 2421 static int
2433 2422 ndata_alloc_memseg(struct memlist *ndata, size_t avail)
2434 2423 {
2435 2424 int nseg;
2436 2425 size_t memseg_sz;
2437 2426 struct memseg *msp;
2438 2427
2439 2428 /*
2440 2429 * The memseg list is for the chunks of physical memory that
2441 2430 * will be managed by the vm system. The number calculated is
2442 2431 * a guess as boot may fragment it more when memory allocations
2443 2432 * are made before kphysm_init().
2444 2433 */
2445 2434 memseg_sz = (avail + 10) * sizeof (struct memseg);
2446 2435 memseg_sz = roundup(memseg_sz, PAGESIZE);
2447 2436 nseg = memseg_sz / sizeof (struct memseg);
2448 2437 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
2449 2438 if (msp == NULL)
2450 2439 return (1);
2451 2440 PRM_DEBUG(memseg_free);
2452 2441
2453 2442 while (nseg--) {
2454 2443 msp->next = memseg_free;
2455 2444 memseg_free = msp;
2456 2445 msp++;
2457 2446 }
2458 2447 return (0);
2459 2448 }
2460 2449
2461 2450 /*
2462 2451 * In the case of architectures that support dynamic addition of
2463 2452 * memory at run-time there are two cases where memsegs need to
2464 2453 * be initialized and added to the memseg list.
2465 2454 * 1) memsegs that are constructed at startup.
2466 2455 * 2) memsegs that are constructed at run-time on
2467 2456 * hot-plug capable architectures.
2468 2457 * This code was originally part of the function kphysm_init().
2469 2458 */
2470 2459
2471 2460 static void
2472 2461 memseg_list_add(struct memseg *memsegp)
2473 2462 {
2474 2463 struct memseg **prev_memsegp;
2475 2464 pgcnt_t num;
2476 2465
2477 2466 /* insert in memseg list, decreasing number of pages order */
2478 2467
2479 2468 num = MSEG_NPAGES(memsegp);
2480 2469
2481 2470 for (prev_memsegp = &memsegs; *prev_memsegp;
2482 2471 prev_memsegp = &((*prev_memsegp)->next)) {
2483 2472 if (num > MSEG_NPAGES(*prev_memsegp))
2484 2473 break;
2485 2474 }
2486 2475
2487 2476 memsegp->next = *prev_memsegp;
2488 2477 *prev_memsegp = memsegp;
2489 2478
2490 2479 if (kpm_enable) {
2491 2480 memsegp->nextpa = (memsegp->next) ?
2492 2481 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
2493 2482
2494 2483 if (prev_memsegp != &memsegs) {
2495 2484 struct memseg *msp;
2496 2485 msp = (struct memseg *)((caddr_t)prev_memsegp -
2497 2486 offsetof(struct memseg, next));
2498 2487 msp->nextpa = va_to_pa(memsegp);
2499 2488 } else {
2500 2489 memsegspa = va_to_pa(memsegs);
2501 2490 }
2502 2491 }
2503 2492 }
2504 2493
2505 2494 /*
2506 2495 * PSM add_physmem_cb(). US-II and newer processors have some
2507 2496 * flavor of the prefetch capability implemented. We exploit
2508 2497 * this capability for optimum performance.
2509 2498 */
2510 2499 #define PREFETCH_BYTES 64
2511 2500
2512 2501 void
2513 2502 add_physmem_cb(page_t *pp, pfn_t pnum)
2514 2503 {
2515 2504 extern void prefetch_page_w(void *);
2516 2505
2517 2506 pp->p_pagenum = pnum;
2518 2507
2519 2508 /*
2520 2509 * Prefetch one more page_t into E$. To prevent future
2521 2510 * mishaps with the sizeof(page_t) changing on us, we
2522 2511 * catch this on debug kernels if we can't bring in the
2523 2512 * entire hpage with 2 PREFETCH_BYTES reads. See
2524 2513 * also, sun4u/cpu/cpu_module.c
2525 2514 */
2526 2515 /*LINTED*/
2527 2516 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
2528 2517 prefetch_page_w((char *)pp);
2529 2518 }
2530 2519
2531 2520 /*
2532 2521 * Find memseg with given pfn
2533 2522 */
2534 2523 static struct memseg *
2535 2524 memseg_find(pfn_t base, pfn_t *next)
2536 2525 {
2537 2526 struct memseg *seg;
2538 2527
2539 2528 if (next != NULL)
2540 2529 *next = LONG_MAX;
2541 2530 for (seg = memsegs; seg != NULL; seg = seg->next) {
2542 2531 if (base >= seg->pages_base && base < seg->pages_end)
2543 2532 return (seg);
2544 2533 if (next != NULL && seg->pages_base > base &&
2545 2534 seg->pages_base < *next)
2546 2535 *next = seg->pages_base;
2547 2536 }
2548 2537 return (NULL);
2549 2538 }
2550 2539
2551 2540 /*
2552 2541 * Put page allocated by OBP on prom_ppages
2553 2542 */
2554 2543 static void
2555 2544 kphysm_erase(uint64_t addr, uint64_t len)
2556 2545 {
2557 2546 struct page *pp;
2558 2547 struct memseg *seg;
2559 2548 pfn_t base = btop(addr), next;
2560 2549 pgcnt_t num = btop(len);
2561 2550
2562 2551 while (num != 0) {
2563 2552 pgcnt_t off, left;
2564 2553
2565 2554 seg = memseg_find(base, &next);
2566 2555 if (seg == NULL) {
2567 2556 if (next == LONG_MAX)
2568 2557 break;
2569 2558 left = MIN(next - base, num);
2570 2559 base += left, num -= left;
2571 2560 continue;
2572 2561 }
2573 2562 off = base - seg->pages_base;
2574 2563 pp = seg->pages + off;
2575 2564 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
2576 2565 while (num != left) {
2577 2566 /*
2578 2567 * init it, lock it, and hashin on prom_pages vp.
2579 2568 *
2580 2569 * Mark it as NONRELOC to let DR know the page
2581 2570 * is locked long term, otherwise DR hangs when
2582 2571 * trying to remove those pages.
2583 2572 *
2584 2573 * XXX vnode offsets on the prom_ppages vnode
2585 2574 * are page numbers (gack) for >32 bit
2586 2575 * physical memory machines.
2587 2576 */
2588 2577 PP_SETNORELOC(pp);
2589 2578 add_physmem_cb(pp, base);
2590 2579 if (page_trylock(pp, SE_EXCL) == 0)
2591 2580 cmn_err(CE_PANIC, "prom page locked");
2592 2581 (void) page_hashin(pp, &promvp,
2593 2582 (offset_t)base, NULL);
2594 2583 (void) page_pp_lock(pp, 0, 1);
2595 2584 pp++, base++, num--;
2596 2585 }
2597 2586 }
2598 2587 }
2599 2588
2600 2589 static page_t *ppnext;
2601 2590 static pgcnt_t ppleft;
2602 2591
2603 2592 static void *kpm_ppnext;
2604 2593 static pgcnt_t kpm_ppleft;
2605 2594
2606 2595 /*
2607 2596 * Create a memseg
2608 2597 */
2609 2598 static void
2610 2599 kphysm_memseg(uint64_t addr, uint64_t len)
2611 2600 {
2612 2601 pfn_t base = btop(addr);
2613 2602 pgcnt_t num = btop(len);
2614 2603 struct memseg *seg;
2615 2604
2616 2605 seg = memseg_free;
2617 2606 memseg_free = seg->next;
2618 2607 ASSERT(seg != NULL);
2619 2608
2620 2609 seg->pages = ppnext;
2621 2610 seg->epages = ppnext + num;
2622 2611 seg->pages_base = base;
2623 2612 seg->pages_end = base + num;
2624 2613 ppnext += num;
2625 2614 ppleft -= num;
2626 2615
2627 2616 if (kpm_enable) {
2628 2617 pgcnt_t kpnum = ptokpmpr(num);
2629 2618
2630 2619 if (kpnum > kpm_ppleft)
2631 2620 panic("kphysm_memseg: kpm_pp overflow");
2632 2621 seg->pagespa = va_to_pa(seg->pages);
2633 2622 seg->epagespa = va_to_pa(seg->epages);
2634 2623 seg->kpm_pbase = kpmptop(ptokpmp(base));
2635 2624 seg->kpm_nkpmpgs = kpnum;
2636 2625 /*
2637 2626 * In the kpm_smallpage case, the kpm array
2638 2627 * is 1-1 wrt the page array
2639 2628 */
2640 2629 if (kpm_smallpages) {
2641 2630 kpm_spage_t *kpm_pp = kpm_ppnext;
2642 2631
2643 2632 kpm_ppnext = kpm_pp + kpnum;
2644 2633 seg->kpm_spages = kpm_pp;
2645 2634 seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
2646 2635 } else {
2647 2636 kpm_page_t *kpm_pp = kpm_ppnext;
2648 2637
2649 2638 kpm_ppnext = kpm_pp + kpnum;
2650 2639 seg->kpm_pages = kpm_pp;
2651 2640 seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
2652 2641 /* ASSERT no kpm overlaps */
2653 2642 ASSERT(
2654 2643 memseg_find(base - pmodkpmp(base), NULL) == NULL);
2655 2644 ASSERT(memseg_find(
2656 2645 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
2657 2646 }
2658 2647 kpm_ppleft -= kpnum;
2659 2648 }
2660 2649
2661 2650 memseg_list_add(seg);
2662 2651 }
2663 2652
2664 2653 /*
2665 2654 * Add range to free list
2666 2655 */
2667 2656 void
2668 2657 kphysm_add(uint64_t addr, uint64_t len, int reclaim)
2669 2658 {
2670 2659 struct page *pp;
2671 2660 struct memseg *seg;
2672 2661 pfn_t base = btop(addr);
2673 2662 pgcnt_t num = btop(len);
2674 2663
2675 2664 seg = memseg_find(base, NULL);
2676 2665 ASSERT(seg != NULL);
2677 2666 pp = seg->pages + (base - seg->pages_base);
2678 2667
2679 2668 if (reclaim) {
2680 2669 struct page *rpp = pp;
2681 2670 struct page *lpp = pp + num;
2682 2671
2683 2672 /*
2684 2673 * page should be locked on prom_ppages
2685 2674 * unhash and unlock it
2686 2675 */
2687 2676 while (rpp < lpp) {
2688 2677 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp);
2689 2678 ASSERT(PP_ISNORELOC(rpp));
2690 2679 PP_CLRNORELOC(rpp);
2691 2680 page_pp_unlock(rpp, 0, 1);
2692 2681 page_hashout(rpp, NULL);
2693 2682 page_unlock(rpp);
2694 2683 rpp++;
2695 2684 }
2696 2685 }
2697 2686
2698 2687 /*
2699 2688 * add_physmem() initializes the PSM part of the page
2700 2689 * struct by calling the PSM back with add_physmem_cb().
2701 2690 * In addition it coalesces pages into larger pages as
2702 2691 * it initializes them.
2703 2692 */
2704 2693 add_physmem(pp, num, base);
2705 2694 }
2706 2695
2707 2696 /*
2708 2697 * kphysm_init() tackles the problem of initializing physical memory.
2709 2698 */
2710 2699 static void
2711 2700 kphysm_init(void)
2712 2701 {
2713 2702 struct memlist *pmem;
2714 2703
2715 2704 ASSERT(page_hash != NULL && page_hashsz != 0);
2716 2705
2717 2706 ppnext = pp_base;
2718 2707 ppleft = npages;
2719 2708 kpm_ppnext = kpm_pp_base;
2720 2709 kpm_ppleft = kpm_npages;
2721 2710
2722 2711 /*
2723 2712 * installed pages not on nopp_memlist go in memseg list
2724 2713 */
2725 2714 diff_memlists(phys_install, nopp_list, kphysm_memseg);
2726 2715
2727 2716 /*
2728 2717 * Free the avail list
2729 2718 */
2730 2719 for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next)
2731 2720 kphysm_add(pmem->ml_address, pmem->ml_size, 0);
2732 2721
2733 2722 /*
2734 2723 * Erase pages that aren't available
2735 2724 */
2736 2725 diff_memlists(phys_install, phys_avail, kphysm_erase);
2737 2726
2738 2727 build_pfn_hash();
2739 2728 }
2740 2729
2741 2730 /*
2742 2731 * Kernel VM initialization.
2743 2732 * Assumptions about kernel address space ordering:
2744 2733 * (1) gap (user space)
2745 2734 * (2) kernel text
2746 2735 * (3) kernel data/bss
2747 2736 * (4) gap
2748 2737 * (5) kernel data structures
2749 2738 * (6) gap
2750 2739 * (7) debugger (optional)
2751 2740 * (8) monitor
2752 2741 * (9) gap (possibly null)
2753 2742 * (10) dvma
2754 2743 * (11) devices
2755 2744 */
2756 2745 static void
2757 2746 kvm_init(void)
2758 2747 {
2759 2748 /*
2760 2749 * Put the kernel segments in kernel address space.
2761 2750 */
2762 2751 rw_enter(&kas.a_lock, RW_WRITER);
2763 2752 as_avlinit(&kas);
2764 2753
2765 2754 (void) seg_attach(&kas, (caddr_t)KERNELBASE,
2766 2755 (size_t)(e_moddata - KERNELBASE), &ktextseg);
2767 2756 (void) segkmem_create(&ktextseg);
2768 2757
2769 2758 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
2770 2759 (size_t)(MMU_PAGESIZE4M), &ktexthole);
2771 2760 (void) segkmem_create(&ktexthole);
2772 2761
2773 2762 (void) seg_attach(&kas, (caddr_t)valloc_base,
2774 2763 (size_t)(econtig32 - valloc_base), &kvalloc);
2775 2764 (void) segkmem_create(&kvalloc);
2776 2765
2777 2766 if (kmem64_base) {
2778 2767 (void) seg_attach(&kas, (caddr_t)kmem64_base,
2779 2768 (size_t)(kmem64_end - kmem64_base), &kmem64);
2780 2769 (void) segkmem_create(&kmem64);
2781 2770 }
2782 2771
2783 2772 /*
2784 2773 * We're about to map out /boot. This is the beginning of the
2785 2774 * system resource management transition. We can no longer
2786 2775 * call into /boot for I/O or memory allocations.
2787 2776 */
2788 2777 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
2789 2778 (void) segkmem_create(&kvseg);
2790 2779 hblk_alloc_dynamic = 1;
2791 2780
2792 2781 /*
2793 2782 * we need to preallocate pages for DR operations before enabling large
2794 2783 * page kernel heap because of memseg_remap_init() hat_unload() hack.
2795 2784 */
2796 2785 memseg_remap_init();
2797 2786
2798 2787 /* at this point we are ready to use large page heap */
2799 2788 segkmem_heap_lp_init();
2800 2789
2801 2790 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
2802 2791 &kvseg32);
2803 2792 (void) segkmem_create(&kvseg32);
2804 2793
2805 2794 /*
2806 2795 * Create a segment for the debugger.
2807 2796 */
2808 2797 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2809 2798 (void) segkmem_create(&kdebugseg);
2810 2799
2811 2800 rw_exit(&kas.a_lock);
2812 2801 }
2813 2802
2814 2803 char obp_tte_str[] =
2815 2804 "h# %x constant MMU_PAGESHIFT "
2816 2805 "h# %x constant TTE8K "
2817 2806 "h# %x constant SFHME_SIZE "
2818 2807 "h# %x constant SFHME_TTE "
2819 2808 "h# %x constant HMEBLK_TAG "
2820 2809 "h# %x constant HMEBLK_NEXT "
2821 2810 "h# %x constant HMEBLK_MISC "
2822 2811 "h# %x constant HMEBLK_HME1 "
2823 2812 "h# %x constant NHMENTS "
2824 2813 "h# %x constant HBLK_SZMASK "
2825 2814 "h# %x constant HBLK_RANGE_SHIFT "
2826 2815 "h# %x constant HMEBP_HBLK "
2827 2816 "h# %x constant HMEBLK_ENDPA "
2828 2817 "h# %x constant HMEBUCKET_SIZE "
2829 2818 "h# %x constant HTAG_SFMMUPSZ "
2830 2819 "h# %x constant HTAG_BSPAGE_SHIFT "
2831 2820 "h# %x constant HTAG_REHASH_SHIFT "
2832 2821 "h# %x constant SFMMU_INVALID_SHMERID "
2833 2822 "h# %x constant mmu_hashcnt "
2834 2823 "h# %p constant uhme_hash "
2835 2824 "h# %p constant khme_hash "
2836 2825 "h# %x constant UHMEHASH_SZ "
2837 2826 "h# %x constant KHMEHASH_SZ "
2838 2827 "h# %p constant KCONTEXT "
2839 2828 "h# %p constant KHATID "
2840 2829 "h# %x constant ASI_MEM "
2841 2830
2842 2831 ": PHYS-X@ ( phys -- data ) "
2843 2832 " ASI_MEM spacex@ "
2844 2833 "; "
2845 2834
2846 2835 ": PHYS-W@ ( phys -- data ) "
2847 2836 " ASI_MEM spacew@ "
2848 2837 "; "
2849 2838
2850 2839 ": PHYS-L@ ( phys -- data ) "
2851 2840 " ASI_MEM spaceL@ "
2852 2841 "; "
2853 2842
2854 2843 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
2855 2844 " 3 * MMU_PAGESHIFT + "
2856 2845 "; "
2857 2846
2858 2847 ": TTE_IS_VALID ( ttep -- flag ) "
2859 2848 " PHYS-X@ 0< "
2860 2849 "; "
2861 2850
2862 2851 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
2863 2852 " dup TTE8K = if "
2864 2853 " drop HBLK_RANGE_SHIFT "
2865 2854 " else "
2866 2855 " TTE_PAGE_SHIFT "
2867 2856 " then "
2868 2857 "; "
2869 2858
2870 2859 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
2871 2860 " tuck >> swap MMU_PAGESHIFT - << "
2872 2861 "; "
2873 2862
2874 2863 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
2875 2864 " >> over xor swap ( hash sfmmup ) "
2876 2865 " KHATID <> if ( hash ) "
2877 2866 " UHMEHASH_SZ and ( bucket ) "
2878 2867 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) "
2879 2868 " else ( hash ) "
2880 2869 " KHMEHASH_SZ and ( bucket ) "
2881 2870 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) "
2882 2871 " then ( hmebp ) "
2883 2872 "; "
2884 2873
2885 2874 ": HME_HASH_TABLE_SEARCH "
2886 2875 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) "
2887 2876 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
2888 2877 " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) "
2889 2878 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) "
2890 2879 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if "
2891 2880 " true ( sfmmup hmeblkp true ) ( r: hblktag ) "
2892 2881 " else "
2893 2882 " hmeblk_next + phys-x@ false "
2894 2883 " ( sfmmup hmeblkp false ) ( r: hblktag ) "
2895 2884 " then "
2896 2885 " else "
2897 2886 " hmeblk_next + phys-x@ false "
2898 2887 " ( sfmmup hmeblkp false ) ( r: hblktag ) "
2899 2888 " then "
2900 2889 " else "
2901 2890 " drop 0 true "
2902 2891 " then "
2903 2892 " until r> drop "
2904 2893 "; "
2905 2894
2906 2895 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
2907 2896 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) "
2908 2897 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )"
2909 2898 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )"
2910 2899 " SFMMU_INVALID_SHMERID or nip ( hblktag ) "
2911 2900 "; "
2912 2901
2913 2902 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
2914 2903 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) "
2915 2904 " TTE8K = if ( hmeblkp addr ) "
2916 2905 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) "
2917 2906 " else ( hmeblkp addr ) "
2918 2907 " drop 0 ( hmeblkp 0 ) "
2919 2908 " then ( hmeblkp hme-index ) "
2920 2909 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) "
2921 2910 " SFHME_TTE + ( ttep ) "
2922 2911 "; "
2923 2912
2924 2913 ": unix-tte ( addr cnum -- false | tte-data true ) "
2925 2914 " KCONTEXT = if ( addr ) "
2926 2915 " KHATID ( addr khatid ) "
2927 2916 " else ( addr ) "
2928 2917 " drop false exit ( false ) "
2929 2918 " then "
2930 2919 " ( addr khatid ) "
2931 2920 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) "
2932 2921 " 2dup swap i HME_HASH_SHIFT "
2933 2922 "( addr sfmmup sfmmup addr hmeshift ) "
2934 2923 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) "
2935 2924 " over i 4 pick "
2936 2925 "( addr sfmmup hmebp sfmmup rehash addr ) "
2937 2926 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) "
2938 2927 " HME_HASH_TABLE_SEARCH "
2939 2928 "( addr sfmmup { null | hmeblkp } ) "
2940 2929 " ?dup if ( addr sfmmup hmeblkp ) "
2941 2930 " nip swap HBLK_TO_TTEP ( ttep ) "
2942 2931 " dup TTE_IS_VALID if ( valid-ttep ) "
2943 2932 " PHYS-X@ true ( tte-data true ) "
2944 2933 " else ( invalid-tte ) "
2945 2934 " drop false ( false ) "
2946 2935 " then ( false | tte-data true ) "
2947 2936 " unloop exit ( false | tte-data true ) "
2948 2937 " then ( addr sfmmup ) "
2949 2938 " loop ( addr sfmmup ) "
2950 2939 " 2drop false ( false ) "
2951 2940 "; "
2952 2941 ;
2953 2942
2954 2943 void
2955 2944 create_va_to_tte(void)
2956 2945 {
2957 2946 char *bp;
2958 2947 extern int khmehash_num, uhmehash_num;
2959 2948 extern struct hmehash_bucket *khme_hash, *uhme_hash;
2960 2949
2961 2950 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field))
2962 2951
2963 2952 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
2964 2953
2965 2954 /*
2966 2955 * Teach obp how to parse our sw ttes.
2967 2956 */
2968 2957 (void) sprintf(bp, obp_tte_str,
2969 2958 MMU_PAGESHIFT,
2970 2959 TTE8K,
2971 2960 sizeof (struct sf_hment),
2972 2961 OFFSET(struct sf_hment, hme_tte),
2973 2962 OFFSET(struct hme_blk, hblk_tag),
2974 2963 OFFSET(struct hme_blk, hblk_nextpa),
2975 2964 OFFSET(struct hme_blk, hblk_misc),
2976 2965 OFFSET(struct hme_blk, hblk_hme),
2977 2966 NHMENTS,
2978 2967 HBLK_SZMASK,
2979 2968 HBLK_RANGE_SHIFT,
2980 2969 OFFSET(struct hmehash_bucket, hmeh_nextpa),
2981 2970 HMEBLK_ENDPA,
2982 2971 sizeof (struct hmehash_bucket),
2983 2972 HTAG_SFMMUPSZ,
2984 2973 HTAG_BSPAGE_SHIFT,
2985 2974 HTAG_REHASH_SHIFT,
2986 2975 SFMMU_INVALID_SHMERID,
2987 2976 mmu_hashcnt,
2988 2977 (caddr_t)va_to_pa((caddr_t)uhme_hash),
2989 2978 (caddr_t)va_to_pa((caddr_t)khme_hash),
2990 2979 UHMEHASH_SZ,
2991 2980 KHMEHASH_SZ,
2992 2981 KCONTEXT,
2993 2982 KHATID,
2994 2983 ASI_MEM);
2995 2984 prom_interpret(bp, 0, 0, 0, 0, 0);
2996 2985
2997 2986 kobj_free(bp, MMU_PAGESIZE);
2998 2987 }
2999 2988
3000 2989 void
3001 2990 install_va_to_tte(void)
3002 2991 {
3003 2992 /*
3004 2993 * advise prom that it can use unix-tte
3005 2994 */
3006 2995 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
3007 2996 }
3008 2997
3009 2998 /*
3010 2999 * Here we add "device-type=console" for /os-io node, for currently
3011 3000 * our kernel console output only supports displaying text and
3012 3001 * performing cursor-positioning operations (through kernel framebuffer
3013 3002 * driver) and it doesn't support other functionalities required for a
3014 3003 * standard "display" device as specified in 1275 spec. The main missing
3015 3004 * interface defined by the 1275 spec is "draw-logo".
3016 3005 * also see the comments above prom_stdout_is_framebuffer().
3017 3006 */
3018 3007 static char *create_node =
3019 3008 "\" /\" find-device "
3020 3009 "new-device "
3021 3010 "\" os-io\" device-name "
3022 3011 "\" "OBP_DISPLAY_CONSOLE"\" device-type "
3023 3012 ": cb-r/w ( adr,len method$ -- #read/#written ) "
3024 3013 " 2>r swap 2 2r> ['] $callback catch if "
3025 3014 " 2drop 3drop 0 "
3026 3015 " then "
3027 3016 "; "
3028 3017 ": read ( adr,len -- #read ) "
3029 3018 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then "
3030 3019 " ( retN ... ret1 N ) "
3031 3020 " ?dup if "
3032 3021 " swap >r 1- 0 ?do drop loop r> "
3033 3022 " else "
3034 3023 " -2 "
3035 3024 " then "
3036 3025 "; "
3037 3026 ": write ( adr,len -- #written ) "
3038 3027 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then "
3039 3028 " ( retN ... ret1 N ) "
3040 3029 " ?dup if "
3041 3030 " swap >r 1- 0 ?do drop loop r> "
3042 3031 " else "
3043 3032 " 0 "
3044 3033 " then "
3045 3034 "; "
3046 3035 ": poll-tty ( -- ) ; "
3047 3036 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; "
3048 3037 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; "
3049 3038 ": cb-give/take ( $method -- ) "
3050 3039 " 0 -rot ['] $callback catch ?dup if "
3051 3040 " >r 2drop 2drop r> throw "
3052 3041 " else "
3053 3042 " 0 ?do drop loop "
3054 3043 " then "
3055 3044 "; "
3056 3045 ": give ( -- ) \" exit-input\" cb-give/take ; "
3057 3046 ": take ( -- ) \" enter-input\" cb-give/take ; "
3058 3047 ": open ( -- ok? ) true ; "
3059 3048 ": close ( -- ) ; "
3060 3049 "finish-device "
3061 3050 "device-end ";
3062 3051
3063 3052 /*
3064 3053 * Create the OBP input/output node (FCode serial driver).
3065 3054 * It is needed for both USB console keyboard and for
3066 3055 * the kernel terminal emulator. It is too early to check for a
3067 3056 * kernel console compatible framebuffer now, so we create this
3068 3057 * so that we're ready if we need to enable kernel terminal emulation.
3069 3058 *
3070 3059 * When the USB software takes over the input device at the time
3071 3060 * consconfig runs, OBP's stdin is redirected to this node.
3072 3061 * Whenever the FORTH user interface is used after this switch,
3073 3062 * the node will call back into the kernel for console input.
3074 3063 * If a serial device such as ttya or a UART with a Type 5 keyboard
3075 3064 * attached is used, OBP takes over the serial device when the system
3076 3065 * goes to the debugger after the system is booted. This sharing
3077 3066 * of the relatively simple serial device is difficult but possible.
3078 3067 * Sharing the USB host controller is impossible due its complexity.
3079 3068 *
3080 3069 * Similarly to USB keyboard input redirection, after consconfig_dacf
|
↓ open down ↓ |
1545 lines elided |
↑ open up ↑ |
3081 3070 * configures a kernel console framebuffer as the standard output
3082 3071 * device, OBP's stdout is switched to to vector through the
3083 3072 * /os-io node into the kernel terminal emulator.
3084 3073 */
3085 3074 static void
3086 3075 startup_create_io_node(void)
3087 3076 {
3088 3077 prom_interpret(create_node, 0, 0, 0, 0, 0);
3089 3078 }
3090 3079
3091 -
3092 -static void
3093 -do_prom_version_check(void)
3094 -{
3095 - int i;
3096 - pnode_t node;
3097 - char buf[64];
3098 - static char drev[] = "Down-rev firmware detected%s\n"
3099 - "\tPlease upgrade to the following minimum version:\n"
3100 - "\t\t%s\n";
3101 -
3102 - i = prom_version_check(buf, sizeof (buf), &node);
3103 -
3104 - if (i == PROM_VER64_OK)
3105 - return;
3106 -
3107 - if (i == PROM_VER64_UPGRADE) {
3108 - cmn_err(CE_WARN, drev, "", buf);
3109 -
3110 -#ifdef DEBUG
3111 - prom_enter_mon(); /* Type 'go' to continue */
3112 - cmn_err(CE_WARN, "Booting with down-rev firmware\n");
3113 - return;
3114 -#else
3115 - halt(0);
3116 -#endif
3117 - }
3118 -
3119 - /*
3120 - * The other possibility is that this is a server running
3121 - * good firmware, but down-rev firmware was detected on at
3122 - * least one other cpu board. We just complain if we see
3123 - * that.
3124 - */
3125 - cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);
3126 -}
3127 -
3128 3080
3129 3081 /*
3130 3082 * Must be defined in platform dependent code.
3131 3083 */
3132 3084 extern caddr_t modtext;
3133 3085 extern size_t modtext_sz;
3134 3086 extern caddr_t moddata;
3135 3087
3136 3088 #define HEAPTEXT_ARENA(addr) \
3137 3089 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
3138 3090 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \
3139 3091 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
3140 3092
3141 3093 #define HEAPTEXT_OVERSIZED(addr) \
3142 3094 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
3143 3095
3144 3096 #define HEAPTEXT_IN_NUCLEUSDATA(addr) \
3145 3097 (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
3146 3098 ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
3147 3099
3148 3100 vmem_t *texthole_source[HEAPTEXT_NARENAS];
3149 3101 vmem_t *texthole_arena[HEAPTEXT_NARENAS];
3150 3102 kmutex_t texthole_lock;
3151 3103
3152 3104 char kern_bootargs[OBP_MAXPATHLEN];
3153 3105 char kern_bootfile[OBP_MAXPATHLEN];
3154 3106
3155 3107 void
3156 3108 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3157 3109 {
3158 3110 uintptr_t addr, limit;
3159 3111
3160 3112 addr = HEAPTEXT_BASE;
3161 3113 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
3162 3114
3163 3115 /*
3164 3116 * Before we initialize the text_arena, we want to punch holes in the
3165 3117 * underlying heaptext_arena. This guarantees that for any text
3166 3118 * address we can find a text hole less than HEAPTEXT_MAPPED away.
3167 3119 */
3168 3120 for (; addr + HEAPTEXT_UNMAPPED <= limit;
3169 3121 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
3170 3122 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
3171 3123 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
3172 3124 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3173 3125 }
3174 3126
3175 3127 /*
3176 3128 * Allocate one page at the oversize to break up the text region
3177 3129 * from the oversized region.
3178 3130 */
3179 3131 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
3180 3132 (void *)limit, (void *)(limit + PAGESIZE),
3181 3133 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3182 3134
3183 3135 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
3184 3136 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
3185 3137 heaptext_arena, 0, VM_SLEEP);
3186 3138 *data_arena = vmem_create("module_data", moddata, MODDATA, 1,
3187 3139 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3188 3140 }
3189 3141
3190 3142 caddr_t
3191 3143 kobj_text_alloc(vmem_t *arena, size_t size)
3192 3144 {
3193 3145 caddr_t rval, better;
3194 3146
3195 3147 /*
3196 3148 * First, try a sleeping allocation.
3197 3149 */
3198 3150 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
3199 3151
3200 3152 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
3201 3153 return (rval);
3202 3154
3203 3155 /*
3204 3156 * We didn't get the area that we wanted. We're going to try to do an
3205 3157 * allocation with explicit constraints.
3206 3158 */
3207 3159 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
3208 3160 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
3209 3161 VM_NOSLEEP | VM_BESTFIT);
3210 3162
3211 3163 if (better != NULL) {
3212 3164 /*
3213 3165 * That worked. Free our first attempt and return.
3214 3166 */
3215 3167 vmem_free(arena, rval, size);
3216 3168 return (better);
3217 3169 }
3218 3170
3219 3171 /*
3220 3172 * That didn't work; we'll have to return our first attempt.
3221 3173 */
3222 3174 return (rval);
3223 3175 }
3224 3176
3225 3177 caddr_t
3226 3178 kobj_texthole_alloc(caddr_t addr, size_t size)
3227 3179 {
3228 3180 int arena = HEAPTEXT_ARENA(addr);
3229 3181 char c[30];
3230 3182 uintptr_t base;
3231 3183
3232 3184 if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
3233 3185 /*
3234 3186 * If this is an oversized allocation or it is allocated in
3235 3187 * the nucleus data page, there is no text hole available for
3236 3188 * it; return NULL.
3237 3189 */
3238 3190 return (NULL);
3239 3191 }
3240 3192
3241 3193 mutex_enter(&texthole_lock);
3242 3194
3243 3195 if (texthole_arena[arena] == NULL) {
3244 3196 ASSERT(texthole_source[arena] == NULL);
3245 3197
3246 3198 if (arena == 0) {
3247 3199 texthole_source[0] = vmem_create("module_text_holesrc",
3248 3200 (void *)(KERNELBASE + MMU_PAGESIZE4M),
3249 3201 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
3250 3202 0, VM_SLEEP);
3251 3203 } else {
3252 3204 base = HEAPTEXT_BASE +
3253 3205 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
3254 3206
3255 3207 (void) snprintf(c, sizeof (c),
3256 3208 "heaptext_holesrc_%d", arena);
3257 3209
3258 3210 texthole_source[arena] = vmem_create(c, (void *)base,
3259 3211 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
3260 3212 0, VM_SLEEP);
3261 3213 }
3262 3214
3263 3215 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
3264 3216
3265 3217 texthole_arena[arena] = vmem_create(c, NULL, 0,
3266 3218 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
3267 3219 texthole_source[arena], 0, VM_SLEEP);
3268 3220 }
3269 3221
3270 3222 mutex_exit(&texthole_lock);
3271 3223
3272 3224 ASSERT(texthole_arena[arena] != NULL);
3273 3225 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3274 3226 return (vmem_alloc(texthole_arena[arena], size,
3275 3227 VM_BESTFIT | VM_NOSLEEP));
3276 3228 }
3277 3229
3278 3230 void
3279 3231 kobj_texthole_free(caddr_t addr, size_t size)
3280 3232 {
3281 3233 int arena = HEAPTEXT_ARENA(addr);
3282 3234
3283 3235 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3284 3236 ASSERT(texthole_arena[arena] != NULL);
3285 3237 vmem_free(texthole_arena[arena], addr, size);
3286 3238 }
3287 3239
3288 3240 void
3289 3241 release_bootstrap(void)
3290 3242 {
3291 3243 if (&cif_init)
3292 3244 cif_init();
3293 3245 }
|
↓ open down ↓ |
156 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX