1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2019 Joyent, Inc.
24 */
25
26 #include <sys/types.h>
27 #include <sys/t_lock.h>
28 #include <sys/param.h>
29 #include <sys/sysmacros.h>
30 #include <sys/tuneable.h>
31 #include <sys/systm.h>
32 #include <sys/vm.h>
33 #include <sys/kmem.h>
34 #include <sys/vmem.h>
35 #include <sys/mman.h>
36 #include <sys/cmn_err.h>
37 #include <sys/debug.h>
38 #include <sys/dumphdr.h>
39 #include <sys/bootconf.h>
40 #include <sys/lgrp.h>
41 #include <vm/seg_kmem.h>
42 #include <vm/hat.h>
43 #include <vm/page.h>
44 #include <vm/vm_dep.h>
45 #include <vm/faultcode.h>
46 #include <sys/promif.h>
47 #include <vm/seg_kp.h>
48 #include <sys/bitmap.h>
49 #include <sys/mem_cage.h>
50
51 #ifdef __sparc
52 #include <sys/ivintr.h>
53 #include <sys/panic.h>
54 #endif
55
56 /*
57 * seg_kmem is the primary kernel memory segment driver. It
58 * maps the kernel heap [kernelheap, ekernelheap), module text,
59 * and all memory which was allocated before the VM was initialized
60 * into kas.
61 *
62 * Pages which belong to seg_kmem are hashed into &kvp vnode at
63 * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1.
64 * They must never be paged out since segkmem_fault() is a no-op to
65 * prevent recursive faults.
66 *
67 * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
68 * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86
69 * supports relocation the #ifdef kludges can be removed.
70 *
71 * seg_kmem pages may be subject to relocation by page_relocate(),
72 * provided that the HAT supports it; if this is so, segkmem_reloc
73 * will be set to a nonzero value. All boot time allocated memory as
74 * well as static memory is considered off limits to relocation.
75 * Pages are "relocatable" if p_state does not have P_NORELOC set, so
76 * we request P_NORELOC pages for memory that isn't safe to relocate.
77 *
78 * The kernel heap is logically divided up into four pieces:
79 *
80 * heap32_arena is for allocations that require 32-bit absolute
81 * virtual addresses (e.g. code that uses 32-bit pointers/offsets).
82 *
83 * heap_core is for allocations that require 2GB *relative*
84 * offsets; in other words all memory from heap_core is within
85 * 2GB of all other memory from the same arena. This is a requirement
86 * of the addressing modes of some processors in supervisor code.
87 *
88 * heap_arena is the general heap arena.
89 *
90 * static_arena is the static memory arena. Allocations from it
91 * are not subject to relocation so it is safe to use the memory
92 * physical address as well as the virtual address (e.g. the VA to
93 * PA translations are static). Caches may import from static_arena;
94 * all other static memory allocations should use static_alloc_arena.
95 *
96 * On some platforms which have limited virtual address space, seg_kmem
97 * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
98 * segkp_bitmap is non-NULL, and each bit represents a page of virtual
99 * address space which is actually seg_kp mapped.
100 */
101
102 extern ulong_t *segkp_bitmap; /* Is set if segkp is from the kernel heap */
103
104 char *kernelheap; /* start of primary kernel heap */
105 char *ekernelheap; /* end of primary kernel heap */
106 struct seg kvseg; /* primary kernel heap segment */
107 struct seg kvseg_core; /* "core" kernel heap segment */
108 struct seg kzioseg; /* Segment for zio mappings */
109 vmem_t *heap_arena; /* primary kernel heap arena */
110 vmem_t *heap_core_arena; /* core kernel heap arena */
111 char *heap_core_base; /* start of core kernel heap arena */
112 char *heap_lp_base; /* start of kernel large page heap arena */
113 char *heap_lp_end; /* end of kernel large page heap arena */
114 vmem_t *hat_memload_arena; /* HAT translation data */
115 struct seg kvseg32; /* 32-bit kernel heap segment */
116 vmem_t *heap32_arena; /* 32-bit kernel heap arena */
117 vmem_t *heaptext_arena; /* heaptext arena */
118 struct as kas; /* kernel address space */
119 int segkmem_reloc; /* enable/disable relocatable segkmem pages */
120 vmem_t *static_arena; /* arena for caches to import static memory */
121 vmem_t *static_alloc_arena; /* arena for allocating static memory */
122 vmem_t *zio_arena = NULL; /* arena for allocating zio memory */
123 vmem_t *zio_alloc_arena = NULL; /* arena for allocating zio memory */
124
125 /*
126 * seg_kmem driver can map part of the kernel heap with large pages.
127 * Currently this functionality is implemented for sparc platforms only.
128 *
129 * The large page size "segkmem_lpsize" for kernel heap is selected in the
130 * platform specific code. It can also be modified via /etc/system file.
131 * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
132 * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
133 * match segkmem_lpsize.
134 *
135 * At boot time we carve from kernel heap arena a range of virtual addresses
136 * that will be used for large page mappings. This range [heap_lp_base,
137 * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
138 * create "kmem_lp_arena" that caches memory already backed up by large
139 * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
140 */
141
142 size_t segkmem_lpsize;
143 static uint_t segkmem_lpshift = PAGESHIFT;
144 int segkmem_lpszc = 0;
145
146 size_t segkmem_kmemlp_quantum = 0x400000; /* 4MB */
147 size_t segkmem_heaplp_quantum;
148 vmem_t *heap_lp_arena;
149 static vmem_t *kmem_lp_arena;
150 static vmem_t *segkmem_ppa_arena;
151 static segkmem_lpcb_t segkmem_lpcb;
152
153 /*
154 * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
155 * consumed by the large page heap. By default this parameter is set to 1/8 of
156 * physmem but can be adjusted through /etc/system either directly or
157 * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
158 * we allow for large page heap.
159 */
160 size_t segkmem_kmemlp_max;
161 static uint_t segkmem_kmemlp_pcnt;
162
163 /*
164 * Getting large pages for kernel heap could be problematic due to
165 * physical memory fragmentation. That's why we allow to preallocate
166 * "segkmem_kmemlp_min" bytes at boot time.
167 */
168 static size_t segkmem_kmemlp_min;
169
170 /*
171 * Throttling is used to avoid expensive tries to allocate large pages
172 * for kernel heap when a lot of succesive attempts to do so fail.
173 */
174 static ulong_t segkmem_lpthrottle_max = 0x400000;
175 static ulong_t segkmem_lpthrottle_start = 0x40;
176 static ulong_t segkmem_use_lpthrottle = 1;
177
178 /*
179 * Freed pages accumulate on a garbage list until segkmem is ready,
180 * at which point we call segkmem_gc() to free it all.
181 */
182 typedef struct segkmem_gc_list {
183 struct segkmem_gc_list *gc_next;
184 vmem_t *gc_arena;
185 size_t gc_size;
186 } segkmem_gc_list_t;
187
188 static segkmem_gc_list_t *segkmem_gc_list;
189
190 /*
191 * Allocations from the hat_memload arena add VM_MEMLOAD to their
192 * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
193 * to take steps to prevent infinite recursion. HAT allocations also
194 * must be non-relocatable to prevent recursive page faults.
195 */
196 static void *
197 hat_memload_alloc(vmem_t *vmp, size_t size, int flags)
198 {
199 flags |= (VM_MEMLOAD | VM_NORELOC);
200 return (segkmem_alloc(vmp, size, flags));
201 }
202
203 /*
204 * Allocations from static_arena arena (or any other arena that uses
205 * segkmem_alloc_permanent()) require non-relocatable (permanently
206 * wired) memory pages, since these pages are referenced by physical
207 * as well as virtual address.
208 */
209 void *
210 segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags)
211 {
212 return (segkmem_alloc(vmp, size, flags | VM_NORELOC));
213 }
214
215 /*
216 * Initialize kernel heap boundaries.
217 */
218 void
219 kernelheap_init(
220 void *heap_start,
221 void *heap_end,
222 char *first_avail,
223 void *core_start,
224 void *core_end)
225 {
226 uintptr_t textbase;
227 size_t core_size;
228 size_t heap_size;
229 vmem_t *heaptext_parent;
230 size_t heap_lp_size = 0;
231 #ifdef __sparc
232 size_t kmem64_sz = kmem64_aligned_end - kmem64_base;
233 #endif /* __sparc */
234
235 kernelheap = heap_start;
236 ekernelheap = heap_end;
237
238 #ifdef __sparc
239 heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4);
240 /*
241 * Bias heap_lp start address by kmem64_sz to reduce collisions
242 * in 4M kernel TSB between kmem64 area and heap_lp
243 */
244 kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M);
245 if (kmem64_sz <= heap_lp_size / 2)
246 heap_lp_size -= kmem64_sz;
247 heap_lp_base = ekernelheap - heap_lp_size;
248 heap_lp_end = heap_lp_base + heap_lp_size;
249 #endif /* __sparc */
250
251 /*
252 * If this platform has a 'core' heap area, then the space for
253 * overflow module text should be carved out of the end of that
254 * heap. Otherwise, it gets carved out of the general purpose
255 * heap.
256 */
257 core_size = (uintptr_t)core_end - (uintptr_t)core_start;
258 if (core_size > 0) {
259 ASSERT(core_size >= HEAPTEXT_SIZE);
260 textbase = (uintptr_t)core_end - HEAPTEXT_SIZE;
261 core_size -= HEAPTEXT_SIZE;
262 }
263 #ifndef __sparc
264 else {
265 ekernelheap -= HEAPTEXT_SIZE;
266 textbase = (uintptr_t)ekernelheap;
267 }
268 #endif
269
270 heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap;
271 heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE,
272 segkmem_alloc, segkmem_free);
273
274 if (core_size > 0) {
275 heap_core_arena = vmem_create("heap_core", core_start,
276 core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
277 heap_core_base = core_start;
278 } else {
279 heap_core_arena = heap_arena;
280 heap_core_base = kernelheap;
281 }
282
283 /*
284 * reserve space for the large page heap. If large pages for kernel
285 * heap is enabled large page heap arean will be created later in the
286 * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
287 * range will be returned back to the heap_arena.
288 */
289 if (heap_lp_size) {
290 (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0,
291 heap_lp_base, heap_lp_end,
292 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
293 }
294
295 /*
296 * Remove the already-spoken-for memory range [kernelheap, first_avail).
297 */
298 (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE,
299 0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
300
301 #ifdef __sparc
302 heap32_arena = vmem_create("heap32", (void *)SYSBASE32,
303 SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL,
304 NULL, NULL, 0, VM_SLEEP);
305 /*
306 * Prom claims the physical and virtual resources used by panicbuf
307 * and inter_vec_table. So reserve space for panicbuf, intr_vec_table,
308 * reserved interrupt vector data structures from 32-bit heap.
309 */
310 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
311 panicbuf, panicbuf + PANICBUFSIZE,
312 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
313
314 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
315 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
316 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
317
318 textbase = SYSLIMIT32 - HEAPTEXT_SIZE;
319 heaptext_parent = NULL;
320 #else /* __sparc */
321 heap32_arena = heap_core_arena;
322 heaptext_parent = heap_core_arena;
323 #endif /* __sparc */
324
325 heaptext_arena = vmem_create("heaptext", (void *)textbase,
326 HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP);
327
328 /*
329 * Create a set of arenas for memory with static translations
330 * (e.g. VA -> PA translations cannot change). Since using
331 * kernel pages by physical address implies it isn't safe to
332 * walk across page boundaries, the static_arena quantum must
333 * be PAGESIZE. Any kmem caches that require static memory
334 * should source from static_arena, while direct allocations
335 * should only use static_alloc_arena.
336 */
337 static_arena = vmem_create("static", NULL, 0, PAGESIZE,
338 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP);
339 static_alloc_arena = vmem_create("static_alloc", NULL, 0,
340 sizeof (uint64_t), vmem_alloc, vmem_free, static_arena,
341 0, VM_SLEEP);
342
343 /*
344 * Create an arena for translation data (ptes, hmes, or hblks).
345 * We need an arena for this because hat_memload() is essential
346 * to vmem_populate() (see comments in common/os/vmem.c).
347 *
348 * Note: any kmem cache that allocates from hat_memload_arena
349 * must be created as a KMC_NOHASH cache (i.e. no external slab
350 * and bufctl structures to allocate) so that slab creation doesn't
351 * require anything more than a single vmem_alloc().
352 */
353 hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE,
354 hat_memload_alloc, segkmem_free, heap_arena, 0,
355 VM_SLEEP | VMC_POPULATOR | VMC_DUMPSAFE);
356 }
357
358 void
359 boot_mapin(caddr_t addr, size_t size)
360 {
361 caddr_t eaddr;
362 page_t *pp;
363 pfn_t pfnum;
364
365 if (page_resv(btop(size), KM_NOSLEEP) == 0)
366 panic("boot_mapin: page_resv failed");
367
368 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
369 pfnum = va_to_pfn(addr);
370 if (pfnum == PFN_INVALID)
371 continue;
372 if ((pp = page_numtopp_nolock(pfnum)) == NULL)
373 panic("boot_mapin(): No pp for pfnum = %lx", pfnum);
374
375 /*
376 * must break up any large pages that may have constituent
377 * pages being utilized for BOP_ALLOC()'s before calling
378 * page_numtopp().The locking code (ie. page_reclaim())
379 * can't handle them
380 */
381 if (pp->p_szc != 0)
382 page_boot_demote(pp);
383
384 pp = page_numtopp(pfnum, SE_EXCL);
385 if (pp == NULL || PP_ISFREE(pp))
386 panic("boot_alloc: pp is NULL or free");
387
388 /*
389 * If the cage is on but doesn't yet contain this page,
390 * mark it as non-relocatable.
391 */
392 if (kcage_on && !PP_ISNORELOC(pp)) {
393 PP_SETNORELOC(pp);
394 PLCNT_XFER_NORELOC(pp);
395 }
396
397 (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL);
398 pp->p_lckcnt = 1;
399 #if defined(__x86)
400 page_downgrade(pp);
401 #else
402 page_unlock(pp);
403 #endif
404 }
405 }
406
407 /*
408 * Get pages from boot and hash them into the kernel's vp.
409 * Used after page structs have been allocated, but before segkmem is ready.
410 */
411 void *
412 boot_alloc(void *inaddr, size_t size, uint_t align)
413 {
414 caddr_t addr = inaddr;
415
416 if (bootops == NULL)
417 prom_panic("boot_alloc: attempt to allocate memory after "
418 "BOP_GONE");
419
420 size = ptob(btopr(size));
421 #ifdef __sparc
422 if (bop_alloc_chunk(addr, size, align) != (caddr_t)addr)
423 panic("boot_alloc: bop_alloc_chunk failed");
424 #else
425 if (BOP_ALLOC(bootops, addr, size, align) != addr)
426 panic("boot_alloc: BOP_ALLOC failed");
427 #endif
428 boot_mapin((caddr_t)addr, size);
429 return (addr);
430 }
431
432 static void
433 segkmem_badop()
434 {
435 panic("segkmem_badop");
436 }
437
438 #define SEGKMEM_BADOP(t) (t(*)())(uintptr_t)segkmem_badop
439
440 /*ARGSUSED*/
441 static faultcode_t
442 segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size,
443 enum fault_type type, enum seg_rw rw)
444 {
445 pgcnt_t npages;
446 spgcnt_t pg;
447 page_t *pp;
448 struct vnode *vp = seg->s_data;
449
450 ASSERT(RW_READ_HELD(&seg->s_as->a_lock));
451
452 if (seg->s_as != &kas || size > seg->s_size ||
453 addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
454 panic("segkmem_fault: bad args");
455
456 /*
457 * If it is one of segkp pages, call segkp_fault.
458 */
459 if (segkp_bitmap && seg == &kvseg &&
460 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
461 return (SEGOP_FAULT(hat, segkp, addr, size, type, rw));
462
463 if (rw != S_READ && rw != S_WRITE && rw != S_OTHER)
464 return (FC_NOSUPPORT);
465
466 npages = btopr(size);
467
468 switch (type) {
469 case F_SOFTLOCK: /* lock down already-loaded translations */
470 for (pg = 0; pg < npages; pg++) {
471 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
472 SE_SHARED);
473 if (pp == NULL) {
474 /*
475 * Hmm, no page. Does a kernel mapping
476 * exist for it?
477 */
478 if (!hat_probe(kas.a_hat, addr)) {
479 addr -= PAGESIZE;
480 while (--pg >= 0) {
481 pp = page_find(vp, (u_offset_t)
482 (uintptr_t)addr);
483 if (pp)
484 page_unlock(pp);
485 addr -= PAGESIZE;
486 }
487 return (FC_NOMAP);
488 }
489 }
490 addr += PAGESIZE;
491 }
492 if (rw == S_OTHER)
493 hat_reserve(seg->s_as, addr, size);
494 return (0);
495 case F_SOFTUNLOCK:
496 while (npages--) {
497 pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
498 if (pp)
499 page_unlock(pp);
500 addr += PAGESIZE;
501 }
502 return (0);
503 default:
504 return (FC_NOSUPPORT);
505 }
506 /*NOTREACHED*/
507 }
508
509 static int
510 segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
511 {
512 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
513
514 if (seg->s_as != &kas || size > seg->s_size ||
515 addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
516 panic("segkmem_setprot: bad args");
517
518 /*
519 * If it is one of segkp pages, call segkp.
520 */
521 if (segkp_bitmap && seg == &kvseg &&
522 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
523 return (SEGOP_SETPROT(segkp, addr, size, prot));
524
525 if (prot == 0)
526 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD);
527 else
528 hat_chgprot(kas.a_hat, addr, size, prot);
529 return (0);
530 }
531
532 /*
533 * This is a dummy segkmem function overloaded to call segkp
534 * when segkp is under the heap.
535 */
536 /* ARGSUSED */
537 static int
538 segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
539 {
540 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
541
542 if (seg->s_as != &kas)
543 segkmem_badop();
544
545 /*
546 * If it is one of segkp pages, call into segkp.
547 */
548 if (segkp_bitmap && seg == &kvseg &&
549 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
550 return (SEGOP_CHECKPROT(segkp, addr, size, prot));
551
552 segkmem_badop();
553 return (0);
554 }
555
556 /*
557 * This is a dummy segkmem function overloaded to call segkp
558 * when segkp is under the heap.
559 */
560 /* ARGSUSED */
561 static int
562 segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
563 {
564 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
565
566 if (seg->s_as != &kas)
567 segkmem_badop();
568
569 /*
570 * If it is one of segkp pages, call into segkp.
571 */
572 if (segkp_bitmap && seg == &kvseg &&
573 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
574 return (SEGOP_KLUSTER(segkp, addr, delta));
575
576 segkmem_badop();
577 return (0);
578 }
579
580 static void
581 segkmem_xdump_range(void *arg, void *start, size_t size)
582 {
583 struct as *as = arg;
584 caddr_t addr = start;
585 caddr_t addr_end = addr + size;
586
587 while (addr < addr_end) {
588 pfn_t pfn = hat_getpfnum(kas.a_hat, addr);
589 if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn))
590 dump_addpage(as, addr, pfn);
591 addr += PAGESIZE;
592 dump_timeleft = dump_timeout;
593 }
594 }
595
596 static void
597 segkmem_dump_range(void *arg, void *start, size_t size)
598 {
599 caddr_t addr = start;
600 caddr_t addr_end = addr + size;
601
602 /*
603 * If we are about to start dumping the range of addresses we
604 * carved out of the kernel heap for the large page heap walk
605 * heap_lp_arena to find what segments are actually populated
606 */
607 if (SEGKMEM_USE_LARGEPAGES &&
608 addr == heap_lp_base && addr_end == heap_lp_end &&
609 vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
610 vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT,
611 segkmem_xdump_range, arg);
612 } else {
613 segkmem_xdump_range(arg, start, size);
614 }
615 }
616
617 static void
618 segkmem_dump(struct seg *seg)
619 {
620 /*
621 * The kernel's heap_arena (represented by kvseg) is a very large
622 * VA space, most of which is typically unused. To speed up dumping
623 * we use vmem_walk() to quickly find the pieces of heap_arena that
624 * are actually in use. We do the same for heap32_arena and
625 * heap_core.
626 *
627 * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
628 * may ultimately need to allocate memory. Reentrant walks are
629 * necessarily imperfect snapshots. The kernel heap continues
630 * to change during a live crash dump, for example. For a normal
631 * crash dump, however, we know that there won't be any other threads
632 * messing with the heap. Therefore, at worst, we may fail to dump
633 * the pages that get allocated by the act of dumping; but we will
634 * always dump every page that was allocated when the walk began.
635 *
636 * The other segkmem segments are dense (fully populated), so there's
637 * no need to use this technique when dumping them.
638 *
639 * Note: when adding special dump handling for any new sparsely-
640 * populated segments, be sure to add similar handling to the ::kgrep
641 * code in mdb.
642 */
643 if (seg == &kvseg) {
644 vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT,
645 segkmem_dump_range, seg->s_as);
646 #ifndef __sparc
647 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
648 segkmem_dump_range, seg->s_as);
649 #endif
650 } else if (seg == &kvseg_core) {
651 vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT,
652 segkmem_dump_range, seg->s_as);
653 } else if (seg == &kvseg32) {
654 vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT,
655 segkmem_dump_range, seg->s_as);
656 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
657 segkmem_dump_range, seg->s_as);
658 } else if (seg == &kzioseg) {
659 /*
660 * We don't want to dump pages attached to kzioseg since they
661 * contain file data from ZFS. If this page's segment is
662 * kzioseg return instead of writing it to the dump device.
663 */
664 return;
665 } else {
666 segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size);
667 }
668 }
669
670 /*
671 * lock/unlock kmem pages over a given range [addr, addr+len).
672 * Returns a shadow list of pages in ppp. If there are holes
673 * in the range (e.g. some of the kernel mappings do not have
674 * underlying page_ts) returns ENOTSUP so that as_pagelock()
675 * will handle the range via as_fault(F_SOFTLOCK).
676 */
677 /*ARGSUSED*/
678 static int
679 segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len,
680 page_t ***ppp, enum lock_type type, enum seg_rw rw)
681 {
682 page_t **pplist, *pp;
683 pgcnt_t npages;
684 spgcnt_t pg;
685 size_t nb;
686 struct vnode *vp = seg->s_data;
687
688 ASSERT(ppp != NULL);
689
690 /*
691 * If it is one of segkp pages, call into segkp.
692 */
693 if (segkp_bitmap && seg == &kvseg &&
694 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
695 return (SEGOP_PAGELOCK(segkp, addr, len, ppp, type, rw));
696
697 npages = btopr(len);
698 nb = sizeof (page_t *) * npages;
699
700 if (type == L_PAGEUNLOCK) {
701 pplist = *ppp;
702 ASSERT(pplist != NULL);
703
704 for (pg = 0; pg < npages; pg++) {
705 pp = pplist[pg];
706 page_unlock(pp);
707 }
708 kmem_free(pplist, nb);
709 return (0);
710 }
711
712 ASSERT(type == L_PAGELOCK);
713
714 pplist = kmem_alloc(nb, KM_NOSLEEP);
715 if (pplist == NULL) {
716 *ppp = NULL;
717 return (ENOTSUP); /* take the slow path */
718 }
719
720 for (pg = 0; pg < npages; pg++) {
721 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED);
722 if (pp == NULL) {
723 while (--pg >= 0)
724 page_unlock(pplist[pg]);
725 kmem_free(pplist, nb);
726 *ppp = NULL;
727 return (ENOTSUP);
728 }
729 pplist[pg] = pp;
730 addr += PAGESIZE;
731 }
732
733 *ppp = pplist;
734 return (0);
735 }
736
737 /*
738 * This is a dummy segkmem function overloaded to call segkp
739 * when segkp is under the heap.
740 */
741 /* ARGSUSED */
742 static int
743 segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
744 {
745 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
746
747 if (seg->s_as != &kas)
748 segkmem_badop();
749
750 /*
751 * If it is one of segkp pages, call into segkp.
752 */
753 if (segkp_bitmap && seg == &kvseg &&
754 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
755 return (SEGOP_GETMEMID(segkp, addr, memidp));
756
757 segkmem_badop();
758 return (0);
759 }
760
761 /*ARGSUSED*/
762 static lgrp_mem_policy_info_t *
763 segkmem_getpolicy(struct seg *seg, caddr_t addr)
764 {
765 return (NULL);
766 }
767
768 /*ARGSUSED*/
769 static int
770 segkmem_capable(struct seg *seg, segcapability_t capability)
771 {
772 if (capability == S_CAPABILITY_NOMINFLT)
773 return (1);
774 return (0);
775 }
776
777 struct seg_ops segkmem_ops = {
778 SEGKMEM_BADOP(int), /* dup */
779 SEGKMEM_BADOP(int), /* unmap */
780 SEGKMEM_BADOP(void), /* free */
781 segkmem_fault,
782 SEGKMEM_BADOP(faultcode_t), /* faulta */
783 segkmem_setprot,
784 segkmem_checkprot,
785 segkmem_kluster,
786 SEGKMEM_BADOP(size_t), /* swapout */
787 SEGKMEM_BADOP(int), /* sync */
788 SEGKMEM_BADOP(size_t), /* incore */
789 SEGKMEM_BADOP(int), /* lockop */
790 SEGKMEM_BADOP(int), /* getprot */
791 SEGKMEM_BADOP(u_offset_t), /* getoffset */
792 SEGKMEM_BADOP(int), /* gettype */
793 SEGKMEM_BADOP(int), /* getvp */
794 SEGKMEM_BADOP(int), /* advise */
795 segkmem_dump,
796 segkmem_pagelock,
797 SEGKMEM_BADOP(int), /* setpgsz */
798 segkmem_getmemid,
799 segkmem_getpolicy, /* getpolicy */
800 segkmem_capable, /* capable */
801 seg_inherit_notsup /* inherit */
802 };
803
804 int
805 segkmem_zio_create(struct seg *seg)
806 {
807 ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
808 seg->s_ops = &segkmem_ops;
809 seg->s_data = &zvp;
810 kas.a_size += seg->s_size;
811 return (0);
812 }
813
814 int
815 segkmem_create(struct seg *seg)
816 {
817 ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
818 seg->s_ops = &segkmem_ops;
819 seg->s_data = &kvp;
820 kas.a_size += seg->s_size;
821 return (0);
822 }
823
824 /*ARGSUSED*/
825 page_t *
826 segkmem_page_create(void *addr, size_t size, int vmflag, void *arg)
827 {
828 struct seg kseg = { 0 };
829 int pgflags = PG_EXCL;
830 struct vnode *vp = arg;
831
832 if (vp == NULL)
833 vp = &kvp;
834
835 kseg.s_as = &kas;
836
837 if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
838 pgflags |= PG_NORELOC;
839 if ((vmflag & VM_NOSLEEP) == 0)
840 pgflags |= PG_WAIT;
841 if (vmflag & VM_PANIC)
842 pgflags |= PG_PANIC;
843 if (vmflag & VM_PUSHPAGE)
844 pgflags |= PG_PUSHPAGE;
845 if (vmflag & VM_NORMALPRI) {
846 ASSERT(vmflag & VM_NOSLEEP);
847 pgflags |= PG_NORMALPRI;
848 }
849
850 return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size,
851 pgflags, &kseg, addr));
852 }
853
854 /*
855 * Allocate pages to back the virtual address range [addr, addr + size).
856 * If addr is NULL, allocate the virtual address space as well.
857 */
858 void *
859 segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr,
860 page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg)
861 {
862 page_t *ppl;
863 caddr_t addr = inaddr;
864 pgcnt_t npages = btopr(size);
865 int allocflag;
866
867 if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
868 return (NULL);
869
870 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
871
872 if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
873 if (inaddr == NULL)
874 vmem_free(vmp, addr, size);
875 return (NULL);
876 }
877
878 ppl = page_create_func(addr, size, vmflag, pcarg);
879 if (ppl == NULL) {
880 if (inaddr == NULL)
881 vmem_free(vmp, addr, size);
882 page_unresv(npages);
883 return (NULL);
884 }
885
886 /*
887 * Under certain conditions, we need to let the HAT layer know
888 * that it cannot safely allocate memory. Allocations from
889 * the hat_memload vmem arena always need this, to prevent
890 * infinite recursion.
891 *
892 * In addition, the x86 hat cannot safely do memory
893 * allocations while in vmem_populate(), because there
894 * is no simple bound on its usage.
895 */
896 if (vmflag & VM_MEMLOAD)
897 allocflag = HAT_NO_KALLOC;
898 #if defined(__x86)
899 else if (vmem_is_populator())
900 allocflag = HAT_NO_KALLOC;
901 #endif
902 else
903 allocflag = 0;
904
905 while (ppl != NULL) {
906 page_t *pp = ppl;
907 page_sub(&ppl, pp);
908 ASSERT(page_iolock_assert(pp));
909 ASSERT(PAGE_EXCL(pp));
910 page_io_unlock(pp);
911 hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp,
912 (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
913 HAT_LOAD_LOCK | allocflag);
914 pp->p_lckcnt = 1;
915 #if defined(__x86)
916 page_downgrade(pp);
917 #else
918 if (vmflag & SEGKMEM_SHARELOCKED)
919 page_downgrade(pp);
920 else
921 page_unlock(pp);
922 #endif
923 }
924
925 return (addr);
926 }
927
928 static void *
929 segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp)
930 {
931 void *addr;
932 segkmem_gc_list_t *gcp, **prev_gcpp;
933
934 ASSERT(vp != NULL);
935
936 if (kvseg.s_base == NULL) {
937 #ifndef __sparc
938 if (bootops->bsys_alloc == NULL)
939 halt("Memory allocation between bop_alloc() and "
940 "kmem_alloc().\n");
941 #endif
942
943 /*
944 * There's not a lot of memory to go around during boot,
945 * so recycle it if we can.
946 */
947 for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL;
948 prev_gcpp = &gcp->gc_next) {
949 if (gcp->gc_arena == vmp && gcp->gc_size == size) {
950 *prev_gcpp = gcp->gc_next;
951 return (gcp);
952 }
953 }
954
955 addr = vmem_alloc(vmp, size, vmflag | VM_PANIC);
956 if (boot_alloc(addr, size, BO_NO_ALIGN) != addr)
957 panic("segkmem_alloc: boot_alloc failed");
958 return (addr);
959 }
960 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
961 segkmem_page_create, vp));
962 }
963
964 void *
965 segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
966 {
967 return (segkmem_alloc_vn(vmp, size, vmflag, &kvp));
968 }
969
970 void *
971 segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag)
972 {
973 return (segkmem_alloc_vn(vmp, size, vmflag, &zvp));
974 }
975
976 /*
977 * Any changes to this routine must also be carried over to
978 * devmap_free_pages() in the seg_dev driver. This is because
979 * we currently don't have a special kernel segment for non-paged
980 * kernel memory that is exported by drivers to user space.
981 */
982 static void
983 segkmem_free_vn(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp,
984 void (*func)(page_t *))
985 {
986 page_t *pp;
987 caddr_t addr = inaddr;
988 caddr_t eaddr;
989 pgcnt_t npages = btopr(size);
990
991 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
992 ASSERT(vp != NULL);
993
994 if (kvseg.s_base == NULL) {
995 segkmem_gc_list_t *gc = inaddr;
996 gc->gc_arena = vmp;
997 gc->gc_size = size;
998 gc->gc_next = segkmem_gc_list;
999 segkmem_gc_list = gc;
1000 return;
1001 }
1002
1003 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1004
1005 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
1006 #if defined(__x86)
1007 pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
1008 if (pp == NULL)
1009 panic("segkmem_free: page not found");
1010 if (!page_tryupgrade(pp)) {
1011 /*
1012 * Some other thread has a sharelock. Wait for
1013 * it to drop the lock so we can free this page.
1014 */
1015 page_unlock(pp);
1016 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
1017 SE_EXCL);
1018 }
1019 #else
1020 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1021 #endif
1022 if (pp == NULL)
1023 panic("segkmem_free: page not found");
1024 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
1025 pp->p_lckcnt = 0;
1026 if (func)
1027 func(pp);
1028 else
1029 page_destroy(pp, 0);
1030 }
1031 if (func == NULL)
1032 page_unresv(npages);
1033
1034 if (vmp != NULL)
1035 vmem_free(vmp, inaddr, size);
1036
1037 }
1038
1039 void
1040 segkmem_xfree(vmem_t *vmp, void *inaddr, size_t size, void (*func)(page_t *))
1041 {
1042 segkmem_free_vn(vmp, inaddr, size, &kvp, func);
1043 }
1044
1045 void
1046 segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
1047 {
1048 segkmem_free_vn(vmp, inaddr, size, &kvp, NULL);
1049 }
1050
1051 void
1052 segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size)
1053 {
1054 segkmem_free_vn(vmp, inaddr, size, &zvp, NULL);
1055 }
1056
1057 void
1058 segkmem_gc(void)
1059 {
1060 ASSERT(kvseg.s_base != NULL);
1061 while (segkmem_gc_list != NULL) {
1062 segkmem_gc_list_t *gc = segkmem_gc_list;
1063 segkmem_gc_list = gc->gc_next;
1064 segkmem_free(gc->gc_arena, gc, gc->gc_size);
1065 }
1066 }
1067
1068 /*
1069 * Legacy entry points from here to end of file.
1070 */
1071 void
1072 segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot,
1073 pfn_t pfn, uint_t flags)
1074 {
1075 hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1076 hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot,
1077 flags | HAT_LOAD_LOCK);
1078 }
1079
1080 void
1081 segkmem_mapout(struct seg *seg, void *addr, size_t size)
1082 {
1083 hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1084 }
1085
1086 void *
1087 kmem_getpages(pgcnt_t npages, int kmflag)
1088 {
1089 return (kmem_alloc(ptob(npages), kmflag));
1090 }
1091
1092 void
1093 kmem_freepages(void *addr, pgcnt_t npages)
1094 {
1095 kmem_free(addr, ptob(npages));
1096 }
1097
1098 /*
1099 * segkmem_page_create_large() allocates a large page to be used for the kmem
1100 * caches. If kpr is enabled we ask for a relocatable page unless requested
1101 * otherwise. If kpr is disabled we have to ask for a non-reloc page
1102 */
1103 static page_t *
1104 segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg)
1105 {
1106 int pgflags;
1107
1108 pgflags = PG_EXCL;
1109
1110 if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
1111 pgflags |= PG_NORELOC;
1112 if (!(vmflag & VM_NOSLEEP))
1113 pgflags |= PG_WAIT;
1114 if (vmflag & VM_PUSHPAGE)
1115 pgflags |= PG_PUSHPAGE;
1116 if (vmflag & VM_NORMALPRI)
1117 pgflags |= PG_NORMALPRI;
1118
1119 return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
1120 pgflags, &kvseg, addr, arg));
1121 }
1122
1123 /*
1124 * Allocate a large page to back the virtual address range
1125 * [addr, addr + size). If addr is NULL, allocate the virtual address
1126 * space as well.
1127 */
1128 static void *
1129 segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
1130 uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
1131 void *pcarg)
1132 {
1133 caddr_t addr = inaddr, pa;
1134 size_t lpsize = segkmem_lpsize;
1135 pgcnt_t npages = btopr(size);
1136 pgcnt_t nbpages = btop(lpsize);
1137 pgcnt_t nlpages = size >> segkmem_lpshift;
1138 size_t ppasize = nbpages * sizeof (page_t *);
1139 page_t *pp, *rootpp, **ppa, *pplist = NULL;
1140 int i;
1141
1142 vmflag |= VM_NOSLEEP;
1143
1144 if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
1145 return (NULL);
1146 }
1147
1148 /*
1149 * allocate an array we need for hat_memload_array.
1150 * we use a separate arena to avoid recursion.
1151 * we will not need this array when hat_memload_array learns pp++
1152 */
1153 if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) {
1154 goto fail_array_alloc;
1155 }
1156
1157 if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
1158 goto fail_vmem_alloc;
1159
1160 ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0);
1161
1162 /* create all the pages */
1163 for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) {
1164 if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL)
1165 goto fail_page_create;
1166 page_list_concat(&pplist, &pp);
1167 }
1168
1169 /* at this point we have all the resource to complete the request */
1170 while ((rootpp = pplist) != NULL) {
1171 for (i = 0; i < nbpages; i++) {
1172 ASSERT(pplist != NULL);
1173 pp = pplist;
1174 page_sub(&pplist, pp);
1175 ASSERT(page_iolock_assert(pp));
1176 page_io_unlock(pp);
1177 ppa[i] = pp;
1178 }
1179 /*
1180 * Load the locked entry. It's OK to preload the entry into the
1181 * TSB since we now support large mappings in the kernel TSB.
1182 */
1183 hat_memload_array(kas.a_hat,
1184 (caddr_t)(uintptr_t)rootpp->p_offset, lpsize,
1185 ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
1186 HAT_LOAD_LOCK);
1187
1188 for (--i; i >= 0; --i) {
1189 ppa[i]->p_lckcnt = 1;
1190 page_unlock(ppa[i]);
1191 }
1192 }
1193
1194 vmem_free(segkmem_ppa_arena, ppa, ppasize);
1195 return (addr);
1196
1197 fail_page_create:
1198 while ((rootpp = pplist) != NULL) {
1199 for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) {
1200 ASSERT(pp != NULL);
1201 page_sub(&pplist, pp);
1202 ASSERT(page_iolock_assert(pp));
1203 page_io_unlock(pp);
1204 }
1205 page_destroy_pages(rootpp);
1206 }
1207
1208 if (inaddr == NULL)
1209 vmem_free(vmp, addr, size);
1210
1211 fail_vmem_alloc:
1212 vmem_free(segkmem_ppa_arena, ppa, ppasize);
1213
1214 fail_array_alloc:
1215 page_unresv(npages);
1216
1217 return (NULL);
1218 }
1219
1220 static void
1221 segkmem_free_one_lp(caddr_t addr, size_t size)
1222 {
1223 page_t *pp, *rootpp = NULL;
1224 pgcnt_t pgs_left = btopr(size);
1225
1226 ASSERT(size == segkmem_lpsize);
1227
1228 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1229
1230 for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) {
1231 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1232 if (pp == NULL)
1233 panic("segkmem_free_one_lp: page not found");
1234 ASSERT(PAGE_EXCL(pp));
1235 pp->p_lckcnt = 0;
1236 if (rootpp == NULL)
1237 rootpp = pp;
1238 }
1239 ASSERT(rootpp != NULL);
1240 page_destroy_pages(rootpp);
1241
1242 /* page_unresv() is done by the caller */
1243 }
1244
1245 /*
1246 * This function is called to import new spans into the vmem arenas like
1247 * kmem_default_arena and kmem_oversize_arena. It first tries to import
1248 * spans from large page arena - kmem_lp_arena. In order to do this it might
1249 * have to "upgrade the requested size" to kmem_lp_arena quantum. If
1250 * it was not able to satisfy the upgraded request it then calls regular
1251 * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
1252 */
1253 /*ARGSUSED*/
1254 void *
1255 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
1256 {
1257 size_t size;
1258 kthread_t *t = curthread;
1259 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1260
1261 ASSERT(sizep != NULL);
1262
1263 size = *sizep;
1264
1265 if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
1266 !(vmflag & SEGKMEM_SHARELOCKED)) {
1267
1268 size_t kmemlp_qnt = segkmem_kmemlp_quantum;
1269 size_t asize = P2ROUNDUP(size, kmemlp_qnt);
1270 void *addr = NULL;
1271 ulong_t *lpthrtp = &lpcb->lp_throttle;
1272 ulong_t lpthrt = *lpthrtp;
1273 int dowakeup = 0;
1274 int doalloc = 1;
1275
1276 ASSERT(kmem_lp_arena != NULL);
1277 ASSERT(asize >= size);
1278
1279 if (lpthrt != 0) {
1280 /* try to update the throttle value */
1281 lpthrt = atomic_inc_ulong_nv(lpthrtp);
1282 if (lpthrt >= segkmem_lpthrottle_max) {
1283 lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
1284 segkmem_lpthrottle_max / 4);
1285 }
1286
1287 /*
1288 * when we get above throttle start do an exponential
1289 * backoff at trying large pages and reaping
1290 */
1291 if (lpthrt > segkmem_lpthrottle_start &&
1292 !ISP2(lpthrt)) {
1293 lpcb->allocs_throttled++;
1294 lpthrt--;
1295 if (ISP2(lpthrt))
1296 kmem_reap();
1297 return (segkmem_alloc(vmp, size, vmflag));
1298 }
1299 }
1300
1301 if (!(vmflag & VM_NOSLEEP) &&
1302 segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
1303 vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
1304 asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
1305
1306 /*
1307 * we are low on free memory in kmem_lp_arena
1308 * we let only one guy to allocate heap_lp
1309 * quantum size chunk that everybody is going to
1310 * share
1311 */
1312 mutex_enter(&lpcb->lp_lock);
1313
1314 if (lpcb->lp_wait) {
1315
1316 /* we are not the first one - wait */
1317 cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
1318 if (vmem_size(kmem_lp_arena, VMEM_FREE) <
1319 kmemlp_qnt) {
1320 doalloc = 0;
1321 }
1322 } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
1323 kmemlp_qnt) {
1324
1325 /*
1326 * we are the first one, make sure we import
1327 * a large page
1328 */
1329 if (asize == kmemlp_qnt)
1330 asize += kmemlp_qnt;
1331 dowakeup = 1;
1332 lpcb->lp_wait = 1;
1333 }
1334
1335 mutex_exit(&lpcb->lp_lock);
1336 }
1337
1338 /*
1339 * VM_ABORT flag prevents sleeps in vmem_xalloc when
1340 * large pages are not available. In that case this allocation
1341 * attempt will fail and we will retry allocation with small
1342 * pages. We also do not want to panic if this allocation fails
1343 * because we are going to retry.
1344 */
1345 if (doalloc) {
1346 addr = vmem_alloc(kmem_lp_arena, asize,
1347 (vmflag | VM_ABORT) & ~VM_PANIC);
1348
1349 if (dowakeup) {
1350 mutex_enter(&lpcb->lp_lock);
1351 ASSERT(lpcb->lp_wait != 0);
1352 lpcb->lp_wait = 0;
1353 cv_broadcast(&lpcb->lp_cv);
1354 mutex_exit(&lpcb->lp_lock);
1355 }
1356 }
1357
1358 if (addr != NULL) {
1359 *sizep = asize;
1360 *lpthrtp = 0;
1361 return (addr);
1362 }
1363
1364 if (vmflag & VM_NOSLEEP)
1365 lpcb->nosleep_allocs_failed++;
1366 else
1367 lpcb->sleep_allocs_failed++;
1368 lpcb->alloc_bytes_failed += size;
1369
1370 /* if large page throttling is not started yet do it */
1371 if (segkmem_use_lpthrottle && lpthrt == 0) {
1372 lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
1373 }
1374 }
1375 return (segkmem_alloc(vmp, size, vmflag));
1376 }
1377
1378 void
1379 segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size)
1380 {
1381 if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) {
1382 segkmem_free(vmp, inaddr, size);
1383 } else {
1384 vmem_free(kmem_lp_arena, inaddr, size);
1385 }
1386 }
1387
1388 /*
1389 * segkmem_alloc_lpi() imports virtual memory from large page heap arena
1390 * into kmem_lp arena. In the process it maps the imported segment with
1391 * large pages
1392 */
1393 static void *
1394 segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag)
1395 {
1396 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1397 void *addr;
1398
1399 ASSERT(size != 0);
1400 ASSERT(vmp == heap_lp_arena);
1401
1402 /* do not allow large page heap grow beyound limits */
1403 if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) {
1404 lpcb->allocs_limited++;
1405 return (NULL);
1406 }
1407
1408 addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0,
1409 segkmem_page_create_large, NULL);
1410 return (addr);
1411 }
1412
1413 /*
1414 * segkmem_free_lpi() returns virtual memory back into large page heap arena
1415 * from kmem_lp arena. Beore doing this it unmaps the segment and frees
1416 * large pages used to map it.
1417 */
1418 static void
1419 segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size)
1420 {
1421 pgcnt_t nlpages = size >> segkmem_lpshift;
1422 size_t lpsize = segkmem_lpsize;
1423 caddr_t addr = inaddr;
1424 pgcnt_t npages = btopr(size);
1425 int i;
1426
1427 ASSERT(vmp == heap_lp_arena);
1428 ASSERT(IS_KMEM_VA_LARGEPAGE(addr));
1429 ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0);
1430
1431 for (i = 0; i < nlpages; i++) {
1432 segkmem_free_one_lp(addr, lpsize);
1433 addr += lpsize;
1434 }
1435
1436 page_unresv(npages);
1437
1438 vmem_free(vmp, inaddr, size);
1439 }
1440
1441 /*
1442 * This function is called at system boot time by kmem_init right after
1443 * /etc/system file has been read. It checks based on hardware configuration
1444 * and /etc/system settings if system is going to use large pages. The
1445 * initialiazation necessary to actually start using large pages
1446 * happens later in the process after segkmem_heap_lp_init() is called.
1447 */
1448 int
1449 segkmem_lpsetup()
1450 {
1451 int use_large_pages = 0;
1452
1453 #ifdef __sparc
1454
1455 size_t memtotal = physmem * PAGESIZE;
1456
1457 if (heap_lp_base == NULL) {
1458 segkmem_lpsize = PAGESIZE;
1459 return (0);
1460 }
1461
1462 /* get a platform dependent value of large page size for kernel heap */
1463 segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize);
1464
1465 if (segkmem_lpsize <= PAGESIZE) {
1466 /*
1467 * put virtual space reserved for the large page kernel
1468 * back to the regular heap
1469 */
1470 vmem_xfree(heap_arena, heap_lp_base,
1471 heap_lp_end - heap_lp_base);
1472 heap_lp_base = NULL;
1473 heap_lp_end = NULL;
1474 segkmem_lpsize = PAGESIZE;
1475 return (0);
1476 }
1477
1478 /* set heap_lp quantum if necessary */
1479 if (segkmem_heaplp_quantum == 0 || !ISP2(segkmem_heaplp_quantum) ||
1480 P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) {
1481 segkmem_heaplp_quantum = segkmem_lpsize;
1482 }
1483
1484 /* set kmem_lp quantum if necessary */
1485 if (segkmem_kmemlp_quantum == 0 || !ISP2(segkmem_kmemlp_quantum) ||
1486 segkmem_kmemlp_quantum > segkmem_heaplp_quantum) {
1487 segkmem_kmemlp_quantum = segkmem_heaplp_quantum;
1488 }
1489
1490 /* set total amount of memory allowed for large page kernel heap */
1491 if (segkmem_kmemlp_max == 0) {
1492 if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100)
1493 segkmem_kmemlp_pcnt = 12;
1494 segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100;
1495 }
1496 segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max,
1497 segkmem_heaplp_quantum);
1498
1499 /* fix lp kmem preallocation request if necesssary */
1500 if (segkmem_kmemlp_min) {
1501 segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min,
1502 segkmem_heaplp_quantum);
1503 if (segkmem_kmemlp_min > segkmem_kmemlp_max)
1504 segkmem_kmemlp_min = segkmem_kmemlp_max;
1505 }
1506
1507 use_large_pages = 1;
1508 segkmem_lpszc = page_szc(segkmem_lpsize);
1509 segkmem_lpshift = page_get_shift(segkmem_lpszc);
1510
1511 #endif
1512 return (use_large_pages);
1513 }
1514
1515 void
1516 segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size)
1517 {
1518 ASSERT(zio_mem_base != NULL);
1519 ASSERT(zio_mem_size != 0);
1520
1521 /*
1522 * To reduce VA space fragmentation, we set up quantum caches for the
1523 * smaller sizes; we chose 32k because that translates to 128k VA
1524 * slabs, which matches nicely with the common 128k zio_data bufs.
1525 */
1526 zio_arena = vmem_create("zfs_file_data", zio_mem_base, zio_mem_size,
1527 PAGESIZE, NULL, NULL, NULL, 32 * 1024, VM_SLEEP);
1528
1529 zio_alloc_arena = vmem_create("zfs_file_data_buf", NULL, 0, PAGESIZE,
1530 segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP);
1531
1532 ASSERT(zio_arena != NULL);
1533 ASSERT(zio_alloc_arena != NULL);
1534 }
1535
1536 #ifdef __sparc
1537
1538
1539 static void *
1540 segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag)
1541 {
1542 size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1543 void *addr;
1544
1545 if (ppaquantum <= PAGESIZE)
1546 return (segkmem_alloc(vmp, size, vmflag));
1547
1548 ASSERT((size & (ppaquantum - 1)) == 0);
1549
1550 addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag);
1551 if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0,
1552 segkmem_page_create, NULL) == NULL) {
1553 vmem_xfree(vmp, addr, size);
1554 addr = NULL;
1555 }
1556
1557 return (addr);
1558 }
1559
1560 static void
1561 segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size)
1562 {
1563 size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1564
1565 ASSERT(addr != NULL);
1566
1567 if (ppaquantum <= PAGESIZE) {
1568 segkmem_free(vmp, addr, size);
1569 } else {
1570 segkmem_free(NULL, addr, size);
1571 vmem_xfree(vmp, addr, size);
1572 }
1573 }
1574
1575 void
1576 segkmem_heap_lp_init()
1577 {
1578 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1579 size_t heap_lp_size = heap_lp_end - heap_lp_base;
1580 size_t lpsize = segkmem_lpsize;
1581 size_t ppaquantum;
1582 void *addr;
1583
1584 if (segkmem_lpsize <= PAGESIZE) {
1585 ASSERT(heap_lp_base == NULL);
1586 ASSERT(heap_lp_end == NULL);
1587 return;
1588 }
1589
1590 ASSERT(segkmem_heaplp_quantum >= lpsize);
1591 ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0);
1592 ASSERT(lpcb->lp_uselp == 0);
1593 ASSERT(heap_lp_base != NULL);
1594 ASSERT(heap_lp_end != NULL);
1595 ASSERT(heap_lp_base < heap_lp_end);
1596 ASSERT(heap_lp_arena == NULL);
1597 ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0);
1598 ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0);
1599
1600 /* create large page heap arena */
1601 heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size,
1602 segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1603
1604 ASSERT(heap_lp_arena != NULL);
1605
1606 /* This arena caches memory already mapped by large pages */
1607 kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum,
1608 segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP);
1609
1610 ASSERT(kmem_lp_arena != NULL);
1611
1612 mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL);
1613 cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL);
1614
1615 /*
1616 * this arena is used for the array of page_t pointers necessary
1617 * to call hat_mem_load_array
1618 */
1619 ppaquantum = btopr(lpsize) * sizeof (page_t *);
1620 segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum,
1621 segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum,
1622 VM_SLEEP);
1623
1624 ASSERT(segkmem_ppa_arena != NULL);
1625
1626 /* prealloacate some memory for the lp kernel heap */
1627 if (segkmem_kmemlp_min) {
1628
1629 ASSERT(P2PHASE(segkmem_kmemlp_min,
1630 segkmem_heaplp_quantum) == 0);
1631
1632 if ((addr = segkmem_alloc_lpi(heap_lp_arena,
1633 segkmem_kmemlp_min, VM_SLEEP)) != NULL) {
1634
1635 addr = vmem_add(kmem_lp_arena, addr,
1636 segkmem_kmemlp_min, VM_SLEEP);
1637 ASSERT(addr != NULL);
1638 }
1639 }
1640
1641 lpcb->lp_uselp = 1;
1642 }
1643
1644 #endif