1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright (c) 2010, Intel Corporation.
26 * All rights reserved.
27 * Copyright 2019, Joyent, Inc.
28 */
29
30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
31 /* All Rights Reserved */
32
33 /*
34 * Portions of this source code were derived from Berkeley 4.3 BSD
35 * under license from the Regents of the University of California.
36 */
37
38 /*
39 * UNIX machine dependent virtual memory support.
40 */
41
42 #include <sys/types.h>
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/user.h>
46 #include <sys/proc.h>
47 #include <sys/kmem.h>
48 #include <sys/vmem.h>
49 #include <sys/buf.h>
50 #include <sys/cpuvar.h>
51 #include <sys/lgrp.h>
52 #include <sys/disp.h>
53 #include <sys/vm.h>
54 #include <sys/mman.h>
55 #include <sys/vnode.h>
56 #include <sys/cred.h>
57 #include <sys/exec.h>
58 #include <sys/exechdr.h>
59 #include <sys/debug.h>
60 #include <sys/vmsystm.h>
61 #include <sys/swap.h>
62 #include <sys/dumphdr.h>
63 #include <sys/random.h>
64
65 #include <vm/hat.h>
66 #include <vm/as.h>
67 #include <vm/seg.h>
68 #include <vm/seg_kp.h>
69 #include <vm/seg_vn.h>
70 #include <vm/page.h>
71 #include <vm/seg_kmem.h>
72 #include <vm/seg_kpm.h>
73 #include <vm/vm_dep.h>
74
75 #include <sys/cpu.h>
76 #include <sys/vm_machparam.h>
77 #include <sys/memlist.h>
78 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */
79 #include <vm/hat_i86.h>
80 #include <sys/x86_archext.h>
81 #include <sys/elf_386.h>
82 #include <sys/cmn_err.h>
83 #include <sys/archsystm.h>
84 #include <sys/machsystm.h>
85 #include <sys/secflags.h>
86
87 #include <sys/vtrace.h>
88 #include <sys/ddidmareq.h>
89 #include <sys/promif.h>
90 #include <sys/memnode.h>
91 #include <sys/stack.h>
92 #include <util/qsort.h>
93 #include <sys/taskq.h>
94
95 #ifdef __xpv
96
97 #include <sys/hypervisor.h>
98 #include <sys/xen_mmu.h>
99 #include <sys/balloon_impl.h>
100
101 /*
102 * domain 0 pages usable for DMA are kept pre-allocated and kept in
103 * distinct lists, ordered by increasing mfn.
104 */
105 static kmutex_t io_pool_lock;
106 static kmutex_t contig_list_lock;
107 static page_t *io_pool_4g; /* pool for 32 bit dma limited devices */
108 static page_t *io_pool_16m; /* pool for 24 bit dma limited legacy devices */
109 static long io_pool_cnt;
110 static long io_pool_cnt_max = 0;
111 #define DEFAULT_IO_POOL_MIN 128
112 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN;
113 static long io_pool_cnt_lowater = 0;
114 static long io_pool_shrink_attempts; /* how many times did we try to shrink */
115 static long io_pool_shrinks; /* how many times did we really shrink */
116 static long io_pool_grows; /* how many times did we grow */
117 static mfn_t start_mfn = 1;
118 static caddr_t io_pool_kva; /* use to alloc pages when needed */
119
120 static int create_contig_pfnlist(uint_t);
121
122 /*
123 * percentage of phys mem to hold in the i/o pool
124 */
125 #define DEFAULT_IO_POOL_PCT 2
126 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT;
127 static void page_io_pool_sub(page_t **, page_t *, page_t *);
128 int ioalloc_dbg = 0;
129
130 #endif /* __xpv */
131
132 uint_t vac_colors = 1;
133
134 int largepagesupport = 0;
135 extern uint_t page_create_new;
136 extern uint_t page_create_exists;
137 extern uint_t page_create_putbacks;
138 /*
139 * Allow users to disable the kernel's use of SSE.
140 */
141 extern int use_sse_pagecopy, use_sse_pagezero;
142
143 /*
144 * combined memory ranges from mnode and memranges[] to manage single
145 * mnode/mtype dimension in the page lists.
146 */
147 typedef struct {
148 pfn_t mnr_pfnlo;
149 pfn_t mnr_pfnhi;
150 int mnr_mnode;
151 int mnr_memrange; /* index into memranges[] */
152 int mnr_next; /* next lower PA mnoderange */
153 int mnr_exists;
154 /* maintain page list stats */
155 pgcnt_t mnr_mt_clpgcnt; /* cache list cnt */
156 pgcnt_t mnr_mt_flpgcnt[MMU_PAGE_SIZES]; /* free list cnt per szc */
157 pgcnt_t mnr_mt_totcnt; /* sum of cache and free lists */
158 #ifdef DEBUG
159 struct mnr_mts { /* mnode/mtype szc stats */
160 pgcnt_t mnr_mts_pgcnt;
161 int mnr_mts_colors;
162 pgcnt_t *mnr_mtsc_pgcnt;
163 } *mnr_mts;
164 #endif
165 } mnoderange_t;
166
167 #define MEMRANGEHI(mtype) \
168 ((mtype > 0) ? memranges[mtype - 1] - 1: physmax)
169 #define MEMRANGELO(mtype) (memranges[mtype])
170
171 #define MTYPE_FREEMEM(mt) (mnoderanges[mt].mnr_mt_totcnt)
172
173 /*
174 * As the PC architecture evolved memory up was clumped into several
175 * ranges for various historical I/O devices to do DMA.
176 * < 16Meg - ISA bus
177 * < 2Gig - ???
178 * < 4Gig - PCI bus or drivers that don't understand PAE mode
179 *
180 * These are listed in reverse order, so that we can skip over unused
181 * ranges on machines with small memories.
182 *
183 * For now under the Hypervisor, we'll only ever have one memrange.
184 */
185 #define PFN_4GIG 0x100000
186 #define PFN_16MEG 0x1000
187 /* Indices into the memory range (arch_memranges) array. */
188 #define MRI_4G 0
189 #define MRI_2G 1
190 #define MRI_16M 2
191 #define MRI_0 3
192 static pfn_t arch_memranges[NUM_MEM_RANGES] = {
193 PFN_4GIG, /* pfn range for 4G and above */
194 0x80000, /* pfn range for 2G-4G */
195 PFN_16MEG, /* pfn range for 16M-2G */
196 0x00000, /* pfn range for 0-16M */
197 };
198 pfn_t *memranges = &arch_memranges[0];
199 int nranges = NUM_MEM_RANGES;
200
201 /*
202 * This combines mem_node_config and memranges into one data
203 * structure to be used for page list management.
204 */
205 static mnoderange_t *mnoderanges;
206 static int mnoderangecnt;
207 static int mtype4g;
208 static int mtype16m;
209 static int mtypetop;
210
211 /*
212 * 4g memory management variables for systems with more than 4g of memory:
213 *
214 * physical memory below 4g is required for 32bit dma devices and, currently,
215 * for kmem memory. On systems with more than 4g of memory, the pool of memory
216 * below 4g can be depleted without any paging activity given that there is
217 * likely to be sufficient memory above 4g.
218 *
219 * physmax4g is set true if the largest pfn is over 4g. The rest of the
220 * 4g memory management code is enabled only when physmax4g is true.
221 *
222 * maxmem4g is the count of the maximum number of pages on the page lists
223 * with physical addresses below 4g. It can be a lot less then 4g given that
224 * BIOS may reserve large chunks of space below 4g for hot plug pci devices,
225 * agp aperture etc.
226 *
227 * freemem4g maintains the count of the number of available pages on the
228 * page lists with physical addresses below 4g.
229 *
230 * DESFREE4G specifies the desired amount of below 4g memory. It defaults to
231 * 6% (desfree4gshift = 4) of maxmem4g.
232 *
233 * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G
234 * and the amount of physical memory above 4g is greater than freemem4g.
235 * In this case, page_get_* routines will restrict below 4g allocations
236 * for requests that don't specifically require it.
237 */
238
239 #define DESFREE4G (maxmem4g >> desfree4gshift)
240
241 #define RESTRICT4G_ALLOC \
242 (physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem))
243
244 static pgcnt_t maxmem4g;
245 static pgcnt_t freemem4g;
246 static int physmax4g;
247 static int desfree4gshift = 4; /* maxmem4g shift to derive DESFREE4G */
248
249 /*
250 * 16m memory management:
251 *
252 * reserve some amount of physical memory below 16m for legacy devices.
253 *
254 * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above
255 * 16m or if the 16m pool drops below DESFREE16M.
256 *
257 * In this case, general page allocations via page_get_{free,cache}list
258 * routines will be restricted from allocating from the 16m pool. Allocations
259 * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations
260 * are not restricted.
261 */
262
263 #define FREEMEM16M MTYPE_FREEMEM(mtype16m)
264 #define DESFREE16M desfree16m
265 #define RESTRICT16M_ALLOC(freemem, pgcnt, flags) \
266 (mtype16m != -1 && (freemem != 0) && ((flags & PG_PANIC) == 0) && \
267 ((freemem >= (FREEMEM16M)) || \
268 (FREEMEM16M < (DESFREE16M + pgcnt))))
269
270 static pgcnt_t desfree16m = 0x380;
271
272 /*
273 * This can be patched via /etc/system to allow old non-PAE aware device
274 * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM.
275 */
276 int restricted_kmemalloc = 0;
277
278 #ifdef VM_STATS
279 struct {
280 ulong_t pga_alloc;
281 ulong_t pga_notfullrange;
282 ulong_t pga_nulldmaattr;
283 ulong_t pga_allocok;
284 ulong_t pga_allocfailed;
285 ulong_t pgma_alloc;
286 ulong_t pgma_allocok;
287 ulong_t pgma_allocfailed;
288 ulong_t pgma_allocempty;
289 } pga_vmstats;
290 #endif
291
292 uint_t mmu_page_sizes;
293
294 /* How many page sizes the users can see */
295 uint_t mmu_exported_page_sizes;
296
297 /* page sizes that legacy applications can see */
298 uint_t mmu_legacy_page_sizes;
299
300 /*
301 * Number of pages in 1 GB. Don't enable automatic large pages if we have
302 * fewer than this many pages.
303 */
304 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
305 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
306
307 /*
308 * Maximum and default segment size tunables for user private
309 * and shared anon memory, and user text and initialized data.
310 * These can be patched via /etc/system to allow large pages
311 * to be used for mapping application private and shared anon memory.
312 */
313 size_t mcntl0_lpsize = MMU_PAGESIZE;
314 size_t max_uheap_lpsize = MMU_PAGESIZE;
315 size_t default_uheap_lpsize = MMU_PAGESIZE;
316 size_t max_ustack_lpsize = MMU_PAGESIZE;
317 size_t default_ustack_lpsize = MMU_PAGESIZE;
318 size_t max_privmap_lpsize = MMU_PAGESIZE;
319 size_t max_uidata_lpsize = MMU_PAGESIZE;
320 size_t max_utext_lpsize = MMU_PAGESIZE;
321 size_t max_shm_lpsize = MMU_PAGESIZE;
322
323
324 /*
325 * initialized by page_coloring_init().
326 */
327 uint_t page_colors;
328 uint_t page_colors_mask;
329 uint_t page_coloring_shift;
330 int cpu_page_colors;
331 static uint_t l2_colors;
332
333 /*
334 * Page freelists and cachelists are dynamically allocated once mnoderangecnt
335 * and page_colors are calculated from the l2 cache n-way set size. Within a
336 * mnode range, the page freelist and cachelist are hashed into bins based on
337 * color. This makes it easier to search for a page within a specific memory
338 * range.
339 */
340 #define PAGE_COLORS_MIN 16
341
342 page_t ****page_freelists;
343 page_t ***page_cachelists;
344
345
346 /*
347 * Used by page layer to know about page sizes
348 */
349 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1];
350
351 kmutex_t *fpc_mutex[NPC_MUTEX];
352 kmutex_t *cpc_mutex[NPC_MUTEX];
353
354 /* Lock to protect mnoderanges array for memory DR operations. */
355 static kmutex_t mnoderange_lock;
356
357 /*
358 * Only let one thread at a time try to coalesce large pages, to
359 * prevent them from working against each other.
360 */
361 static kmutex_t contig_lock;
362 #define CONTIG_LOCK() mutex_enter(&contig_lock);
363 #define CONTIG_UNLOCK() mutex_exit(&contig_lock);
364
365 #define PFN_16M (mmu_btop((uint64_t)0x1000000))
366
367 caddr_t
368 i86devmap(pfn_t pf, pgcnt_t pgcnt, uint_t prot)
369 {
370 caddr_t addr;
371 caddr_t addr1;
372 page_t *pp;
373
374 addr1 = addr = vmem_alloc(heap_arena, mmu_ptob(pgcnt), VM_SLEEP);
375
376 for (; pgcnt != 0; addr += MMU_PAGESIZE, ++pf, --pgcnt) {
377 pp = page_numtopp_nolock(pf);
378 if (pp == NULL) {
379 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pf,
380 prot | HAT_NOSYNC, HAT_LOAD_LOCK);
381 } else {
382 hat_memload(kas.a_hat, addr, pp,
383 prot | HAT_NOSYNC, HAT_LOAD_LOCK);
384 }
385 }
386
387 return (addr1);
388 }
389
390 /*
391 * This routine is like page_numtopp, but accepts only free pages, which
392 * it allocates (unfrees) and returns with the exclusive lock held.
393 * It is used by machdep.c/dma_init() to find contiguous free pages.
394 */
395 page_t *
396 page_numtopp_alloc(pfn_t pfnum)
397 {
398 page_t *pp;
399
400 retry:
401 pp = page_numtopp_nolock(pfnum);
402 if (pp == NULL) {
403 return (NULL);
404 }
405
406 if (!page_trylock(pp, SE_EXCL)) {
407 return (NULL);
408 }
409
410 if (page_pptonum(pp) != pfnum) {
411 page_unlock(pp);
412 goto retry;
413 }
414
415 if (!PP_ISFREE(pp)) {
416 page_unlock(pp);
417 return (NULL);
418 }
419 if (pp->p_szc) {
420 page_demote_free_pages(pp);
421 page_unlock(pp);
422 goto retry;
423 }
424
425 /* If associated with a vnode, destroy mappings */
426
427 if (pp->p_vnode) {
428
429 page_destroy_free(pp);
430
431 if (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_NO_RECLAIM)) {
432 return (NULL);
433 }
434
435 if (page_pptonum(pp) != pfnum) {
436 page_unlock(pp);
437 goto retry;
438 }
439 }
440
441 if (!PP_ISFREE(pp)) {
442 page_unlock(pp);
443 return (NULL);
444 }
445
446 if (!page_reclaim(pp, (kmutex_t *)NULL))
447 return (NULL);
448
449 return (pp);
450 }
451
452 /*
453 * Return the optimum page size for a given mapping
454 */
455 /*ARGSUSED*/
456 size_t
457 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl)
458 {
459 level_t l = 0;
460 size_t pgsz = MMU_PAGESIZE;
461 size_t max_lpsize;
462 uint_t mszc;
463
464 ASSERT(maptype != MAPPGSZ_VA);
465
466 if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) {
467 return (MMU_PAGESIZE);
468 }
469
470 switch (maptype) {
471 case MAPPGSZ_HEAP:
472 case MAPPGSZ_STK:
473 max_lpsize = memcntl ? mcntl0_lpsize : (maptype ==
474 MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize);
475 if (max_lpsize == MMU_PAGESIZE) {
476 return (MMU_PAGESIZE);
477 }
478 if (len == 0) {
479 len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase +
480 p->p_brksize - p->p_bssbase : p->p_stksize;
481 }
482 len = (maptype == MAPPGSZ_HEAP) ? MAX(len,
483 default_uheap_lpsize) : MAX(len, default_ustack_lpsize);
484
485 /*
486 * use the pages size that best fits len
487 */
488 for (l = mmu.umax_page_level; l > 0; --l) {
489 if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) {
490 continue;
491 } else {
492 pgsz = LEVEL_SIZE(l);
493 }
494 break;
495 }
496
497 mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc :
498 p->p_stkpageszc);
499 if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) {
500 pgsz = hw_page_array[mszc].hp_size;
501 }
502 return (pgsz);
503
504 case MAPPGSZ_ISM:
505 for (l = mmu.umax_page_level; l > 0; --l) {
506 if (len >= LEVEL_SIZE(l))
507 return (LEVEL_SIZE(l));
508 }
509 return (LEVEL_SIZE(0));
510 }
511 return (pgsz);
512 }
513
514 static uint_t
515 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize,
516 size_t min_physmem)
517 {
518 caddr_t eaddr = addr + size;
519 uint_t szcvec = 0;
520 caddr_t raddr;
521 caddr_t readdr;
522 size_t pgsz;
523 int i;
524
525 if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) {
526 return (0);
527 }
528
529 for (i = mmu_exported_page_sizes - 1; i > 0; i--) {
530 pgsz = page_get_pagesize(i);
531 if (pgsz > max_lpsize) {
532 continue;
533 }
534 raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
535 readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
536 if (raddr < addr || raddr >= readdr) {
537 continue;
538 }
539 if (P2PHASE((uintptr_t)addr ^ off, pgsz)) {
540 continue;
541 }
542 /*
543 * Set szcvec to the remaining page sizes.
544 */
545 szcvec = ((1 << (i + 1)) - 1) & ~1;
546 break;
547 }
548 return (szcvec);
549 }
550
551 /*
552 * Return a bit vector of large page size codes that
553 * can be used to map [addr, addr + len) region.
554 */
555 /*ARGSUSED*/
556 uint_t
557 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type,
558 int memcntl)
559 {
560 size_t max_lpsize = mcntl0_lpsize;
561
562 if (mmu.max_page_level == 0)
563 return (0);
564
565 if (flags & MAP_TEXT) {
566 if (!memcntl)
567 max_lpsize = max_utext_lpsize;
568 return (map_szcvec(addr, size, off, max_lpsize,
569 shm_lpg_min_physmem));
570
571 } else if (flags & MAP_INITDATA) {
572 if (!memcntl)
573 max_lpsize = max_uidata_lpsize;
574 return (map_szcvec(addr, size, off, max_lpsize,
575 privm_lpg_min_physmem));
576
577 } else if (type == MAPPGSZC_SHM) {
578 if (!memcntl)
579 max_lpsize = max_shm_lpsize;
580 return (map_szcvec(addr, size, off, max_lpsize,
581 shm_lpg_min_physmem));
582
583 } else if (type == MAPPGSZC_HEAP) {
584 if (!memcntl)
585 max_lpsize = max_uheap_lpsize;
586 return (map_szcvec(addr, size, off, max_lpsize,
587 privm_lpg_min_physmem));
588
589 } else if (type == MAPPGSZC_STACK) {
590 if (!memcntl)
591 max_lpsize = max_ustack_lpsize;
592 return (map_szcvec(addr, size, off, max_lpsize,
593 privm_lpg_min_physmem));
594
595 } else {
596 if (!memcntl)
597 max_lpsize = max_privmap_lpsize;
598 return (map_szcvec(addr, size, off, max_lpsize,
599 privm_lpg_min_physmem));
600 }
601 }
602
603 /*
604 * Handle a pagefault.
605 */
606 faultcode_t
607 pagefault(
608 caddr_t addr,
609 enum fault_type type,
610 enum seg_rw rw,
611 int iskernel)
612 {
613 struct as *as;
614 struct hat *hat;
615 struct proc *p;
616 kthread_t *t;
617 faultcode_t res;
618 caddr_t base;
619 size_t len;
620 int err;
621 int mapped_red;
622 uintptr_t ea;
623
624 ASSERT_STACK_ALIGNED();
625
626 if (INVALID_VADDR(addr))
627 return (FC_NOMAP);
628
629 mapped_red = segkp_map_red();
630
631 if (iskernel) {
632 as = &kas;
633 hat = as->a_hat;
634 } else {
635 t = curthread;
636 p = ttoproc(t);
637 as = p->p_as;
638 hat = as->a_hat;
639 }
640
641 /*
642 * Dispatch pagefault.
643 */
644 res = as_fault(hat, as, addr, 1, type, rw);
645
646 /*
647 * If this isn't a potential unmapped hole in the user's
648 * UNIX data or stack segments, just return status info.
649 */
650 if (res != FC_NOMAP || iskernel)
651 goto out;
652
653 /*
654 * Check to see if we happened to faulted on a currently unmapped
655 * part of the UNIX data or stack segments. If so, create a zfod
656 * mapping there and then try calling the fault routine again.
657 */
658 base = p->p_brkbase;
659 len = p->p_brksize;
660
661 if (addr < base || addr >= base + len) { /* data seg? */
662 base = (caddr_t)p->p_usrstack - p->p_stksize;
663 len = p->p_stksize;
664 if (addr < base || addr >= p->p_usrstack) { /* stack seg? */
665 /* not in either UNIX data or stack segments */
666 res = FC_NOMAP;
667 goto out;
668 }
669 }
670
671 /*
672 * the rest of this function implements a 3.X 4.X 5.X compatibility
673 * This code is probably not needed anymore
674 */
675 if (p->p_model == DATAMODEL_ILP32) {
676
677 /* expand the gap to the page boundaries on each side */
678 ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE);
679 base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE);
680 len = ea - (uintptr_t)base;
681
682 as_rangelock(as);
683 if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) ==
684 0) {
685 err = as_map(as, base, len, segvn_create, zfod_argsp);
686 as_rangeunlock(as);
687 if (err) {
688 res = FC_MAKE_ERR(err);
689 goto out;
690 }
691 } else {
692 /*
693 * This page is already mapped by another thread after
694 * we returned from as_fault() above. We just fall
695 * through as_fault() below.
696 */
697 as_rangeunlock(as);
698 }
699
700 res = as_fault(hat, as, addr, 1, F_INVAL, rw);
701 }
702
703 out:
704 if (mapped_red)
705 segkp_unmap_red();
706
707 return (res);
708 }
709
710 void
711 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags)
712 {
713 struct proc *p = curproc;
714 caddr_t userlimit = (flags & _MAP_LOW32) ?
715 (caddr_t)_userlimit32 : p->p_as->a_userlimit;
716
717 map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags);
718 }
719
720 /*ARGSUSED*/
721 int
722 map_addr_vacalign_check(caddr_t addr, u_offset_t off)
723 {
724 return (0);
725 }
726
727 /*
728 * The maximum amount a randomized mapping will be slewed. We should perhaps
729 * arrange things so these tunables can be separate for mmap, mmapobj, and
730 * ld.so
731 */
732 size_t aslr_max_map_skew = 256 * 1024 * 1024; /* 256MB */
733
734 /*
735 * map_addr_proc() is the routine called when the system is to
736 * choose an address for the user. We will pick an address
737 * range which is the highest available below userlimit.
738 *
739 * Every mapping will have a redzone of a single page on either side of
740 * the request. This is done to leave one page unmapped between segments.
741 * This is not required, but it's useful for the user because if their
742 * program strays across a segment boundary, it will catch a fault
743 * immediately making debugging a little easier. Currently the redzone
744 * is mandatory.
745 *
746 * addrp is a value/result parameter.
747 * On input it is a hint from the user to be used in a completely
748 * machine dependent fashion. We decide to completely ignore this hint.
749 * If MAP_ALIGN was specified, addrp contains the minimal alignment, which
750 * must be some "power of two" multiple of pagesize.
751 *
752 * On output it is NULL if no address can be found in the current
753 * processes address space or else an address that is currently
754 * not mapped for len bytes with a page of red zone on either side.
755 *
756 * vacalign is not needed on x86 (it's for viturally addressed caches)
757 */
758 /*ARGSUSED*/
759 void
760 map_addr_proc(
761 caddr_t *addrp,
762 size_t len,
763 offset_t off,
764 int vacalign,
765 caddr_t userlimit,
766 struct proc *p,
767 uint_t flags)
768 {
769 struct as *as = p->p_as;
770 caddr_t addr;
771 caddr_t base;
772 size_t slen;
773 size_t align_amount;
774
775 ASSERT32(userlimit == as->a_userlimit);
776
777 base = p->p_brkbase;
778 #if defined(__amd64)
779 if (p->p_model == DATAMODEL_NATIVE) {
780 if (userlimit < as->a_userlimit) {
781 /*
782 * This happens when a program wants to map
783 * something in a range that's accessible to a
784 * program in a smaller address space. For example,
785 * a 64-bit program calling mmap32(2) to guarantee
786 * that the returned address is below 4Gbytes.
787 */
788 ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff));
789
790 if (userlimit > base)
791 slen = userlimit - base;
792 else {
793 *addrp = NULL;
794 return;
795 }
796 } else {
797 /*
798 * With the stack positioned at a higher address than
799 * the heap for 64-bit processes, it is necessary to be
800 * mindful of its location and potential size.
801 *
802 * Unallocated space above the top of the stack (that
803 * is, at a lower address) but still within the bounds
804 * of the stack limit should be considered unavailable.
805 *
806 * As the 64-bit stack guard is mapped in immediately
807 * adjacent to the stack limit boundary, this prevents
808 * new mappings from having accidentally dangerous
809 * proximity to the stack.
810 */
811 slen = p->p_usrstack - base -
812 ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK);
813 }
814 } else
815 #endif /* defined(__amd64) */
816 slen = userlimit - base;
817
818 /* Make len be a multiple of PAGESIZE */
819 len = (len + PAGEOFFSET) & PAGEMASK;
820
821 /*
822 * figure out what the alignment should be
823 *
824 * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same????
825 */
826 if (len <= ELF_386_MAXPGSZ) {
827 /*
828 * Align virtual addresses to ensure that ELF shared libraries
829 * are mapped with the appropriate alignment constraints by
830 * the run-time linker.
831 */
832 align_amount = ELF_386_MAXPGSZ;
833 } else {
834 /*
835 * For 32-bit processes, only those which have specified
836 * MAP_ALIGN and an addr will be aligned on a larger page size.
837 * Not doing so can potentially waste up to 1G of process
838 * address space.
839 */
840 int lvl = (p->p_model == DATAMODEL_ILP32) ? 1 :
841 mmu.umax_page_level;
842
843 while (lvl && len < LEVEL_SIZE(lvl))
844 --lvl;
845
846 align_amount = LEVEL_SIZE(lvl);
847 }
848 if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount))
849 align_amount = (uintptr_t)*addrp;
850
851 ASSERT(ISP2(align_amount));
852 ASSERT(align_amount == 0 || align_amount >= PAGESIZE);
853
854 off = off & (align_amount - 1);
855
856 /*
857 * Look for a large enough hole starting below userlimit.
858 * After finding it, use the upper part.
859 */
860 if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount,
861 PAGESIZE, off) == 0) {
862 caddr_t as_addr;
863
864 /*
865 * addr is the highest possible address to use since we have
866 * a PAGESIZE redzone at the beginning and end.
867 */
868 addr = base + slen - (PAGESIZE + len);
869 as_addr = addr;
870 /*
871 * Round address DOWN to the alignment amount and
872 * add the offset in.
873 * If addr is greater than as_addr, len would not be large
874 * enough to include the redzone, so we must adjust down
875 * by the alignment amount.
876 */
877 addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1)));
878 addr += (uintptr_t)off;
879 if (addr > as_addr) {
880 addr -= align_amount;
881 }
882
883 /*
884 * If randomization is requested, slew the allocation
885 * backwards, within the same gap, by a random amount.
886 */
887 if (flags & _MAP_RANDOMIZE) {
888 uint32_t slew;
889
890 (void) random_get_pseudo_bytes((uint8_t *)&slew,
891 sizeof (slew));
892
893 slew = slew % MIN(aslr_max_map_skew, (addr - base));
894 addr -= P2ALIGN(slew, align_amount);
895 }
896
897 ASSERT(addr > base);
898 ASSERT(addr + len < base + slen);
899 ASSERT(((uintptr_t)addr & (align_amount - 1)) ==
900 ((uintptr_t)(off)));
901 *addrp = addr;
902 } else {
903 *addrp = NULL; /* no more virtual space */
904 }
905 }
906
907 int valid_va_range_aligned_wraparound;
908
909 /*
910 * Determine whether [*basep, *basep + *lenp) contains a mappable range of
911 * addresses at least "minlen" long, where the base of the range is at "off"
912 * phase from an "align" boundary and there is space for a "redzone"-sized
913 * redzone on either side of the range. On success, 1 is returned and *basep
914 * and *lenp are adjusted to describe the acceptable range (including
915 * the redzone). On failure, 0 is returned.
916 */
917 /*ARGSUSED3*/
918 int
919 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir,
920 size_t align, size_t redzone, size_t off)
921 {
922 uintptr_t hi, lo;
923 size_t tot_len;
924
925 ASSERT(align == 0 ? off == 0 : off < align);
926 ASSERT(ISP2(align));
927 ASSERT(align == 0 || align >= PAGESIZE);
928
929 lo = (uintptr_t)*basep;
930 hi = lo + *lenp;
931 tot_len = minlen + 2 * redzone; /* need at least this much space */
932
933 /*
934 * If hi rolled over the top, try cutting back.
935 */
936 if (hi < lo) {
937 *lenp = 0UL - lo - 1UL;
938 /* See if this really happens. If so, then we figure out why */
939 valid_va_range_aligned_wraparound++;
940 hi = lo + *lenp;
941 }
942 if (*lenp < tot_len) {
943 return (0);
944 }
945
946 #if defined(__amd64)
947 /*
948 * Deal with a possible hole in the address range between
949 * hole_start and hole_end that should never be mapped.
950 */
951 if (lo < hole_start) {
952 if (hi > hole_start) {
953 if (hi < hole_end) {
954 hi = hole_start;
955 } else {
956 /* lo < hole_start && hi >= hole_end */
957 if (dir == AH_LO) {
958 /*
959 * prefer lowest range
960 */
961 if (hole_start - lo >= tot_len)
962 hi = hole_start;
963 else if (hi - hole_end >= tot_len)
964 lo = hole_end;
965 else
966 return (0);
967 } else {
968 /*
969 * prefer highest range
970 */
971 if (hi - hole_end >= tot_len)
972 lo = hole_end;
973 else if (hole_start - lo >= tot_len)
974 hi = hole_start;
975 else
976 return (0);
977 }
978 }
979 }
980 } else {
981 /* lo >= hole_start */
982 if (hi < hole_end)
983 return (0);
984 if (lo < hole_end)
985 lo = hole_end;
986 }
987 #endif
988
989 if (hi - lo < tot_len)
990 return (0);
991
992 if (align > 1) {
993 uintptr_t tlo = lo + redzone;
994 uintptr_t thi = hi - redzone;
995 tlo = (uintptr_t)P2PHASEUP(tlo, align, off);
996 if (tlo < lo + redzone) {
997 return (0);
998 }
999 if (thi < tlo || thi - tlo < minlen) {
1000 return (0);
1001 }
1002 }
1003
1004 *basep = (caddr_t)lo;
1005 *lenp = hi - lo;
1006 return (1);
1007 }
1008
1009 /*
1010 * Determine whether [*basep, *basep + *lenp) contains a mappable range of
1011 * addresses at least "minlen" long. On success, 1 is returned and *basep
1012 * and *lenp are adjusted to describe the acceptable range. On failure, 0
1013 * is returned.
1014 */
1015 int
1016 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir)
1017 {
1018 return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0));
1019 }
1020
1021 /*
1022 * Default to forbidding the first 64k of address space. This protects most
1023 * reasonably sized structures from dereferences through NULL:
1024 * ((foo_t *)0)->bar
1025 */
1026 uintptr_t forbidden_null_mapping_sz = 0x10000;
1027
1028 /*
1029 * Determine whether [addr, addr+len] are valid user addresses.
1030 */
1031 /*ARGSUSED*/
1032 int
1033 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as,
1034 caddr_t userlimit)
1035 {
1036 caddr_t eaddr = addr + len;
1037
1038 if (eaddr <= addr || addr >= userlimit || eaddr > userlimit)
1039 return (RANGE_BADADDR);
1040
1041 if ((addr <= (caddr_t)forbidden_null_mapping_sz) &&
1042 as->a_proc != NULL &&
1043 secflag_enabled(as->a_proc, PROC_SEC_FORBIDNULLMAP))
1044 return (RANGE_BADADDR);
1045
1046 #if defined(__amd64)
1047 /*
1048 * Check for the VA hole
1049 */
1050 if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end)
1051 return (RANGE_BADADDR);
1052 #endif
1053
1054 return (RANGE_OKAY);
1055 }
1056
1057 /*
1058 * Return 1 if the page frame is onboard memory, else 0.
1059 */
1060 int
1061 pf_is_memory(pfn_t pf)
1062 {
1063 if (pfn_is_foreign(pf))
1064 return (0);
1065 return (address_in_memlist(phys_install, pfn_to_pa(pf), 1));
1066 }
1067
1068 /*
1069 * return the memrange containing pfn
1070 */
1071 int
1072 memrange_num(pfn_t pfn)
1073 {
1074 int n;
1075
1076 for (n = 0; n < nranges - 1; ++n) {
1077 if (pfn >= memranges[n])
1078 break;
1079 }
1080 return (n);
1081 }
1082
1083 /*
1084 * return the mnoderange containing pfn
1085 */
1086 /*ARGSUSED*/
1087 int
1088 pfn_2_mtype(pfn_t pfn)
1089 {
1090 #if defined(__xpv)
1091 return (0);
1092 #else
1093 int n;
1094
1095 /* Always start from highest pfn and work our way down */
1096 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1097 if (pfn >= mnoderanges[n].mnr_pfnlo) {
1098 break;
1099 }
1100 }
1101 return (n);
1102 #endif
1103 }
1104
1105 #if !defined(__xpv)
1106 /*
1107 * is_contigpage_free:
1108 * returns a page list of contiguous pages. It minimally has to return
1109 * minctg pages. Caller determines minctg based on the scatter-gather
1110 * list length.
1111 *
1112 * pfnp is set to the next page frame to search on return.
1113 */
1114 static page_t *
1115 is_contigpage_free(
1116 pfn_t *pfnp,
1117 pgcnt_t *pgcnt,
1118 pgcnt_t minctg,
1119 uint64_t pfnseg,
1120 int iolock)
1121 {
1122 int i = 0;
1123 pfn_t pfn = *pfnp;
1124 page_t *pp;
1125 page_t *plist = NULL;
1126
1127 /*
1128 * fail if pfn + minctg crosses a segment boundary.
1129 * Adjust for next starting pfn to begin at segment boundary.
1130 */
1131
1132 if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) {
1133 *pfnp = roundup(*pfnp, pfnseg + 1);
1134 return (NULL);
1135 }
1136
1137 do {
1138 retry:
1139 pp = page_numtopp_nolock(pfn + i);
1140 if ((pp == NULL) || IS_DUMP_PAGE(pp) ||
1141 (page_trylock(pp, SE_EXCL) == 0)) {
1142 (*pfnp)++;
1143 break;
1144 }
1145 if (page_pptonum(pp) != pfn + i) {
1146 page_unlock(pp);
1147 goto retry;
1148 }
1149
1150 if (!(PP_ISFREE(pp))) {
1151 page_unlock(pp);
1152 (*pfnp)++;
1153 break;
1154 }
1155
1156 if (!PP_ISAGED(pp)) {
1157 page_list_sub(pp, PG_CACHE_LIST);
1158 page_hashout(pp, (kmutex_t *)NULL);
1159 } else {
1160 page_list_sub(pp, PG_FREE_LIST);
1161 }
1162
1163 if (iolock)
1164 page_io_lock(pp);
1165 page_list_concat(&plist, &pp);
1166
1167 /*
1168 * exit loop when pgcnt satisfied or segment boundary reached.
1169 */
1170
1171 } while ((++i < *pgcnt) && ((pfn + i) & pfnseg));
1172
1173 *pfnp += i; /* set to next pfn to search */
1174
1175 if (i >= minctg) {
1176 *pgcnt -= i;
1177 return (plist);
1178 }
1179
1180 /*
1181 * failure: minctg not satisfied.
1182 *
1183 * if next request crosses segment boundary, set next pfn
1184 * to search from the segment boundary.
1185 */
1186 if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg))
1187 *pfnp = roundup(*pfnp, pfnseg + 1);
1188
1189 /* clean up any pages already allocated */
1190
1191 while (plist) {
1192 pp = plist;
1193 page_sub(&plist, pp);
1194 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
1195 if (iolock)
1196 page_io_unlock(pp);
1197 page_unlock(pp);
1198 }
1199
1200 return (NULL);
1201 }
1202 #endif /* !__xpv */
1203
1204 /*
1205 * verify that pages being returned from allocator have correct DMA attribute
1206 */
1207 #ifndef DEBUG
1208 #define check_dma(a, b, c) (void)(0)
1209 #else
1210 static void
1211 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt)
1212 {
1213 if (dma_attr == NULL)
1214 return;
1215
1216 while (cnt-- > 0) {
1217 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) <
1218 dma_attr->dma_attr_addr_lo)
1219 panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp);
1220 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >=
1221 dma_attr->dma_attr_addr_hi)
1222 panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp);
1223 pp = pp->p_next;
1224 }
1225 }
1226 #endif
1227
1228 #if !defined(__xpv)
1229 static page_t *
1230 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock)
1231 {
1232 pfn_t pfn;
1233 int sgllen;
1234 uint64_t pfnseg;
1235 pgcnt_t minctg;
1236 page_t *pplist = NULL, *plist;
1237 uint64_t lo, hi;
1238 pgcnt_t pfnalign = 0;
1239 static pfn_t startpfn;
1240 static pgcnt_t lastctgcnt;
1241 uintptr_t align;
1242
1243 CONTIG_LOCK();
1244
1245 if (mattr) {
1246 lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET));
1247 hi = mmu_btop(mattr->dma_attr_addr_hi);
1248 if (hi >= physmax)
1249 hi = physmax - 1;
1250 sgllen = mattr->dma_attr_sgllen;
1251 pfnseg = mmu_btop(mattr->dma_attr_seg);
1252
1253 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
1254 if (align > MMU_PAGESIZE)
1255 pfnalign = mmu_btop(align);
1256
1257 /*
1258 * in order to satisfy the request, must minimally
1259 * acquire minctg contiguous pages
1260 */
1261 minctg = howmany(*pgcnt, sgllen);
1262
1263 ASSERT(hi >= lo);
1264
1265 /*
1266 * start from where last searched if the minctg >= lastctgcnt
1267 */
1268 if (minctg < lastctgcnt || startpfn < lo || startpfn > hi)
1269 startpfn = lo;
1270 } else {
1271 hi = physmax - 1;
1272 lo = 0;
1273 sgllen = 1;
1274 pfnseg = mmu.highest_pfn;
1275 minctg = *pgcnt;
1276
1277 if (minctg < lastctgcnt)
1278 startpfn = lo;
1279 }
1280 lastctgcnt = minctg;
1281
1282 ASSERT(pfnseg + 1 >= (uint64_t)minctg);
1283
1284 /* conserve 16m memory - start search above 16m when possible */
1285 if (hi > PFN_16M && startpfn < PFN_16M)
1286 startpfn = PFN_16M;
1287
1288 pfn = startpfn;
1289 if (pfnalign)
1290 pfn = P2ROUNDUP(pfn, pfnalign);
1291
1292 while (pfn + minctg - 1 <= hi) {
1293
1294 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1295 if (plist) {
1296 page_list_concat(&pplist, &plist);
1297 sgllen--;
1298 /*
1299 * return when contig pages no longer needed
1300 */
1301 if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1302 startpfn = pfn;
1303 CONTIG_UNLOCK();
1304 check_dma(mattr, pplist, *pgcnt);
1305 return (pplist);
1306 }
1307 minctg = howmany(*pgcnt, sgllen);
1308 }
1309 if (pfnalign)
1310 pfn = P2ROUNDUP(pfn, pfnalign);
1311 }
1312
1313 /* cannot find contig pages in specified range */
1314 if (startpfn == lo) {
1315 CONTIG_UNLOCK();
1316 return (NULL);
1317 }
1318
1319 /* did not start with lo previously */
1320 pfn = lo;
1321 if (pfnalign)
1322 pfn = P2ROUNDUP(pfn, pfnalign);
1323
1324 /* allow search to go above startpfn */
1325 while (pfn < startpfn) {
1326
1327 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1328 if (plist != NULL) {
1329
1330 page_list_concat(&pplist, &plist);
1331 sgllen--;
1332
1333 /*
1334 * return when contig pages no longer needed
1335 */
1336 if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1337 startpfn = pfn;
1338 CONTIG_UNLOCK();
1339 check_dma(mattr, pplist, *pgcnt);
1340 return (pplist);
1341 }
1342 minctg = howmany(*pgcnt, sgllen);
1343 }
1344 if (pfnalign)
1345 pfn = P2ROUNDUP(pfn, pfnalign);
1346 }
1347 CONTIG_UNLOCK();
1348 return (NULL);
1349 }
1350 #endif /* !__xpv */
1351
1352 /*
1353 * mnode_range_cnt() calculates the number of memory ranges for mnode and
1354 * memranges[]. Used to determine the size of page lists and mnoderanges.
1355 */
1356 int
1357 mnode_range_cnt(int mnode)
1358 {
1359 #if defined(__xpv)
1360 ASSERT(mnode == 0);
1361 return (1);
1362 #else /* __xpv */
1363 int mri;
1364 int mnrcnt = 0;
1365
1366 if (mem_node_config[mnode].exists != 0) {
1367 mri = nranges - 1;
1368
1369 /* find the memranges index below contained in mnode range */
1370
1371 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1372 mri--;
1373
1374 /*
1375 * increment mnode range counter when memranges or mnode
1376 * boundary is reached.
1377 */
1378 while (mri >= 0 &&
1379 mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1380 mnrcnt++;
1381 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1382 mri--;
1383 else
1384 break;
1385 }
1386 }
1387 ASSERT(mnrcnt <= MAX_MNODE_MRANGES);
1388 return (mnrcnt);
1389 #endif /* __xpv */
1390 }
1391
1392 static int
1393 mnoderange_cmp(const void *v1, const void *v2)
1394 {
1395 const mnoderange_t *m1 = v1;
1396 const mnoderange_t *m2 = v2;
1397
1398 if (m1->mnr_pfnlo < m2->mnr_pfnlo)
1399 return (-1);
1400 return (m1->mnr_pfnlo > m2->mnr_pfnlo);
1401 }
1402
1403 void
1404 mnode_range_setup(mnoderange_t *mnoderanges)
1405 {
1406 mnoderange_t *mp;
1407 size_t nr_ranges;
1408 size_t mnode;
1409
1410 for (mnode = 0, nr_ranges = 0, mp = mnoderanges;
1411 mnode < max_mem_nodes; mnode++) {
1412 size_t mri = nranges - 1;
1413
1414 if (mem_node_config[mnode].exists == 0)
1415 continue;
1416
1417 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1418 mri--;
1419
1420 while (mri >= 0 && mem_node_config[mnode].physmax >=
1421 MEMRANGELO(mri)) {
1422 mp->mnr_pfnlo = MAX(MEMRANGELO(mri),
1423 mem_node_config[mnode].physbase);
1424 mp->mnr_pfnhi = MIN(MEMRANGEHI(mri),
1425 mem_node_config[mnode].physmax);
1426 mp->mnr_mnode = mnode;
1427 mp->mnr_memrange = mri;
1428 mp->mnr_next = -1;
1429 mp->mnr_exists = 1;
1430 mp++;
1431 nr_ranges++;
1432 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1433 mri--;
1434 else
1435 break;
1436 }
1437 }
1438
1439 /*
1440 * mnoderangecnt can be larger than nr_ranges when memory DR is
1441 * supposedly supported.
1442 */
1443 VERIFY3U(nr_ranges, <=, mnoderangecnt);
1444
1445 qsort(mnoderanges, nr_ranges, sizeof (mnoderange_t), mnoderange_cmp);
1446
1447 /*
1448 * If some intrepid soul takes the axe to the memory DR code, we can
1449 * remove ->mnr_next altogether, as we just sorted by ->mnr_pfnlo order.
1450 *
1451 * The VERIFY3U() above can be "==" then too.
1452 */
1453 for (size_t i = 1; i < nr_ranges; i++)
1454 mnoderanges[i].mnr_next = i - 1;
1455
1456 mtypetop = nr_ranges - 1;
1457 mtype16m = pfn_2_mtype(PFN_16MEG - 1); /* Can be -1 ... */
1458 if (physmax4g)
1459 mtype4g = pfn_2_mtype(0xfffff);
1460 }
1461
1462 #ifndef __xpv
1463 /*
1464 * Update mnoderanges for memory hot-add DR operations.
1465 */
1466 static void
1467 mnode_range_add(int mnode)
1468 {
1469 int *prev;
1470 int n, mri;
1471 pfn_t start, end;
1472 extern void membar_sync(void);
1473
1474 ASSERT(0 <= mnode && mnode < max_mem_nodes);
1475 ASSERT(mem_node_config[mnode].exists);
1476 start = mem_node_config[mnode].physbase;
1477 end = mem_node_config[mnode].physmax;
1478 ASSERT(start <= end);
1479 mutex_enter(&mnoderange_lock);
1480
1481 #ifdef DEBUG
1482 /* Check whether it interleaves with other memory nodes. */
1483 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1484 ASSERT(mnoderanges[n].mnr_exists);
1485 if (mnoderanges[n].mnr_mnode == mnode)
1486 continue;
1487 ASSERT(start > mnoderanges[n].mnr_pfnhi ||
1488 end < mnoderanges[n].mnr_pfnlo);
1489 }
1490 #endif /* DEBUG */
1491
1492 mri = nranges - 1;
1493 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1494 mri--;
1495 while (mri >= 0 && mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1496 /* Check whether mtype already exists. */
1497 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1498 if (mnoderanges[n].mnr_mnode == mnode &&
1499 mnoderanges[n].mnr_memrange == mri) {
1500 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri),
1501 start);
1502 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri),
1503 end);
1504 break;
1505 }
1506 }
1507
1508 /* Add a new entry if it doesn't exist yet. */
1509 if (n == -1) {
1510 /* Try to find an unused entry in mnoderanges array. */
1511 for (n = 0; n < mnoderangecnt; n++) {
1512 if (mnoderanges[n].mnr_exists == 0)
1513 break;
1514 }
1515 ASSERT(n < mnoderangecnt);
1516 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), start);
1517 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), end);
1518 mnoderanges[n].mnr_mnode = mnode;
1519 mnoderanges[n].mnr_memrange = mri;
1520 mnoderanges[n].mnr_exists = 1;
1521 /* Page 0 should always be present. */
1522 for (prev = &mtypetop;
1523 mnoderanges[*prev].mnr_pfnlo > start;
1524 prev = &mnoderanges[*prev].mnr_next) {
1525 ASSERT(mnoderanges[*prev].mnr_next >= 0);
1526 ASSERT(mnoderanges[*prev].mnr_pfnlo > end);
1527 }
1528 mnoderanges[n].mnr_next = *prev;
1529 membar_sync();
1530 *prev = n;
1531 }
1532
1533 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1534 mri--;
1535 else
1536 break;
1537 }
1538
1539 mutex_exit(&mnoderange_lock);
1540 }
1541
1542 /*
1543 * Update mnoderanges for memory hot-removal DR operations.
1544 */
1545 static void
1546 mnode_range_del(int mnode)
1547 {
1548 _NOTE(ARGUNUSED(mnode));
1549 ASSERT(0 <= mnode && mnode < max_mem_nodes);
1550 /* TODO: support deletion operation. */
1551 ASSERT(0);
1552 }
1553
1554 void
1555 plat_slice_add(pfn_t start, pfn_t end)
1556 {
1557 mem_node_add_slice(start, end);
1558 if (plat_dr_enabled()) {
1559 mnode_range_add(PFN_2_MEM_NODE(start));
1560 }
1561 }
1562
1563 void
1564 plat_slice_del(pfn_t start, pfn_t end)
1565 {
1566 ASSERT(PFN_2_MEM_NODE(start) == PFN_2_MEM_NODE(end));
1567 ASSERT(plat_dr_enabled());
1568 mnode_range_del(PFN_2_MEM_NODE(start));
1569 mem_node_del_slice(start, end);
1570 }
1571 #endif /* __xpv */
1572
1573 /*ARGSUSED*/
1574 int
1575 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz)
1576 {
1577 int mtype = mtypetop;
1578
1579 #if !defined(__xpv)
1580 #if defined(__i386)
1581 /*
1582 * set the mtype range
1583 * - kmem requests need to be below 4g if restricted_kmemalloc is set.
1584 * - for non kmem requests, set range to above 4g if memory below 4g
1585 * runs low.
1586 */
1587 if (restricted_kmemalloc && VN_ISKAS(vp) &&
1588 (caddr_t)(vaddr) >= kernelheap &&
1589 (caddr_t)(vaddr) < ekernelheap) {
1590 ASSERT(physmax4g);
1591 mtype = mtype4g;
1592 if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz),
1593 btop(pgsz), *flags)) {
1594 *flags |= PGI_MT_RANGE16M;
1595 } else {
1596 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1597 VM_STAT_COND_ADD((*flags & PG_PANIC),
1598 vmm_vmstats.pgpanicalloc);
1599 *flags |= PGI_MT_RANGE0;
1600 }
1601 return (mtype);
1602 }
1603 #endif /* __i386 */
1604
1605 if (RESTRICT4G_ALLOC) {
1606 VM_STAT_ADD(vmm_vmstats.restrict4gcnt);
1607 /* here only for > 4g systems */
1608 *flags |= PGI_MT_RANGE4G;
1609 } else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) {
1610 *flags |= PGI_MT_RANGE16M;
1611 } else {
1612 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1613 VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc);
1614 *flags |= PGI_MT_RANGE0;
1615 }
1616 #endif /* !__xpv */
1617 return (mtype);
1618 }
1619
1620
1621 /* mtype init for page_get_replacement_page */
1622 /*ARGSUSED*/
1623 int
1624 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt)
1625 {
1626 int mtype = mtypetop;
1627 #if !defined(__xpv)
1628 if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) {
1629 *flags |= PGI_MT_RANGE16M;
1630 } else {
1631 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1632 *flags |= PGI_MT_RANGE0;
1633 }
1634 #endif
1635 return (mtype);
1636 }
1637
1638 /*
1639 * Determine if the mnode range specified in mtype contains memory belonging
1640 * to memory node mnode. If flags & PGI_MT_RANGE is set then mtype contains
1641 * the range from high pfn to 0, 16m or 4g.
1642 *
1643 * Return first mnode range type index found otherwise return -1 if none found.
1644 */
1645 int
1646 mtype_func(int mnode, int mtype, uint_t flags)
1647 {
1648 if (flags & PGI_MT_RANGE) {
1649 int mnr_lim = MRI_0;
1650
1651 if (flags & PGI_MT_NEXT) {
1652 mtype = mnoderanges[mtype].mnr_next;
1653 }
1654 if (flags & PGI_MT_RANGE4G)
1655 mnr_lim = MRI_4G; /* exclude 0-4g range */
1656 else if (flags & PGI_MT_RANGE16M)
1657 mnr_lim = MRI_16M; /* exclude 0-16m range */
1658 while (mtype != -1 &&
1659 mnoderanges[mtype].mnr_memrange <= mnr_lim) {
1660 if (mnoderanges[mtype].mnr_mnode == mnode)
1661 return (mtype);
1662 mtype = mnoderanges[mtype].mnr_next;
1663 }
1664 } else if (mnoderanges[mtype].mnr_mnode == mnode) {
1665 return (mtype);
1666 }
1667 return (-1);
1668 }
1669
1670 /*
1671 * Update the page list max counts with the pfn range specified by the
1672 * input parameters.
1673 */
1674 void
1675 mtype_modify_max(pfn_t startpfn, long cnt)
1676 {
1677 int mtype;
1678 pgcnt_t inc;
1679 spgcnt_t scnt = (spgcnt_t)(cnt);
1680 pgcnt_t acnt = ABS(scnt);
1681 pfn_t endpfn = startpfn + acnt;
1682 pfn_t pfn, lo;
1683
1684 if (!physmax4g)
1685 return;
1686
1687 mtype = mtypetop;
1688 for (pfn = endpfn; pfn > startpfn; ) {
1689 ASSERT(mtype != -1);
1690 lo = mnoderanges[mtype].mnr_pfnlo;
1691 if (pfn > lo) {
1692 if (startpfn >= lo) {
1693 inc = pfn - startpfn;
1694 } else {
1695 inc = pfn - lo;
1696 }
1697 if (mnoderanges[mtype].mnr_memrange != MRI_4G) {
1698 if (scnt > 0)
1699 maxmem4g += inc;
1700 else
1701 maxmem4g -= inc;
1702 }
1703 pfn -= inc;
1704 }
1705 mtype = mnoderanges[mtype].mnr_next;
1706 }
1707 }
1708
1709 int
1710 mtype_2_mrange(int mtype)
1711 {
1712 return (mnoderanges[mtype].mnr_memrange);
1713 }
1714
1715 void
1716 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi)
1717 {
1718 _NOTE(ARGUNUSED(mnode));
1719 ASSERT(mnoderanges[mtype].mnr_mnode == mnode);
1720 *pfnlo = mnoderanges[mtype].mnr_pfnlo;
1721 *pfnhi = mnoderanges[mtype].mnr_pfnhi;
1722 }
1723
1724 size_t
1725 plcnt_sz(size_t ctrs_sz)
1726 {
1727 #ifdef DEBUG
1728 int szc, colors;
1729
1730 ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes;
1731 for (szc = 0; szc < mmu_page_sizes; szc++) {
1732 colors = page_get_pagecolors(szc);
1733 ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors;
1734 }
1735 #endif
1736 return (ctrs_sz);
1737 }
1738
1739 caddr_t
1740 plcnt_init(caddr_t addr)
1741 {
1742 #ifdef DEBUG
1743 int mt, szc, colors;
1744
1745 for (mt = 0; mt < mnoderangecnt; mt++) {
1746 mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr;
1747 addr += (sizeof (struct mnr_mts) * mmu_page_sizes);
1748 for (szc = 0; szc < mmu_page_sizes; szc++) {
1749 colors = page_get_pagecolors(szc);
1750 mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors;
1751 mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt =
1752 (pgcnt_t *)addr;
1753 addr += (sizeof (pgcnt_t) * colors);
1754 }
1755 }
1756 #endif
1757 return (addr);
1758 }
1759
1760 void
1761 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags)
1762 {
1763 _NOTE(ARGUNUSED(pp));
1764 #ifdef DEBUG
1765 int bin = PP_2_BIN(pp);
1766
1767 atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt);
1768 atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin],
1769 cnt);
1770 #endif
1771 ASSERT(mtype == PP_2_MTYPE(pp));
1772 if (physmax4g && mnoderanges[mtype].mnr_memrange != MRI_4G)
1773 atomic_add_long(&freemem4g, cnt);
1774 if (flags & PG_CACHE_LIST)
1775 atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt);
1776 else
1777 atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt);
1778 atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt);
1779 }
1780
1781 /*
1782 * Returns the free page count for mnode
1783 */
1784 int
1785 mnode_pgcnt(int mnode)
1786 {
1787 int mtype = mtypetop;
1788 int flags = PGI_MT_RANGE0;
1789 pgcnt_t pgcnt = 0;
1790
1791 mtype = mtype_func(mnode, mtype, flags);
1792
1793 while (mtype != -1) {
1794 pgcnt += MTYPE_FREEMEM(mtype);
1795 mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT);
1796 }
1797 return (pgcnt);
1798 }
1799
1800 /*
1801 * Initialize page coloring variables based on the l2 cache parameters.
1802 * Calculate and return memory needed for page coloring data structures.
1803 */
1804 size_t
1805 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc)
1806 {
1807 _NOTE(ARGUNUSED(l2_linesz));
1808 size_t colorsz = 0;
1809 int i;
1810 int colors;
1811
1812 #if defined(__xpv)
1813 /*
1814 * Hypervisor domains currently don't have any concept of NUMA.
1815 * Hence we'll act like there is only 1 memrange.
1816 */
1817 i = memrange_num(1);
1818 #else /* !__xpv */
1819 /*
1820 * Reduce the memory ranges lists if we don't have large amounts
1821 * of memory. This avoids searching known empty free lists.
1822 * To support memory DR operations, we need to keep memory ranges
1823 * for possible memory hot-add operations.
1824 */
1825 if (plat_dr_physmax > physmax)
1826 i = memrange_num(plat_dr_physmax);
1827 else
1828 i = memrange_num(physmax);
1829 #if defined(__i386)
1830 if (i > MRI_4G)
1831 restricted_kmemalloc = 0;
1832 #endif
1833 /* physmax greater than 4g */
1834 if (i == MRI_4G)
1835 physmax4g = 1;
1836 #endif /* !__xpv */
1837 memranges += i;
1838 nranges -= i;
1839
1840 ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES);
1841
1842 ASSERT(ISP2(l2_linesz));
1843 ASSERT(l2_sz > MMU_PAGESIZE);
1844
1845 /* l2_assoc is 0 for fully associative l2 cache */
1846 if (l2_assoc)
1847 l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE));
1848 else
1849 l2_colors = 1;
1850
1851 ASSERT(ISP2(l2_colors));
1852
1853 /* for scalability, configure at least PAGE_COLORS_MIN color bins */
1854 page_colors = MAX(l2_colors, PAGE_COLORS_MIN);
1855
1856 /*
1857 * cpu_page_colors is non-zero when a page color may be spread across
1858 * multiple bins.
1859 */
1860 if (l2_colors < page_colors)
1861 cpu_page_colors = l2_colors;
1862
1863 ASSERT(ISP2(page_colors));
1864
1865 page_colors_mask = page_colors - 1;
1866
1867 ASSERT(ISP2(CPUSETSIZE()));
1868 page_coloring_shift = lowbit(CPUSETSIZE());
1869
1870 /* initialize number of colors per page size */
1871 for (i = 0; i <= mmu.max_page_level; i++) {
1872 hw_page_array[i].hp_size = LEVEL_SIZE(i);
1873 hw_page_array[i].hp_shift = LEVEL_SHIFT(i);
1874 hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0);
1875 hw_page_array[i].hp_colors = (page_colors_mask >>
1876 (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift))
1877 + 1;
1878 colorequivszc[i] = 0;
1879 }
1880
1881 /*
1882 * The value of cpu_page_colors determines if additional color bins
1883 * need to be checked for a particular color in the page_get routines.
1884 */
1885 if (cpu_page_colors != 0) {
1886
1887 int a = lowbit(page_colors) - lowbit(cpu_page_colors);
1888 ASSERT(a > 0);
1889 ASSERT(a < 16);
1890
1891 for (i = 0; i <= mmu.max_page_level; i++) {
1892 if ((colors = hw_page_array[i].hp_colors) <= 1) {
1893 colorequivszc[i] = 0;
1894 continue;
1895 }
1896 while ((colors >> a) == 0)
1897 a--;
1898 ASSERT(a >= 0);
1899
1900 /* higher 4 bits encodes color equiv mask */
1901 colorequivszc[i] = (a << 4);
1902 }
1903 }
1904
1905 /* factor in colorequiv to check additional 'equivalent' bins. */
1906 if (colorequiv > 1) {
1907
1908 int a = lowbit(colorequiv) - 1;
1909 if (a > 15)
1910 a = 15;
1911
1912 for (i = 0; i <= mmu.max_page_level; i++) {
1913 if ((colors = hw_page_array[i].hp_colors) <= 1) {
1914 continue;
1915 }
1916 while ((colors >> a) == 0)
1917 a--;
1918 if ((a << 4) > colorequivszc[i]) {
1919 colorequivszc[i] = (a << 4);
1920 }
1921 }
1922 }
1923
1924 /* size for mnoderanges */
1925 for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++)
1926 mnoderangecnt += mnode_range_cnt(i);
1927 if (plat_dr_support_memory()) {
1928 /*
1929 * Reserve enough space for memory DR operations.
1930 * Two extra mnoderanges for possbile fragmentations,
1931 * one for the 2G boundary and the other for the 4G boundary.
1932 * We don't expect a memory board crossing the 16M boundary
1933 * for memory hot-add operations on x86 platforms.
1934 */
1935 mnoderangecnt += 2 + max_mem_nodes - lgrp_plat_node_cnt;
1936 }
1937 colorsz = mnoderangecnt * sizeof (mnoderange_t);
1938
1939 /* size for fpc_mutex and cpc_mutex */
1940 colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX);
1941
1942 /* size of page_freelists */
1943 colorsz += mnoderangecnt * sizeof (page_t ***);
1944 colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **);
1945
1946 for (i = 0; i < mmu_page_sizes; i++) {
1947 colors = page_get_pagecolors(i);
1948 colorsz += mnoderangecnt * colors * sizeof (page_t *);
1949 }
1950
1951 /* size of page_cachelists */
1952 colorsz += mnoderangecnt * sizeof (page_t **);
1953 colorsz += mnoderangecnt * page_colors * sizeof (page_t *);
1954
1955 return (colorsz);
1956 }
1957
1958 /*
1959 * Called once at startup to configure page_coloring data structures and
1960 * does the 1st page_free()/page_freelist_add().
1961 */
1962 void
1963 page_coloring_setup(caddr_t pcmemaddr)
1964 {
1965 int i;
1966 int j;
1967 int k;
1968 caddr_t addr;
1969 int colors;
1970
1971 /*
1972 * do page coloring setup
1973 */
1974 addr = pcmemaddr;
1975
1976 mnoderanges = (mnoderange_t *)addr;
1977 addr += (mnoderangecnt * sizeof (mnoderange_t));
1978
1979 mnode_range_setup(mnoderanges);
1980
1981 for (k = 0; k < NPC_MUTEX; k++) {
1982 fpc_mutex[k] = (kmutex_t *)addr;
1983 addr += (max_mem_nodes * sizeof (kmutex_t));
1984 }
1985 for (k = 0; k < NPC_MUTEX; k++) {
1986 cpc_mutex[k] = (kmutex_t *)addr;
1987 addr += (max_mem_nodes * sizeof (kmutex_t));
1988 }
1989 page_freelists = (page_t ****)addr;
1990 addr += (mnoderangecnt * sizeof (page_t ***));
1991
1992 page_cachelists = (page_t ***)addr;
1993 addr += (mnoderangecnt * sizeof (page_t **));
1994
1995 for (i = 0; i < mnoderangecnt; i++) {
1996 page_freelists[i] = (page_t ***)addr;
1997 addr += (mmu_page_sizes * sizeof (page_t **));
1998
1999 for (j = 0; j < mmu_page_sizes; j++) {
2000 colors = page_get_pagecolors(j);
2001 page_freelists[i][j] = (page_t **)addr;
2002 addr += (colors * sizeof (page_t *));
2003 }
2004 page_cachelists[i] = (page_t **)addr;
2005 addr += (page_colors * sizeof (page_t *));
2006 }
2007 }
2008
2009 #if defined(__xpv)
2010 /*
2011 * Give back 10% of the io_pool pages to the free list.
2012 * Don't shrink the pool below some absolute minimum.
2013 */
2014 static void
2015 page_io_pool_shrink()
2016 {
2017 int retcnt;
2018 page_t *pp, *pp_first, *pp_last, **curpool;
2019 mfn_t mfn;
2020 int bothpools = 0;
2021
2022 mutex_enter(&io_pool_lock);
2023 io_pool_shrink_attempts++; /* should be a kstat? */
2024 retcnt = io_pool_cnt / 10;
2025 if (io_pool_cnt - retcnt < io_pool_cnt_min)
2026 retcnt = io_pool_cnt - io_pool_cnt_min;
2027 if (retcnt <= 0)
2028 goto done;
2029 io_pool_shrinks++; /* should be a kstat? */
2030 curpool = &io_pool_4g;
2031 domore:
2032 /*
2033 * Loop through taking pages from the end of the list
2034 * (highest mfns) till amount to return reached.
2035 */
2036 for (pp = *curpool; pp && retcnt > 0; ) {
2037 pp_first = pp_last = pp->p_prev;
2038 if (pp_first == *curpool)
2039 break;
2040 retcnt--;
2041 io_pool_cnt--;
2042 page_io_pool_sub(curpool, pp_first, pp_last);
2043 if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn)
2044 start_mfn = mfn;
2045 page_free(pp_first, 1);
2046 pp = *curpool;
2047 }
2048 if (retcnt != 0 && !bothpools) {
2049 /*
2050 * If not enough found in less constrained pool try the
2051 * more constrained one.
2052 */
2053 curpool = &io_pool_16m;
2054 bothpools = 1;
2055 goto domore;
2056 }
2057 done:
2058 mutex_exit(&io_pool_lock);
2059 }
2060
2061 #endif /* __xpv */
2062
2063 uint_t
2064 page_create_update_flags_x86(uint_t flags)
2065 {
2066 #if defined(__xpv)
2067 /*
2068 * Check this is an urgent allocation and free pages are depleted.
2069 */
2070 if (!(flags & PG_WAIT) && freemem < desfree)
2071 page_io_pool_shrink();
2072 #else /* !__xpv */
2073 /*
2074 * page_create_get_something may call this because 4g memory may be
2075 * depleted. Set flags to allow for relocation of base page below
2076 * 4g if necessary.
2077 */
2078 if (physmax4g)
2079 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
2080 #endif /* __xpv */
2081 return (flags);
2082 }
2083
2084 /*ARGSUSED*/
2085 int
2086 bp_color(struct buf *bp)
2087 {
2088 return (0);
2089 }
2090
2091 #if defined(__xpv)
2092
2093 /*
2094 * Take pages out of an io_pool
2095 */
2096 static void
2097 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last)
2098 {
2099 if (*poolp == pp_first) {
2100 *poolp = pp_last->p_next;
2101 if (*poolp == pp_first)
2102 *poolp = NULL;
2103 }
2104 pp_first->p_prev->p_next = pp_last->p_next;
2105 pp_last->p_next->p_prev = pp_first->p_prev;
2106 pp_first->p_prev = pp_last;
2107 pp_last->p_next = pp_first;
2108 }
2109
2110 /*
2111 * Put a page on the io_pool list. The list is ordered by increasing MFN.
2112 */
2113 static void
2114 page_io_pool_add(page_t **poolp, page_t *pp)
2115 {
2116 page_t *look;
2117 mfn_t mfn = mfn_list[pp->p_pagenum];
2118
2119 if (*poolp == NULL) {
2120 *poolp = pp;
2121 pp->p_next = pp;
2122 pp->p_prev = pp;
2123 return;
2124 }
2125
2126 /*
2127 * Since we try to take pages from the high end of the pool
2128 * chances are good that the pages to be put on the list will
2129 * go at or near the end of the list. so start at the end and
2130 * work backwards.
2131 */
2132 look = (*poolp)->p_prev;
2133 while (mfn < mfn_list[look->p_pagenum]) {
2134 look = look->p_prev;
2135 if (look == (*poolp)->p_prev)
2136 break; /* backed all the way to front of list */
2137 }
2138
2139 /* insert after look */
2140 pp->p_prev = look;
2141 pp->p_next = look->p_next;
2142 pp->p_next->p_prev = pp;
2143 look->p_next = pp;
2144 if (mfn < mfn_list[(*poolp)->p_pagenum]) {
2145 /*
2146 * we inserted a new first list element
2147 * adjust pool pointer to newly inserted element
2148 */
2149 *poolp = pp;
2150 }
2151 }
2152
2153 /*
2154 * Add a page to the io_pool. Setting the force flag will force the page
2155 * into the io_pool no matter what.
2156 */
2157 static void
2158 add_page_to_pool(page_t *pp, int force)
2159 {
2160 page_t *highest;
2161 page_t *freep = NULL;
2162
2163 mutex_enter(&io_pool_lock);
2164 /*
2165 * Always keep the scarce low memory pages
2166 */
2167 if (mfn_list[pp->p_pagenum] < PFN_16MEG) {
2168 ++io_pool_cnt;
2169 page_io_pool_add(&io_pool_16m, pp);
2170 goto done;
2171 }
2172 if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) {
2173 ++io_pool_cnt;
2174 page_io_pool_add(&io_pool_4g, pp);
2175 } else {
2176 highest = io_pool_4g->p_prev;
2177 if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) {
2178 page_io_pool_sub(&io_pool_4g, highest, highest);
2179 page_io_pool_add(&io_pool_4g, pp);
2180 freep = highest;
2181 } else {
2182 freep = pp;
2183 }
2184 }
2185 done:
2186 mutex_exit(&io_pool_lock);
2187 if (freep)
2188 page_free(freep, 1);
2189 }
2190
2191
2192 int contig_pfn_cnt; /* no of pfns in the contig pfn list */
2193 int contig_pfn_max; /* capacity of the contig pfn list */
2194 int next_alloc_pfn; /* next position in list to start a contig search */
2195 int contig_pfnlist_updates; /* pfn list update count */
2196 int contig_pfnlist_builds; /* how many times have we (re)built list */
2197 int contig_pfnlist_buildfailed; /* how many times has list build failed */
2198 int create_contig_pending; /* nonzero means taskq creating contig list */
2199 pfn_t *contig_pfn_list = NULL; /* list of contig pfns in ascending mfn order */
2200
2201 /*
2202 * Function to use in sorting a list of pfns by their underlying mfns.
2203 */
2204 static int
2205 mfn_compare(const void *pfnp1, const void *pfnp2)
2206 {
2207 mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1];
2208 mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2];
2209
2210 if (mfn1 > mfn2)
2211 return (1);
2212 if (mfn1 < mfn2)
2213 return (-1);
2214 return (0);
2215 }
2216
2217 /*
2218 * Compact the contig_pfn_list by tossing all the non-contiguous
2219 * elements from the list.
2220 */
2221 static void
2222 compact_contig_pfn_list(void)
2223 {
2224 pfn_t pfn, lapfn, prev_lapfn;
2225 mfn_t mfn;
2226 int i, newcnt = 0;
2227
2228 prev_lapfn = 0;
2229 for (i = 0; i < contig_pfn_cnt - 1; i++) {
2230 pfn = contig_pfn_list[i];
2231 lapfn = contig_pfn_list[i + 1];
2232 mfn = mfn_list[pfn];
2233 /*
2234 * See if next pfn is for a contig mfn
2235 */
2236 if (mfn_list[lapfn] != mfn + 1)
2237 continue;
2238 /*
2239 * pfn and lookahead are both put in list
2240 * unless pfn is the previous lookahead.
2241 */
2242 if (pfn != prev_lapfn)
2243 contig_pfn_list[newcnt++] = pfn;
2244 contig_pfn_list[newcnt++] = lapfn;
2245 prev_lapfn = lapfn;
2246 }
2247 for (i = newcnt; i < contig_pfn_cnt; i++)
2248 contig_pfn_list[i] = 0;
2249 contig_pfn_cnt = newcnt;
2250 }
2251
2252 /*ARGSUSED*/
2253 static void
2254 call_create_contiglist(void *arg)
2255 {
2256 (void) create_contig_pfnlist(PG_WAIT);
2257 }
2258
2259 /*
2260 * Create list of freelist pfns that have underlying
2261 * contiguous mfns. The list is kept in ascending mfn order.
2262 * returns 1 if list created else 0.
2263 */
2264 static int
2265 create_contig_pfnlist(uint_t flags)
2266 {
2267 pfn_t pfn;
2268 page_t *pp;
2269 int ret = 1;
2270
2271 mutex_enter(&contig_list_lock);
2272 if (contig_pfn_list != NULL)
2273 goto out;
2274 contig_pfn_max = freemem + (freemem / 10);
2275 contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t),
2276 (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP);
2277 if (contig_pfn_list == NULL) {
2278 /*
2279 * If we could not create the contig list (because
2280 * we could not sleep for memory). Dispatch a taskq that can
2281 * sleep to get the memory.
2282 */
2283 if (!create_contig_pending) {
2284 if (taskq_dispatch(system_taskq, call_create_contiglist,
2285 NULL, TQ_NOSLEEP) != TASKQID_INVALID)
2286 create_contig_pending = 1;
2287 }
2288 contig_pfnlist_buildfailed++; /* count list build failures */
2289 ret = 0;
2290 goto out;
2291 }
2292 create_contig_pending = 0;
2293 ASSERT(contig_pfn_cnt == 0);
2294 for (pfn = 0; pfn < mfn_count; pfn++) {
2295 pp = page_numtopp_nolock(pfn);
2296 if (pp == NULL || !PP_ISFREE(pp))
2297 continue;
2298 contig_pfn_list[contig_pfn_cnt] = pfn;
2299 if (++contig_pfn_cnt == contig_pfn_max)
2300 break;
2301 }
2302 /*
2303 * Sanity check the new list.
2304 */
2305 if (contig_pfn_cnt < 2) { /* no contig pfns */
2306 contig_pfn_cnt = 0;
2307 contig_pfnlist_buildfailed++;
2308 kmem_free(contig_pfn_list, contig_pfn_max * sizeof (pfn_t));
2309 contig_pfn_list = NULL;
2310 contig_pfn_max = 0;
2311 ret = 0;
2312 goto out;
2313 }
2314 qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare);
2315 compact_contig_pfn_list();
2316 /*
2317 * Make sure next search of the newly created contiguous pfn
2318 * list starts at the beginning of the list.
2319 */
2320 next_alloc_pfn = 0;
2321 contig_pfnlist_builds++; /* count list builds */
2322 out:
2323 mutex_exit(&contig_list_lock);
2324 return (ret);
2325 }
2326
2327
2328 /*
2329 * Toss the current contig pfnlist. Someone is about to do a massive
2330 * update to pfn<->mfn mappings. So we have them destroy the list and lock
2331 * it till they are done with their update.
2332 */
2333 void
2334 clear_and_lock_contig_pfnlist()
2335 {
2336 pfn_t *listp = NULL;
2337 size_t listsize;
2338
2339 mutex_enter(&contig_list_lock);
2340 if (contig_pfn_list != NULL) {
2341 listp = contig_pfn_list;
2342 listsize = contig_pfn_max * sizeof (pfn_t);
2343 contig_pfn_list = NULL;
2344 contig_pfn_max = contig_pfn_cnt = 0;
2345 }
2346 if (listp != NULL)
2347 kmem_free(listp, listsize);
2348 }
2349
2350 /*
2351 * Unlock the contig_pfn_list. The next attempted use of it will cause
2352 * it to be re-created.
2353 */
2354 void
2355 unlock_contig_pfnlist()
2356 {
2357 mutex_exit(&contig_list_lock);
2358 }
2359
2360 /*
2361 * Update the contiguous pfn list in response to a pfn <-> mfn reassignment
2362 */
2363 void
2364 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn)
2365 {
2366 int probe_hi, probe_lo, probe_pos, insert_after, insert_point;
2367 pfn_t probe_pfn;
2368 mfn_t probe_mfn;
2369 int drop_lock = 0;
2370
2371 if (mutex_owner(&contig_list_lock) != curthread) {
2372 drop_lock = 1;
2373 mutex_enter(&contig_list_lock);
2374 }
2375 if (contig_pfn_list == NULL)
2376 goto done;
2377 contig_pfnlist_updates++;
2378 /*
2379 * Find the pfn in the current list. Use a binary chop to locate it.
2380 */
2381 probe_hi = contig_pfn_cnt - 1;
2382 probe_lo = 0;
2383 probe_pos = (probe_hi + probe_lo) / 2;
2384 while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) {
2385 if (probe_pos == probe_lo) { /* pfn not in list */
2386 probe_pos = -1;
2387 break;
2388 }
2389 if (pfn_to_mfn(probe_pfn) <= oldmfn)
2390 probe_lo = probe_pos;
2391 else
2392 probe_hi = probe_pos;
2393 probe_pos = (probe_hi + probe_lo) / 2;
2394 }
2395 if (probe_pos >= 0) {
2396 /*
2397 * Remove pfn from list and ensure next alloc
2398 * position stays in bounds.
2399 */
2400 if (--contig_pfn_cnt <= next_alloc_pfn)
2401 next_alloc_pfn = 0;
2402 if (contig_pfn_cnt < 2) { /* no contig pfns */
2403 contig_pfn_cnt = 0;
2404 kmem_free(contig_pfn_list,
2405 contig_pfn_max * sizeof (pfn_t));
2406 contig_pfn_list = NULL;
2407 contig_pfn_max = 0;
2408 goto done;
2409 }
2410 ovbcopy(&contig_pfn_list[probe_pos + 1],
2411 &contig_pfn_list[probe_pos],
2412 (contig_pfn_cnt - probe_pos) * sizeof (pfn_t));
2413 }
2414 if (newmfn == MFN_INVALID)
2415 goto done;
2416 /*
2417 * Check if new mfn has adjacent mfns in the list
2418 */
2419 probe_hi = contig_pfn_cnt - 1;
2420 probe_lo = 0;
2421 insert_after = -2;
2422 do {
2423 probe_pos = (probe_hi + probe_lo) / 2;
2424 probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]);
2425 if (newmfn == probe_mfn + 1)
2426 insert_after = probe_pos;
2427 else if (newmfn == probe_mfn - 1)
2428 insert_after = probe_pos - 1;
2429 if (probe_pos == probe_lo)
2430 break;
2431 if (probe_mfn <= newmfn)
2432 probe_lo = probe_pos;
2433 else
2434 probe_hi = probe_pos;
2435 } while (insert_after == -2);
2436 /*
2437 * If there is space in the list and there are adjacent mfns
2438 * insert the pfn in to its proper place in the list.
2439 */
2440 if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) {
2441 insert_point = insert_after + 1;
2442 ovbcopy(&contig_pfn_list[insert_point],
2443 &contig_pfn_list[insert_point + 1],
2444 (contig_pfn_cnt - insert_point) * sizeof (pfn_t));
2445 contig_pfn_list[insert_point] = pfn;
2446 contig_pfn_cnt++;
2447 }
2448 done:
2449 if (drop_lock)
2450 mutex_exit(&contig_list_lock);
2451 }
2452
2453 /*
2454 * Called to (re-)populate the io_pool from the free page lists.
2455 */
2456 long
2457 populate_io_pool(void)
2458 {
2459 pfn_t pfn;
2460 mfn_t mfn, max_mfn;
2461 page_t *pp;
2462
2463 /*
2464 * Figure out the bounds of the pool on first invocation.
2465 * We use a percentage of memory for the io pool size.
2466 * we allow that to shrink, but not to less than a fixed minimum
2467 */
2468 if (io_pool_cnt_max == 0) {
2469 io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct);
2470 io_pool_cnt_lowater = io_pool_cnt_max;
2471 /*
2472 * This is the first time in populate_io_pool, grab a va to use
2473 * when we need to allocate pages.
2474 */
2475 io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2476 }
2477 /*
2478 * If we are out of pages in the pool, then grow the size of the pool
2479 */
2480 if (io_pool_cnt == 0) {
2481 /*
2482 * Grow the max size of the io pool by 5%, but never more than
2483 * 25% of physical memory.
2484 */
2485 if (io_pool_cnt_max < physmem / 4)
2486 io_pool_cnt_max += io_pool_cnt_max / 20;
2487 }
2488 io_pool_grows++; /* should be a kstat? */
2489
2490 /*
2491 * Get highest mfn on this platform, but limit to the 32 bit DMA max.
2492 */
2493 (void) mfn_to_pfn(start_mfn);
2494 max_mfn = MIN(cached_max_mfn, PFN_4GIG);
2495 for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) {
2496 pfn = mfn_to_pfn(mfn);
2497 if (pfn & PFN_IS_FOREIGN_MFN)
2498 continue;
2499 /*
2500 * try to allocate it from free pages
2501 */
2502 pp = page_numtopp_alloc(pfn);
2503 if (pp == NULL)
2504 continue;
2505 PP_CLRFREE(pp);
2506 add_page_to_pool(pp, 1);
2507 if (io_pool_cnt >= io_pool_cnt_max)
2508 break;
2509 }
2510
2511 return (io_pool_cnt);
2512 }
2513
2514 /*
2515 * Destroy a page that was being used for DMA I/O. It may or
2516 * may not actually go back to the io_pool.
2517 */
2518 void
2519 page_destroy_io(page_t *pp)
2520 {
2521 mfn_t mfn = mfn_list[pp->p_pagenum];
2522
2523 /*
2524 * When the page was alloc'd a reservation was made, release it now
2525 */
2526 page_unresv(1);
2527 /*
2528 * Unload translations, if any, then hash out the
2529 * page to erase its identity.
2530 */
2531 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2532 page_hashout(pp, NULL);
2533
2534 /*
2535 * If the page came from the free lists, just put it back to them.
2536 * DomU pages always go on the free lists as well.
2537 */
2538 if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) {
2539 page_free(pp, 1);
2540 return;
2541 }
2542
2543 add_page_to_pool(pp, 0);
2544 }
2545
2546
2547 long contig_searches; /* count of times contig pages requested */
2548 long contig_search_restarts; /* count of contig ranges tried */
2549 long contig_search_failed; /* count of contig alloc failures */
2550
2551 /*
2552 * Free partial page list
2553 */
2554 static void
2555 free_partial_list(page_t **pplist)
2556 {
2557 page_t *pp;
2558
2559 while (*pplist != NULL) {
2560 pp = *pplist;
2561 page_io_pool_sub(pplist, pp, pp);
2562 page_free(pp, 1);
2563 }
2564 }
2565
2566 /*
2567 * Look thru the contiguous pfns that are not part of the io_pool for
2568 * contiguous free pages. Return a list of the found pages or NULL.
2569 */
2570 page_t *
2571 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg,
2572 pgcnt_t pfnalign)
2573 {
2574 page_t *pp, *plist = NULL;
2575 mfn_t mfn, prev_mfn, start_mfn;
2576 pfn_t pfn;
2577 int pages_needed, pages_requested;
2578 int search_start;
2579
2580 /*
2581 * create the contig pfn list if not already done
2582 */
2583 retry:
2584 mutex_enter(&contig_list_lock);
2585 if (contig_pfn_list == NULL) {
2586 mutex_exit(&contig_list_lock);
2587 if (!create_contig_pfnlist(flags)) {
2588 return (NULL);
2589 }
2590 goto retry;
2591 }
2592 contig_searches++;
2593 /*
2594 * Search contiguous pfn list for physically contiguous pages not in
2595 * the io_pool. Start the search where the last search left off.
2596 */
2597 pages_requested = pages_needed = npages;
2598 search_start = next_alloc_pfn;
2599 start_mfn = prev_mfn = 0;
2600 while (pages_needed) {
2601 pfn = contig_pfn_list[next_alloc_pfn];
2602 mfn = pfn_to_mfn(pfn);
2603 /*
2604 * Check if mfn is first one or contig to previous one and
2605 * if page corresponding to mfn is free and that mfn
2606 * range is not crossing a segment boundary.
2607 */
2608 if ((prev_mfn == 0 || mfn == prev_mfn + 1) &&
2609 (pp = page_numtopp_alloc(pfn)) != NULL &&
2610 !((mfn & pfnseg) < (start_mfn & pfnseg))) {
2611 PP_CLRFREE(pp);
2612 page_io_pool_add(&plist, pp);
2613 pages_needed--;
2614 if (prev_mfn == 0) {
2615 if (pfnalign &&
2616 mfn != P2ROUNDUP(mfn, pfnalign)) {
2617 /*
2618 * not properly aligned
2619 */
2620 contig_search_restarts++;
2621 free_partial_list(&plist);
2622 pages_needed = pages_requested;
2623 start_mfn = prev_mfn = 0;
2624 goto skip;
2625 }
2626 start_mfn = mfn;
2627 }
2628 prev_mfn = mfn;
2629 } else {
2630 contig_search_restarts++;
2631 free_partial_list(&plist);
2632 pages_needed = pages_requested;
2633 start_mfn = prev_mfn = 0;
2634 }
2635 skip:
2636 if (++next_alloc_pfn == contig_pfn_cnt)
2637 next_alloc_pfn = 0;
2638 if (next_alloc_pfn == search_start)
2639 break; /* all pfns searched */
2640 }
2641 mutex_exit(&contig_list_lock);
2642 if (pages_needed) {
2643 contig_search_failed++;
2644 /*
2645 * Failed to find enough contig pages.
2646 * free partial page list
2647 */
2648 free_partial_list(&plist);
2649 }
2650 return (plist);
2651 }
2652
2653 /*
2654 * Search the reserved io pool pages for a page range with the
2655 * desired characteristics.
2656 */
2657 page_t *
2658 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg)
2659 {
2660 page_t *pp_first, *pp_last;
2661 page_t *pp, **poolp;
2662 pgcnt_t nwanted, pfnalign;
2663 uint64_t pfnseg;
2664 mfn_t mfn, tmfn, hi_mfn, lo_mfn;
2665 int align, attempt = 0;
2666
2667 if (minctg == 1)
2668 contig = 0;
2669 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2670 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2671 pfnseg = mmu_btop(mattr->dma_attr_seg);
2672 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2673 if (align > MMU_PAGESIZE)
2674 pfnalign = mmu_btop(align);
2675 else
2676 pfnalign = 0;
2677
2678 try_again:
2679 /*
2680 * See if we want pages for a legacy device
2681 */
2682 if (hi_mfn < PFN_16MEG)
2683 poolp = &io_pool_16m;
2684 else
2685 poolp = &io_pool_4g;
2686 try_smaller:
2687 /*
2688 * Take pages from I/O pool. We'll use pages from the highest
2689 * MFN range possible.
2690 */
2691 pp_first = pp_last = NULL;
2692 mutex_enter(&io_pool_lock);
2693 nwanted = minctg;
2694 for (pp = *poolp; pp && nwanted > 0; ) {
2695 pp = pp->p_prev;
2696
2697 /*
2698 * skip pages above allowable range
2699 */
2700 mfn = mfn_list[pp->p_pagenum];
2701 if (hi_mfn < mfn)
2702 goto skip;
2703
2704 /*
2705 * stop at pages below allowable range
2706 */
2707 if (lo_mfn > mfn)
2708 break;
2709 restart:
2710 if (pp_last == NULL) {
2711 /*
2712 * Check alignment
2713 */
2714 tmfn = mfn - (minctg - 1);
2715 if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign))
2716 goto skip; /* not properly aligned */
2717 /*
2718 * Check segment
2719 */
2720 if ((mfn & pfnseg) < (tmfn & pfnseg))
2721 goto skip; /* crosses seg boundary */
2722 /*
2723 * Start building page list
2724 */
2725 pp_first = pp_last = pp;
2726 nwanted--;
2727 } else {
2728 /*
2729 * check physical contiguity if required
2730 */
2731 if (contig &&
2732 mfn_list[pp_first->p_pagenum] != mfn + 1) {
2733 /*
2734 * not a contiguous page, restart list.
2735 */
2736 pp_last = NULL;
2737 nwanted = minctg;
2738 goto restart;
2739 } else { /* add page to list */
2740 pp_first = pp;
2741 nwanted--;
2742 }
2743 }
2744 skip:
2745 if (pp == *poolp)
2746 break;
2747 }
2748
2749 /*
2750 * If we didn't find memory. Try the more constrained pool, then
2751 * sweep free pages into the DMA pool and try again.
2752 */
2753 if (nwanted != 0) {
2754 mutex_exit(&io_pool_lock);
2755 /*
2756 * If we were looking in the less constrained pool and
2757 * didn't find pages, try the more constrained pool.
2758 */
2759 if (poolp == &io_pool_4g) {
2760 poolp = &io_pool_16m;
2761 goto try_smaller;
2762 }
2763 kmem_reap();
2764 if (++attempt < 4) {
2765 /*
2766 * Grab some more io_pool pages
2767 */
2768 (void) populate_io_pool();
2769 goto try_again; /* go around and retry */
2770 }
2771 return (NULL);
2772 }
2773 /*
2774 * Found the pages, now snip them from the list
2775 */
2776 page_io_pool_sub(poolp, pp_first, pp_last);
2777 io_pool_cnt -= minctg;
2778 /*
2779 * reset low water mark
2780 */
2781 if (io_pool_cnt < io_pool_cnt_lowater)
2782 io_pool_cnt_lowater = io_pool_cnt;
2783 mutex_exit(&io_pool_lock);
2784 return (pp_first);
2785 }
2786
2787 page_t *
2788 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr,
2789 ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg)
2790 {
2791 uint_t kflags;
2792 int order, extra, extpages, i, contig, nbits, extents;
2793 page_t *pp, *expp, *pp_first, **pplist = NULL;
2794 mfn_t *mfnlist = NULL;
2795
2796 contig = flags & PG_PHYSCONTIG;
2797 if (minctg == 1)
2798 contig = 0;
2799 flags &= ~PG_PHYSCONTIG;
2800 kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP;
2801 /*
2802 * Hypervisor will allocate extents, if we want contig
2803 * pages extent must be >= minctg
2804 */
2805 if (contig) {
2806 order = highbit(minctg) - 1;
2807 if (minctg & ((1 << order) - 1))
2808 order++;
2809 extpages = 1 << order;
2810 } else {
2811 order = 0;
2812 extpages = minctg;
2813 }
2814 if (extpages > minctg) {
2815 extra = extpages - minctg;
2816 if (!page_resv(extra, kflags))
2817 return (NULL);
2818 }
2819 pp_first = NULL;
2820 pplist = kmem_alloc(extpages * sizeof (page_t *), kflags);
2821 if (pplist == NULL)
2822 goto balloon_fail;
2823 mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags);
2824 if (mfnlist == NULL)
2825 goto balloon_fail;
2826 pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr);
2827 if (pp == NULL)
2828 goto balloon_fail;
2829 pp_first = pp;
2830 if (extpages > minctg) {
2831 /*
2832 * fill out the rest of extent pages to swap
2833 * with the hypervisor
2834 */
2835 for (i = 0; i < extra; i++) {
2836 expp = page_create_va(vp,
2837 (u_offset_t)(uintptr_t)io_pool_kva,
2838 PAGESIZE, flags, &kvseg, io_pool_kva);
2839 if (expp == NULL)
2840 goto balloon_fail;
2841 (void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD);
2842 page_io_unlock(expp);
2843 page_hashout(expp, NULL);
2844 page_io_lock(expp);
2845 /*
2846 * add page to end of list
2847 */
2848 expp->p_prev = pp_first->p_prev;
2849 expp->p_next = pp_first;
2850 expp->p_prev->p_next = expp;
2851 pp_first->p_prev = expp;
2852 }
2853
2854 }
2855 for (i = 0; i < extpages; i++) {
2856 pplist[i] = pp;
2857 pp = pp->p_next;
2858 }
2859 nbits = highbit(mattr->dma_attr_addr_hi);
2860 extents = contig ? 1 : minctg;
2861 if (balloon_replace_pages(extents, pplist, nbits, order,
2862 mfnlist) != extents) {
2863 if (ioalloc_dbg)
2864 cmn_err(CE_NOTE, "request to hypervisor"
2865 " for %d pages, maxaddr %" PRIx64 " failed",
2866 extpages, mattr->dma_attr_addr_hi);
2867 goto balloon_fail;
2868 }
2869
2870 kmem_free(pplist, extpages * sizeof (page_t *));
2871 kmem_free(mfnlist, extpages * sizeof (mfn_t));
2872 /*
2873 * Return any excess pages to free list
2874 */
2875 if (extpages > minctg) {
2876 for (i = 0; i < extra; i++) {
2877 pp = pp_first->p_prev;
2878 page_sub(&pp_first, pp);
2879 page_io_unlock(pp);
2880 page_unresv(1);
2881 page_free(pp, 1);
2882 }
2883 }
2884 return (pp_first);
2885 balloon_fail:
2886 /*
2887 * Return pages to free list and return failure
2888 */
2889 while (pp_first != NULL) {
2890 pp = pp_first;
2891 page_sub(&pp_first, pp);
2892 page_io_unlock(pp);
2893 if (pp->p_vnode != NULL)
2894 page_hashout(pp, NULL);
2895 page_free(pp, 1);
2896 }
2897 if (pplist)
2898 kmem_free(pplist, extpages * sizeof (page_t *));
2899 if (mfnlist)
2900 kmem_free(mfnlist, extpages * sizeof (mfn_t));
2901 page_unresv(extpages - minctg);
2902 return (NULL);
2903 }
2904
2905 static void
2906 return_partial_alloc(page_t *plist)
2907 {
2908 page_t *pp;
2909
2910 while (plist != NULL) {
2911 pp = plist;
2912 page_sub(&plist, pp);
2913 page_io_unlock(pp);
2914 page_destroy_io(pp);
2915 }
2916 }
2917
2918 static page_t *
2919 page_get_contigpages(
2920 struct vnode *vp,
2921 u_offset_t off,
2922 int *npagesp,
2923 uint_t flags,
2924 caddr_t vaddr,
2925 ddi_dma_attr_t *mattr)
2926 {
2927 mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2928 page_t *plist; /* list to return */
2929 page_t *pp, *mcpl;
2930 int contig, anyaddr, npages, getone = 0;
2931 mfn_t lo_mfn;
2932 mfn_t hi_mfn;
2933 pgcnt_t pfnalign = 0;
2934 int align, sgllen;
2935 uint64_t pfnseg;
2936 pgcnt_t minctg;
2937
2938 npages = *npagesp;
2939 ASSERT(mattr != NULL);
2940 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2941 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2942 sgllen = mattr->dma_attr_sgllen;
2943 pfnseg = mmu_btop(mattr->dma_attr_seg);
2944 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2945 if (align > MMU_PAGESIZE)
2946 pfnalign = mmu_btop(align);
2947
2948 contig = flags & PG_PHYSCONTIG;
2949 if (npages == -1) {
2950 npages = 1;
2951 pfnalign = 0;
2952 }
2953 /*
2954 * Clear the contig flag if only one page is needed.
2955 */
2956 if (npages == 1) {
2957 getone = 1;
2958 contig = 0;
2959 }
2960
2961 /*
2962 * Check if any page in the system is fine.
2963 */
2964 anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn;
2965 if (!contig && anyaddr && !pfnalign) {
2966 flags &= ~PG_PHYSCONTIG;
2967 plist = page_create_va(vp, off, npages * MMU_PAGESIZE,
2968 flags, &kvseg, vaddr);
2969 if (plist != NULL) {
2970 *npagesp = 0;
2971 return (plist);
2972 }
2973 }
2974 plist = NULL;
2975 minctg = howmany(npages, sgllen);
2976 while (npages > sgllen || getone) {
2977 if (minctg > npages)
2978 minctg = npages;
2979 mcpl = NULL;
2980 /*
2981 * We could want contig pages with no address range limits.
2982 */
2983 if (anyaddr && contig) {
2984 /*
2985 * Look for free contig pages to satisfy the request.
2986 */
2987 mcpl = find_contig_free(minctg, flags, pfnseg,
2988 pfnalign);
2989 }
2990 /*
2991 * Try the reserved io pools next
2992 */
2993 if (mcpl == NULL)
2994 mcpl = page_io_pool_alloc(mattr, contig, minctg);
2995 if (mcpl != NULL) {
2996 pp = mcpl;
2997 do {
2998 if (!page_hashin(pp, vp, off, NULL)) {
2999 panic("page_get_contigpages:"
3000 " hashin failed"
3001 " pp %p, vp %p, off %llx",
3002 (void *)pp, (void *)vp, off);
3003 }
3004 off += MMU_PAGESIZE;
3005 PP_CLRFREE(pp);
3006 PP_CLRAGED(pp);
3007 page_set_props(pp, P_REF);
3008 page_io_lock(pp);
3009 pp = pp->p_next;
3010 } while (pp != mcpl);
3011 } else {
3012 /*
3013 * Hypervisor exchange doesn't handle segment or
3014 * alignment constraints
3015 */
3016 if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi ||
3017 pfnalign)
3018 goto fail;
3019 /*
3020 * Try exchanging pages with the hypervisor
3021 */
3022 mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr,
3023 flags, minctg);
3024 if (mcpl == NULL)
3025 goto fail;
3026 off += minctg * MMU_PAGESIZE;
3027 }
3028 check_dma(mattr, mcpl, minctg);
3029 /*
3030 * Here with a minctg run of contiguous pages, add them to the
3031 * list we will return for this request.
3032 */
3033 page_list_concat(&plist, &mcpl);
3034 npages -= minctg;
3035 *npagesp = npages;
3036 sgllen--;
3037 if (getone)
3038 break;
3039 }
3040 return (plist);
3041 fail:
3042 return_partial_alloc(plist);
3043 return (NULL);
3044 }
3045
3046 /*
3047 * Allocator for domain 0 I/O pages. We match the required
3048 * DMA attributes and contiguity constraints.
3049 */
3050 /*ARGSUSED*/
3051 page_t *
3052 page_create_io(
3053 struct vnode *vp,
3054 u_offset_t off,
3055 uint_t bytes,
3056 uint_t flags,
3057 struct as *as,
3058 caddr_t vaddr,
3059 ddi_dma_attr_t *mattr)
3060 {
3061 page_t *plist = NULL, *pp;
3062 int npages = 0, contig, anyaddr, pages_req;
3063 mfn_t lo_mfn;
3064 mfn_t hi_mfn;
3065 pgcnt_t pfnalign = 0;
3066 int align;
3067 int is_domu = 0;
3068 int dummy, bytes_got;
3069 mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
3070
3071 ASSERT(mattr != NULL);
3072 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
3073 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
3074 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
3075 if (align > MMU_PAGESIZE)
3076 pfnalign = mmu_btop(align);
3077
3078 /*
3079 * Clear the contig flag if only one page is needed or the scatter
3080 * gather list length is >= npages.
3081 */
3082 pages_req = npages = mmu_btopr(bytes);
3083 contig = (flags & PG_PHYSCONTIG);
3084 bytes = P2ROUNDUP(bytes, MMU_PAGESIZE);
3085 if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages)
3086 contig = 0;
3087
3088 /*
3089 * Check if any old page in the system is fine.
3090 * DomU should always go down this path.
3091 */
3092 is_domu = !DOMAIN_IS_INITDOMAIN(xen_info);
3093 anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
3094 if ((!contig && anyaddr) || is_domu) {
3095 flags &= ~PG_PHYSCONTIG;
3096 plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr);
3097 if (plist != NULL)
3098 return (plist);
3099 else if (is_domu)
3100 return (NULL); /* no memory available */
3101 }
3102 /*
3103 * DomU should never reach here
3104 */
3105 if (contig) {
3106 plist = page_get_contigpages(vp, off, &npages, flags, vaddr,
3107 mattr);
3108 if (plist == NULL)
3109 goto fail;
3110 bytes_got = (pages_req - npages) << MMU_PAGESHIFT;
3111 vaddr += bytes_got;
3112 off += bytes_got;
3113 /*
3114 * We now have all the contiguous pages we need, but
3115 * we may still need additional non-contiguous pages.
3116 */
3117 }
3118 /*
3119 * now loop collecting the requested number of pages, these do
3120 * not have to be contiguous pages but we will use the contig
3121 * page alloc code to get the pages since it will honor any
3122 * other constraints the pages may have.
3123 */
3124 while (npages--) {
3125 dummy = -1;
3126 pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr);
3127 if (pp == NULL)
3128 goto fail;
3129 page_add(&plist, pp);
3130 vaddr += MMU_PAGESIZE;
3131 off += MMU_PAGESIZE;
3132 }
3133 return (plist);
3134 fail:
3135 /*
3136 * Failed to get enough pages, return ones we did get
3137 */
3138 return_partial_alloc(plist);
3139 return (NULL);
3140 }
3141
3142 /*
3143 * Lock and return the page with the highest mfn that we can find. last_mfn
3144 * holds the last one found, so the next search can start from there. We
3145 * also keep a counter so that we don't loop forever if the machine has no
3146 * free pages.
3147 *
3148 * This is called from the balloon thread to find pages to give away. new_high
3149 * is used when new mfn's have been added to the system - we will reset our
3150 * search if the new mfn's are higher than our current search position.
3151 */
3152 page_t *
3153 page_get_high_mfn(mfn_t new_high)
3154 {
3155 static mfn_t last_mfn = 0;
3156 pfn_t pfn;
3157 page_t *pp;
3158 ulong_t loop_count = 0;
3159
3160 if (new_high > last_mfn)
3161 last_mfn = new_high;
3162
3163 for (; loop_count < mfn_count; loop_count++, last_mfn--) {
3164 if (last_mfn == 0) {
3165 last_mfn = cached_max_mfn;
3166 }
3167
3168 pfn = mfn_to_pfn(last_mfn);
3169 if (pfn & PFN_IS_FOREIGN_MFN)
3170 continue;
3171
3172 /* See if the page is free. If so, lock it. */
3173 pp = page_numtopp_alloc(pfn);
3174 if (pp == NULL)
3175 continue;
3176 PP_CLRFREE(pp);
3177
3178 ASSERT(PAGE_EXCL(pp));
3179 ASSERT(pp->p_vnode == NULL);
3180 ASSERT(!hat_page_is_mapped(pp));
3181 last_mfn--;
3182 return (pp);
3183 }
3184 return (NULL);
3185 }
3186
3187 #else /* !__xpv */
3188
3189 /*
3190 * get a page from any list with the given mnode
3191 */
3192 static page_t *
3193 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags,
3194 int mnode, int mtype, ddi_dma_attr_t *dma_attr)
3195 {
3196 kmutex_t *pcm;
3197 int i;
3198 page_t *pp;
3199 page_t *first_pp;
3200 uint64_t pgaddr;
3201 ulong_t bin;
3202 int mtypestart;
3203 int plw_initialized;
3204 page_list_walker_t plw;
3205
3206 VM_STAT_ADD(pga_vmstats.pgma_alloc);
3207
3208 ASSERT((flags & PG_MATCH_COLOR) == 0);
3209 ASSERT(szc == 0);
3210 ASSERT(dma_attr != NULL);
3211
3212 MTYPE_START(mnode, mtype, flags);
3213 if (mtype < 0) {
3214 VM_STAT_ADD(pga_vmstats.pgma_allocempty);
3215 return (NULL);
3216 }
3217
3218 mtypestart = mtype;
3219
3220 bin = origbin;
3221
3222 /*
3223 * check up to page_colors + 1 bins - origbin may be checked twice
3224 * because of BIN_STEP skip
3225 */
3226 do {
3227 plw_initialized = 0;
3228
3229 for (plw.plw_count = 0;
3230 plw.plw_count < page_colors; plw.plw_count++) {
3231
3232 if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL)
3233 goto nextfreebin;
3234
3235 pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST);
3236 mutex_enter(pcm);
3237 pp = PAGE_FREELISTS(mnode, szc, bin, mtype);
3238 first_pp = pp;
3239 while (pp != NULL) {
3240 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3241 SE_EXCL) == 0) {
3242 pp = pp->p_next;
3243 if (pp == first_pp) {
3244 pp = NULL;
3245 }
3246 continue;
3247 }
3248
3249 ASSERT(PP_ISFREE(pp));
3250 ASSERT(PP_ISAGED(pp));
3251 ASSERT(pp->p_vnode == NULL);
3252 ASSERT(pp->p_hash == NULL);
3253 ASSERT(pp->p_offset == (u_offset_t)-1);
3254 ASSERT(pp->p_szc == szc);
3255 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3256 /* check if page within DMA attributes */
3257 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3258 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3259 (pgaddr + MMU_PAGESIZE - 1 <=
3260 dma_attr->dma_attr_addr_hi)) {
3261 break;
3262 }
3263
3264 /* continue looking */
3265 page_unlock(pp);
3266 pp = pp->p_next;
3267 if (pp == first_pp)
3268 pp = NULL;
3269
3270 }
3271 if (pp != NULL) {
3272 ASSERT(mtype == PP_2_MTYPE(pp));
3273 ASSERT(pp->p_szc == 0);
3274
3275 /* found a page with specified DMA attributes */
3276 page_sub(&PAGE_FREELISTS(mnode, szc, bin,
3277 mtype), pp);
3278 page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST);
3279
3280 if ((PP_ISFREE(pp) == 0) ||
3281 (PP_ISAGED(pp) == 0)) {
3282 cmn_err(CE_PANIC, "page %p is not free",
3283 (void *)pp);
3284 }
3285
3286 mutex_exit(pcm);
3287 check_dma(dma_attr, pp, 1);
3288 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3289 return (pp);
3290 }
3291 mutex_exit(pcm);
3292 nextfreebin:
3293 if (plw_initialized == 0) {
3294 page_list_walk_init(szc, 0, bin, 1, 0, &plw);
3295 ASSERT(plw.plw_ceq_dif == page_colors);
3296 plw_initialized = 1;
3297 }
3298
3299 if (plw.plw_do_split) {
3300 pp = page_freelist_split(szc, bin, mnode,
3301 mtype,
3302 mmu_btop(dma_attr->dma_attr_addr_lo),
3303 mmu_btop(dma_attr->dma_attr_addr_hi + 1),
3304 &plw);
3305 if (pp != NULL) {
3306 check_dma(dma_attr, pp, 1);
3307 return (pp);
3308 }
3309 }
3310
3311 bin = page_list_walk_next_bin(szc, bin, &plw);
3312 }
3313
3314 MTYPE_NEXT(mnode, mtype, flags);
3315 } while (mtype >= 0);
3316
3317 /* failed to find a page in the freelist; try it in the cachelist */
3318
3319 /* reset mtype start for cachelist search */
3320 mtype = mtypestart;
3321 ASSERT(mtype >= 0);
3322
3323 /* start with the bin of matching color */
3324 bin = origbin;
3325
3326 do {
3327 for (i = 0; i <= page_colors; i++) {
3328 if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL)
3329 goto nextcachebin;
3330 pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST);
3331 mutex_enter(pcm);
3332 pp = PAGE_CACHELISTS(mnode, bin, mtype);
3333 first_pp = pp;
3334 while (pp != NULL) {
3335 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3336 SE_EXCL) == 0) {
3337 pp = pp->p_next;
3338 if (pp == first_pp)
3339 pp = NULL;
3340 continue;
3341 }
3342 ASSERT(pp->p_vnode);
3343 ASSERT(PP_ISAGED(pp) == 0);
3344 ASSERT(pp->p_szc == 0);
3345 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3346
3347 /* check if page within DMA attributes */
3348
3349 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3350 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3351 (pgaddr + MMU_PAGESIZE - 1 <=
3352 dma_attr->dma_attr_addr_hi)) {
3353 break;
3354 }
3355
3356 /* continue looking */
3357 page_unlock(pp);
3358 pp = pp->p_next;
3359 if (pp == first_pp)
3360 pp = NULL;
3361 }
3362
3363 if (pp != NULL) {
3364 ASSERT(mtype == PP_2_MTYPE(pp));
3365 ASSERT(pp->p_szc == 0);
3366
3367 /* found a page with specified DMA attributes */
3368 page_sub(&PAGE_CACHELISTS(mnode, bin,
3369 mtype), pp);
3370 page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST);
3371
3372 mutex_exit(pcm);
3373 ASSERT(pp->p_vnode);
3374 ASSERT(PP_ISAGED(pp) == 0);
3375 check_dma(dma_attr, pp, 1);
3376 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3377 return (pp);
3378 }
3379 mutex_exit(pcm);
3380 nextcachebin:
3381 bin += (i == 0) ? BIN_STEP : 1;
3382 bin &= page_colors_mask;
3383 }
3384 MTYPE_NEXT(mnode, mtype, flags);
3385 } while (mtype >= 0);
3386
3387 VM_STAT_ADD(pga_vmstats.pgma_allocfailed);
3388 return (NULL);
3389 }
3390
3391 /*
3392 * This function is similar to page_get_freelist()/page_get_cachelist()
3393 * but it searches both the lists to find a page with the specified
3394 * color (or no color) and DMA attributes. The search is done in the
3395 * freelist first and then in the cache list within the highest memory
3396 * range (based on DMA attributes) before searching in the lower
3397 * memory ranges.
3398 *
3399 * Note: This function is called only by page_create_io().
3400 */
3401 /*ARGSUSED*/
3402 static page_t *
3403 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr,
3404 size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t *lgrp)
3405 {
3406 uint_t bin;
3407 int mtype;
3408 page_t *pp;
3409 int n;
3410 int m;
3411 int szc;
3412 int fullrange;
3413 int mnode;
3414 int local_failed_stat = 0;
3415 lgrp_mnode_cookie_t lgrp_cookie;
3416
3417 VM_STAT_ADD(pga_vmstats.pga_alloc);
3418
3419 /* only base pagesize currently supported */
3420 if (size != MMU_PAGESIZE)
3421 return (NULL);
3422
3423 /*
3424 * If we're passed a specific lgroup, we use it. Otherwise,
3425 * assume first-touch placement is desired.
3426 */
3427 if (!LGRP_EXISTS(lgrp))
3428 lgrp = lgrp_home_lgrp();
3429
3430 /* LINTED */
3431 AS_2_BIN(as, seg, vp, vaddr, bin, 0);
3432
3433 /*
3434 * Only hold one freelist or cachelist lock at a time, that way we
3435 * can start anywhere and not have to worry about lock
3436 * ordering.
3437 */
3438 if (dma_attr == NULL) {
3439 n = mtype16m;
3440 m = mtypetop;
3441 fullrange = 1;
3442 VM_STAT_ADD(pga_vmstats.pga_nulldmaattr);
3443 } else {
3444 pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo);
3445 pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi);
3446
3447 /*
3448 * We can guarantee alignment only for page boundary.
3449 */
3450 if (dma_attr->dma_attr_align > MMU_PAGESIZE)
3451 return (NULL);
3452
3453 /* Sanity check the dma_attr */
3454 if (pfnlo > pfnhi)
3455 return (NULL);
3456
3457 n = pfn_2_mtype(pfnlo);
3458 m = pfn_2_mtype(pfnhi);
3459
3460 fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) &&
3461 (pfnhi >= mnoderanges[m].mnr_pfnhi));
3462 }
3463 VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange);
3464
3465 szc = 0;
3466
3467 /* cylcing thru mtype handled by RANGE0 if n == mtype16m */
3468 if (n == mtype16m) {
3469 flags |= PGI_MT_RANGE0;
3470 n = m;
3471 }
3472
3473 /*
3474 * Try local memory node first, but try remote if we can't
3475 * get a page of the right color.
3476 */
3477 LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER);
3478 while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) {
3479 /*
3480 * allocate pages from high pfn to low.
3481 */
3482 mtype = m;
3483 do {
3484 if (fullrange != 0) {
3485 pp = page_get_mnode_freelist(mnode,
3486 bin, mtype, szc, flags);
3487 if (pp == NULL) {
3488 pp = page_get_mnode_cachelist(
3489 bin, flags, mnode, mtype);
3490 }
3491 } else {
3492 pp = page_get_mnode_anylist(bin, szc,
3493 flags, mnode, mtype, dma_attr);
3494 }
3495 if (pp != NULL) {
3496 VM_STAT_ADD(pga_vmstats.pga_allocok);
3497 check_dma(dma_attr, pp, 1);
3498 return (pp);
3499 }
3500 } while (mtype != n &&
3501 (mtype = mnoderanges[mtype].mnr_next) != -1);
3502 if (!local_failed_stat) {
3503 lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1);
3504 local_failed_stat = 1;
3505 }
3506 }
3507 VM_STAT_ADD(pga_vmstats.pga_allocfailed);
3508
3509 return (NULL);
3510 }
3511
3512 /*
3513 * page_create_io()
3514 *
3515 * This function is a copy of page_create_va() with an additional
3516 * argument 'mattr' that specifies DMA memory requirements to
3517 * the page list functions. This function is used by the segkmem
3518 * allocator so it is only to create new pages (i.e PG_EXCL is
3519 * set).
3520 *
3521 * Note: This interface is currently used by x86 PSM only and is
3522 * not fully specified so the commitment level is only for
3523 * private interface specific to x86. This interface uses PSM
3524 * specific page_get_anylist() interface.
3525 */
3526
3527 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \
3528 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
3529 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
3530 break; \
3531 } \
3532 }
3533
3534
3535 page_t *
3536 page_create_io(
3537 struct vnode *vp,
3538 u_offset_t off,
3539 uint_t bytes,
3540 uint_t flags,
3541 struct as *as,
3542 caddr_t vaddr,
3543 ddi_dma_attr_t *mattr) /* DMA memory attributes if any */
3544 {
3545 page_t *plist = NULL;
3546 uint_t plist_len = 0;
3547 pgcnt_t npages;
3548 page_t *npp = NULL;
3549 uint_t pages_req;
3550 page_t *pp;
3551 kmutex_t *phm = NULL;
3552 uint_t index;
3553
3554 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
3555 "page_create_start:vp %p off %llx bytes %u flags %x",
3556 vp, off, bytes, flags);
3557
3558 ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0);
3559
3560 pages_req = npages = mmu_btopr(bytes);
3561
3562 /*
3563 * Do the freemem and pcf accounting.
3564 */
3565 if (!page_create_wait(npages, flags)) {
3566 return (NULL);
3567 }
3568
3569 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
3570 "page_create_success:vp %p off %llx", vp, off);
3571
3572 /*
3573 * If satisfying this request has left us with too little
3574 * memory, start the wheels turning to get some back. The
3575 * first clause of the test prevents waking up the pageout
3576 * daemon in situations where it would decide that there's
3577 * nothing to do.
3578 */
3579 if (nscan < desscan && freemem < minfree) {
3580 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
3581 "pageout_cv_signal:freemem %ld", freemem);
3582 cv_signal(&proc_pageout->p_cv);
3583 }
3584
3585 if (flags & PG_PHYSCONTIG) {
3586
3587 plist = page_get_contigpage(&npages, mattr, 1);
3588 if (plist == NULL) {
3589 page_create_putback(npages);
3590 return (NULL);
3591 }
3592
3593 pp = plist;
3594
3595 do {
3596 if (!page_hashin(pp, vp, off, NULL)) {
3597 panic("pg_creat_io: hashin failed %p %p %llx",
3598 (void *)pp, (void *)vp, off);
3599 }
3600 VM_STAT_ADD(page_create_new);
3601 off += MMU_PAGESIZE;
3602 PP_CLRFREE(pp);
3603 PP_CLRAGED(pp);
3604 page_set_props(pp, P_REF);
3605 pp = pp->p_next;
3606 } while (pp != plist);
3607
3608 if (!npages) {
3609 check_dma(mattr, plist, pages_req);
3610 return (plist);
3611 } else {
3612 vaddr += (pages_req - npages) << MMU_PAGESHIFT;
3613 }
3614
3615 /*
3616 * fall-thru:
3617 *
3618 * page_get_contigpage returns when npages <= sgllen.
3619 * Grab the rest of the non-contig pages below from anylist.
3620 */
3621 }
3622
3623 /*
3624 * Loop around collecting the requested number of pages.
3625 * Most of the time, we have to `create' a new page. With
3626 * this in mind, pull the page off the free list before
3627 * getting the hash lock. This will minimize the hash
3628 * lock hold time, nesting, and the like. If it turns
3629 * out we don't need the page, we put it back at the end.
3630 */
3631 while (npages--) {
3632 phm = NULL;
3633
3634 index = PAGE_HASH_FUNC(vp, off);
3635 top:
3636 ASSERT(phm == NULL);
3637 ASSERT(index == PAGE_HASH_FUNC(vp, off));
3638 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3639
3640 if (npp == NULL) {
3641 /*
3642 * Try to get the page of any color either from
3643 * the freelist or from the cache list.
3644 */
3645 npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE,
3646 flags & ~PG_MATCH_COLOR, mattr, NULL);
3647 if (npp == NULL) {
3648 if (mattr == NULL) {
3649 /*
3650 * Not looking for a special page;
3651 * panic!
3652 */
3653 panic("no page found %d", (int)npages);
3654 }
3655 /*
3656 * No page found! This can happen
3657 * if we are looking for a page
3658 * within a specific memory range
3659 * for DMA purposes. If PG_WAIT is
3660 * specified then we wait for a
3661 * while and then try again. The
3662 * wait could be forever if we
3663 * don't get the page(s) we need.
3664 *
3665 * Note: XXX We really need a mechanism
3666 * to wait for pages in the desired
3667 * range. For now, we wait for any
3668 * pages and see if we can use it.
3669 */
3670
3671 if ((mattr != NULL) && (flags & PG_WAIT)) {
3672 delay(10);
3673 goto top;
3674 }
3675 goto fail; /* undo accounting stuff */
3676 }
3677
3678 if (PP_ISAGED(npp) == 0) {
3679 /*
3680 * Since this page came from the
3681 * cachelist, we must destroy the
3682 * old vnode association.
3683 */
3684 page_hashout(npp, (kmutex_t *)NULL);
3685 }
3686 }
3687
3688 /*
3689 * We own this page!
3690 */
3691 ASSERT(PAGE_EXCL(npp));
3692 ASSERT(npp->p_vnode == NULL);
3693 ASSERT(!hat_page_is_mapped(npp));
3694 PP_CLRFREE(npp);
3695 PP_CLRAGED(npp);
3696
3697 /*
3698 * Here we have a page in our hot little mits and are
3699 * just waiting to stuff it on the appropriate lists.
3700 * Get the mutex and check to see if it really does
3701 * not exist.
3702 */
3703 phm = PAGE_HASH_MUTEX(index);
3704 mutex_enter(phm);
3705 PAGE_HASH_SEARCH(index, pp, vp, off);
3706 if (pp == NULL) {
3707 VM_STAT_ADD(page_create_new);
3708 pp = npp;
3709 npp = NULL;
3710 if (!page_hashin(pp, vp, off, phm)) {
3711 /*
3712 * Since we hold the page hash mutex and
3713 * just searched for this page, page_hashin
3714 * had better not fail. If it does, that
3715 * means somethread did not follow the
3716 * page hash mutex rules. Panic now and
3717 * get it over with. As usual, go down
3718 * holding all the locks.
3719 */
3720 ASSERT(MUTEX_HELD(phm));
3721 panic("page_create: hashin fail %p %p %llx %p",
3722 (void *)pp, (void *)vp, off, (void *)phm);
3723
3724 }
3725 ASSERT(MUTEX_HELD(phm));
3726 mutex_exit(phm);
3727 phm = NULL;
3728
3729 /*
3730 * Hat layer locking need not be done to set
3731 * the following bits since the page is not hashed
3732 * and was on the free list (i.e., had no mappings).
3733 *
3734 * Set the reference bit to protect
3735 * against immediate pageout
3736 *
3737 * XXXmh modify freelist code to set reference
3738 * bit so we don't have to do it here.
3739 */
3740 page_set_props(pp, P_REF);
3741 } else {
3742 ASSERT(MUTEX_HELD(phm));
3743 mutex_exit(phm);
3744 phm = NULL;
3745 /*
3746 * NOTE: This should not happen for pages associated
3747 * with kernel vnode 'kvp'.
3748 */
3749 /* XX64 - to debug why this happens! */
3750 ASSERT(!VN_ISKAS(vp));
3751 if (VN_ISKAS(vp))
3752 cmn_err(CE_NOTE,
3753 "page_create: page not expected "
3754 "in hash list for kernel vnode - pp 0x%p",
3755 (void *)pp);
3756 VM_STAT_ADD(page_create_exists);
3757 goto fail;
3758 }
3759
3760 /*
3761 * Got a page! It is locked. Acquire the i/o
3762 * lock since we are going to use the p_next and
3763 * p_prev fields to link the requested pages together.
3764 */
3765 page_io_lock(pp);
3766 page_add(&plist, pp);
3767 plist = plist->p_next;
3768 off += MMU_PAGESIZE;
3769 vaddr += MMU_PAGESIZE;
3770 }
3771
3772 check_dma(mattr, plist, pages_req);
3773 return (plist);
3774
3775 fail:
3776 if (npp != NULL) {
3777 /*
3778 * Did not need this page after all.
3779 * Put it back on the free list.
3780 */
3781 VM_STAT_ADD(page_create_putbacks);
3782 PP_SETFREE(npp);
3783 PP_SETAGED(npp);
3784 npp->p_offset = (u_offset_t)-1;
3785 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
3786 page_unlock(npp);
3787 }
3788
3789 /*
3790 * Give up the pages we already got.
3791 */
3792 while (plist != NULL) {
3793 pp = plist;
3794 page_sub(&plist, pp);
3795 page_io_unlock(pp);
3796 plist_len++;
3797 /*LINTED: constant in conditional ctx*/
3798 VN_DISPOSE(pp, B_INVAL, 0, kcred);
3799 }
3800
3801 /*
3802 * VN_DISPOSE does freemem accounting for the pages in plist
3803 * by calling page_free. So, we need to undo the pcf accounting
3804 * for only the remaining pages.
3805 */
3806 VM_STAT_ADD(page_create_putbacks);
3807 page_create_putback(pages_req - plist_len);
3808
3809 return (NULL);
3810 }
3811 #endif /* !__xpv */
3812
3813
3814 /*
3815 * Copy the data from the physical page represented by "frompp" to
3816 * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and
3817 * CPU->cpu_caddr2. It assumes that no one uses either map at interrupt
3818 * level and no one sleeps with an active mapping there.
3819 *
3820 * Note that the ref/mod bits in the page_t's are not affected by
3821 * this operation, hence it is up to the caller to update them appropriately.
3822 */
3823 int
3824 ppcopy(page_t *frompp, page_t *topp)
3825 {
3826 caddr_t pp_addr1;
3827 caddr_t pp_addr2;
3828 hat_mempte_t pte1;
3829 hat_mempte_t pte2;
3830 kmutex_t *ppaddr_mutex;
3831 label_t ljb;
3832 int ret = 1;
3833
3834 ASSERT_STACK_ALIGNED();
3835 ASSERT(PAGE_LOCKED(frompp));
3836 ASSERT(PAGE_LOCKED(topp));
3837
3838 if (kpm_enable) {
3839 pp_addr1 = hat_kpm_page2va(frompp, 0);
3840 pp_addr2 = hat_kpm_page2va(topp, 0);
3841 kpreempt_disable();
3842 } else {
3843 /*
3844 * disable pre-emption so that CPU can't change
3845 */
3846 kpreempt_disable();
3847
3848 pp_addr1 = CPU->cpu_caddr1;
3849 pp_addr2 = CPU->cpu_caddr2;
3850 pte1 = CPU->cpu_caddr1pte;
3851 pte2 = CPU->cpu_caddr2pte;
3852
3853 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3854 mutex_enter(ppaddr_mutex);
3855
3856 hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1,
3857 PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST);
3858 hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2,
3859 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3860 HAT_LOAD_NOCONSIST);
3861 }
3862
3863 if (on_fault(&ljb)) {
3864 ret = 0;
3865 goto faulted;
3866 }
3867 if (use_sse_pagecopy)
3868 #ifdef __xpv
3869 page_copy_no_xmm(pp_addr2, pp_addr1);
3870 #else
3871 hwblkpagecopy(pp_addr1, pp_addr2);
3872 #endif
3873 else
3874 bcopy(pp_addr1, pp_addr2, PAGESIZE);
3875
3876 no_fault();
3877 faulted:
3878 if (!kpm_enable) {
3879 #ifdef __xpv
3880 /*
3881 * We can't leave unused mappings laying about under the
3882 * hypervisor, so blow them away.
3883 */
3884 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0,
3885 UVMF_INVLPG | UVMF_LOCAL) < 0)
3886 panic("HYPERVISOR_update_va_mapping() failed");
3887 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3888 UVMF_INVLPG | UVMF_LOCAL) < 0)
3889 panic("HYPERVISOR_update_va_mapping() failed");
3890 #endif
3891 mutex_exit(ppaddr_mutex);
3892 }
3893 kpreempt_enable();
3894 return (ret);
3895 }
3896
3897 void
3898 pagezero(page_t *pp, uint_t off, uint_t len)
3899 {
3900 ASSERT(PAGE_LOCKED(pp));
3901 pfnzero(page_pptonum(pp), off, len);
3902 }
3903
3904 /*
3905 * Zero the physical page from off to off + len given by pfn
3906 * without changing the reference and modified bits of page.
3907 *
3908 * We use this using CPU private page address #2, see ppcopy() for more info.
3909 * pfnzero() must not be called at interrupt level.
3910 */
3911 void
3912 pfnzero(pfn_t pfn, uint_t off, uint_t len)
3913 {
3914 caddr_t pp_addr2;
3915 hat_mempte_t pte2;
3916 kmutex_t *ppaddr_mutex = NULL;
3917
3918 ASSERT_STACK_ALIGNED();
3919 ASSERT(len <= MMU_PAGESIZE);
3920 ASSERT(off <= MMU_PAGESIZE);
3921 ASSERT(off + len <= MMU_PAGESIZE);
3922
3923 if (kpm_enable && !pfn_is_foreign(pfn)) {
3924 pp_addr2 = hat_kpm_pfn2va(pfn);
3925 kpreempt_disable();
3926 } else {
3927 kpreempt_disable();
3928
3929 pp_addr2 = CPU->cpu_caddr2;
3930 pte2 = CPU->cpu_caddr2pte;
3931
3932 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3933 mutex_enter(ppaddr_mutex);
3934
3935 hat_mempte_remap(pfn, pp_addr2, pte2,
3936 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3937 HAT_LOAD_NOCONSIST);
3938 }
3939
3940 if (use_sse_pagezero) {
3941 #ifdef __xpv
3942 uint_t rem;
3943
3944 /*
3945 * zero a byte at a time until properly aligned for
3946 * block_zero_no_xmm().
3947 */
3948 while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0)
3949 pp_addr2[off++] = 0;
3950
3951 /*
3952 * Now use faster block_zero_no_xmm() for any range
3953 * that is properly aligned and sized.
3954 */
3955 rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN));
3956 len -= rem;
3957 if (len != 0) {
3958 block_zero_no_xmm(pp_addr2 + off, len);
3959 off += len;
3960 }
3961
3962 /*
3963 * zero remainder with byte stores.
3964 */
3965 while (rem-- > 0)
3966 pp_addr2[off++] = 0;
3967 #else
3968 hwblkclr(pp_addr2 + off, len);
3969 #endif
3970 } else {
3971 bzero(pp_addr2 + off, len);
3972 }
3973
3974 if (!kpm_enable || pfn_is_foreign(pfn)) {
3975 #ifdef __xpv
3976 /*
3977 * On the hypervisor this page might get used for a page
3978 * table before any intervening change to this mapping,
3979 * so blow it away.
3980 */
3981 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3982 UVMF_INVLPG) < 0)
3983 panic("HYPERVISOR_update_va_mapping() failed");
3984 #endif
3985 mutex_exit(ppaddr_mutex);
3986 }
3987
3988 kpreempt_enable();
3989 }
3990
3991 /*
3992 * Platform-dependent page scrub call.
3993 */
3994 void
3995 pagescrub(page_t *pp, uint_t off, uint_t len)
3996 {
3997 /*
3998 * For now, we rely on the fact that pagezero() will
3999 * always clear UEs.
4000 */
4001 pagezero(pp, off, len);
4002 }
4003
4004 /*
4005 * set up two private addresses for use on a given CPU for use in ppcopy()
4006 */
4007 void
4008 setup_vaddr_for_ppcopy(struct cpu *cpup)
4009 {
4010 void *addr;
4011 hat_mempte_t pte_pa;
4012
4013 addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
4014 pte_pa = hat_mempte_setup(addr);
4015 cpup->cpu_caddr1 = addr;
4016 cpup->cpu_caddr1pte = pte_pa;
4017
4018 addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
4019 pte_pa = hat_mempte_setup(addr);
4020 cpup->cpu_caddr2 = addr;
4021 cpup->cpu_caddr2pte = pte_pa;
4022
4023 mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL);
4024 }
4025
4026 /*
4027 * Undo setup_vaddr_for_ppcopy
4028 */
4029 void
4030 teardown_vaddr_for_ppcopy(struct cpu *cpup)
4031 {
4032 mutex_destroy(&cpup->cpu_ppaddr_mutex);
4033
4034 hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte);
4035 cpup->cpu_caddr2pte = 0;
4036 vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1));
4037 cpup->cpu_caddr2 = 0;
4038
4039 hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte);
4040 cpup->cpu_caddr1pte = 0;
4041 vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1));
4042 cpup->cpu_caddr1 = 0;
4043 }
4044
4045 /*
4046 * Function for flushing D-cache when performing module relocations
4047 * to an alternate mapping. Unnecessary on Intel / AMD platforms.
4048 */
4049 void
4050 dcache_flushall()
4051 {}
4052
4053 /*
4054 * Allocate a memory page. The argument 'seed' can be any pseudo-random
4055 * number to vary where the pages come from. This is quite a hacked up
4056 * method -- it works for now, but really needs to be fixed up a bit.
4057 *
4058 * We currently use page_create_va() on the kvp with fake offsets,
4059 * segments and virt address. This is pretty bogus, but was copied from the
4060 * old hat_i86.c code. A better approach would be to specify either mnode
4061 * random or mnode local and takes a page from whatever color has the MOST
4062 * available - this would have a minimal impact on page coloring.
4063 */
4064 page_t *
4065 page_get_physical(uintptr_t seed)
4066 {
4067 page_t *pp;
4068 u_offset_t offset;
4069 static struct seg tmpseg;
4070 static uintptr_t ctr = 0;
4071
4072 /*
4073 * This code is gross, we really need a simpler page allocator.
4074 *
4075 * We need to assign an offset for the page to call page_create_va()
4076 * To avoid conflicts with other pages, we get creative with the offset.
4077 * For 32 bits, we need an offset > 4Gig
4078 * For 64 bits, need an offset somewhere in the VA hole.
4079 */
4080 offset = seed;
4081 if (offset > kernelbase)
4082 offset -= kernelbase;
4083 offset <<= MMU_PAGESHIFT;
4084 #if defined(__amd64)
4085 offset += mmu.hole_start; /* something in VA hole */
4086 #else
4087 offset += 1ULL << 40; /* something > 4 Gig */
4088 #endif
4089
4090 if (page_resv(1, KM_NOSLEEP) == 0)
4091 return (NULL);
4092
4093 #ifdef DEBUG
4094 pp = page_exists(&kvp, offset);
4095 if (pp != NULL)
4096 panic("page already exists %p", (void *)pp);
4097 #endif
4098
4099 pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL,
4100 &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE)); /* changing VA usage */
4101 if (pp != NULL) {
4102 page_io_unlock(pp);
4103 page_downgrade(pp);
4104 }
4105 return (pp);
4106 }