1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright (c) 2010, Intel Corporation.
26 * All rights reserved.
27 * Copyright 2016 Joyent, Inc.
28 */
29
30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
31 /* All Rights Reserved */
32
33 /*
34 * Portions of this source code were derived from Berkeley 4.3 BSD
35 * under license from the Regents of the University of California.
36 */
37
38 /*
39 * UNIX machine dependent virtual memory support.
40 */
41
42 #include <sys/types.h>
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/user.h>
46 #include <sys/proc.h>
47 #include <sys/kmem.h>
48 #include <sys/vmem.h>
49 #include <sys/buf.h>
50 #include <sys/cpuvar.h>
51 #include <sys/lgrp.h>
52 #include <sys/disp.h>
53 #include <sys/vm.h>
54 #include <sys/mman.h>
55 #include <sys/vnode.h>
56 #include <sys/cred.h>
57 #include <sys/exec.h>
58 #include <sys/exechdr.h>
59 #include <sys/debug.h>
60 #include <sys/vmsystm.h>
61 #include <sys/swap.h>
62 #include <sys/dumphdr.h>
63 #include <sys/random.h>
64
65 #include <vm/hat.h>
66 #include <vm/as.h>
67 #include <vm/seg.h>
68 #include <vm/seg_kp.h>
69 #include <vm/seg_vn.h>
70 #include <vm/page.h>
71 #include <vm/seg_kmem.h>
72 #include <vm/seg_kpm.h>
73 #include <vm/vm_dep.h>
74
75 #include <sys/cpu.h>
76 #include <sys/vm_machparam.h>
77 #include <sys/memlist.h>
78 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */
79 #include <vm/hat_i86.h>
80 #include <sys/x86_archext.h>
81 #include <sys/elf_386.h>
82 #include <sys/cmn_err.h>
83 #include <sys/archsystm.h>
84 #include <sys/machsystm.h>
85 #include <sys/secflags.h>
86
87 #include <sys/vtrace.h>
88 #include <sys/ddidmareq.h>
89 #include <sys/promif.h>
90 #include <sys/memnode.h>
91 #include <sys/stack.h>
92 #include <util/qsort.h>
93 #include <sys/taskq.h>
94
95 #ifdef __xpv
96
97 #include <sys/hypervisor.h>
98 #include <sys/xen_mmu.h>
99 #include <sys/balloon_impl.h>
100
101 /*
102 * domain 0 pages usable for DMA are kept pre-allocated and kept in
103 * distinct lists, ordered by increasing mfn.
104 */
105 static kmutex_t io_pool_lock;
106 static kmutex_t contig_list_lock;
107 static page_t *io_pool_4g; /* pool for 32 bit dma limited devices */
108 static page_t *io_pool_16m; /* pool for 24 bit dma limited legacy devices */
109 static long io_pool_cnt;
110 static long io_pool_cnt_max = 0;
111 #define DEFAULT_IO_POOL_MIN 128
112 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN;
113 static long io_pool_cnt_lowater = 0;
114 static long io_pool_shrink_attempts; /* how many times did we try to shrink */
115 static long io_pool_shrinks; /* how many times did we really shrink */
116 static long io_pool_grows; /* how many times did we grow */
117 static mfn_t start_mfn = 1;
118 static caddr_t io_pool_kva; /* use to alloc pages when needed */
119
120 static int create_contig_pfnlist(uint_t);
121
122 /*
123 * percentage of phys mem to hold in the i/o pool
124 */
125 #define DEFAULT_IO_POOL_PCT 2
126 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT;
127 static void page_io_pool_sub(page_t **, page_t *, page_t *);
128 int ioalloc_dbg = 0;
129
130 #endif /* __xpv */
131
132 uint_t vac_colors = 1;
133
134 int largepagesupport = 0;
135 extern uint_t page_create_new;
136 extern uint_t page_create_exists;
137 extern uint_t page_create_putbacks;
138 /*
139 * Allow users to disable the kernel's use of SSE.
140 */
141 extern int use_sse_pagecopy, use_sse_pagezero;
142
143 /*
144 * combined memory ranges from mnode and memranges[] to manage single
145 * mnode/mtype dimension in the page lists.
146 */
147 typedef struct {
148 pfn_t mnr_pfnlo;
149 pfn_t mnr_pfnhi;
150 int mnr_mnode;
151 int mnr_memrange; /* index into memranges[] */
152 int mnr_next; /* next lower PA mnoderange */
153 int mnr_exists;
154 /* maintain page list stats */
155 pgcnt_t mnr_mt_clpgcnt; /* cache list cnt */
156 pgcnt_t mnr_mt_flpgcnt[MMU_PAGE_SIZES]; /* free list cnt per szc */
157 pgcnt_t mnr_mt_totcnt; /* sum of cache and free lists */
158 #ifdef DEBUG
159 struct mnr_mts { /* mnode/mtype szc stats */
160 pgcnt_t mnr_mts_pgcnt;
161 int mnr_mts_colors;
162 pgcnt_t *mnr_mtsc_pgcnt;
163 } *mnr_mts;
164 #endif
165 } mnoderange_t;
166
167 #define MEMRANGEHI(mtype) \
168 ((mtype > 0) ? memranges[mtype - 1] - 1: physmax)
169 #define MEMRANGELO(mtype) (memranges[mtype])
170
171 #define MTYPE_FREEMEM(mt) (mnoderanges[mt].mnr_mt_totcnt)
172
173 /*
174 * As the PC architecture evolved memory up was clumped into several
175 * ranges for various historical I/O devices to do DMA.
176 * < 16Meg - ISA bus
177 * < 2Gig - ???
178 * < 4Gig - PCI bus or drivers that don't understand PAE mode
179 *
180 * These are listed in reverse order, so that we can skip over unused
181 * ranges on machines with small memories.
182 *
183 * For now under the Hypervisor, we'll only ever have one memrange.
184 */
185 #define PFN_4GIG 0x100000
186 #define PFN_16MEG 0x1000
187 /* Indices into the memory range (arch_memranges) array. */
188 #define MRI_4G 0
189 #define MRI_2G 1
190 #define MRI_16M 2
191 #define MRI_0 3
192 static pfn_t arch_memranges[NUM_MEM_RANGES] = {
193 PFN_4GIG, /* pfn range for 4G and above */
194 0x80000, /* pfn range for 2G-4G */
195 PFN_16MEG, /* pfn range for 16M-2G */
196 0x00000, /* pfn range for 0-16M */
197 };
198 pfn_t *memranges = &arch_memranges[0];
199 int nranges = NUM_MEM_RANGES;
200
201 /*
202 * This combines mem_node_config and memranges into one data
203 * structure to be used for page list management.
204 */
205 mnoderange_t *mnoderanges;
206 int mnoderangecnt;
207 int mtype4g;
208 int mtype16m;
209 int mtypetop; /* index of highest pfn'ed mnoderange */
210
211 /*
212 * 4g memory management variables for systems with more than 4g of memory:
213 *
214 * physical memory below 4g is required for 32bit dma devices and, currently,
215 * for kmem memory. On systems with more than 4g of memory, the pool of memory
216 * below 4g can be depleted without any paging activity given that there is
217 * likely to be sufficient memory above 4g.
218 *
219 * physmax4g is set true if the largest pfn is over 4g. The rest of the
220 * 4g memory management code is enabled only when physmax4g is true.
221 *
222 * maxmem4g is the count of the maximum number of pages on the page lists
223 * with physical addresses below 4g. It can be a lot less then 4g given that
224 * BIOS may reserve large chunks of space below 4g for hot plug pci devices,
225 * agp aperture etc.
226 *
227 * freemem4g maintains the count of the number of available pages on the
228 * page lists with physical addresses below 4g.
229 *
230 * DESFREE4G specifies the desired amount of below 4g memory. It defaults to
231 * 6% (desfree4gshift = 4) of maxmem4g.
232 *
233 * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G
234 * and the amount of physical memory above 4g is greater than freemem4g.
235 * In this case, page_get_* routines will restrict below 4g allocations
236 * for requests that don't specifically require it.
237 */
238
239 #define DESFREE4G (maxmem4g >> desfree4gshift)
240
241 #define RESTRICT4G_ALLOC \
242 (physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem))
243
244 static pgcnt_t maxmem4g;
245 static pgcnt_t freemem4g;
246 static int physmax4g;
247 static int desfree4gshift = 4; /* maxmem4g shift to derive DESFREE4G */
248
249 /*
250 * 16m memory management:
251 *
252 * reserve some amount of physical memory below 16m for legacy devices.
253 *
254 * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above
255 * 16m or if the 16m pool drops below DESFREE16M.
256 *
257 * In this case, general page allocations via page_get_{free,cache}list
258 * routines will be restricted from allocating from the 16m pool. Allocations
259 * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations
260 * are not restricted.
261 */
262
263 #define FREEMEM16M MTYPE_FREEMEM(mtype16m)
264 #define DESFREE16M desfree16m
265 #define RESTRICT16M_ALLOC(freemem, pgcnt, flags) \
266 ((freemem != 0) && ((flags & PG_PANIC) == 0) && \
267 ((freemem >= (FREEMEM16M)) || \
268 (FREEMEM16M < (DESFREE16M + pgcnt))))
269
270 static pgcnt_t desfree16m = 0x380;
271
272 /*
273 * This can be patched via /etc/system to allow old non-PAE aware device
274 * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM.
275 */
276 int restricted_kmemalloc = 0;
277
278 #ifdef VM_STATS
279 struct {
280 ulong_t pga_alloc;
281 ulong_t pga_notfullrange;
282 ulong_t pga_nulldmaattr;
283 ulong_t pga_allocok;
284 ulong_t pga_allocfailed;
285 ulong_t pgma_alloc;
286 ulong_t pgma_allocok;
287 ulong_t pgma_allocfailed;
288 ulong_t pgma_allocempty;
289 } pga_vmstats;
290 #endif
291
292 uint_t mmu_page_sizes;
293
294 /* How many page sizes the users can see */
295 uint_t mmu_exported_page_sizes;
296
297 /* page sizes that legacy applications can see */
298 uint_t mmu_legacy_page_sizes;
299
300 /*
301 * Number of pages in 1 GB. Don't enable automatic large pages if we have
302 * fewer than this many pages.
303 */
304 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
305 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
306
307 /*
308 * Maximum and default segment size tunables for user private
309 * and shared anon memory, and user text and initialized data.
310 * These can be patched via /etc/system to allow large pages
311 * to be used for mapping application private and shared anon memory.
312 */
313 size_t mcntl0_lpsize = MMU_PAGESIZE;
314 size_t max_uheap_lpsize = MMU_PAGESIZE;
315 size_t default_uheap_lpsize = MMU_PAGESIZE;
316 size_t max_ustack_lpsize = MMU_PAGESIZE;
317 size_t default_ustack_lpsize = MMU_PAGESIZE;
318 size_t max_privmap_lpsize = MMU_PAGESIZE;
319 size_t max_uidata_lpsize = MMU_PAGESIZE;
320 size_t max_utext_lpsize = MMU_PAGESIZE;
321 size_t max_shm_lpsize = MMU_PAGESIZE;
322
323
324 /*
325 * initialized by page_coloring_init().
326 */
327 uint_t page_colors;
328 uint_t page_colors_mask;
329 uint_t page_coloring_shift;
330 int cpu_page_colors;
331 static uint_t l2_colors;
332
333 /*
334 * Page freelists and cachelists are dynamically allocated once mnoderangecnt
335 * and page_colors are calculated from the l2 cache n-way set size. Within a
336 * mnode range, the page freelist and cachelist are hashed into bins based on
337 * color. This makes it easier to search for a page within a specific memory
338 * range.
339 */
340 #define PAGE_COLORS_MIN 16
341
342 page_t ****page_freelists;
343 page_t ***page_cachelists;
344
345
346 /*
347 * Used by page layer to know about page sizes
348 */
349 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1];
350
351 kmutex_t *fpc_mutex[NPC_MUTEX];
352 kmutex_t *cpc_mutex[NPC_MUTEX];
353
354 /* Lock to protect mnoderanges array for memory DR operations. */
355 static kmutex_t mnoderange_lock;
356
357 /*
358 * Only let one thread at a time try to coalesce large pages, to
359 * prevent them from working against each other.
360 */
361 static kmutex_t contig_lock;
362 #define CONTIG_LOCK() mutex_enter(&contig_lock);
363 #define CONTIG_UNLOCK() mutex_exit(&contig_lock);
364
365 #define PFN_16M (mmu_btop((uint64_t)0x1000000))
366
367 /*
368 * Return the optimum page size for a given mapping
369 */
370 /*ARGSUSED*/
371 size_t
372 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl)
373 {
374 level_t l = 0;
375 size_t pgsz = MMU_PAGESIZE;
376 size_t max_lpsize;
377 uint_t mszc;
378
379 ASSERT(maptype != MAPPGSZ_VA);
380
381 if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) {
382 return (MMU_PAGESIZE);
383 }
384
385 switch (maptype) {
386 case MAPPGSZ_HEAP:
387 case MAPPGSZ_STK:
388 max_lpsize = memcntl ? mcntl0_lpsize : (maptype ==
389 MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize);
390 if (max_lpsize == MMU_PAGESIZE) {
391 return (MMU_PAGESIZE);
392 }
393 if (len == 0) {
394 len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase +
395 p->p_brksize - p->p_bssbase : p->p_stksize;
396 }
397 len = (maptype == MAPPGSZ_HEAP) ? MAX(len,
398 default_uheap_lpsize) : MAX(len, default_ustack_lpsize);
399
400 /*
401 * use the pages size that best fits len
402 */
403 for (l = mmu.umax_page_level; l > 0; --l) {
404 if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) {
405 continue;
406 } else {
407 pgsz = LEVEL_SIZE(l);
408 }
409 break;
410 }
411
412 mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc :
413 p->p_stkpageszc);
414 if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) {
415 pgsz = hw_page_array[mszc].hp_size;
416 }
417 return (pgsz);
418
419 case MAPPGSZ_ISM:
420 for (l = mmu.umax_page_level; l > 0; --l) {
421 if (len >= LEVEL_SIZE(l))
422 return (LEVEL_SIZE(l));
423 }
424 return (LEVEL_SIZE(0));
425 }
426 return (pgsz);
427 }
428
429 static uint_t
430 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize,
431 size_t min_physmem)
432 {
433 caddr_t eaddr = addr + size;
434 uint_t szcvec = 0;
435 caddr_t raddr;
436 caddr_t readdr;
437 size_t pgsz;
438 int i;
439
440 if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) {
441 return (0);
442 }
443
444 for (i = mmu_exported_page_sizes - 1; i > 0; i--) {
445 pgsz = page_get_pagesize(i);
446 if (pgsz > max_lpsize) {
447 continue;
448 }
449 raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
450 readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
451 if (raddr < addr || raddr >= readdr) {
452 continue;
453 }
454 if (P2PHASE((uintptr_t)addr ^ off, pgsz)) {
455 continue;
456 }
457 /*
458 * Set szcvec to the remaining page sizes.
459 */
460 szcvec = ((1 << (i + 1)) - 1) & ~1;
461 break;
462 }
463 return (szcvec);
464 }
465
466 /*
467 * Return a bit vector of large page size codes that
468 * can be used to map [addr, addr + len) region.
469 */
470 /*ARGSUSED*/
471 uint_t
472 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type,
473 int memcntl)
474 {
475 size_t max_lpsize = mcntl0_lpsize;
476
477 if (mmu.max_page_level == 0)
478 return (0);
479
480 if (flags & MAP_TEXT) {
481 if (!memcntl)
482 max_lpsize = max_utext_lpsize;
483 return (map_szcvec(addr, size, off, max_lpsize,
484 shm_lpg_min_physmem));
485
486 } else if (flags & MAP_INITDATA) {
487 if (!memcntl)
488 max_lpsize = max_uidata_lpsize;
489 return (map_szcvec(addr, size, off, max_lpsize,
490 privm_lpg_min_physmem));
491
492 } else if (type == MAPPGSZC_SHM) {
493 if (!memcntl)
494 max_lpsize = max_shm_lpsize;
495 return (map_szcvec(addr, size, off, max_lpsize,
496 shm_lpg_min_physmem));
497
498 } else if (type == MAPPGSZC_HEAP) {
499 if (!memcntl)
500 max_lpsize = max_uheap_lpsize;
501 return (map_szcvec(addr, size, off, max_lpsize,
502 privm_lpg_min_physmem));
503
504 } else if (type == MAPPGSZC_STACK) {
505 if (!memcntl)
506 max_lpsize = max_ustack_lpsize;
507 return (map_szcvec(addr, size, off, max_lpsize,
508 privm_lpg_min_physmem));
509
510 } else {
511 if (!memcntl)
512 max_lpsize = max_privmap_lpsize;
513 return (map_szcvec(addr, size, off, max_lpsize,
514 privm_lpg_min_physmem));
515 }
516 }
517
518 /*
519 * Handle a pagefault.
520 */
521 faultcode_t
522 pagefault(
523 caddr_t addr,
524 enum fault_type type,
525 enum seg_rw rw,
526 int iskernel)
527 {
528 struct as *as;
529 struct hat *hat;
530 struct proc *p;
531 kthread_t *t;
532 faultcode_t res;
533 caddr_t base;
534 size_t len;
535 int err;
536 int mapped_red;
537 uintptr_t ea;
538
539 ASSERT_STACK_ALIGNED();
540
541 if (INVALID_VADDR(addr))
542 return (FC_NOMAP);
543
544 mapped_red = segkp_map_red();
545
546 if (iskernel) {
547 as = &kas;
548 hat = as->a_hat;
549 } else {
550 t = curthread;
551 p = ttoproc(t);
552 as = p->p_as;
553 hat = as->a_hat;
554 }
555
556 /*
557 * Dispatch pagefault.
558 */
559 res = as_fault(hat, as, addr, 1, type, rw);
560
561 /*
562 * If this isn't a potential unmapped hole in the user's
563 * UNIX data or stack segments, just return status info.
564 */
565 if (res != FC_NOMAP || iskernel)
566 goto out;
567
568 /*
569 * Check to see if we happened to faulted on a currently unmapped
570 * part of the UNIX data or stack segments. If so, create a zfod
571 * mapping there and then try calling the fault routine again.
572 */
573 base = p->p_brkbase;
574 len = p->p_brksize;
575
576 if (addr < base || addr >= base + len) { /* data seg? */
577 base = (caddr_t)p->p_usrstack - p->p_stksize;
578 len = p->p_stksize;
579 if (addr < base || addr >= p->p_usrstack) { /* stack seg? */
580 /* not in either UNIX data or stack segments */
581 res = FC_NOMAP;
582 goto out;
583 }
584 }
585
586 /*
587 * the rest of this function implements a 3.X 4.X 5.X compatibility
588 * This code is probably not needed anymore
589 */
590 if (p->p_model == DATAMODEL_ILP32) {
591
592 /* expand the gap to the page boundaries on each side */
593 ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE);
594 base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE);
595 len = ea - (uintptr_t)base;
596
597 as_rangelock(as);
598 if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) ==
599 0) {
600 err = as_map(as, base, len, segvn_create, zfod_argsp);
601 as_rangeunlock(as);
602 if (err) {
603 res = FC_MAKE_ERR(err);
604 goto out;
605 }
606 } else {
607 /*
608 * This page is already mapped by another thread after
609 * we returned from as_fault() above. We just fall
610 * through as_fault() below.
611 */
612 as_rangeunlock(as);
613 }
614
615 res = as_fault(hat, as, addr, 1, F_INVAL, rw);
616 }
617
618 out:
619 if (mapped_red)
620 segkp_unmap_red();
621
622 return (res);
623 }
624
625 void
626 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags)
627 {
628 struct proc *p = curproc;
629 caddr_t userlimit = (flags & _MAP_LOW32) ?
630 (caddr_t)_userlimit32 : p->p_as->a_userlimit;
631
632 map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags);
633 }
634
635 /*ARGSUSED*/
636 int
637 map_addr_vacalign_check(caddr_t addr, u_offset_t off)
638 {
639 return (0);
640 }
641
642 /*
643 * The maximum amount a randomized mapping will be slewed. We should perhaps
644 * arrange things so these tunables can be separate for mmap, mmapobj, and
645 * ld.so
646 */
647 size_t aslr_max_map_skew = 256 * 1024 * 1024; /* 256MB */
648
649 /*
650 * map_addr_proc() is the routine called when the system is to
651 * choose an address for the user. We will pick an address
652 * range which is the highest available below userlimit.
653 *
654 * Every mapping will have a redzone of a single page on either side of
655 * the request. This is done to leave one page unmapped between segments.
656 * This is not required, but it's useful for the user because if their
657 * program strays across a segment boundary, it will catch a fault
658 * immediately making debugging a little easier. Currently the redzone
659 * is mandatory.
660 *
661 * addrp is a value/result parameter.
662 * On input it is a hint from the user to be used in a completely
663 * machine dependent fashion. We decide to completely ignore this hint.
664 * If MAP_ALIGN was specified, addrp contains the minimal alignment, which
665 * must be some "power of two" multiple of pagesize.
666 *
667 * On output it is NULL if no address can be found in the current
668 * processes address space or else an address that is currently
669 * not mapped for len bytes with a page of red zone on either side.
670 *
671 * vacalign is not needed on x86 (it's for viturally addressed caches)
672 */
673 /*ARGSUSED*/
674 void
675 map_addr_proc(
676 caddr_t *addrp,
677 size_t len,
678 offset_t off,
679 int vacalign,
680 caddr_t userlimit,
681 struct proc *p,
682 uint_t flags)
683 {
684 struct as *as = p->p_as;
685 caddr_t addr;
686 caddr_t base;
687 size_t slen;
688 size_t align_amount;
689
690 ASSERT32(userlimit == as->a_userlimit);
691
692 base = p->p_brkbase;
693 #if defined(__amd64)
694 /*
695 * XX64 Yes, this needs more work.
696 */
697 if (p->p_model == DATAMODEL_NATIVE) {
698 if (userlimit < as->a_userlimit) {
699 /*
700 * This happens when a program wants to map
701 * something in a range that's accessible to a
702 * program in a smaller address space. For example,
703 * a 64-bit program calling mmap32(2) to guarantee
704 * that the returned address is below 4Gbytes.
705 */
706 ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff));
707
708 if (userlimit > base)
709 slen = userlimit - base;
710 else {
711 *addrp = NULL;
712 return;
713 }
714 } else {
715 /*
716 * XX64 This layout is probably wrong .. but in
717 * the event we make the amd64 address space look
718 * like sparcv9 i.e. with the stack -above- the
719 * heap, this bit of code might even be correct.
720 */
721 slen = p->p_usrstack - base -
722 ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK);
723 }
724 } else
725 #endif
726 slen = userlimit - base;
727
728 /* Make len be a multiple of PAGESIZE */
729 len = (len + PAGEOFFSET) & PAGEMASK;
730
731 /*
732 * figure out what the alignment should be
733 *
734 * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same????
735 */
736 if (len <= ELF_386_MAXPGSZ) {
737 /*
738 * Align virtual addresses to ensure that ELF shared libraries
739 * are mapped with the appropriate alignment constraints by
740 * the run-time linker.
741 */
742 align_amount = ELF_386_MAXPGSZ;
743 } else {
744 /*
745 * For 32-bit processes, only those which have specified
746 * MAP_ALIGN and an addr will be aligned on a larger page size.
747 * Not doing so can potentially waste up to 1G of process
748 * address space.
749 */
750 int lvl = (p->p_model == DATAMODEL_ILP32) ? 1 :
751 mmu.umax_page_level;
752
753 while (lvl && len < LEVEL_SIZE(lvl))
754 --lvl;
755
756 align_amount = LEVEL_SIZE(lvl);
757 }
758 if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount))
759 align_amount = (uintptr_t)*addrp;
760
761 ASSERT(ISP2(align_amount));
762 ASSERT(align_amount == 0 || align_amount >= PAGESIZE);
763
764 off = off & (align_amount - 1);
765
766 /*
767 * Look for a large enough hole starting below userlimit.
768 * After finding it, use the upper part.
769 */
770 if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount,
771 PAGESIZE, off) == 0) {
772 caddr_t as_addr;
773
774 /*
775 * addr is the highest possible address to use since we have
776 * a PAGESIZE redzone at the beginning and end.
777 */
778 addr = base + slen - (PAGESIZE + len);
779 as_addr = addr;
780 /*
781 * Round address DOWN to the alignment amount and
782 * add the offset in.
783 * If addr is greater than as_addr, len would not be large
784 * enough to include the redzone, so we must adjust down
785 * by the alignment amount.
786 */
787 addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1)));
788 addr += (uintptr_t)off;
789 if (addr > as_addr) {
790 addr -= align_amount;
791 }
792
793 /*
794 * If randomization is requested, slew the allocation
795 * backwards, within the same gap, by a random amount.
796 */
797 if (flags & _MAP_RANDOMIZE) {
798 uint32_t slew;
799
800 (void) random_get_pseudo_bytes((uint8_t *)&slew,
801 sizeof (slew));
802
803 slew = slew % MIN(aslr_max_map_skew, (addr - base));
804 addr -= P2ALIGN(slew, align_amount);
805 }
806
807 ASSERT(addr > base);
808 ASSERT(addr + len < base + slen);
809 ASSERT(((uintptr_t)addr & (align_amount - 1)) ==
810 ((uintptr_t)(off)));
811 *addrp = addr;
812 } else {
813 *addrp = NULL; /* no more virtual space */
814 }
815 }
816
817 int valid_va_range_aligned_wraparound;
818
819 /*
820 * Determine whether [*basep, *basep + *lenp) contains a mappable range of
821 * addresses at least "minlen" long, where the base of the range is at "off"
822 * phase from an "align" boundary and there is space for a "redzone"-sized
823 * redzone on either side of the range. On success, 1 is returned and *basep
824 * and *lenp are adjusted to describe the acceptable range (including
825 * the redzone). On failure, 0 is returned.
826 */
827 /*ARGSUSED3*/
828 int
829 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir,
830 size_t align, size_t redzone, size_t off)
831 {
832 uintptr_t hi, lo;
833 size_t tot_len;
834
835 ASSERT(align == 0 ? off == 0 : off < align);
836 ASSERT(ISP2(align));
837 ASSERT(align == 0 || align >= PAGESIZE);
838
839 lo = (uintptr_t)*basep;
840 hi = lo + *lenp;
841 tot_len = minlen + 2 * redzone; /* need at least this much space */
842
843 /*
844 * If hi rolled over the top, try cutting back.
845 */
846 if (hi < lo) {
847 *lenp = 0UL - lo - 1UL;
848 /* See if this really happens. If so, then we figure out why */
849 valid_va_range_aligned_wraparound++;
850 hi = lo + *lenp;
851 }
852 if (*lenp < tot_len) {
853 return (0);
854 }
855
856 #if defined(__amd64)
857 /*
858 * Deal with a possible hole in the address range between
859 * hole_start and hole_end that should never be mapped.
860 */
861 if (lo < hole_start) {
862 if (hi > hole_start) {
863 if (hi < hole_end) {
864 hi = hole_start;
865 } else {
866 /* lo < hole_start && hi >= hole_end */
867 if (dir == AH_LO) {
868 /*
869 * prefer lowest range
870 */
871 if (hole_start - lo >= tot_len)
872 hi = hole_start;
873 else if (hi - hole_end >= tot_len)
874 lo = hole_end;
875 else
876 return (0);
877 } else {
878 /*
879 * prefer highest range
880 */
881 if (hi - hole_end >= tot_len)
882 lo = hole_end;
883 else if (hole_start - lo >= tot_len)
884 hi = hole_start;
885 else
886 return (0);
887 }
888 }
889 }
890 } else {
891 /* lo >= hole_start */
892 if (hi < hole_end)
893 return (0);
894 if (lo < hole_end)
895 lo = hole_end;
896 }
897 #endif
898
899 if (hi - lo < tot_len)
900 return (0);
901
902 if (align > 1) {
903 uintptr_t tlo = lo + redzone;
904 uintptr_t thi = hi - redzone;
905 tlo = (uintptr_t)P2PHASEUP(tlo, align, off);
906 if (tlo < lo + redzone) {
907 return (0);
908 }
909 if (thi < tlo || thi - tlo < minlen) {
910 return (0);
911 }
912 }
913
914 *basep = (caddr_t)lo;
915 *lenp = hi - lo;
916 return (1);
917 }
918
919 /*
920 * Determine whether [*basep, *basep + *lenp) contains a mappable range of
921 * addresses at least "minlen" long. On success, 1 is returned and *basep
922 * and *lenp are adjusted to describe the acceptable range. On failure, 0
923 * is returned.
924 */
925 int
926 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir)
927 {
928 return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0));
929 }
930
931 /*
932 * Default to forbidding the first 64k of address space. This protects most
933 * reasonably sized structures from dereferences through NULL:
934 * ((foo_t *)0)->bar
935 */
936 uintptr_t forbidden_null_mapping_sz = 0x10000;
937
938 /*
939 * Determine whether [addr, addr+len] are valid user addresses.
940 */
941 /*ARGSUSED*/
942 int
943 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as,
944 caddr_t userlimit)
945 {
946 caddr_t eaddr = addr + len;
947
948 if (eaddr <= addr || addr >= userlimit || eaddr > userlimit)
949 return (RANGE_BADADDR);
950
951 if ((addr <= (caddr_t)forbidden_null_mapping_sz) &&
952 as->a_proc != NULL &&
953 secflag_enabled(as->a_proc, PROC_SEC_FORBIDNULLMAP))
954 return (RANGE_BADADDR);
955
956 #if defined(__amd64)
957 /*
958 * Check for the VA hole
959 */
960 if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end)
961 return (RANGE_BADADDR);
962 #endif
963
964 return (RANGE_OKAY);
965 }
966
967 /*
968 * Return 1 if the page frame is onboard memory, else 0.
969 */
970 int
971 pf_is_memory(pfn_t pf)
972 {
973 if (pfn_is_foreign(pf))
974 return (0);
975 return (address_in_memlist(phys_install, pfn_to_pa(pf), 1));
976 }
977
978 /*
979 * return the memrange containing pfn
980 */
981 int
982 memrange_num(pfn_t pfn)
983 {
984 int n;
985
986 for (n = 0; n < nranges - 1; ++n) {
987 if (pfn >= memranges[n])
988 break;
989 }
990 return (n);
991 }
992
993 /*
994 * return the mnoderange containing pfn
995 */
996 /*ARGSUSED*/
997 int
998 pfn_2_mtype(pfn_t pfn)
999 {
1000 #if defined(__xpv)
1001 return (0);
1002 #else
1003 int n;
1004
1005 /* Always start from highest pfn and work our way down */
1006 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1007 if (pfn >= mnoderanges[n].mnr_pfnlo) {
1008 break;
1009 }
1010 }
1011 return (n);
1012 #endif
1013 }
1014
1015 #if !defined(__xpv)
1016 /*
1017 * is_contigpage_free:
1018 * returns a page list of contiguous pages. It minimally has to return
1019 * minctg pages. Caller determines minctg based on the scatter-gather
1020 * list length.
1021 *
1022 * pfnp is set to the next page frame to search on return.
1023 */
1024 static page_t *
1025 is_contigpage_free(
1026 pfn_t *pfnp,
1027 pgcnt_t *pgcnt,
1028 pgcnt_t minctg,
1029 uint64_t pfnseg,
1030 int iolock)
1031 {
1032 int i = 0;
1033 pfn_t pfn = *pfnp;
1034 page_t *pp;
1035 page_t *plist = NULL;
1036
1037 /*
1038 * fail if pfn + minctg crosses a segment boundary.
1039 * Adjust for next starting pfn to begin at segment boundary.
1040 */
1041
1042 if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) {
1043 *pfnp = roundup(*pfnp, pfnseg + 1);
1044 return (NULL);
1045 }
1046
1047 do {
1048 retry:
1049 pp = page_numtopp_nolock(pfn + i);
1050 if ((pp == NULL) || IS_DUMP_PAGE(pp) ||
1051 (page_trylock(pp, SE_EXCL) == 0)) {
1052 (*pfnp)++;
1053 break;
1054 }
1055 if (page_pptonum(pp) != pfn + i) {
1056 page_unlock(pp);
1057 goto retry;
1058 }
1059
1060 if (!(PP_ISFREE(pp))) {
1061 page_unlock(pp);
1062 (*pfnp)++;
1063 break;
1064 }
1065
1066 if (!PP_ISAGED(pp)) {
1067 page_list_sub(pp, PG_CACHE_LIST);
1068 page_hashout(pp, (kmutex_t *)NULL);
1069 } else {
1070 page_list_sub(pp, PG_FREE_LIST);
1071 }
1072
1073 if (iolock)
1074 page_io_lock(pp);
1075 page_list_concat(&plist, &pp);
1076
1077 /*
1078 * exit loop when pgcnt satisfied or segment boundary reached.
1079 */
1080
1081 } while ((++i < *pgcnt) && ((pfn + i) & pfnseg));
1082
1083 *pfnp += i; /* set to next pfn to search */
1084
1085 if (i >= minctg) {
1086 *pgcnt -= i;
1087 return (plist);
1088 }
1089
1090 /*
1091 * failure: minctg not satisfied.
1092 *
1093 * if next request crosses segment boundary, set next pfn
1094 * to search from the segment boundary.
1095 */
1096 if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg))
1097 *pfnp = roundup(*pfnp, pfnseg + 1);
1098
1099 /* clean up any pages already allocated */
1100
1101 while (plist) {
1102 pp = plist;
1103 page_sub(&plist, pp);
1104 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
1105 if (iolock)
1106 page_io_unlock(pp);
1107 page_unlock(pp);
1108 }
1109
1110 return (NULL);
1111 }
1112 #endif /* !__xpv */
1113
1114 /*
1115 * verify that pages being returned from allocator have correct DMA attribute
1116 */
1117 #ifndef DEBUG
1118 #define check_dma(a, b, c) (void)(0)
1119 #else
1120 static void
1121 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt)
1122 {
1123 if (dma_attr == NULL)
1124 return;
1125
1126 while (cnt-- > 0) {
1127 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) <
1128 dma_attr->dma_attr_addr_lo)
1129 panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp);
1130 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >=
1131 dma_attr->dma_attr_addr_hi)
1132 panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp);
1133 pp = pp->p_next;
1134 }
1135 }
1136 #endif
1137
1138 #if !defined(__xpv)
1139 static page_t *
1140 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock)
1141 {
1142 pfn_t pfn;
1143 int sgllen;
1144 uint64_t pfnseg;
1145 pgcnt_t minctg;
1146 page_t *pplist = NULL, *plist;
1147 uint64_t lo, hi;
1148 pgcnt_t pfnalign = 0;
1149 static pfn_t startpfn;
1150 static pgcnt_t lastctgcnt;
1151 uintptr_t align;
1152
1153 CONTIG_LOCK();
1154
1155 if (mattr) {
1156 lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET));
1157 hi = mmu_btop(mattr->dma_attr_addr_hi);
1158 if (hi >= physmax)
1159 hi = physmax - 1;
1160 sgllen = mattr->dma_attr_sgllen;
1161 pfnseg = mmu_btop(mattr->dma_attr_seg);
1162
1163 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
1164 if (align > MMU_PAGESIZE)
1165 pfnalign = mmu_btop(align);
1166
1167 /*
1168 * in order to satisfy the request, must minimally
1169 * acquire minctg contiguous pages
1170 */
1171 minctg = howmany(*pgcnt, sgllen);
1172
1173 ASSERT(hi >= lo);
1174
1175 /*
1176 * start from where last searched if the minctg >= lastctgcnt
1177 */
1178 if (minctg < lastctgcnt || startpfn < lo || startpfn > hi)
1179 startpfn = lo;
1180 } else {
1181 hi = physmax - 1;
1182 lo = 0;
1183 sgllen = 1;
1184 pfnseg = mmu.highest_pfn;
1185 minctg = *pgcnt;
1186
1187 if (minctg < lastctgcnt)
1188 startpfn = lo;
1189 }
1190 lastctgcnt = minctg;
1191
1192 ASSERT(pfnseg + 1 >= (uint64_t)minctg);
1193
1194 /* conserve 16m memory - start search above 16m when possible */
1195 if (hi > PFN_16M && startpfn < PFN_16M)
1196 startpfn = PFN_16M;
1197
1198 pfn = startpfn;
1199 if (pfnalign)
1200 pfn = P2ROUNDUP(pfn, pfnalign);
1201
1202 while (pfn + minctg - 1 <= hi) {
1203
1204 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1205 if (plist) {
1206 page_list_concat(&pplist, &plist);
1207 sgllen--;
1208 /*
1209 * return when contig pages no longer needed
1210 */
1211 if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1212 startpfn = pfn;
1213 CONTIG_UNLOCK();
1214 check_dma(mattr, pplist, *pgcnt);
1215 return (pplist);
1216 }
1217 minctg = howmany(*pgcnt, sgllen);
1218 }
1219 if (pfnalign)
1220 pfn = P2ROUNDUP(pfn, pfnalign);
1221 }
1222
1223 /* cannot find contig pages in specified range */
1224 if (startpfn == lo) {
1225 CONTIG_UNLOCK();
1226 return (NULL);
1227 }
1228
1229 /* did not start with lo previously */
1230 pfn = lo;
1231 if (pfnalign)
1232 pfn = P2ROUNDUP(pfn, pfnalign);
1233
1234 /* allow search to go above startpfn */
1235 while (pfn < startpfn) {
1236
1237 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1238 if (plist != NULL) {
1239
1240 page_list_concat(&pplist, &plist);
1241 sgllen--;
1242
1243 /*
1244 * return when contig pages no longer needed
1245 */
1246 if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1247 startpfn = pfn;
1248 CONTIG_UNLOCK();
1249 check_dma(mattr, pplist, *pgcnt);
1250 return (pplist);
1251 }
1252 minctg = howmany(*pgcnt, sgllen);
1253 }
1254 if (pfnalign)
1255 pfn = P2ROUNDUP(pfn, pfnalign);
1256 }
1257 CONTIG_UNLOCK();
1258 return (NULL);
1259 }
1260 #endif /* !__xpv */
1261
1262 /*
1263 * mnode_range_cnt() calculates the number of memory ranges for mnode and
1264 * memranges[]. Used to determine the size of page lists and mnoderanges.
1265 */
1266 int
1267 mnode_range_cnt(int mnode)
1268 {
1269 #if defined(__xpv)
1270 ASSERT(mnode == 0);
1271 return (1);
1272 #else /* __xpv */
1273 int mri;
1274 int mnrcnt = 0;
1275
1276 if (mem_node_config[mnode].exists != 0) {
1277 mri = nranges - 1;
1278
1279 /* find the memranges index below contained in mnode range */
1280
1281 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1282 mri--;
1283
1284 /*
1285 * increment mnode range counter when memranges or mnode
1286 * boundary is reached.
1287 */
1288 while (mri >= 0 &&
1289 mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1290 mnrcnt++;
1291 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1292 mri--;
1293 else
1294 break;
1295 }
1296 }
1297 ASSERT(mnrcnt <= MAX_MNODE_MRANGES);
1298 return (mnrcnt);
1299 #endif /* __xpv */
1300 }
1301
1302 /*
1303 * mnode_range_setup() initializes mnoderanges.
1304 */
1305 void
1306 mnode_range_setup(mnoderange_t *mnoderanges)
1307 {
1308 mnoderange_t *mp = mnoderanges;
1309 int mnode, mri;
1310 int mindex = 0; /* current index into mnoderanges array */
1311 int i, j;
1312 pfn_t hipfn;
1313 int last, hi;
1314
1315 for (mnode = 0; mnode < max_mem_nodes; mnode++) {
1316 if (mem_node_config[mnode].exists == 0)
1317 continue;
1318
1319 mri = nranges - 1;
1320
1321 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1322 mri--;
1323
1324 while (mri >= 0 && mem_node_config[mnode].physmax >=
1325 MEMRANGELO(mri)) {
1326 mnoderanges->mnr_pfnlo = MAX(MEMRANGELO(mri),
1327 mem_node_config[mnode].physbase);
1328 mnoderanges->mnr_pfnhi = MIN(MEMRANGEHI(mri),
1329 mem_node_config[mnode].physmax);
1330 mnoderanges->mnr_mnode = mnode;
1331 mnoderanges->mnr_memrange = mri;
1332 mnoderanges->mnr_exists = 1;
1333 mnoderanges++;
1334 mindex++;
1335 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1336 mri--;
1337 else
1338 break;
1339 }
1340 }
1341
1342 /*
1343 * For now do a simple sort of the mnoderanges array to fill in
1344 * the mnr_next fields. Since mindex is expected to be relatively
1345 * small, using a simple O(N^2) algorithm.
1346 */
1347 for (i = 0; i < mindex; i++) {
1348 if (mp[i].mnr_pfnlo == 0) /* find lowest */
1349 break;
1350 }
1351 ASSERT(i < mindex);
1352 last = i;
1353 mtype16m = last;
1354 mp[last].mnr_next = -1;
1355 for (i = 0; i < mindex - 1; i++) {
1356 hipfn = (pfn_t)(-1);
1357 hi = -1;
1358 /* find next highest mnode range */
1359 for (j = 0; j < mindex; j++) {
1360 if (mp[j].mnr_pfnlo > mp[last].mnr_pfnlo &&
1361 mp[j].mnr_pfnlo < hipfn) {
1362 hipfn = mp[j].mnr_pfnlo;
1363 hi = j;
1364 }
1365 }
1366 mp[hi].mnr_next = last;
1367 last = hi;
1368 }
1369 mtypetop = last;
1370 }
1371
1372 #ifndef __xpv
1373 /*
1374 * Update mnoderanges for memory hot-add DR operations.
1375 */
1376 static void
1377 mnode_range_add(int mnode)
1378 {
1379 int *prev;
1380 int n, mri;
1381 pfn_t start, end;
1382 extern void membar_sync(void);
1383
1384 ASSERT(0 <= mnode && mnode < max_mem_nodes);
1385 ASSERT(mem_node_config[mnode].exists);
1386 start = mem_node_config[mnode].physbase;
1387 end = mem_node_config[mnode].physmax;
1388 ASSERT(start <= end);
1389 mutex_enter(&mnoderange_lock);
1390
1391 #ifdef DEBUG
1392 /* Check whether it interleaves with other memory nodes. */
1393 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1394 ASSERT(mnoderanges[n].mnr_exists);
1395 if (mnoderanges[n].mnr_mnode == mnode)
1396 continue;
1397 ASSERT(start > mnoderanges[n].mnr_pfnhi ||
1398 end < mnoderanges[n].mnr_pfnlo);
1399 }
1400 #endif /* DEBUG */
1401
1402 mri = nranges - 1;
1403 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1404 mri--;
1405 while (mri >= 0 && mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1406 /* Check whether mtype already exists. */
1407 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1408 if (mnoderanges[n].mnr_mnode == mnode &&
1409 mnoderanges[n].mnr_memrange == mri) {
1410 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri),
1411 start);
1412 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri),
1413 end);
1414 break;
1415 }
1416 }
1417
1418 /* Add a new entry if it doesn't exist yet. */
1419 if (n == -1) {
1420 /* Try to find an unused entry in mnoderanges array. */
1421 for (n = 0; n < mnoderangecnt; n++) {
1422 if (mnoderanges[n].mnr_exists == 0)
1423 break;
1424 }
1425 ASSERT(n < mnoderangecnt);
1426 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), start);
1427 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), end);
1428 mnoderanges[n].mnr_mnode = mnode;
1429 mnoderanges[n].mnr_memrange = mri;
1430 mnoderanges[n].mnr_exists = 1;
1431 /* Page 0 should always be present. */
1432 for (prev = &mtypetop;
1433 mnoderanges[*prev].mnr_pfnlo > start;
1434 prev = &mnoderanges[*prev].mnr_next) {
1435 ASSERT(mnoderanges[*prev].mnr_next >= 0);
1436 ASSERT(mnoderanges[*prev].mnr_pfnlo > end);
1437 }
1438 mnoderanges[n].mnr_next = *prev;
1439 membar_sync();
1440 *prev = n;
1441 }
1442
1443 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1444 mri--;
1445 else
1446 break;
1447 }
1448
1449 mutex_exit(&mnoderange_lock);
1450 }
1451
1452 /*
1453 * Update mnoderanges for memory hot-removal DR operations.
1454 */
1455 static void
1456 mnode_range_del(int mnode)
1457 {
1458 _NOTE(ARGUNUSED(mnode));
1459 ASSERT(0 <= mnode && mnode < max_mem_nodes);
1460 /* TODO: support deletion operation. */
1461 ASSERT(0);
1462 }
1463
1464 void
1465 plat_slice_add(pfn_t start, pfn_t end)
1466 {
1467 mem_node_add_slice(start, end);
1468 if (plat_dr_enabled()) {
1469 mnode_range_add(PFN_2_MEM_NODE(start));
1470 }
1471 }
1472
1473 void
1474 plat_slice_del(pfn_t start, pfn_t end)
1475 {
1476 ASSERT(PFN_2_MEM_NODE(start) == PFN_2_MEM_NODE(end));
1477 ASSERT(plat_dr_enabled());
1478 mnode_range_del(PFN_2_MEM_NODE(start));
1479 mem_node_del_slice(start, end);
1480 }
1481 #endif /* __xpv */
1482
1483 /*ARGSUSED*/
1484 int
1485 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz)
1486 {
1487 int mtype = mtypetop;
1488
1489 #if !defined(__xpv)
1490 #if defined(__i386)
1491 /*
1492 * set the mtype range
1493 * - kmem requests need to be below 4g if restricted_kmemalloc is set.
1494 * - for non kmem requests, set range to above 4g if memory below 4g
1495 * runs low.
1496 */
1497 if (restricted_kmemalloc && VN_ISKAS(vp) &&
1498 (caddr_t)(vaddr) >= kernelheap &&
1499 (caddr_t)(vaddr) < ekernelheap) {
1500 ASSERT(physmax4g);
1501 mtype = mtype4g;
1502 if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz),
1503 btop(pgsz), *flags)) {
1504 *flags |= PGI_MT_RANGE16M;
1505 } else {
1506 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1507 VM_STAT_COND_ADD((*flags & PG_PANIC),
1508 vmm_vmstats.pgpanicalloc);
1509 *flags |= PGI_MT_RANGE0;
1510 }
1511 return (mtype);
1512 }
1513 #endif /* __i386 */
1514
1515 if (RESTRICT4G_ALLOC) {
1516 VM_STAT_ADD(vmm_vmstats.restrict4gcnt);
1517 /* here only for > 4g systems */
1518 *flags |= PGI_MT_RANGE4G;
1519 } else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) {
1520 *flags |= PGI_MT_RANGE16M;
1521 } else {
1522 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1523 VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc);
1524 *flags |= PGI_MT_RANGE0;
1525 }
1526 #endif /* !__xpv */
1527 return (mtype);
1528 }
1529
1530
1531 /* mtype init for page_get_replacement_page */
1532 /*ARGSUSED*/
1533 int
1534 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt)
1535 {
1536 int mtype = mtypetop;
1537 #if !defined(__xpv)
1538 if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) {
1539 *flags |= PGI_MT_RANGE16M;
1540 } else {
1541 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1542 *flags |= PGI_MT_RANGE0;
1543 }
1544 #endif
1545 return (mtype);
1546 }
1547
1548 /*
1549 * Determine if the mnode range specified in mtype contains memory belonging
1550 * to memory node mnode. If flags & PGI_MT_RANGE is set then mtype contains
1551 * the range from high pfn to 0, 16m or 4g.
1552 *
1553 * Return first mnode range type index found otherwise return -1 if none found.
1554 */
1555 int
1556 mtype_func(int mnode, int mtype, uint_t flags)
1557 {
1558 if (flags & PGI_MT_RANGE) {
1559 int mnr_lim = MRI_0;
1560
1561 if (flags & PGI_MT_NEXT) {
1562 mtype = mnoderanges[mtype].mnr_next;
1563 }
1564 if (flags & PGI_MT_RANGE4G)
1565 mnr_lim = MRI_4G; /* exclude 0-4g range */
1566 else if (flags & PGI_MT_RANGE16M)
1567 mnr_lim = MRI_16M; /* exclude 0-16m range */
1568 while (mtype != -1 &&
1569 mnoderanges[mtype].mnr_memrange <= mnr_lim) {
1570 if (mnoderanges[mtype].mnr_mnode == mnode)
1571 return (mtype);
1572 mtype = mnoderanges[mtype].mnr_next;
1573 }
1574 } else if (mnoderanges[mtype].mnr_mnode == mnode) {
1575 return (mtype);
1576 }
1577 return (-1);
1578 }
1579
1580 /*
1581 * Update the page list max counts with the pfn range specified by the
1582 * input parameters.
1583 */
1584 void
1585 mtype_modify_max(pfn_t startpfn, long cnt)
1586 {
1587 int mtype;
1588 pgcnt_t inc;
1589 spgcnt_t scnt = (spgcnt_t)(cnt);
1590 pgcnt_t acnt = ABS(scnt);
1591 pfn_t endpfn = startpfn + acnt;
1592 pfn_t pfn, lo;
1593
1594 if (!physmax4g)
1595 return;
1596
1597 mtype = mtypetop;
1598 for (pfn = endpfn; pfn > startpfn; ) {
1599 ASSERT(mtype != -1);
1600 lo = mnoderanges[mtype].mnr_pfnlo;
1601 if (pfn > lo) {
1602 if (startpfn >= lo) {
1603 inc = pfn - startpfn;
1604 } else {
1605 inc = pfn - lo;
1606 }
1607 if (mnoderanges[mtype].mnr_memrange != MRI_4G) {
1608 if (scnt > 0)
1609 maxmem4g += inc;
1610 else
1611 maxmem4g -= inc;
1612 }
1613 pfn -= inc;
1614 }
1615 mtype = mnoderanges[mtype].mnr_next;
1616 }
1617 }
1618
1619 int
1620 mtype_2_mrange(int mtype)
1621 {
1622 return (mnoderanges[mtype].mnr_memrange);
1623 }
1624
1625 void
1626 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi)
1627 {
1628 _NOTE(ARGUNUSED(mnode));
1629 ASSERT(mnoderanges[mtype].mnr_mnode == mnode);
1630 *pfnlo = mnoderanges[mtype].mnr_pfnlo;
1631 *pfnhi = mnoderanges[mtype].mnr_pfnhi;
1632 }
1633
1634 size_t
1635 plcnt_sz(size_t ctrs_sz)
1636 {
1637 #ifdef DEBUG
1638 int szc, colors;
1639
1640 ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes;
1641 for (szc = 0; szc < mmu_page_sizes; szc++) {
1642 colors = page_get_pagecolors(szc);
1643 ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors;
1644 }
1645 #endif
1646 return (ctrs_sz);
1647 }
1648
1649 caddr_t
1650 plcnt_init(caddr_t addr)
1651 {
1652 #ifdef DEBUG
1653 int mt, szc, colors;
1654
1655 for (mt = 0; mt < mnoderangecnt; mt++) {
1656 mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr;
1657 addr += (sizeof (struct mnr_mts) * mmu_page_sizes);
1658 for (szc = 0; szc < mmu_page_sizes; szc++) {
1659 colors = page_get_pagecolors(szc);
1660 mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors;
1661 mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt =
1662 (pgcnt_t *)addr;
1663 addr += (sizeof (pgcnt_t) * colors);
1664 }
1665 }
1666 #endif
1667 return (addr);
1668 }
1669
1670 void
1671 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags)
1672 {
1673 _NOTE(ARGUNUSED(pp));
1674 #ifdef DEBUG
1675 int bin = PP_2_BIN(pp);
1676
1677 atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt);
1678 atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin],
1679 cnt);
1680 #endif
1681 ASSERT(mtype == PP_2_MTYPE(pp));
1682 if (physmax4g && mnoderanges[mtype].mnr_memrange != MRI_4G)
1683 atomic_add_long(&freemem4g, cnt);
1684 if (flags & PG_CACHE_LIST)
1685 atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt);
1686 else
1687 atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt);
1688 atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt);
1689 }
1690
1691 /*
1692 * Returns the free page count for mnode
1693 */
1694 int
1695 mnode_pgcnt(int mnode)
1696 {
1697 int mtype = mtypetop;
1698 int flags = PGI_MT_RANGE0;
1699 pgcnt_t pgcnt = 0;
1700
1701 mtype = mtype_func(mnode, mtype, flags);
1702
1703 while (mtype != -1) {
1704 pgcnt += MTYPE_FREEMEM(mtype);
1705 mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT);
1706 }
1707 return (pgcnt);
1708 }
1709
1710 /*
1711 * Initialize page coloring variables based on the l2 cache parameters.
1712 * Calculate and return memory needed for page coloring data structures.
1713 */
1714 size_t
1715 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc)
1716 {
1717 _NOTE(ARGUNUSED(l2_linesz));
1718 size_t colorsz = 0;
1719 int i;
1720 int colors;
1721
1722 #if defined(__xpv)
1723 /*
1724 * Hypervisor domains currently don't have any concept of NUMA.
1725 * Hence we'll act like there is only 1 memrange.
1726 */
1727 i = memrange_num(1);
1728 #else /* !__xpv */
1729 /*
1730 * Reduce the memory ranges lists if we don't have large amounts
1731 * of memory. This avoids searching known empty free lists.
1732 * To support memory DR operations, we need to keep memory ranges
1733 * for possible memory hot-add operations.
1734 */
1735 if (plat_dr_physmax > physmax)
1736 i = memrange_num(plat_dr_physmax);
1737 else
1738 i = memrange_num(physmax);
1739 #if defined(__i386)
1740 if (i > MRI_4G)
1741 restricted_kmemalloc = 0;
1742 #endif
1743 /* physmax greater than 4g */
1744 if (i == MRI_4G)
1745 physmax4g = 1;
1746 #endif /* !__xpv */
1747 memranges += i;
1748 nranges -= i;
1749
1750 ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES);
1751
1752 ASSERT(ISP2(l2_linesz));
1753 ASSERT(l2_sz > MMU_PAGESIZE);
1754
1755 /* l2_assoc is 0 for fully associative l2 cache */
1756 if (l2_assoc)
1757 l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE));
1758 else
1759 l2_colors = 1;
1760
1761 ASSERT(ISP2(l2_colors));
1762
1763 /* for scalability, configure at least PAGE_COLORS_MIN color bins */
1764 page_colors = MAX(l2_colors, PAGE_COLORS_MIN);
1765
1766 /*
1767 * cpu_page_colors is non-zero when a page color may be spread across
1768 * multiple bins.
1769 */
1770 if (l2_colors < page_colors)
1771 cpu_page_colors = l2_colors;
1772
1773 ASSERT(ISP2(page_colors));
1774
1775 page_colors_mask = page_colors - 1;
1776
1777 ASSERT(ISP2(CPUSETSIZE()));
1778 page_coloring_shift = lowbit(CPUSETSIZE());
1779
1780 /* initialize number of colors per page size */
1781 for (i = 0; i <= mmu.max_page_level; i++) {
1782 hw_page_array[i].hp_size = LEVEL_SIZE(i);
1783 hw_page_array[i].hp_shift = LEVEL_SHIFT(i);
1784 hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0);
1785 hw_page_array[i].hp_colors = (page_colors_mask >>
1786 (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift))
1787 + 1;
1788 colorequivszc[i] = 0;
1789 }
1790
1791 /*
1792 * The value of cpu_page_colors determines if additional color bins
1793 * need to be checked for a particular color in the page_get routines.
1794 */
1795 if (cpu_page_colors != 0) {
1796
1797 int a = lowbit(page_colors) - lowbit(cpu_page_colors);
1798 ASSERT(a > 0);
1799 ASSERT(a < 16);
1800
1801 for (i = 0; i <= mmu.max_page_level; i++) {
1802 if ((colors = hw_page_array[i].hp_colors) <= 1) {
1803 colorequivszc[i] = 0;
1804 continue;
1805 }
1806 while ((colors >> a) == 0)
1807 a--;
1808 ASSERT(a >= 0);
1809
1810 /* higher 4 bits encodes color equiv mask */
1811 colorequivszc[i] = (a << 4);
1812 }
1813 }
1814
1815 /* factor in colorequiv to check additional 'equivalent' bins. */
1816 if (colorequiv > 1) {
1817
1818 int a = lowbit(colorequiv) - 1;
1819 if (a > 15)
1820 a = 15;
1821
1822 for (i = 0; i <= mmu.max_page_level; i++) {
1823 if ((colors = hw_page_array[i].hp_colors) <= 1) {
1824 continue;
1825 }
1826 while ((colors >> a) == 0)
1827 a--;
1828 if ((a << 4) > colorequivszc[i]) {
1829 colorequivszc[i] = (a << 4);
1830 }
1831 }
1832 }
1833
1834 /* size for mnoderanges */
1835 for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++)
1836 mnoderangecnt += mnode_range_cnt(i);
1837 if (plat_dr_support_memory()) {
1838 /*
1839 * Reserve enough space for memory DR operations.
1840 * Two extra mnoderanges for possbile fragmentations,
1841 * one for the 2G boundary and the other for the 4G boundary.
1842 * We don't expect a memory board crossing the 16M boundary
1843 * for memory hot-add operations on x86 platforms.
1844 */
1845 mnoderangecnt += 2 + max_mem_nodes - lgrp_plat_node_cnt;
1846 }
1847 colorsz = mnoderangecnt * sizeof (mnoderange_t);
1848
1849 /* size for fpc_mutex and cpc_mutex */
1850 colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX);
1851
1852 /* size of page_freelists */
1853 colorsz += mnoderangecnt * sizeof (page_t ***);
1854 colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **);
1855
1856 for (i = 0; i < mmu_page_sizes; i++) {
1857 colors = page_get_pagecolors(i);
1858 colorsz += mnoderangecnt * colors * sizeof (page_t *);
1859 }
1860
1861 /* size of page_cachelists */
1862 colorsz += mnoderangecnt * sizeof (page_t **);
1863 colorsz += mnoderangecnt * page_colors * sizeof (page_t *);
1864
1865 return (colorsz);
1866 }
1867
1868 /*
1869 * Called once at startup to configure page_coloring data structures and
1870 * does the 1st page_free()/page_freelist_add().
1871 */
1872 void
1873 page_coloring_setup(caddr_t pcmemaddr)
1874 {
1875 int i;
1876 int j;
1877 int k;
1878 caddr_t addr;
1879 int colors;
1880
1881 /*
1882 * do page coloring setup
1883 */
1884 addr = pcmemaddr;
1885
1886 mnoderanges = (mnoderange_t *)addr;
1887 addr += (mnoderangecnt * sizeof (mnoderange_t));
1888
1889 mnode_range_setup(mnoderanges);
1890
1891 if (physmax4g)
1892 mtype4g = pfn_2_mtype(0xfffff);
1893
1894 for (k = 0; k < NPC_MUTEX; k++) {
1895 fpc_mutex[k] = (kmutex_t *)addr;
1896 addr += (max_mem_nodes * sizeof (kmutex_t));
1897 }
1898 for (k = 0; k < NPC_MUTEX; k++) {
1899 cpc_mutex[k] = (kmutex_t *)addr;
1900 addr += (max_mem_nodes * sizeof (kmutex_t));
1901 }
1902 page_freelists = (page_t ****)addr;
1903 addr += (mnoderangecnt * sizeof (page_t ***));
1904
1905 page_cachelists = (page_t ***)addr;
1906 addr += (mnoderangecnt * sizeof (page_t **));
1907
1908 for (i = 0; i < mnoderangecnt; i++) {
1909 page_freelists[i] = (page_t ***)addr;
1910 addr += (mmu_page_sizes * sizeof (page_t **));
1911
1912 for (j = 0; j < mmu_page_sizes; j++) {
1913 colors = page_get_pagecolors(j);
1914 page_freelists[i][j] = (page_t **)addr;
1915 addr += (colors * sizeof (page_t *));
1916 }
1917 page_cachelists[i] = (page_t **)addr;
1918 addr += (page_colors * sizeof (page_t *));
1919 }
1920 }
1921
1922 #if defined(__xpv)
1923 /*
1924 * Give back 10% of the io_pool pages to the free list.
1925 * Don't shrink the pool below some absolute minimum.
1926 */
1927 static void
1928 page_io_pool_shrink()
1929 {
1930 int retcnt;
1931 page_t *pp, *pp_first, *pp_last, **curpool;
1932 mfn_t mfn;
1933 int bothpools = 0;
1934
1935 mutex_enter(&io_pool_lock);
1936 io_pool_shrink_attempts++; /* should be a kstat? */
1937 retcnt = io_pool_cnt / 10;
1938 if (io_pool_cnt - retcnt < io_pool_cnt_min)
1939 retcnt = io_pool_cnt - io_pool_cnt_min;
1940 if (retcnt <= 0)
1941 goto done;
1942 io_pool_shrinks++; /* should be a kstat? */
1943 curpool = &io_pool_4g;
1944 domore:
1945 /*
1946 * Loop through taking pages from the end of the list
1947 * (highest mfns) till amount to return reached.
1948 */
1949 for (pp = *curpool; pp && retcnt > 0; ) {
1950 pp_first = pp_last = pp->p_prev;
1951 if (pp_first == *curpool)
1952 break;
1953 retcnt--;
1954 io_pool_cnt--;
1955 page_io_pool_sub(curpool, pp_first, pp_last);
1956 if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn)
1957 start_mfn = mfn;
1958 page_free(pp_first, 1);
1959 pp = *curpool;
1960 }
1961 if (retcnt != 0 && !bothpools) {
1962 /*
1963 * If not enough found in less constrained pool try the
1964 * more constrained one.
1965 */
1966 curpool = &io_pool_16m;
1967 bothpools = 1;
1968 goto domore;
1969 }
1970 done:
1971 mutex_exit(&io_pool_lock);
1972 }
1973
1974 #endif /* __xpv */
1975
1976 uint_t
1977 page_create_update_flags_x86(uint_t flags)
1978 {
1979 #if defined(__xpv)
1980 /*
1981 * Check this is an urgent allocation and free pages are depleted.
1982 */
1983 if (!(flags & PG_WAIT) && freemem < desfree)
1984 page_io_pool_shrink();
1985 #else /* !__xpv */
1986 /*
1987 * page_create_get_something may call this because 4g memory may be
1988 * depleted. Set flags to allow for relocation of base page below
1989 * 4g if necessary.
1990 */
1991 if (physmax4g)
1992 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
1993 #endif /* __xpv */
1994 return (flags);
1995 }
1996
1997 /*ARGSUSED*/
1998 int
1999 bp_color(struct buf *bp)
2000 {
2001 return (0);
2002 }
2003
2004 #if defined(__xpv)
2005
2006 /*
2007 * Take pages out of an io_pool
2008 */
2009 static void
2010 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last)
2011 {
2012 if (*poolp == pp_first) {
2013 *poolp = pp_last->p_next;
2014 if (*poolp == pp_first)
2015 *poolp = NULL;
2016 }
2017 pp_first->p_prev->p_next = pp_last->p_next;
2018 pp_last->p_next->p_prev = pp_first->p_prev;
2019 pp_first->p_prev = pp_last;
2020 pp_last->p_next = pp_first;
2021 }
2022
2023 /*
2024 * Put a page on the io_pool list. The list is ordered by increasing MFN.
2025 */
2026 static void
2027 page_io_pool_add(page_t **poolp, page_t *pp)
2028 {
2029 page_t *look;
2030 mfn_t mfn = mfn_list[pp->p_pagenum];
2031
2032 if (*poolp == NULL) {
2033 *poolp = pp;
2034 pp->p_next = pp;
2035 pp->p_prev = pp;
2036 return;
2037 }
2038
2039 /*
2040 * Since we try to take pages from the high end of the pool
2041 * chances are good that the pages to be put on the list will
2042 * go at or near the end of the list. so start at the end and
2043 * work backwards.
2044 */
2045 look = (*poolp)->p_prev;
2046 while (mfn < mfn_list[look->p_pagenum]) {
2047 look = look->p_prev;
2048 if (look == (*poolp)->p_prev)
2049 break; /* backed all the way to front of list */
2050 }
2051
2052 /* insert after look */
2053 pp->p_prev = look;
2054 pp->p_next = look->p_next;
2055 pp->p_next->p_prev = pp;
2056 look->p_next = pp;
2057 if (mfn < mfn_list[(*poolp)->p_pagenum]) {
2058 /*
2059 * we inserted a new first list element
2060 * adjust pool pointer to newly inserted element
2061 */
2062 *poolp = pp;
2063 }
2064 }
2065
2066 /*
2067 * Add a page to the io_pool. Setting the force flag will force the page
2068 * into the io_pool no matter what.
2069 */
2070 static void
2071 add_page_to_pool(page_t *pp, int force)
2072 {
2073 page_t *highest;
2074 page_t *freep = NULL;
2075
2076 mutex_enter(&io_pool_lock);
2077 /*
2078 * Always keep the scarce low memory pages
2079 */
2080 if (mfn_list[pp->p_pagenum] < PFN_16MEG) {
2081 ++io_pool_cnt;
2082 page_io_pool_add(&io_pool_16m, pp);
2083 goto done;
2084 }
2085 if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) {
2086 ++io_pool_cnt;
2087 page_io_pool_add(&io_pool_4g, pp);
2088 } else {
2089 highest = io_pool_4g->p_prev;
2090 if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) {
2091 page_io_pool_sub(&io_pool_4g, highest, highest);
2092 page_io_pool_add(&io_pool_4g, pp);
2093 freep = highest;
2094 } else {
2095 freep = pp;
2096 }
2097 }
2098 done:
2099 mutex_exit(&io_pool_lock);
2100 if (freep)
2101 page_free(freep, 1);
2102 }
2103
2104
2105 int contig_pfn_cnt; /* no of pfns in the contig pfn list */
2106 int contig_pfn_max; /* capacity of the contig pfn list */
2107 int next_alloc_pfn; /* next position in list to start a contig search */
2108 int contig_pfnlist_updates; /* pfn list update count */
2109 int contig_pfnlist_builds; /* how many times have we (re)built list */
2110 int contig_pfnlist_buildfailed; /* how many times has list build failed */
2111 int create_contig_pending; /* nonzero means taskq creating contig list */
2112 pfn_t *contig_pfn_list = NULL; /* list of contig pfns in ascending mfn order */
2113
2114 /*
2115 * Function to use in sorting a list of pfns by their underlying mfns.
2116 */
2117 static int
2118 mfn_compare(const void *pfnp1, const void *pfnp2)
2119 {
2120 mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1];
2121 mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2];
2122
2123 if (mfn1 > mfn2)
2124 return (1);
2125 if (mfn1 < mfn2)
2126 return (-1);
2127 return (0);
2128 }
2129
2130 /*
2131 * Compact the contig_pfn_list by tossing all the non-contiguous
2132 * elements from the list.
2133 */
2134 static void
2135 compact_contig_pfn_list(void)
2136 {
2137 pfn_t pfn, lapfn, prev_lapfn;
2138 mfn_t mfn;
2139 int i, newcnt = 0;
2140
2141 prev_lapfn = 0;
2142 for (i = 0; i < contig_pfn_cnt - 1; i++) {
2143 pfn = contig_pfn_list[i];
2144 lapfn = contig_pfn_list[i + 1];
2145 mfn = mfn_list[pfn];
2146 /*
2147 * See if next pfn is for a contig mfn
2148 */
2149 if (mfn_list[lapfn] != mfn + 1)
2150 continue;
2151 /*
2152 * pfn and lookahead are both put in list
2153 * unless pfn is the previous lookahead.
2154 */
2155 if (pfn != prev_lapfn)
2156 contig_pfn_list[newcnt++] = pfn;
2157 contig_pfn_list[newcnt++] = lapfn;
2158 prev_lapfn = lapfn;
2159 }
2160 for (i = newcnt; i < contig_pfn_cnt; i++)
2161 contig_pfn_list[i] = 0;
2162 contig_pfn_cnt = newcnt;
2163 }
2164
2165 /*ARGSUSED*/
2166 static void
2167 call_create_contiglist(void *arg)
2168 {
2169 (void) create_contig_pfnlist(PG_WAIT);
2170 }
2171
2172 /*
2173 * Create list of freelist pfns that have underlying
2174 * contiguous mfns. The list is kept in ascending mfn order.
2175 * returns 1 if list created else 0.
2176 */
2177 static int
2178 create_contig_pfnlist(uint_t flags)
2179 {
2180 pfn_t pfn;
2181 page_t *pp;
2182 int ret = 1;
2183
2184 mutex_enter(&contig_list_lock);
2185 if (contig_pfn_list != NULL)
2186 goto out;
2187 contig_pfn_max = freemem + (freemem / 10);
2188 contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t),
2189 (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP);
2190 if (contig_pfn_list == NULL) {
2191 /*
2192 * If we could not create the contig list (because
2193 * we could not sleep for memory). Dispatch a taskq that can
2194 * sleep to get the memory.
2195 */
2196 if (!create_contig_pending) {
2197 if (taskq_dispatch(system_taskq, call_create_contiglist,
2198 NULL, TQ_NOSLEEP) != NULL)
2199 create_contig_pending = 1;
2200 }
2201 contig_pfnlist_buildfailed++; /* count list build failures */
2202 ret = 0;
2203 goto out;
2204 }
2205 create_contig_pending = 0;
2206 ASSERT(contig_pfn_cnt == 0);
2207 for (pfn = 0; pfn < mfn_count; pfn++) {
2208 pp = page_numtopp_nolock(pfn);
2209 if (pp == NULL || !PP_ISFREE(pp))
2210 continue;
2211 contig_pfn_list[contig_pfn_cnt] = pfn;
2212 if (++contig_pfn_cnt == contig_pfn_max)
2213 break;
2214 }
2215 /*
2216 * Sanity check the new list.
2217 */
2218 if (contig_pfn_cnt < 2) { /* no contig pfns */
2219 contig_pfn_cnt = 0;
2220 contig_pfnlist_buildfailed++;
2221 kmem_free(contig_pfn_list, contig_pfn_max * sizeof (pfn_t));
2222 contig_pfn_list = NULL;
2223 contig_pfn_max = 0;
2224 ret = 0;
2225 goto out;
2226 }
2227 qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare);
2228 compact_contig_pfn_list();
2229 /*
2230 * Make sure next search of the newly created contiguous pfn
2231 * list starts at the beginning of the list.
2232 */
2233 next_alloc_pfn = 0;
2234 contig_pfnlist_builds++; /* count list builds */
2235 out:
2236 mutex_exit(&contig_list_lock);
2237 return (ret);
2238 }
2239
2240
2241 /*
2242 * Toss the current contig pfnlist. Someone is about to do a massive
2243 * update to pfn<->mfn mappings. So we have them destroy the list and lock
2244 * it till they are done with their update.
2245 */
2246 void
2247 clear_and_lock_contig_pfnlist()
2248 {
2249 pfn_t *listp = NULL;
2250 size_t listsize;
2251
2252 mutex_enter(&contig_list_lock);
2253 if (contig_pfn_list != NULL) {
2254 listp = contig_pfn_list;
2255 listsize = contig_pfn_max * sizeof (pfn_t);
2256 contig_pfn_list = NULL;
2257 contig_pfn_max = contig_pfn_cnt = 0;
2258 }
2259 if (listp != NULL)
2260 kmem_free(listp, listsize);
2261 }
2262
2263 /*
2264 * Unlock the contig_pfn_list. The next attempted use of it will cause
2265 * it to be re-created.
2266 */
2267 void
2268 unlock_contig_pfnlist()
2269 {
2270 mutex_exit(&contig_list_lock);
2271 }
2272
2273 /*
2274 * Update the contiguous pfn list in response to a pfn <-> mfn reassignment
2275 */
2276 void
2277 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn)
2278 {
2279 int probe_hi, probe_lo, probe_pos, insert_after, insert_point;
2280 pfn_t probe_pfn;
2281 mfn_t probe_mfn;
2282 int drop_lock = 0;
2283
2284 if (mutex_owner(&contig_list_lock) != curthread) {
2285 drop_lock = 1;
2286 mutex_enter(&contig_list_lock);
2287 }
2288 if (contig_pfn_list == NULL)
2289 goto done;
2290 contig_pfnlist_updates++;
2291 /*
2292 * Find the pfn in the current list. Use a binary chop to locate it.
2293 */
2294 probe_hi = contig_pfn_cnt - 1;
2295 probe_lo = 0;
2296 probe_pos = (probe_hi + probe_lo) / 2;
2297 while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) {
2298 if (probe_pos == probe_lo) { /* pfn not in list */
2299 probe_pos = -1;
2300 break;
2301 }
2302 if (pfn_to_mfn(probe_pfn) <= oldmfn)
2303 probe_lo = probe_pos;
2304 else
2305 probe_hi = probe_pos;
2306 probe_pos = (probe_hi + probe_lo) / 2;
2307 }
2308 if (probe_pos >= 0) {
2309 /*
2310 * Remove pfn from list and ensure next alloc
2311 * position stays in bounds.
2312 */
2313 if (--contig_pfn_cnt <= next_alloc_pfn)
2314 next_alloc_pfn = 0;
2315 if (contig_pfn_cnt < 2) { /* no contig pfns */
2316 contig_pfn_cnt = 0;
2317 kmem_free(contig_pfn_list,
2318 contig_pfn_max * sizeof (pfn_t));
2319 contig_pfn_list = NULL;
2320 contig_pfn_max = 0;
2321 goto done;
2322 }
2323 ovbcopy(&contig_pfn_list[probe_pos + 1],
2324 &contig_pfn_list[probe_pos],
2325 (contig_pfn_cnt - probe_pos) * sizeof (pfn_t));
2326 }
2327 if (newmfn == MFN_INVALID)
2328 goto done;
2329 /*
2330 * Check if new mfn has adjacent mfns in the list
2331 */
2332 probe_hi = contig_pfn_cnt - 1;
2333 probe_lo = 0;
2334 insert_after = -2;
2335 do {
2336 probe_pos = (probe_hi + probe_lo) / 2;
2337 probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]);
2338 if (newmfn == probe_mfn + 1)
2339 insert_after = probe_pos;
2340 else if (newmfn == probe_mfn - 1)
2341 insert_after = probe_pos - 1;
2342 if (probe_pos == probe_lo)
2343 break;
2344 if (probe_mfn <= newmfn)
2345 probe_lo = probe_pos;
2346 else
2347 probe_hi = probe_pos;
2348 } while (insert_after == -2);
2349 /*
2350 * If there is space in the list and there are adjacent mfns
2351 * insert the pfn in to its proper place in the list.
2352 */
2353 if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) {
2354 insert_point = insert_after + 1;
2355 ovbcopy(&contig_pfn_list[insert_point],
2356 &contig_pfn_list[insert_point + 1],
2357 (contig_pfn_cnt - insert_point) * sizeof (pfn_t));
2358 contig_pfn_list[insert_point] = pfn;
2359 contig_pfn_cnt++;
2360 }
2361 done:
2362 if (drop_lock)
2363 mutex_exit(&contig_list_lock);
2364 }
2365
2366 /*
2367 * Called to (re-)populate the io_pool from the free page lists.
2368 */
2369 long
2370 populate_io_pool(void)
2371 {
2372 pfn_t pfn;
2373 mfn_t mfn, max_mfn;
2374 page_t *pp;
2375
2376 /*
2377 * Figure out the bounds of the pool on first invocation.
2378 * We use a percentage of memory for the io pool size.
2379 * we allow that to shrink, but not to less than a fixed minimum
2380 */
2381 if (io_pool_cnt_max == 0) {
2382 io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct);
2383 io_pool_cnt_lowater = io_pool_cnt_max;
2384 /*
2385 * This is the first time in populate_io_pool, grab a va to use
2386 * when we need to allocate pages.
2387 */
2388 io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2389 }
2390 /*
2391 * If we are out of pages in the pool, then grow the size of the pool
2392 */
2393 if (io_pool_cnt == 0) {
2394 /*
2395 * Grow the max size of the io pool by 5%, but never more than
2396 * 25% of physical memory.
2397 */
2398 if (io_pool_cnt_max < physmem / 4)
2399 io_pool_cnt_max += io_pool_cnt_max / 20;
2400 }
2401 io_pool_grows++; /* should be a kstat? */
2402
2403 /*
2404 * Get highest mfn on this platform, but limit to the 32 bit DMA max.
2405 */
2406 (void) mfn_to_pfn(start_mfn);
2407 max_mfn = MIN(cached_max_mfn, PFN_4GIG);
2408 for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) {
2409 pfn = mfn_to_pfn(mfn);
2410 if (pfn & PFN_IS_FOREIGN_MFN)
2411 continue;
2412 /*
2413 * try to allocate it from free pages
2414 */
2415 pp = page_numtopp_alloc(pfn);
2416 if (pp == NULL)
2417 continue;
2418 PP_CLRFREE(pp);
2419 add_page_to_pool(pp, 1);
2420 if (io_pool_cnt >= io_pool_cnt_max)
2421 break;
2422 }
2423
2424 return (io_pool_cnt);
2425 }
2426
2427 /*
2428 * Destroy a page that was being used for DMA I/O. It may or
2429 * may not actually go back to the io_pool.
2430 */
2431 void
2432 page_destroy_io(page_t *pp)
2433 {
2434 mfn_t mfn = mfn_list[pp->p_pagenum];
2435
2436 /*
2437 * When the page was alloc'd a reservation was made, release it now
2438 */
2439 page_unresv(1);
2440 /*
2441 * Unload translations, if any, then hash out the
2442 * page to erase its identity.
2443 */
2444 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2445 page_hashout(pp, NULL);
2446
2447 /*
2448 * If the page came from the free lists, just put it back to them.
2449 * DomU pages always go on the free lists as well.
2450 */
2451 if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) {
2452 page_free(pp, 1);
2453 return;
2454 }
2455
2456 add_page_to_pool(pp, 0);
2457 }
2458
2459
2460 long contig_searches; /* count of times contig pages requested */
2461 long contig_search_restarts; /* count of contig ranges tried */
2462 long contig_search_failed; /* count of contig alloc failures */
2463
2464 /*
2465 * Free partial page list
2466 */
2467 static void
2468 free_partial_list(page_t **pplist)
2469 {
2470 page_t *pp;
2471
2472 while (*pplist != NULL) {
2473 pp = *pplist;
2474 page_io_pool_sub(pplist, pp, pp);
2475 page_free(pp, 1);
2476 }
2477 }
2478
2479 /*
2480 * Look thru the contiguous pfns that are not part of the io_pool for
2481 * contiguous free pages. Return a list of the found pages or NULL.
2482 */
2483 page_t *
2484 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg,
2485 pgcnt_t pfnalign)
2486 {
2487 page_t *pp, *plist = NULL;
2488 mfn_t mfn, prev_mfn, start_mfn;
2489 pfn_t pfn;
2490 int pages_needed, pages_requested;
2491 int search_start;
2492
2493 /*
2494 * create the contig pfn list if not already done
2495 */
2496 retry:
2497 mutex_enter(&contig_list_lock);
2498 if (contig_pfn_list == NULL) {
2499 mutex_exit(&contig_list_lock);
2500 if (!create_contig_pfnlist(flags)) {
2501 return (NULL);
2502 }
2503 goto retry;
2504 }
2505 contig_searches++;
2506 /*
2507 * Search contiguous pfn list for physically contiguous pages not in
2508 * the io_pool. Start the search where the last search left off.
2509 */
2510 pages_requested = pages_needed = npages;
2511 search_start = next_alloc_pfn;
2512 start_mfn = prev_mfn = 0;
2513 while (pages_needed) {
2514 pfn = contig_pfn_list[next_alloc_pfn];
2515 mfn = pfn_to_mfn(pfn);
2516 /*
2517 * Check if mfn is first one or contig to previous one and
2518 * if page corresponding to mfn is free and that mfn
2519 * range is not crossing a segment boundary.
2520 */
2521 if ((prev_mfn == 0 || mfn == prev_mfn + 1) &&
2522 (pp = page_numtopp_alloc(pfn)) != NULL &&
2523 !((mfn & pfnseg) < (start_mfn & pfnseg))) {
2524 PP_CLRFREE(pp);
2525 page_io_pool_add(&plist, pp);
2526 pages_needed--;
2527 if (prev_mfn == 0) {
2528 if (pfnalign &&
2529 mfn != P2ROUNDUP(mfn, pfnalign)) {
2530 /*
2531 * not properly aligned
2532 */
2533 contig_search_restarts++;
2534 free_partial_list(&plist);
2535 pages_needed = pages_requested;
2536 start_mfn = prev_mfn = 0;
2537 goto skip;
2538 }
2539 start_mfn = mfn;
2540 }
2541 prev_mfn = mfn;
2542 } else {
2543 contig_search_restarts++;
2544 free_partial_list(&plist);
2545 pages_needed = pages_requested;
2546 start_mfn = prev_mfn = 0;
2547 }
2548 skip:
2549 if (++next_alloc_pfn == contig_pfn_cnt)
2550 next_alloc_pfn = 0;
2551 if (next_alloc_pfn == search_start)
2552 break; /* all pfns searched */
2553 }
2554 mutex_exit(&contig_list_lock);
2555 if (pages_needed) {
2556 contig_search_failed++;
2557 /*
2558 * Failed to find enough contig pages.
2559 * free partial page list
2560 */
2561 free_partial_list(&plist);
2562 }
2563 return (plist);
2564 }
2565
2566 /*
2567 * Search the reserved io pool pages for a page range with the
2568 * desired characteristics.
2569 */
2570 page_t *
2571 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg)
2572 {
2573 page_t *pp_first, *pp_last;
2574 page_t *pp, **poolp;
2575 pgcnt_t nwanted, pfnalign;
2576 uint64_t pfnseg;
2577 mfn_t mfn, tmfn, hi_mfn, lo_mfn;
2578 int align, attempt = 0;
2579
2580 if (minctg == 1)
2581 contig = 0;
2582 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2583 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2584 pfnseg = mmu_btop(mattr->dma_attr_seg);
2585 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2586 if (align > MMU_PAGESIZE)
2587 pfnalign = mmu_btop(align);
2588 else
2589 pfnalign = 0;
2590
2591 try_again:
2592 /*
2593 * See if we want pages for a legacy device
2594 */
2595 if (hi_mfn < PFN_16MEG)
2596 poolp = &io_pool_16m;
2597 else
2598 poolp = &io_pool_4g;
2599 try_smaller:
2600 /*
2601 * Take pages from I/O pool. We'll use pages from the highest
2602 * MFN range possible.
2603 */
2604 pp_first = pp_last = NULL;
2605 mutex_enter(&io_pool_lock);
2606 nwanted = minctg;
2607 for (pp = *poolp; pp && nwanted > 0; ) {
2608 pp = pp->p_prev;
2609
2610 /*
2611 * skip pages above allowable range
2612 */
2613 mfn = mfn_list[pp->p_pagenum];
2614 if (hi_mfn < mfn)
2615 goto skip;
2616
2617 /*
2618 * stop at pages below allowable range
2619 */
2620 if (lo_mfn > mfn)
2621 break;
2622 restart:
2623 if (pp_last == NULL) {
2624 /*
2625 * Check alignment
2626 */
2627 tmfn = mfn - (minctg - 1);
2628 if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign))
2629 goto skip; /* not properly aligned */
2630 /*
2631 * Check segment
2632 */
2633 if ((mfn & pfnseg) < (tmfn & pfnseg))
2634 goto skip; /* crosses seg boundary */
2635 /*
2636 * Start building page list
2637 */
2638 pp_first = pp_last = pp;
2639 nwanted--;
2640 } else {
2641 /*
2642 * check physical contiguity if required
2643 */
2644 if (contig &&
2645 mfn_list[pp_first->p_pagenum] != mfn + 1) {
2646 /*
2647 * not a contiguous page, restart list.
2648 */
2649 pp_last = NULL;
2650 nwanted = minctg;
2651 goto restart;
2652 } else { /* add page to list */
2653 pp_first = pp;
2654 nwanted--;
2655 }
2656 }
2657 skip:
2658 if (pp == *poolp)
2659 break;
2660 }
2661
2662 /*
2663 * If we didn't find memory. Try the more constrained pool, then
2664 * sweep free pages into the DMA pool and try again.
2665 */
2666 if (nwanted != 0) {
2667 mutex_exit(&io_pool_lock);
2668 /*
2669 * If we were looking in the less constrained pool and
2670 * didn't find pages, try the more constrained pool.
2671 */
2672 if (poolp == &io_pool_4g) {
2673 poolp = &io_pool_16m;
2674 goto try_smaller;
2675 }
2676 kmem_reap();
2677 if (++attempt < 4) {
2678 /*
2679 * Grab some more io_pool pages
2680 */
2681 (void) populate_io_pool();
2682 goto try_again; /* go around and retry */
2683 }
2684 return (NULL);
2685 }
2686 /*
2687 * Found the pages, now snip them from the list
2688 */
2689 page_io_pool_sub(poolp, pp_first, pp_last);
2690 io_pool_cnt -= minctg;
2691 /*
2692 * reset low water mark
2693 */
2694 if (io_pool_cnt < io_pool_cnt_lowater)
2695 io_pool_cnt_lowater = io_pool_cnt;
2696 mutex_exit(&io_pool_lock);
2697 return (pp_first);
2698 }
2699
2700 page_t *
2701 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr,
2702 ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg)
2703 {
2704 uint_t kflags;
2705 int order, extra, extpages, i, contig, nbits, extents;
2706 page_t *pp, *expp, *pp_first, **pplist = NULL;
2707 mfn_t *mfnlist = NULL;
2708
2709 contig = flags & PG_PHYSCONTIG;
2710 if (minctg == 1)
2711 contig = 0;
2712 flags &= ~PG_PHYSCONTIG;
2713 kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP;
2714 /*
2715 * Hypervisor will allocate extents, if we want contig
2716 * pages extent must be >= minctg
2717 */
2718 if (contig) {
2719 order = highbit(minctg) - 1;
2720 if (minctg & ((1 << order) - 1))
2721 order++;
2722 extpages = 1 << order;
2723 } else {
2724 order = 0;
2725 extpages = minctg;
2726 }
2727 if (extpages > minctg) {
2728 extra = extpages - minctg;
2729 if (!page_resv(extra, kflags))
2730 return (NULL);
2731 }
2732 pp_first = NULL;
2733 pplist = kmem_alloc(extpages * sizeof (page_t *), kflags);
2734 if (pplist == NULL)
2735 goto balloon_fail;
2736 mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags);
2737 if (mfnlist == NULL)
2738 goto balloon_fail;
2739 pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr);
2740 if (pp == NULL)
2741 goto balloon_fail;
2742 pp_first = pp;
2743 if (extpages > minctg) {
2744 /*
2745 * fill out the rest of extent pages to swap
2746 * with the hypervisor
2747 */
2748 for (i = 0; i < extra; i++) {
2749 expp = page_create_va(vp,
2750 (u_offset_t)(uintptr_t)io_pool_kva,
2751 PAGESIZE, flags, &kvseg, io_pool_kva);
2752 if (expp == NULL)
2753 goto balloon_fail;
2754 (void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD);
2755 page_io_unlock(expp);
2756 page_hashout(expp, NULL);
2757 page_io_lock(expp);
2758 /*
2759 * add page to end of list
2760 */
2761 expp->p_prev = pp_first->p_prev;
2762 expp->p_next = pp_first;
2763 expp->p_prev->p_next = expp;
2764 pp_first->p_prev = expp;
2765 }
2766
2767 }
2768 for (i = 0; i < extpages; i++) {
2769 pplist[i] = pp;
2770 pp = pp->p_next;
2771 }
2772 nbits = highbit(mattr->dma_attr_addr_hi);
2773 extents = contig ? 1 : minctg;
2774 if (balloon_replace_pages(extents, pplist, nbits, order,
2775 mfnlist) != extents) {
2776 if (ioalloc_dbg)
2777 cmn_err(CE_NOTE, "request to hypervisor"
2778 " for %d pages, maxaddr %" PRIx64 " failed",
2779 extpages, mattr->dma_attr_addr_hi);
2780 goto balloon_fail;
2781 }
2782
2783 kmem_free(pplist, extpages * sizeof (page_t *));
2784 kmem_free(mfnlist, extpages * sizeof (mfn_t));
2785 /*
2786 * Return any excess pages to free list
2787 */
2788 if (extpages > minctg) {
2789 for (i = 0; i < extra; i++) {
2790 pp = pp_first->p_prev;
2791 page_sub(&pp_first, pp);
2792 page_io_unlock(pp);
2793 page_unresv(1);
2794 page_free(pp, 1);
2795 }
2796 }
2797 return (pp_first);
2798 balloon_fail:
2799 /*
2800 * Return pages to free list and return failure
2801 */
2802 while (pp_first != NULL) {
2803 pp = pp_first;
2804 page_sub(&pp_first, pp);
2805 page_io_unlock(pp);
2806 if (pp->p_vnode != NULL)
2807 page_hashout(pp, NULL);
2808 page_free(pp, 1);
2809 }
2810 if (pplist)
2811 kmem_free(pplist, extpages * sizeof (page_t *));
2812 if (mfnlist)
2813 kmem_free(mfnlist, extpages * sizeof (mfn_t));
2814 page_unresv(extpages - minctg);
2815 return (NULL);
2816 }
2817
2818 static void
2819 return_partial_alloc(page_t *plist)
2820 {
2821 page_t *pp;
2822
2823 while (plist != NULL) {
2824 pp = plist;
2825 page_sub(&plist, pp);
2826 page_io_unlock(pp);
2827 page_destroy_io(pp);
2828 }
2829 }
2830
2831 static page_t *
2832 page_get_contigpages(
2833 struct vnode *vp,
2834 u_offset_t off,
2835 int *npagesp,
2836 uint_t flags,
2837 caddr_t vaddr,
2838 ddi_dma_attr_t *mattr)
2839 {
2840 mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2841 page_t *plist; /* list to return */
2842 page_t *pp, *mcpl;
2843 int contig, anyaddr, npages, getone = 0;
2844 mfn_t lo_mfn;
2845 mfn_t hi_mfn;
2846 pgcnt_t pfnalign = 0;
2847 int align, sgllen;
2848 uint64_t pfnseg;
2849 pgcnt_t minctg;
2850
2851 npages = *npagesp;
2852 ASSERT(mattr != NULL);
2853 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2854 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2855 sgllen = mattr->dma_attr_sgllen;
2856 pfnseg = mmu_btop(mattr->dma_attr_seg);
2857 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2858 if (align > MMU_PAGESIZE)
2859 pfnalign = mmu_btop(align);
2860
2861 contig = flags & PG_PHYSCONTIG;
2862 if (npages == -1) {
2863 npages = 1;
2864 pfnalign = 0;
2865 }
2866 /*
2867 * Clear the contig flag if only one page is needed.
2868 */
2869 if (npages == 1) {
2870 getone = 1;
2871 contig = 0;
2872 }
2873
2874 /*
2875 * Check if any page in the system is fine.
2876 */
2877 anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn;
2878 if (!contig && anyaddr && !pfnalign) {
2879 flags &= ~PG_PHYSCONTIG;
2880 plist = page_create_va(vp, off, npages * MMU_PAGESIZE,
2881 flags, &kvseg, vaddr);
2882 if (plist != NULL) {
2883 *npagesp = 0;
2884 return (plist);
2885 }
2886 }
2887 plist = NULL;
2888 minctg = howmany(npages, sgllen);
2889 while (npages > sgllen || getone) {
2890 if (minctg > npages)
2891 minctg = npages;
2892 mcpl = NULL;
2893 /*
2894 * We could want contig pages with no address range limits.
2895 */
2896 if (anyaddr && contig) {
2897 /*
2898 * Look for free contig pages to satisfy the request.
2899 */
2900 mcpl = find_contig_free(minctg, flags, pfnseg,
2901 pfnalign);
2902 }
2903 /*
2904 * Try the reserved io pools next
2905 */
2906 if (mcpl == NULL)
2907 mcpl = page_io_pool_alloc(mattr, contig, minctg);
2908 if (mcpl != NULL) {
2909 pp = mcpl;
2910 do {
2911 if (!page_hashin(pp, vp, off, NULL)) {
2912 panic("page_get_contigpages:"
2913 " hashin failed"
2914 " pp %p, vp %p, off %llx",
2915 (void *)pp, (void *)vp, off);
2916 }
2917 off += MMU_PAGESIZE;
2918 PP_CLRFREE(pp);
2919 PP_CLRAGED(pp);
2920 page_set_props(pp, P_REF);
2921 page_io_lock(pp);
2922 pp = pp->p_next;
2923 } while (pp != mcpl);
2924 } else {
2925 /*
2926 * Hypervisor exchange doesn't handle segment or
2927 * alignment constraints
2928 */
2929 if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi ||
2930 pfnalign)
2931 goto fail;
2932 /*
2933 * Try exchanging pages with the hypervisor
2934 */
2935 mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr,
2936 flags, minctg);
2937 if (mcpl == NULL)
2938 goto fail;
2939 off += minctg * MMU_PAGESIZE;
2940 }
2941 check_dma(mattr, mcpl, minctg);
2942 /*
2943 * Here with a minctg run of contiguous pages, add them to the
2944 * list we will return for this request.
2945 */
2946 page_list_concat(&plist, &mcpl);
2947 npages -= minctg;
2948 *npagesp = npages;
2949 sgllen--;
2950 if (getone)
2951 break;
2952 }
2953 return (plist);
2954 fail:
2955 return_partial_alloc(plist);
2956 return (NULL);
2957 }
2958
2959 /*
2960 * Allocator for domain 0 I/O pages. We match the required
2961 * DMA attributes and contiguity constraints.
2962 */
2963 /*ARGSUSED*/
2964 page_t *
2965 page_create_io(
2966 struct vnode *vp,
2967 u_offset_t off,
2968 uint_t bytes,
2969 uint_t flags,
2970 struct as *as,
2971 caddr_t vaddr,
2972 ddi_dma_attr_t *mattr)
2973 {
2974 page_t *plist = NULL, *pp;
2975 int npages = 0, contig, anyaddr, pages_req;
2976 mfn_t lo_mfn;
2977 mfn_t hi_mfn;
2978 pgcnt_t pfnalign = 0;
2979 int align;
2980 int is_domu = 0;
2981 int dummy, bytes_got;
2982 mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2983
2984 ASSERT(mattr != NULL);
2985 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2986 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2987 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2988 if (align > MMU_PAGESIZE)
2989 pfnalign = mmu_btop(align);
2990
2991 /*
2992 * Clear the contig flag if only one page is needed or the scatter
2993 * gather list length is >= npages.
2994 */
2995 pages_req = npages = mmu_btopr(bytes);
2996 contig = (flags & PG_PHYSCONTIG);
2997 bytes = P2ROUNDUP(bytes, MMU_PAGESIZE);
2998 if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages)
2999 contig = 0;
3000
3001 /*
3002 * Check if any old page in the system is fine.
3003 * DomU should always go down this path.
3004 */
3005 is_domu = !DOMAIN_IS_INITDOMAIN(xen_info);
3006 anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
3007 if ((!contig && anyaddr) || is_domu) {
3008 flags &= ~PG_PHYSCONTIG;
3009 plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr);
3010 if (plist != NULL)
3011 return (plist);
3012 else if (is_domu)
3013 return (NULL); /* no memory available */
3014 }
3015 /*
3016 * DomU should never reach here
3017 */
3018 if (contig) {
3019 plist = page_get_contigpages(vp, off, &npages, flags, vaddr,
3020 mattr);
3021 if (plist == NULL)
3022 goto fail;
3023 bytes_got = (pages_req - npages) << MMU_PAGESHIFT;
3024 vaddr += bytes_got;
3025 off += bytes_got;
3026 /*
3027 * We now have all the contiguous pages we need, but
3028 * we may still need additional non-contiguous pages.
3029 */
3030 }
3031 /*
3032 * now loop collecting the requested number of pages, these do
3033 * not have to be contiguous pages but we will use the contig
3034 * page alloc code to get the pages since it will honor any
3035 * other constraints the pages may have.
3036 */
3037 while (npages--) {
3038 dummy = -1;
3039 pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr);
3040 if (pp == NULL)
3041 goto fail;
3042 page_add(&plist, pp);
3043 vaddr += MMU_PAGESIZE;
3044 off += MMU_PAGESIZE;
3045 }
3046 return (plist);
3047 fail:
3048 /*
3049 * Failed to get enough pages, return ones we did get
3050 */
3051 return_partial_alloc(plist);
3052 return (NULL);
3053 }
3054
3055 /*
3056 * Lock and return the page with the highest mfn that we can find. last_mfn
3057 * holds the last one found, so the next search can start from there. We
3058 * also keep a counter so that we don't loop forever if the machine has no
3059 * free pages.
3060 *
3061 * This is called from the balloon thread to find pages to give away. new_high
3062 * is used when new mfn's have been added to the system - we will reset our
3063 * search if the new mfn's are higher than our current search position.
3064 */
3065 page_t *
3066 page_get_high_mfn(mfn_t new_high)
3067 {
3068 static mfn_t last_mfn = 0;
3069 pfn_t pfn;
3070 page_t *pp;
3071 ulong_t loop_count = 0;
3072
3073 if (new_high > last_mfn)
3074 last_mfn = new_high;
3075
3076 for (; loop_count < mfn_count; loop_count++, last_mfn--) {
3077 if (last_mfn == 0) {
3078 last_mfn = cached_max_mfn;
3079 }
3080
3081 pfn = mfn_to_pfn(last_mfn);
3082 if (pfn & PFN_IS_FOREIGN_MFN)
3083 continue;
3084
3085 /* See if the page is free. If so, lock it. */
3086 pp = page_numtopp_alloc(pfn);
3087 if (pp == NULL)
3088 continue;
3089 PP_CLRFREE(pp);
3090
3091 ASSERT(PAGE_EXCL(pp));
3092 ASSERT(pp->p_vnode == NULL);
3093 ASSERT(!hat_page_is_mapped(pp));
3094 last_mfn--;
3095 return (pp);
3096 }
3097 return (NULL);
3098 }
3099
3100 #else /* !__xpv */
3101
3102 /*
3103 * get a page from any list with the given mnode
3104 */
3105 static page_t *
3106 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags,
3107 int mnode, int mtype, ddi_dma_attr_t *dma_attr)
3108 {
3109 kmutex_t *pcm;
3110 int i;
3111 page_t *pp;
3112 page_t *first_pp;
3113 uint64_t pgaddr;
3114 ulong_t bin;
3115 int mtypestart;
3116 int plw_initialized;
3117 page_list_walker_t plw;
3118
3119 VM_STAT_ADD(pga_vmstats.pgma_alloc);
3120
3121 ASSERT((flags & PG_MATCH_COLOR) == 0);
3122 ASSERT(szc == 0);
3123 ASSERT(dma_attr != NULL);
3124
3125 MTYPE_START(mnode, mtype, flags);
3126 if (mtype < 0) {
3127 VM_STAT_ADD(pga_vmstats.pgma_allocempty);
3128 return (NULL);
3129 }
3130
3131 mtypestart = mtype;
3132
3133 bin = origbin;
3134
3135 /*
3136 * check up to page_colors + 1 bins - origbin may be checked twice
3137 * because of BIN_STEP skip
3138 */
3139 do {
3140 plw_initialized = 0;
3141
3142 for (plw.plw_count = 0;
3143 plw.plw_count < page_colors; plw.plw_count++) {
3144
3145 if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL)
3146 goto nextfreebin;
3147
3148 pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST);
3149 mutex_enter(pcm);
3150 pp = PAGE_FREELISTS(mnode, szc, bin, mtype);
3151 first_pp = pp;
3152 while (pp != NULL) {
3153 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3154 SE_EXCL) == 0) {
3155 pp = pp->p_next;
3156 if (pp == first_pp) {
3157 pp = NULL;
3158 }
3159 continue;
3160 }
3161
3162 ASSERT(PP_ISFREE(pp));
3163 ASSERT(PP_ISAGED(pp));
3164 ASSERT(pp->p_vnode == NULL);
3165 ASSERT(pp->p_hash == NULL);
3166 ASSERT(pp->p_offset == (u_offset_t)-1);
3167 ASSERT(pp->p_szc == szc);
3168 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3169 /* check if page within DMA attributes */
3170 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3171 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3172 (pgaddr + MMU_PAGESIZE - 1 <=
3173 dma_attr->dma_attr_addr_hi)) {
3174 break;
3175 }
3176
3177 /* continue looking */
3178 page_unlock(pp);
3179 pp = pp->p_next;
3180 if (pp == first_pp)
3181 pp = NULL;
3182
3183 }
3184 if (pp != NULL) {
3185 ASSERT(mtype == PP_2_MTYPE(pp));
3186 ASSERT(pp->p_szc == 0);
3187
3188 /* found a page with specified DMA attributes */
3189 page_sub(&PAGE_FREELISTS(mnode, szc, bin,
3190 mtype), pp);
3191 page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST);
3192
3193 if ((PP_ISFREE(pp) == 0) ||
3194 (PP_ISAGED(pp) == 0)) {
3195 cmn_err(CE_PANIC, "page %p is not free",
3196 (void *)pp);
3197 }
3198
3199 mutex_exit(pcm);
3200 check_dma(dma_attr, pp, 1);
3201 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3202 return (pp);
3203 }
3204 mutex_exit(pcm);
3205 nextfreebin:
3206 if (plw_initialized == 0) {
3207 page_list_walk_init(szc, 0, bin, 1, 0, &plw);
3208 ASSERT(plw.plw_ceq_dif == page_colors);
3209 plw_initialized = 1;
3210 }
3211
3212 if (plw.plw_do_split) {
3213 pp = page_freelist_split(szc, bin, mnode,
3214 mtype,
3215 mmu_btop(dma_attr->dma_attr_addr_lo),
3216 mmu_btop(dma_attr->dma_attr_addr_hi + 1),
3217 &plw);
3218 if (pp != NULL) {
3219 check_dma(dma_attr, pp, 1);
3220 return (pp);
3221 }
3222 }
3223
3224 bin = page_list_walk_next_bin(szc, bin, &plw);
3225 }
3226
3227 MTYPE_NEXT(mnode, mtype, flags);
3228 } while (mtype >= 0);
3229
3230 /* failed to find a page in the freelist; try it in the cachelist */
3231
3232 /* reset mtype start for cachelist search */
3233 mtype = mtypestart;
3234 ASSERT(mtype >= 0);
3235
3236 /* start with the bin of matching color */
3237 bin = origbin;
3238
3239 do {
3240 for (i = 0; i <= page_colors; i++) {
3241 if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL)
3242 goto nextcachebin;
3243 pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST);
3244 mutex_enter(pcm);
3245 pp = PAGE_CACHELISTS(mnode, bin, mtype);
3246 first_pp = pp;
3247 while (pp != NULL) {
3248 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3249 SE_EXCL) == 0) {
3250 pp = pp->p_next;
3251 if (pp == first_pp)
3252 pp = NULL;
3253 continue;
3254 }
3255 ASSERT(pp->p_vnode);
3256 ASSERT(PP_ISAGED(pp) == 0);
3257 ASSERT(pp->p_szc == 0);
3258 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3259
3260 /* check if page within DMA attributes */
3261
3262 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3263 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3264 (pgaddr + MMU_PAGESIZE - 1 <=
3265 dma_attr->dma_attr_addr_hi)) {
3266 break;
3267 }
3268
3269 /* continue looking */
3270 page_unlock(pp);
3271 pp = pp->p_next;
3272 if (pp == first_pp)
3273 pp = NULL;
3274 }
3275
3276 if (pp != NULL) {
3277 ASSERT(mtype == PP_2_MTYPE(pp));
3278 ASSERT(pp->p_szc == 0);
3279
3280 /* found a page with specified DMA attributes */
3281 page_sub(&PAGE_CACHELISTS(mnode, bin,
3282 mtype), pp);
3283 page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST);
3284
3285 mutex_exit(pcm);
3286 ASSERT(pp->p_vnode);
3287 ASSERT(PP_ISAGED(pp) == 0);
3288 check_dma(dma_attr, pp, 1);
3289 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3290 return (pp);
3291 }
3292 mutex_exit(pcm);
3293 nextcachebin:
3294 bin += (i == 0) ? BIN_STEP : 1;
3295 bin &= page_colors_mask;
3296 }
3297 MTYPE_NEXT(mnode, mtype, flags);
3298 } while (mtype >= 0);
3299
3300 VM_STAT_ADD(pga_vmstats.pgma_allocfailed);
3301 return (NULL);
3302 }
3303
3304 /*
3305 * This function is similar to page_get_freelist()/page_get_cachelist()
3306 * but it searches both the lists to find a page with the specified
3307 * color (or no color) and DMA attributes. The search is done in the
3308 * freelist first and then in the cache list within the highest memory
3309 * range (based on DMA attributes) before searching in the lower
3310 * memory ranges.
3311 *
3312 * Note: This function is called only by page_create_io().
3313 */
3314 /*ARGSUSED*/
3315 static page_t *
3316 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr,
3317 size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t *lgrp)
3318 {
3319 uint_t bin;
3320 int mtype;
3321 page_t *pp;
3322 int n;
3323 int m;
3324 int szc;
3325 int fullrange;
3326 int mnode;
3327 int local_failed_stat = 0;
3328 lgrp_mnode_cookie_t lgrp_cookie;
3329
3330 VM_STAT_ADD(pga_vmstats.pga_alloc);
3331
3332 /* only base pagesize currently supported */
3333 if (size != MMU_PAGESIZE)
3334 return (NULL);
3335
3336 /*
3337 * If we're passed a specific lgroup, we use it. Otherwise,
3338 * assume first-touch placement is desired.
3339 */
3340 if (!LGRP_EXISTS(lgrp))
3341 lgrp = lgrp_home_lgrp();
3342
3343 /* LINTED */
3344 AS_2_BIN(as, seg, vp, vaddr, bin, 0);
3345
3346 /*
3347 * Only hold one freelist or cachelist lock at a time, that way we
3348 * can start anywhere and not have to worry about lock
3349 * ordering.
3350 */
3351 if (dma_attr == NULL) {
3352 n = mtype16m;
3353 m = mtypetop;
3354 fullrange = 1;
3355 VM_STAT_ADD(pga_vmstats.pga_nulldmaattr);
3356 } else {
3357 pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo);
3358 pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi);
3359
3360 /*
3361 * We can guarantee alignment only for page boundary.
3362 */
3363 if (dma_attr->dma_attr_align > MMU_PAGESIZE)
3364 return (NULL);
3365
3366 /* Sanity check the dma_attr */
3367 if (pfnlo > pfnhi)
3368 return (NULL);
3369
3370 n = pfn_2_mtype(pfnlo);
3371 m = pfn_2_mtype(pfnhi);
3372
3373 fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) &&
3374 (pfnhi >= mnoderanges[m].mnr_pfnhi));
3375 }
3376 VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange);
3377
3378 szc = 0;
3379
3380 /* cylcing thru mtype handled by RANGE0 if n == mtype16m */
3381 if (n == mtype16m) {
3382 flags |= PGI_MT_RANGE0;
3383 n = m;
3384 }
3385
3386 /*
3387 * Try local memory node first, but try remote if we can't
3388 * get a page of the right color.
3389 */
3390 LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER);
3391 while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) {
3392 /*
3393 * allocate pages from high pfn to low.
3394 */
3395 mtype = m;
3396 do {
3397 if (fullrange != 0) {
3398 pp = page_get_mnode_freelist(mnode,
3399 bin, mtype, szc, flags);
3400 if (pp == NULL) {
3401 pp = page_get_mnode_cachelist(
3402 bin, flags, mnode, mtype);
3403 }
3404 } else {
3405 pp = page_get_mnode_anylist(bin, szc,
3406 flags, mnode, mtype, dma_attr);
3407 }
3408 if (pp != NULL) {
3409 VM_STAT_ADD(pga_vmstats.pga_allocok);
3410 check_dma(dma_attr, pp, 1);
3411 return (pp);
3412 }
3413 } while (mtype != n &&
3414 (mtype = mnoderanges[mtype].mnr_next) != -1);
3415 if (!local_failed_stat) {
3416 lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1);
3417 local_failed_stat = 1;
3418 }
3419 }
3420 VM_STAT_ADD(pga_vmstats.pga_allocfailed);
3421
3422 return (NULL);
3423 }
3424
3425 /*
3426 * page_create_io()
3427 *
3428 * This function is a copy of page_create_va() with an additional
3429 * argument 'mattr' that specifies DMA memory requirements to
3430 * the page list functions. This function is used by the segkmem
3431 * allocator so it is only to create new pages (i.e PG_EXCL is
3432 * set).
3433 *
3434 * Note: This interface is currently used by x86 PSM only and is
3435 * not fully specified so the commitment level is only for
3436 * private interface specific to x86. This interface uses PSM
3437 * specific page_get_anylist() interface.
3438 */
3439
3440 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \
3441 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
3442 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
3443 break; \
3444 } \
3445 }
3446
3447
3448 page_t *
3449 page_create_io(
3450 struct vnode *vp,
3451 u_offset_t off,
3452 uint_t bytes,
3453 uint_t flags,
3454 struct as *as,
3455 caddr_t vaddr,
3456 ddi_dma_attr_t *mattr) /* DMA memory attributes if any */
3457 {
3458 page_t *plist = NULL;
3459 uint_t plist_len = 0;
3460 pgcnt_t npages;
3461 page_t *npp = NULL;
3462 uint_t pages_req;
3463 page_t *pp;
3464 kmutex_t *phm = NULL;
3465 uint_t index;
3466
3467 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
3468 "page_create_start:vp %p off %llx bytes %u flags %x",
3469 vp, off, bytes, flags);
3470
3471 ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0);
3472
3473 pages_req = npages = mmu_btopr(bytes);
3474
3475 /*
3476 * Do the freemem and pcf accounting.
3477 */
3478 if (!page_create_wait(npages, flags)) {
3479 return (NULL);
3480 }
3481
3482 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
3483 "page_create_success:vp %p off %llx", vp, off);
3484
3485 /*
3486 * If satisfying this request has left us with too little
3487 * memory, start the wheels turning to get some back. The
3488 * first clause of the test prevents waking up the pageout
3489 * daemon in situations where it would decide that there's
3490 * nothing to do.
3491 */
3492 if (nscan < desscan && freemem < minfree) {
3493 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
3494 "pageout_cv_signal:freemem %ld", freemem);
3495 cv_signal(&proc_pageout->p_cv);
3496 }
3497
3498 if (flags & PG_PHYSCONTIG) {
3499
3500 plist = page_get_contigpage(&npages, mattr, 1);
3501 if (plist == NULL) {
3502 page_create_putback(npages);
3503 return (NULL);
3504 }
3505
3506 pp = plist;
3507
3508 do {
3509 if (!page_hashin(pp, vp, off, NULL)) {
3510 panic("pg_creat_io: hashin failed %p %p %llx",
3511 (void *)pp, (void *)vp, off);
3512 }
3513 VM_STAT_ADD(page_create_new);
3514 off += MMU_PAGESIZE;
3515 PP_CLRFREE(pp);
3516 PP_CLRAGED(pp);
3517 page_set_props(pp, P_REF);
3518 pp = pp->p_next;
3519 } while (pp != plist);
3520
3521 if (!npages) {
3522 check_dma(mattr, plist, pages_req);
3523 return (plist);
3524 } else {
3525 vaddr += (pages_req - npages) << MMU_PAGESHIFT;
3526 }
3527
3528 /*
3529 * fall-thru:
3530 *
3531 * page_get_contigpage returns when npages <= sgllen.
3532 * Grab the rest of the non-contig pages below from anylist.
3533 */
3534 }
3535
3536 /*
3537 * Loop around collecting the requested number of pages.
3538 * Most of the time, we have to `create' a new page. With
3539 * this in mind, pull the page off the free list before
3540 * getting the hash lock. This will minimize the hash
3541 * lock hold time, nesting, and the like. If it turns
3542 * out we don't need the page, we put it back at the end.
3543 */
3544 while (npages--) {
3545 phm = NULL;
3546
3547 index = PAGE_HASH_FUNC(vp, off);
3548 top:
3549 ASSERT(phm == NULL);
3550 ASSERT(index == PAGE_HASH_FUNC(vp, off));
3551 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3552
3553 if (npp == NULL) {
3554 /*
3555 * Try to get the page of any color either from
3556 * the freelist or from the cache list.
3557 */
3558 npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE,
3559 flags & ~PG_MATCH_COLOR, mattr, NULL);
3560 if (npp == NULL) {
3561 if (mattr == NULL) {
3562 /*
3563 * Not looking for a special page;
3564 * panic!
3565 */
3566 panic("no page found %d", (int)npages);
3567 }
3568 /*
3569 * No page found! This can happen
3570 * if we are looking for a page
3571 * within a specific memory range
3572 * for DMA purposes. If PG_WAIT is
3573 * specified then we wait for a
3574 * while and then try again. The
3575 * wait could be forever if we
3576 * don't get the page(s) we need.
3577 *
3578 * Note: XXX We really need a mechanism
3579 * to wait for pages in the desired
3580 * range. For now, we wait for any
3581 * pages and see if we can use it.
3582 */
3583
3584 if ((mattr != NULL) && (flags & PG_WAIT)) {
3585 delay(10);
3586 goto top;
3587 }
3588 goto fail; /* undo accounting stuff */
3589 }
3590
3591 if (PP_ISAGED(npp) == 0) {
3592 /*
3593 * Since this page came from the
3594 * cachelist, we must destroy the
3595 * old vnode association.
3596 */
3597 page_hashout(npp, (kmutex_t *)NULL);
3598 }
3599 }
3600
3601 /*
3602 * We own this page!
3603 */
3604 ASSERT(PAGE_EXCL(npp));
3605 ASSERT(npp->p_vnode == NULL);
3606 ASSERT(!hat_page_is_mapped(npp));
3607 PP_CLRFREE(npp);
3608 PP_CLRAGED(npp);
3609
3610 /*
3611 * Here we have a page in our hot little mits and are
3612 * just waiting to stuff it on the appropriate lists.
3613 * Get the mutex and check to see if it really does
3614 * not exist.
3615 */
3616 phm = PAGE_HASH_MUTEX(index);
3617 mutex_enter(phm);
3618 PAGE_HASH_SEARCH(index, pp, vp, off);
3619 if (pp == NULL) {
3620 VM_STAT_ADD(page_create_new);
3621 pp = npp;
3622 npp = NULL;
3623 if (!page_hashin(pp, vp, off, phm)) {
3624 /*
3625 * Since we hold the page hash mutex and
3626 * just searched for this page, page_hashin
3627 * had better not fail. If it does, that
3628 * means somethread did not follow the
3629 * page hash mutex rules. Panic now and
3630 * get it over with. As usual, go down
3631 * holding all the locks.
3632 */
3633 ASSERT(MUTEX_HELD(phm));
3634 panic("page_create: hashin fail %p %p %llx %p",
3635 (void *)pp, (void *)vp, off, (void *)phm);
3636
3637 }
3638 ASSERT(MUTEX_HELD(phm));
3639 mutex_exit(phm);
3640 phm = NULL;
3641
3642 /*
3643 * Hat layer locking need not be done to set
3644 * the following bits since the page is not hashed
3645 * and was on the free list (i.e., had no mappings).
3646 *
3647 * Set the reference bit to protect
3648 * against immediate pageout
3649 *
3650 * XXXmh modify freelist code to set reference
3651 * bit so we don't have to do it here.
3652 */
3653 page_set_props(pp, P_REF);
3654 } else {
3655 ASSERT(MUTEX_HELD(phm));
3656 mutex_exit(phm);
3657 phm = NULL;
3658 /*
3659 * NOTE: This should not happen for pages associated
3660 * with kernel vnode 'kvp'.
3661 */
3662 /* XX64 - to debug why this happens! */
3663 ASSERT(!VN_ISKAS(vp));
3664 if (VN_ISKAS(vp))
3665 cmn_err(CE_NOTE,
3666 "page_create: page not expected "
3667 "in hash list for kernel vnode - pp 0x%p",
3668 (void *)pp);
3669 VM_STAT_ADD(page_create_exists);
3670 goto fail;
3671 }
3672
3673 /*
3674 * Got a page! It is locked. Acquire the i/o
3675 * lock since we are going to use the p_next and
3676 * p_prev fields to link the requested pages together.
3677 */
3678 page_io_lock(pp);
3679 page_add(&plist, pp);
3680 plist = plist->p_next;
3681 off += MMU_PAGESIZE;
3682 vaddr += MMU_PAGESIZE;
3683 }
3684
3685 check_dma(mattr, plist, pages_req);
3686 return (plist);
3687
3688 fail:
3689 if (npp != NULL) {
3690 /*
3691 * Did not need this page after all.
3692 * Put it back on the free list.
3693 */
3694 VM_STAT_ADD(page_create_putbacks);
3695 PP_SETFREE(npp);
3696 PP_SETAGED(npp);
3697 npp->p_offset = (u_offset_t)-1;
3698 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
3699 page_unlock(npp);
3700 }
3701
3702 /*
3703 * Give up the pages we already got.
3704 */
3705 while (plist != NULL) {
3706 pp = plist;
3707 page_sub(&plist, pp);
3708 page_io_unlock(pp);
3709 plist_len++;
3710 /*LINTED: constant in conditional ctx*/
3711 VN_DISPOSE(pp, B_INVAL, 0, kcred);
3712 }
3713
3714 /*
3715 * VN_DISPOSE does freemem accounting for the pages in plist
3716 * by calling page_free. So, we need to undo the pcf accounting
3717 * for only the remaining pages.
3718 */
3719 VM_STAT_ADD(page_create_putbacks);
3720 page_create_putback(pages_req - plist_len);
3721
3722 return (NULL);
3723 }
3724 #endif /* !__xpv */
3725
3726
3727 /*
3728 * Copy the data from the physical page represented by "frompp" to
3729 * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and
3730 * CPU->cpu_caddr2. It assumes that no one uses either map at interrupt
3731 * level and no one sleeps with an active mapping there.
3732 *
3733 * Note that the ref/mod bits in the page_t's are not affected by
3734 * this operation, hence it is up to the caller to update them appropriately.
3735 */
3736 int
3737 ppcopy(page_t *frompp, page_t *topp)
3738 {
3739 caddr_t pp_addr1;
3740 caddr_t pp_addr2;
3741 hat_mempte_t pte1;
3742 hat_mempte_t pte2;
3743 kmutex_t *ppaddr_mutex;
3744 label_t ljb;
3745 int ret = 1;
3746
3747 ASSERT_STACK_ALIGNED();
3748 ASSERT(PAGE_LOCKED(frompp));
3749 ASSERT(PAGE_LOCKED(topp));
3750
3751 if (kpm_enable) {
3752 pp_addr1 = hat_kpm_page2va(frompp, 0);
3753 pp_addr2 = hat_kpm_page2va(topp, 0);
3754 kpreempt_disable();
3755 } else {
3756 /*
3757 * disable pre-emption so that CPU can't change
3758 */
3759 kpreempt_disable();
3760
3761 pp_addr1 = CPU->cpu_caddr1;
3762 pp_addr2 = CPU->cpu_caddr2;
3763 pte1 = CPU->cpu_caddr1pte;
3764 pte2 = CPU->cpu_caddr2pte;
3765
3766 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3767 mutex_enter(ppaddr_mutex);
3768
3769 hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1,
3770 PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST);
3771 hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2,
3772 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3773 HAT_LOAD_NOCONSIST);
3774 }
3775
3776 if (on_fault(&ljb)) {
3777 ret = 0;
3778 goto faulted;
3779 }
3780 if (use_sse_pagecopy)
3781 #ifdef __xpv
3782 page_copy_no_xmm(pp_addr2, pp_addr1);
3783 #else
3784 hwblkpagecopy(pp_addr1, pp_addr2);
3785 #endif
3786 else
3787 bcopy(pp_addr1, pp_addr2, PAGESIZE);
3788
3789 no_fault();
3790 faulted:
3791 if (!kpm_enable) {
3792 #ifdef __xpv
3793 /*
3794 * We can't leave unused mappings laying about under the
3795 * hypervisor, so blow them away.
3796 */
3797 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0,
3798 UVMF_INVLPG | UVMF_LOCAL) < 0)
3799 panic("HYPERVISOR_update_va_mapping() failed");
3800 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3801 UVMF_INVLPG | UVMF_LOCAL) < 0)
3802 panic("HYPERVISOR_update_va_mapping() failed");
3803 #endif
3804 mutex_exit(ppaddr_mutex);
3805 }
3806 kpreempt_enable();
3807 return (ret);
3808 }
3809
3810 void
3811 pagezero(page_t *pp, uint_t off, uint_t len)
3812 {
3813 ASSERT(PAGE_LOCKED(pp));
3814 pfnzero(page_pptonum(pp), off, len);
3815 }
3816
3817 /*
3818 * Zero the physical page from off to off + len given by pfn
3819 * without changing the reference and modified bits of page.
3820 *
3821 * We use this using CPU private page address #2, see ppcopy() for more info.
3822 * pfnzero() must not be called at interrupt level.
3823 */
3824 void
3825 pfnzero(pfn_t pfn, uint_t off, uint_t len)
3826 {
3827 caddr_t pp_addr2;
3828 hat_mempte_t pte2;
3829 kmutex_t *ppaddr_mutex = NULL;
3830
3831 ASSERT_STACK_ALIGNED();
3832 ASSERT(len <= MMU_PAGESIZE);
3833 ASSERT(off <= MMU_PAGESIZE);
3834 ASSERT(off + len <= MMU_PAGESIZE);
3835
3836 if (kpm_enable && !pfn_is_foreign(pfn)) {
3837 pp_addr2 = hat_kpm_pfn2va(pfn);
3838 kpreempt_disable();
3839 } else {
3840 kpreempt_disable();
3841
3842 pp_addr2 = CPU->cpu_caddr2;
3843 pte2 = CPU->cpu_caddr2pte;
3844
3845 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3846 mutex_enter(ppaddr_mutex);
3847
3848 hat_mempte_remap(pfn, pp_addr2, pte2,
3849 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3850 HAT_LOAD_NOCONSIST);
3851 }
3852
3853 if (use_sse_pagezero) {
3854 #ifdef __xpv
3855 uint_t rem;
3856
3857 /*
3858 * zero a byte at a time until properly aligned for
3859 * block_zero_no_xmm().
3860 */
3861 while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0)
3862 pp_addr2[off++] = 0;
3863
3864 /*
3865 * Now use faster block_zero_no_xmm() for any range
3866 * that is properly aligned and sized.
3867 */
3868 rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN));
3869 len -= rem;
3870 if (len != 0) {
3871 block_zero_no_xmm(pp_addr2 + off, len);
3872 off += len;
3873 }
3874
3875 /*
3876 * zero remainder with byte stores.
3877 */
3878 while (rem-- > 0)
3879 pp_addr2[off++] = 0;
3880 #else
3881 hwblkclr(pp_addr2 + off, len);
3882 #endif
3883 } else {
3884 bzero(pp_addr2 + off, len);
3885 }
3886
3887 if (!kpm_enable || pfn_is_foreign(pfn)) {
3888 #ifdef __xpv
3889 /*
3890 * On the hypervisor this page might get used for a page
3891 * table before any intervening change to this mapping,
3892 * so blow it away.
3893 */
3894 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3895 UVMF_INVLPG) < 0)
3896 panic("HYPERVISOR_update_va_mapping() failed");
3897 #endif
3898 mutex_exit(ppaddr_mutex);
3899 }
3900
3901 kpreempt_enable();
3902 }
3903
3904 /*
3905 * Platform-dependent page scrub call.
3906 */
3907 void
3908 pagescrub(page_t *pp, uint_t off, uint_t len)
3909 {
3910 /*
3911 * For now, we rely on the fact that pagezero() will
3912 * always clear UEs.
3913 */
3914 pagezero(pp, off, len);
3915 }
3916
3917 /*
3918 * set up two private addresses for use on a given CPU for use in ppcopy()
3919 */
3920 void
3921 setup_vaddr_for_ppcopy(struct cpu *cpup)
3922 {
3923 void *addr;
3924 hat_mempte_t pte_pa;
3925
3926 addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3927 pte_pa = hat_mempte_setup(addr);
3928 cpup->cpu_caddr1 = addr;
3929 cpup->cpu_caddr1pte = pte_pa;
3930
3931 addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3932 pte_pa = hat_mempte_setup(addr);
3933 cpup->cpu_caddr2 = addr;
3934 cpup->cpu_caddr2pte = pte_pa;
3935
3936 mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL);
3937 }
3938
3939 /*
3940 * Undo setup_vaddr_for_ppcopy
3941 */
3942 void
3943 teardown_vaddr_for_ppcopy(struct cpu *cpup)
3944 {
3945 mutex_destroy(&cpup->cpu_ppaddr_mutex);
3946
3947 hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte);
3948 cpup->cpu_caddr2pte = 0;
3949 vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1));
3950 cpup->cpu_caddr2 = 0;
3951
3952 hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte);
3953 cpup->cpu_caddr1pte = 0;
3954 vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1));
3955 cpup->cpu_caddr1 = 0;
3956 }
3957
3958 /*
3959 * Function for flushing D-cache when performing module relocations
3960 * to an alternate mapping. Unnecessary on Intel / AMD platforms.
3961 */
3962 void
3963 dcache_flushall()
3964 {}
3965
3966 /*
3967 * Allocate a memory page. The argument 'seed' can be any pseudo-random
3968 * number to vary where the pages come from. This is quite a hacked up
3969 * method -- it works for now, but really needs to be fixed up a bit.
3970 *
3971 * We currently use page_create_va() on the kvp with fake offsets,
3972 * segments and virt address. This is pretty bogus, but was copied from the
3973 * old hat_i86.c code. A better approach would be to specify either mnode
3974 * random or mnode local and takes a page from whatever color has the MOST
3975 * available - this would have a minimal impact on page coloring.
3976 */
3977 page_t *
3978 page_get_physical(uintptr_t seed)
3979 {
3980 page_t *pp;
3981 u_offset_t offset;
3982 static struct seg tmpseg;
3983 static uintptr_t ctr = 0;
3984
3985 /*
3986 * This code is gross, we really need a simpler page allocator.
3987 *
3988 * We need to assign an offset for the page to call page_create_va()
3989 * To avoid conflicts with other pages, we get creative with the offset.
3990 * For 32 bits, we need an offset > 4Gig
3991 * For 64 bits, need an offset somewhere in the VA hole.
3992 */
3993 offset = seed;
3994 if (offset > kernelbase)
3995 offset -= kernelbase;
3996 offset <<= MMU_PAGESHIFT;
3997 #if defined(__amd64)
3998 offset += mmu.hole_start; /* something in VA hole */
3999 #else
4000 offset += 1ULL << 40; /* something > 4 Gig */
4001 #endif
4002
4003 if (page_resv(1, KM_NOSLEEP) == 0)
4004 return (NULL);
4005
4006 #ifdef DEBUG
4007 pp = page_exists(&kvp, offset);
4008 if (pp != NULL)
4009 panic("page already exists %p", (void *)pp);
4010 #endif
4011
4012 pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL,
4013 &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE)); /* changing VA usage */
4014 if (pp != NULL) {
4015 page_io_unlock(pp);
4016 page_downgrade(pp);
4017 }
4018 return (pp);
4019 }