1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 /*
  25  * Copyright (c) 2010, Intel Corporation.
  26  * All rights reserved.
  27  * Copyright 2016 Joyent, Inc.
  28  */
  29 
  30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
  31 /*      All Rights Reserved   */
  32 
  33 /*
  34  * Portions of this source code were derived from Berkeley 4.3 BSD
  35  * under license from the Regents of the University of California.
  36  */
  37 
  38 /*
  39  * UNIX machine dependent virtual memory support.
  40  */
  41 
  42 #include <sys/types.h>
  43 #include <sys/param.h>
  44 #include <sys/systm.h>
  45 #include <sys/user.h>
  46 #include <sys/proc.h>
  47 #include <sys/kmem.h>
  48 #include <sys/vmem.h>
  49 #include <sys/buf.h>
  50 #include <sys/cpuvar.h>
  51 #include <sys/lgrp.h>
  52 #include <sys/disp.h>
  53 #include <sys/vm.h>
  54 #include <sys/mman.h>
  55 #include <sys/vnode.h>
  56 #include <sys/cred.h>
  57 #include <sys/exec.h>
  58 #include <sys/exechdr.h>
  59 #include <sys/debug.h>
  60 #include <sys/vmsystm.h>
  61 #include <sys/swap.h>
  62 #include <sys/dumphdr.h>
  63 #include <sys/random.h>
  64 
  65 #include <vm/hat.h>
  66 #include <vm/as.h>
  67 #include <vm/seg.h>
  68 #include <vm/seg_kp.h>
  69 #include <vm/seg_vn.h>
  70 #include <vm/page.h>
  71 #include <vm/seg_kmem.h>
  72 #include <vm/seg_kpm.h>
  73 #include <vm/vm_dep.h>
  74 
  75 #include <sys/cpu.h>
  76 #include <sys/vm_machparam.h>
  77 #include <sys/memlist.h>
  78 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */
  79 #include <vm/hat_i86.h>
  80 #include <sys/x86_archext.h>
  81 #include <sys/elf_386.h>
  82 #include <sys/cmn_err.h>
  83 #include <sys/archsystm.h>
  84 #include <sys/machsystm.h>
  85 #include <sys/secflags.h>
  86 
  87 #include <sys/vtrace.h>
  88 #include <sys/ddidmareq.h>
  89 #include <sys/promif.h>
  90 #include <sys/memnode.h>
  91 #include <sys/stack.h>
  92 #include <util/qsort.h>
  93 #include <sys/taskq.h>
  94 
  95 #ifdef __xpv
  96 
  97 #include <sys/hypervisor.h>
  98 #include <sys/xen_mmu.h>
  99 #include <sys/balloon_impl.h>
 100 
 101 /*
 102  * domain 0 pages usable for DMA are kept pre-allocated and kept in
 103  * distinct lists, ordered by increasing mfn.
 104  */
 105 static kmutex_t io_pool_lock;
 106 static kmutex_t contig_list_lock;
 107 static page_t *io_pool_4g;      /* pool for 32 bit dma limited devices */
 108 static page_t *io_pool_16m;     /* pool for 24 bit dma limited legacy devices */
 109 static long io_pool_cnt;
 110 static long io_pool_cnt_max = 0;
 111 #define DEFAULT_IO_POOL_MIN     128
 112 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN;
 113 static long io_pool_cnt_lowater = 0;
 114 static long io_pool_shrink_attempts; /* how many times did we try to shrink */
 115 static long io_pool_shrinks;    /* how many times did we really shrink */
 116 static long io_pool_grows;      /* how many times did we grow */
 117 static mfn_t start_mfn = 1;
 118 static caddr_t io_pool_kva;     /* use to alloc pages when needed */
 119 
 120 static int create_contig_pfnlist(uint_t);
 121 
 122 /*
 123  * percentage of phys mem to hold in the i/o pool
 124  */
 125 #define DEFAULT_IO_POOL_PCT     2
 126 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT;
 127 static void page_io_pool_sub(page_t **, page_t *, page_t *);
 128 int ioalloc_dbg = 0;
 129 
 130 #endif /* __xpv */
 131 
 132 uint_t vac_colors = 1;
 133 
 134 int largepagesupport = 0;
 135 extern uint_t page_create_new;
 136 extern uint_t page_create_exists;
 137 extern uint_t page_create_putbacks;
 138 /*
 139  * Allow users to disable the kernel's use of SSE.
 140  */
 141 extern int use_sse_pagecopy, use_sse_pagezero;
 142 
 143 /*
 144  * combined memory ranges from mnode and memranges[] to manage single
 145  * mnode/mtype dimension in the page lists.
 146  */
 147 typedef struct {
 148         pfn_t   mnr_pfnlo;
 149         pfn_t   mnr_pfnhi;
 150         int     mnr_mnode;
 151         int     mnr_memrange;           /* index into memranges[] */
 152         int     mnr_next;               /* next lower PA mnoderange */
 153         int     mnr_exists;
 154         /* maintain page list stats */
 155         pgcnt_t mnr_mt_clpgcnt;         /* cache list cnt */
 156         pgcnt_t mnr_mt_flpgcnt[MMU_PAGE_SIZES]; /* free list cnt per szc */
 157         pgcnt_t mnr_mt_totcnt;          /* sum of cache and free lists */
 158 #ifdef DEBUG
 159         struct mnr_mts {                /* mnode/mtype szc stats */
 160                 pgcnt_t mnr_mts_pgcnt;
 161                 int     mnr_mts_colors;
 162                 pgcnt_t *mnr_mtsc_pgcnt;
 163         }       *mnr_mts;
 164 #endif
 165 } mnoderange_t;
 166 
 167 #define MEMRANGEHI(mtype)                                               \
 168         ((mtype > 0) ? memranges[mtype - 1] - 1: physmax)
 169 #define MEMRANGELO(mtype)       (memranges[mtype])
 170 
 171 #define MTYPE_FREEMEM(mt)       (mnoderanges[mt].mnr_mt_totcnt)
 172 
 173 /*
 174  * As the PC architecture evolved memory up was clumped into several
 175  * ranges for various historical I/O devices to do DMA.
 176  * < 16Meg - ISA bus
 177  * < 2Gig - ???
 178  * < 4Gig - PCI bus or drivers that don't understand PAE mode
 179  *
 180  * These are listed in reverse order, so that we can skip over unused
 181  * ranges on machines with small memories.
 182  *
 183  * For now under the Hypervisor, we'll only ever have one memrange.
 184  */
 185 #define PFN_4GIG        0x100000
 186 #define PFN_16MEG       0x1000
 187 /* Indices into the memory range (arch_memranges) array. */
 188 #define MRI_4G          0
 189 #define MRI_2G          1
 190 #define MRI_16M         2
 191 #define MRI_0           3
 192 static pfn_t arch_memranges[NUM_MEM_RANGES] = {
 193     PFN_4GIG,   /* pfn range for 4G and above */
 194     0x80000,    /* pfn range for 2G-4G */
 195     PFN_16MEG,  /* pfn range for 16M-2G */
 196     0x00000,    /* pfn range for 0-16M */
 197 };
 198 pfn_t *memranges = &arch_memranges[0];
 199 int nranges = NUM_MEM_RANGES;
 200 
 201 /*
 202  * This combines mem_node_config and memranges into one data
 203  * structure to be used for page list management.
 204  */
 205 mnoderange_t    *mnoderanges;
 206 int             mnoderangecnt;
 207 int             mtype4g;
 208 int             mtype16m;
 209 int             mtypetop;       /* index of highest pfn'ed mnoderange */
 210 
 211 /*
 212  * 4g memory management variables for systems with more than 4g of memory:
 213  *
 214  * physical memory below 4g is required for 32bit dma devices and, currently,
 215  * for kmem memory. On systems with more than 4g of memory, the pool of memory
 216  * below 4g can be depleted without any paging activity given that there is
 217  * likely to be sufficient memory above 4g.
 218  *
 219  * physmax4g is set true if the largest pfn is over 4g. The rest of the
 220  * 4g memory management code is enabled only when physmax4g is true.
 221  *
 222  * maxmem4g is the count of the maximum number of pages on the page lists
 223  * with physical addresses below 4g. It can be a lot less then 4g given that
 224  * BIOS may reserve large chunks of space below 4g for hot plug pci devices,
 225  * agp aperture etc.
 226  *
 227  * freemem4g maintains the count of the number of available pages on the
 228  * page lists with physical addresses below 4g.
 229  *
 230  * DESFREE4G specifies the desired amount of below 4g memory. It defaults to
 231  * 6% (desfree4gshift = 4) of maxmem4g.
 232  *
 233  * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G
 234  * and the amount of physical memory above 4g is greater than freemem4g.
 235  * In this case, page_get_* routines will restrict below 4g allocations
 236  * for requests that don't specifically require it.
 237  */
 238 
 239 #define DESFREE4G       (maxmem4g >> desfree4gshift)
 240 
 241 #define RESTRICT4G_ALLOC                                        \
 242         (physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem))
 243 
 244 static pgcnt_t  maxmem4g;
 245 static pgcnt_t  freemem4g;
 246 static int      physmax4g;
 247 static int      desfree4gshift = 4;     /* maxmem4g shift to derive DESFREE4G */
 248 
 249 /*
 250  * 16m memory management:
 251  *
 252  * reserve some amount of physical memory below 16m for legacy devices.
 253  *
 254  * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above
 255  * 16m or if the 16m pool drops below DESFREE16M.
 256  *
 257  * In this case, general page allocations via page_get_{free,cache}list
 258  * routines will be restricted from allocating from the 16m pool. Allocations
 259  * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations
 260  * are not restricted.
 261  */
 262 
 263 #define FREEMEM16M      MTYPE_FREEMEM(mtype16m)
 264 #define DESFREE16M      desfree16m
 265 #define RESTRICT16M_ALLOC(freemem, pgcnt, flags)                \
 266         ((freemem != 0) && ((flags & PG_PANIC) == 0) &&             \
 267             ((freemem >= (FREEMEM16M)) ||                    \
 268             (FREEMEM16M  < (DESFREE16M + pgcnt))))
 269 
 270 static pgcnt_t  desfree16m = 0x380;
 271 
 272 /*
 273  * This can be patched via /etc/system to allow old non-PAE aware device
 274  * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM.
 275  */
 276 int restricted_kmemalloc = 0;
 277 
 278 #ifdef VM_STATS
 279 struct {
 280         ulong_t pga_alloc;
 281         ulong_t pga_notfullrange;
 282         ulong_t pga_nulldmaattr;
 283         ulong_t pga_allocok;
 284         ulong_t pga_allocfailed;
 285         ulong_t pgma_alloc;
 286         ulong_t pgma_allocok;
 287         ulong_t pgma_allocfailed;
 288         ulong_t pgma_allocempty;
 289 } pga_vmstats;
 290 #endif
 291 
 292 uint_t mmu_page_sizes;
 293 
 294 /* How many page sizes the users can see */
 295 uint_t mmu_exported_page_sizes;
 296 
 297 /* page sizes that legacy applications can see */
 298 uint_t mmu_legacy_page_sizes;
 299 
 300 /*
 301  * Number of pages in 1 GB.  Don't enable automatic large pages if we have
 302  * fewer than this many pages.
 303  */
 304 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
 305 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
 306 
 307 /*
 308  * Maximum and default segment size tunables for user private
 309  * and shared anon memory, and user text and initialized data.
 310  * These can be patched via /etc/system to allow large pages
 311  * to be used for mapping application private and shared anon memory.
 312  */
 313 size_t mcntl0_lpsize = MMU_PAGESIZE;
 314 size_t max_uheap_lpsize = MMU_PAGESIZE;
 315 size_t default_uheap_lpsize = MMU_PAGESIZE;
 316 size_t max_ustack_lpsize = MMU_PAGESIZE;
 317 size_t default_ustack_lpsize = MMU_PAGESIZE;
 318 size_t max_privmap_lpsize = MMU_PAGESIZE;
 319 size_t max_uidata_lpsize = MMU_PAGESIZE;
 320 size_t max_utext_lpsize = MMU_PAGESIZE;
 321 size_t max_shm_lpsize = MMU_PAGESIZE;
 322 
 323 
 324 /*
 325  * initialized by page_coloring_init().
 326  */
 327 uint_t  page_colors;
 328 uint_t  page_colors_mask;
 329 uint_t  page_coloring_shift;
 330 int     cpu_page_colors;
 331 static uint_t   l2_colors;
 332 
 333 /*
 334  * Page freelists and cachelists are dynamically allocated once mnoderangecnt
 335  * and page_colors are calculated from the l2 cache n-way set size.  Within a
 336  * mnode range, the page freelist and cachelist are hashed into bins based on
 337  * color. This makes it easier to search for a page within a specific memory
 338  * range.
 339  */
 340 #define PAGE_COLORS_MIN 16
 341 
 342 page_t ****page_freelists;
 343 page_t ***page_cachelists;
 344 
 345 
 346 /*
 347  * Used by page layer to know about page sizes
 348  */
 349 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1];
 350 
 351 kmutex_t        *fpc_mutex[NPC_MUTEX];
 352 kmutex_t        *cpc_mutex[NPC_MUTEX];
 353 
 354 /* Lock to protect mnoderanges array for memory DR operations. */
 355 static kmutex_t mnoderange_lock;
 356 
 357 /*
 358  * Only let one thread at a time try to coalesce large pages, to
 359  * prevent them from working against each other.
 360  */
 361 static kmutex_t contig_lock;
 362 #define CONTIG_LOCK()   mutex_enter(&contig_lock);
 363 #define CONTIG_UNLOCK() mutex_exit(&contig_lock);
 364 
 365 #define PFN_16M         (mmu_btop((uint64_t)0x1000000))
 366 
 367 /*
 368  * Return the optimum page size for a given mapping
 369  */
 370 /*ARGSUSED*/
 371 size_t
 372 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl)
 373 {
 374         level_t l = 0;
 375         size_t pgsz = MMU_PAGESIZE;
 376         size_t max_lpsize;
 377         uint_t mszc;
 378 
 379         ASSERT(maptype != MAPPGSZ_VA);
 380 
 381         if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) {
 382                 return (MMU_PAGESIZE);
 383         }
 384 
 385         switch (maptype) {
 386         case MAPPGSZ_HEAP:
 387         case MAPPGSZ_STK:
 388                 max_lpsize = memcntl ? mcntl0_lpsize : (maptype ==
 389                     MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize);
 390                 if (max_lpsize == MMU_PAGESIZE) {
 391                         return (MMU_PAGESIZE);
 392                 }
 393                 if (len == 0) {
 394                         len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase +
 395                             p->p_brksize - p->p_bssbase : p->p_stksize;
 396                 }
 397                 len = (maptype == MAPPGSZ_HEAP) ? MAX(len,
 398                     default_uheap_lpsize) : MAX(len, default_ustack_lpsize);
 399 
 400                 /*
 401                  * use the pages size that best fits len
 402                  */
 403                 for (l = mmu.umax_page_level; l > 0; --l) {
 404                         if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) {
 405                                 continue;
 406                         } else {
 407                                 pgsz = LEVEL_SIZE(l);
 408                         }
 409                         break;
 410                 }
 411 
 412                 mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc :
 413                     p->p_stkpageszc);
 414                 if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) {
 415                         pgsz = hw_page_array[mszc].hp_size;
 416                 }
 417                 return (pgsz);
 418 
 419         case MAPPGSZ_ISM:
 420                 for (l = mmu.umax_page_level; l > 0; --l) {
 421                         if (len >= LEVEL_SIZE(l))
 422                                 return (LEVEL_SIZE(l));
 423                 }
 424                 return (LEVEL_SIZE(0));
 425         }
 426         return (pgsz);
 427 }
 428 
 429 static uint_t
 430 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize,
 431     size_t min_physmem)
 432 {
 433         caddr_t eaddr = addr + size;
 434         uint_t szcvec = 0;
 435         caddr_t raddr;
 436         caddr_t readdr;
 437         size_t  pgsz;
 438         int i;
 439 
 440         if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) {
 441                 return (0);
 442         }
 443 
 444         for (i = mmu_exported_page_sizes - 1; i > 0; i--) {
 445                 pgsz = page_get_pagesize(i);
 446                 if (pgsz > max_lpsize) {
 447                         continue;
 448                 }
 449                 raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
 450                 readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
 451                 if (raddr < addr || raddr >= readdr) {
 452                         continue;
 453                 }
 454                 if (P2PHASE((uintptr_t)addr ^ off, pgsz)) {
 455                         continue;
 456                 }
 457                 /*
 458                  * Set szcvec to the remaining page sizes.
 459                  */
 460                 szcvec = ((1 << (i + 1)) - 1) & ~1;
 461                 break;
 462         }
 463         return (szcvec);
 464 }
 465 
 466 /*
 467  * Return a bit vector of large page size codes that
 468  * can be used to map [addr, addr + len) region.
 469  */
 470 /*ARGSUSED*/
 471 uint_t
 472 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type,
 473     int memcntl)
 474 {
 475         size_t max_lpsize = mcntl0_lpsize;
 476 
 477         if (mmu.max_page_level == 0)
 478                 return (0);
 479 
 480         if (flags & MAP_TEXT) {
 481                 if (!memcntl)
 482                         max_lpsize = max_utext_lpsize;
 483                 return (map_szcvec(addr, size, off, max_lpsize,
 484                     shm_lpg_min_physmem));
 485 
 486         } else if (flags & MAP_INITDATA) {
 487                 if (!memcntl)
 488                         max_lpsize = max_uidata_lpsize;
 489                 return (map_szcvec(addr, size, off, max_lpsize,
 490                     privm_lpg_min_physmem));
 491 
 492         } else if (type == MAPPGSZC_SHM) {
 493                 if (!memcntl)
 494                         max_lpsize = max_shm_lpsize;
 495                 return (map_szcvec(addr, size, off, max_lpsize,
 496                     shm_lpg_min_physmem));
 497 
 498         } else if (type == MAPPGSZC_HEAP) {
 499                 if (!memcntl)
 500                         max_lpsize = max_uheap_lpsize;
 501                 return (map_szcvec(addr, size, off, max_lpsize,
 502                     privm_lpg_min_physmem));
 503 
 504         } else if (type == MAPPGSZC_STACK) {
 505                 if (!memcntl)
 506                         max_lpsize = max_ustack_lpsize;
 507                 return (map_szcvec(addr, size, off, max_lpsize,
 508                     privm_lpg_min_physmem));
 509 
 510         } else {
 511                 if (!memcntl)
 512                         max_lpsize = max_privmap_lpsize;
 513                 return (map_szcvec(addr, size, off, max_lpsize,
 514                     privm_lpg_min_physmem));
 515         }
 516 }
 517 
 518 /*
 519  * Handle a pagefault.
 520  */
 521 faultcode_t
 522 pagefault(
 523         caddr_t addr,
 524         enum fault_type type,
 525         enum seg_rw rw,
 526         int iskernel)
 527 {
 528         struct as *as;
 529         struct hat *hat;
 530         struct proc *p;
 531         kthread_t *t;
 532         faultcode_t res;
 533         caddr_t base;
 534         size_t len;
 535         int err;
 536         int mapped_red;
 537         uintptr_t ea;
 538 
 539         ASSERT_STACK_ALIGNED();
 540 
 541         if (INVALID_VADDR(addr))
 542                 return (FC_NOMAP);
 543 
 544         mapped_red = segkp_map_red();
 545 
 546         if (iskernel) {
 547                 as = &kas;
 548                 hat = as->a_hat;
 549         } else {
 550                 t = curthread;
 551                 p = ttoproc(t);
 552                 as = p->p_as;
 553                 hat = as->a_hat;
 554         }
 555 
 556         /*
 557          * Dispatch pagefault.
 558          */
 559         res = as_fault(hat, as, addr, 1, type, rw);
 560 
 561         /*
 562          * If this isn't a potential unmapped hole in the user's
 563          * UNIX data or stack segments, just return status info.
 564          */
 565         if (res != FC_NOMAP || iskernel)
 566                 goto out;
 567 
 568         /*
 569          * Check to see if we happened to faulted on a currently unmapped
 570          * part of the UNIX data or stack segments.  If so, create a zfod
 571          * mapping there and then try calling the fault routine again.
 572          */
 573         base = p->p_brkbase;
 574         len = p->p_brksize;
 575 
 576         if (addr < base || addr >= base + len) {          /* data seg? */
 577                 base = (caddr_t)p->p_usrstack - p->p_stksize;
 578                 len = p->p_stksize;
 579                 if (addr < base || addr >= p->p_usrstack) {    /* stack seg? */
 580                         /* not in either UNIX data or stack segments */
 581                         res = FC_NOMAP;
 582                         goto out;
 583                 }
 584         }
 585 
 586         /*
 587          * the rest of this function implements a 3.X 4.X 5.X compatibility
 588          * This code is probably not needed anymore
 589          */
 590         if (p->p_model == DATAMODEL_ILP32) {
 591 
 592                 /* expand the gap to the page boundaries on each side */
 593                 ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE);
 594                 base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE);
 595                 len = ea - (uintptr_t)base;
 596 
 597                 as_rangelock(as);
 598                 if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) ==
 599                     0) {
 600                         err = as_map(as, base, len, segvn_create, zfod_argsp);
 601                         as_rangeunlock(as);
 602                         if (err) {
 603                                 res = FC_MAKE_ERR(err);
 604                                 goto out;
 605                         }
 606                 } else {
 607                         /*
 608                          * This page is already mapped by another thread after
 609                          * we returned from as_fault() above.  We just fall
 610                          * through as_fault() below.
 611                          */
 612                         as_rangeunlock(as);
 613                 }
 614 
 615                 res = as_fault(hat, as, addr, 1, F_INVAL, rw);
 616         }
 617 
 618 out:
 619         if (mapped_red)
 620                 segkp_unmap_red();
 621 
 622         return (res);
 623 }
 624 
 625 void
 626 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags)
 627 {
 628         struct proc *p = curproc;
 629         caddr_t userlimit = (flags & _MAP_LOW32) ?
 630             (caddr_t)_userlimit32 : p->p_as->a_userlimit;
 631 
 632         map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags);
 633 }
 634 
 635 /*ARGSUSED*/
 636 int
 637 map_addr_vacalign_check(caddr_t addr, u_offset_t off)
 638 {
 639         return (0);
 640 }
 641 
 642 /*
 643  * The maximum amount a randomized mapping will be slewed.  We should perhaps
 644  * arrange things so these tunables can be separate for mmap, mmapobj, and
 645  * ld.so
 646  */
 647 size_t aslr_max_map_skew = 256 * 1024 * 1024; /* 256MB */
 648 
 649 /*
 650  * map_addr_proc() is the routine called when the system is to
 651  * choose an address for the user.  We will pick an address
 652  * range which is the highest available below userlimit.
 653  *
 654  * Every mapping will have a redzone of a single page on either side of
 655  * the request. This is done to leave one page unmapped between segments.
 656  * This is not required, but it's useful for the user because if their
 657  * program strays across a segment boundary, it will catch a fault
 658  * immediately making debugging a little easier.  Currently the redzone
 659  * is mandatory.
 660  *
 661  * addrp is a value/result parameter.
 662  *      On input it is a hint from the user to be used in a completely
 663  *      machine dependent fashion.  We decide to completely ignore this hint.
 664  *      If MAP_ALIGN was specified, addrp contains the minimal alignment, which
 665  *      must be some "power of two" multiple of pagesize.
 666  *
 667  *      On output it is NULL if no address can be found in the current
 668  *      processes address space or else an address that is currently
 669  *      not mapped for len bytes with a page of red zone on either side.
 670  *
 671  *      vacalign is not needed on x86 (it's for viturally addressed caches)
 672  */
 673 /*ARGSUSED*/
 674 void
 675 map_addr_proc(
 676         caddr_t *addrp,
 677         size_t len,
 678         offset_t off,
 679         int vacalign,
 680         caddr_t userlimit,
 681         struct proc *p,
 682         uint_t flags)
 683 {
 684         struct as *as = p->p_as;
 685         caddr_t addr;
 686         caddr_t base;
 687         size_t slen;
 688         size_t align_amount;
 689 
 690         ASSERT32(userlimit == as->a_userlimit);
 691 
 692         base = p->p_brkbase;
 693 #if defined(__amd64)
 694         /*
 695          * XX64 Yes, this needs more work.
 696          */
 697         if (p->p_model == DATAMODEL_NATIVE) {
 698                 if (userlimit < as->a_userlimit) {
 699                         /*
 700                          * This happens when a program wants to map
 701                          * something in a range that's accessible to a
 702                          * program in a smaller address space.  For example,
 703                          * a 64-bit program calling mmap32(2) to guarantee
 704                          * that the returned address is below 4Gbytes.
 705                          */
 706                         ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff));
 707 
 708                         if (userlimit > base)
 709                                 slen = userlimit - base;
 710                         else {
 711                                 *addrp = NULL;
 712                                 return;
 713                         }
 714                 } else {
 715                         /*
 716                          * XX64 This layout is probably wrong .. but in
 717                          * the event we make the amd64 address space look
 718                          * like sparcv9 i.e. with the stack -above- the
 719                          * heap, this bit of code might even be correct.
 720                          */
 721                         slen = p->p_usrstack - base -
 722                             ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK);
 723                 }
 724         } else
 725 #endif
 726                 slen = userlimit - base;
 727 
 728         /* Make len be a multiple of PAGESIZE */
 729         len = (len + PAGEOFFSET) & PAGEMASK;
 730 
 731         /*
 732          * figure out what the alignment should be
 733          *
 734          * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same????
 735          */
 736         if (len <= ELF_386_MAXPGSZ) {
 737                 /*
 738                  * Align virtual addresses to ensure that ELF shared libraries
 739                  * are mapped with the appropriate alignment constraints by
 740                  * the run-time linker.
 741                  */
 742                 align_amount = ELF_386_MAXPGSZ;
 743         } else {
 744                 /*
 745                  * For 32-bit processes, only those which have specified
 746                  * MAP_ALIGN and an addr will be aligned on a larger page size.
 747                  * Not doing so can potentially waste up to 1G of process
 748                  * address space.
 749                  */
 750                 int lvl = (p->p_model == DATAMODEL_ILP32) ? 1 :
 751                     mmu.umax_page_level;
 752 
 753                 while (lvl && len < LEVEL_SIZE(lvl))
 754                         --lvl;
 755 
 756                 align_amount = LEVEL_SIZE(lvl);
 757         }
 758         if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount))
 759                 align_amount = (uintptr_t)*addrp;
 760 
 761         ASSERT(ISP2(align_amount));
 762         ASSERT(align_amount == 0 || align_amount >= PAGESIZE);
 763 
 764         off = off & (align_amount - 1);
 765 
 766         /*
 767          * Look for a large enough hole starting below userlimit.
 768          * After finding it, use the upper part.
 769          */
 770         if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount,
 771             PAGESIZE, off) == 0) {
 772                 caddr_t as_addr;
 773 
 774                 /*
 775                  * addr is the highest possible address to use since we have
 776                  * a PAGESIZE redzone at the beginning and end.
 777                  */
 778                 addr = base + slen - (PAGESIZE + len);
 779                 as_addr = addr;
 780                 /*
 781                  * Round address DOWN to the alignment amount and
 782                  * add the offset in.
 783                  * If addr is greater than as_addr, len would not be large
 784                  * enough to include the redzone, so we must adjust down
 785                  * by the alignment amount.
 786                  */
 787                 addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1)));
 788                 addr += (uintptr_t)off;
 789                 if (addr > as_addr) {
 790                         addr -= align_amount;
 791                 }
 792 
 793                 /*
 794                  * If randomization is requested, slew the allocation
 795                  * backwards, within the same gap, by a random amount.
 796                  */
 797                 if (flags & _MAP_RANDOMIZE) {
 798                         uint32_t slew;
 799 
 800                         (void) random_get_pseudo_bytes((uint8_t *)&slew,
 801                             sizeof (slew));
 802 
 803                         slew = slew % MIN(aslr_max_map_skew, (addr - base));
 804                         addr -= P2ALIGN(slew, align_amount);
 805                 }
 806 
 807                 ASSERT(addr > base);
 808                 ASSERT(addr + len < base + slen);
 809                 ASSERT(((uintptr_t)addr & (align_amount - 1)) ==
 810                     ((uintptr_t)(off)));
 811                 *addrp = addr;
 812         } else {
 813                 *addrp = NULL;  /* no more virtual space */
 814         }
 815 }
 816 
 817 int valid_va_range_aligned_wraparound;
 818 
 819 /*
 820  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
 821  * addresses at least "minlen" long, where the base of the range is at "off"
 822  * phase from an "align" boundary and there is space for a "redzone"-sized
 823  * redzone on either side of the range.  On success, 1 is returned and *basep
 824  * and *lenp are adjusted to describe the acceptable range (including
 825  * the redzone).  On failure, 0 is returned.
 826  */
 827 /*ARGSUSED3*/
 828 int
 829 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir,
 830     size_t align, size_t redzone, size_t off)
 831 {
 832         uintptr_t hi, lo;
 833         size_t tot_len;
 834 
 835         ASSERT(align == 0 ? off == 0 : off < align);
 836         ASSERT(ISP2(align));
 837         ASSERT(align == 0 || align >= PAGESIZE);
 838 
 839         lo = (uintptr_t)*basep;
 840         hi = lo + *lenp;
 841         tot_len = minlen + 2 * redzone; /* need at least this much space */
 842 
 843         /*
 844          * If hi rolled over the top, try cutting back.
 845          */
 846         if (hi < lo) {
 847                 *lenp = 0UL - lo - 1UL;
 848                 /* See if this really happens. If so, then we figure out why */
 849                 valid_va_range_aligned_wraparound++;
 850                 hi = lo + *lenp;
 851         }
 852         if (*lenp < tot_len) {
 853                 return (0);
 854         }
 855 
 856 #if defined(__amd64)
 857         /*
 858          * Deal with a possible hole in the address range between
 859          * hole_start and hole_end that should never be mapped.
 860          */
 861         if (lo < hole_start) {
 862                 if (hi > hole_start) {
 863                         if (hi < hole_end) {
 864                                 hi = hole_start;
 865                         } else {
 866                                 /* lo < hole_start && hi >= hole_end */
 867                                 if (dir == AH_LO) {
 868                                         /*
 869                                          * prefer lowest range
 870                                          */
 871                                         if (hole_start - lo >= tot_len)
 872                                                 hi = hole_start;
 873                                         else if (hi - hole_end >= tot_len)
 874                                                 lo = hole_end;
 875                                         else
 876                                                 return (0);
 877                                 } else {
 878                                         /*
 879                                          * prefer highest range
 880                                          */
 881                                         if (hi - hole_end >= tot_len)
 882                                                 lo = hole_end;
 883                                         else if (hole_start - lo >= tot_len)
 884                                                 hi = hole_start;
 885                                         else
 886                                                 return (0);
 887                                 }
 888                         }
 889                 }
 890         } else {
 891                 /* lo >= hole_start */
 892                 if (hi < hole_end)
 893                         return (0);
 894                 if (lo < hole_end)
 895                         lo = hole_end;
 896         }
 897 #endif
 898 
 899         if (hi - lo < tot_len)
 900                 return (0);
 901 
 902         if (align > 1) {
 903                 uintptr_t tlo = lo + redzone;
 904                 uintptr_t thi = hi - redzone;
 905                 tlo = (uintptr_t)P2PHASEUP(tlo, align, off);
 906                 if (tlo < lo + redzone) {
 907                         return (0);
 908                 }
 909                 if (thi < tlo || thi - tlo < minlen) {
 910                         return (0);
 911                 }
 912         }
 913 
 914         *basep = (caddr_t)lo;
 915         *lenp = hi - lo;
 916         return (1);
 917 }
 918 
 919 /*
 920  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
 921  * addresses at least "minlen" long.  On success, 1 is returned and *basep
 922  * and *lenp are adjusted to describe the acceptable range.  On failure, 0
 923  * is returned.
 924  */
 925 int
 926 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir)
 927 {
 928         return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0));
 929 }
 930 
 931 /*
 932  * Default to forbidding the first 64k of address space.  This protects most
 933  * reasonably sized structures from dereferences through NULL:
 934  *     ((foo_t *)0)->bar
 935  */
 936 uintptr_t forbidden_null_mapping_sz = 0x10000;
 937 
 938 /*
 939  * Determine whether [addr, addr+len] are valid user addresses.
 940  */
 941 /*ARGSUSED*/
 942 int
 943 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as,
 944     caddr_t userlimit)
 945 {
 946         caddr_t eaddr = addr + len;
 947 
 948         if (eaddr <= addr || addr >= userlimit || eaddr > userlimit)
 949                 return (RANGE_BADADDR);
 950 
 951         if ((addr <= (caddr_t)forbidden_null_mapping_sz) &&
 952             as->a_proc != NULL &&
 953             secflag_enabled(as->a_proc, PROC_SEC_FORBIDNULLMAP))
 954                 return (RANGE_BADADDR);
 955 
 956 #if defined(__amd64)
 957         /*
 958          * Check for the VA hole
 959          */
 960         if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end)
 961                 return (RANGE_BADADDR);
 962 #endif
 963 
 964         return (RANGE_OKAY);
 965 }
 966 
 967 /*
 968  * Return 1 if the page frame is onboard memory, else 0.
 969  */
 970 int
 971 pf_is_memory(pfn_t pf)
 972 {
 973         if (pfn_is_foreign(pf))
 974                 return (0);
 975         return (address_in_memlist(phys_install, pfn_to_pa(pf), 1));
 976 }
 977 
 978 /*
 979  * return the memrange containing pfn
 980  */
 981 int
 982 memrange_num(pfn_t pfn)
 983 {
 984         int n;
 985 
 986         for (n = 0; n < nranges - 1; ++n) {
 987                 if (pfn >= memranges[n])
 988                         break;
 989         }
 990         return (n);
 991 }
 992 
 993 /*
 994  * return the mnoderange containing pfn
 995  */
 996 /*ARGSUSED*/
 997 int
 998 pfn_2_mtype(pfn_t pfn)
 999 {
1000 #if defined(__xpv)
1001         return (0);
1002 #else
1003         int     n;
1004 
1005         /* Always start from highest pfn and work our way down */
1006         for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1007                 if (pfn >= mnoderanges[n].mnr_pfnlo) {
1008                         break;
1009                 }
1010         }
1011         return (n);
1012 #endif
1013 }
1014 
1015 #if !defined(__xpv)
1016 /*
1017  * is_contigpage_free:
1018  *      returns a page list of contiguous pages. It minimally has to return
1019  *      minctg pages. Caller determines minctg based on the scatter-gather
1020  *      list length.
1021  *
1022  *      pfnp is set to the next page frame to search on return.
1023  */
1024 static page_t *
1025 is_contigpage_free(
1026         pfn_t *pfnp,
1027         pgcnt_t *pgcnt,
1028         pgcnt_t minctg,
1029         uint64_t pfnseg,
1030         int iolock)
1031 {
1032         int     i = 0;
1033         pfn_t   pfn = *pfnp;
1034         page_t  *pp;
1035         page_t  *plist = NULL;
1036 
1037         /*
1038          * fail if pfn + minctg crosses a segment boundary.
1039          * Adjust for next starting pfn to begin at segment boundary.
1040          */
1041 
1042         if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) {
1043                 *pfnp = roundup(*pfnp, pfnseg + 1);
1044                 return (NULL);
1045         }
1046 
1047         do {
1048 retry:
1049                 pp = page_numtopp_nolock(pfn + i);
1050                 if ((pp == NULL) || IS_DUMP_PAGE(pp) ||
1051                     (page_trylock(pp, SE_EXCL) == 0)) {
1052                         (*pfnp)++;
1053                         break;
1054                 }
1055                 if (page_pptonum(pp) != pfn + i) {
1056                         page_unlock(pp);
1057                         goto retry;
1058                 }
1059 
1060                 if (!(PP_ISFREE(pp))) {
1061                         page_unlock(pp);
1062                         (*pfnp)++;
1063                         break;
1064                 }
1065 
1066                 if (!PP_ISAGED(pp)) {
1067                         page_list_sub(pp, PG_CACHE_LIST);
1068                         page_hashout(pp, (kmutex_t *)NULL);
1069                 } else {
1070                         page_list_sub(pp, PG_FREE_LIST);
1071                 }
1072 
1073                 if (iolock)
1074                         page_io_lock(pp);
1075                 page_list_concat(&plist, &pp);
1076 
1077                 /*
1078                  * exit loop when pgcnt satisfied or segment boundary reached.
1079                  */
1080 
1081         } while ((++i < *pgcnt) && ((pfn + i) & pfnseg));
1082 
1083         *pfnp += i;             /* set to next pfn to search */
1084 
1085         if (i >= minctg) {
1086                 *pgcnt -= i;
1087                 return (plist);
1088         }
1089 
1090         /*
1091          * failure: minctg not satisfied.
1092          *
1093          * if next request crosses segment boundary, set next pfn
1094          * to search from the segment boundary.
1095          */
1096         if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg))
1097                 *pfnp = roundup(*pfnp, pfnseg + 1);
1098 
1099         /* clean up any pages already allocated */
1100 
1101         while (plist) {
1102                 pp = plist;
1103                 page_sub(&plist, pp);
1104                 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
1105                 if (iolock)
1106                         page_io_unlock(pp);
1107                 page_unlock(pp);
1108         }
1109 
1110         return (NULL);
1111 }
1112 #endif  /* !__xpv */
1113 
1114 /*
1115  * verify that pages being returned from allocator have correct DMA attribute
1116  */
1117 #ifndef DEBUG
1118 #define check_dma(a, b, c) (void)(0)
1119 #else
1120 static void
1121 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt)
1122 {
1123         if (dma_attr == NULL)
1124                 return;
1125 
1126         while (cnt-- > 0) {
1127                 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) <
1128                     dma_attr->dma_attr_addr_lo)
1129                         panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp);
1130                 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >=
1131                     dma_attr->dma_attr_addr_hi)
1132                         panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp);
1133                 pp = pp->p_next;
1134         }
1135 }
1136 #endif
1137 
1138 #if !defined(__xpv)
1139 static page_t *
1140 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock)
1141 {
1142         pfn_t           pfn;
1143         int             sgllen;
1144         uint64_t        pfnseg;
1145         pgcnt_t         minctg;
1146         page_t          *pplist = NULL, *plist;
1147         uint64_t        lo, hi;
1148         pgcnt_t         pfnalign = 0;
1149         static pfn_t    startpfn;
1150         static pgcnt_t  lastctgcnt;
1151         uintptr_t       align;
1152 
1153         CONTIG_LOCK();
1154 
1155         if (mattr) {
1156                 lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET));
1157                 hi = mmu_btop(mattr->dma_attr_addr_hi);
1158                 if (hi >= physmax)
1159                         hi = physmax - 1;
1160                 sgllen = mattr->dma_attr_sgllen;
1161                 pfnseg = mmu_btop(mattr->dma_attr_seg);
1162 
1163                 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
1164                 if (align > MMU_PAGESIZE)
1165                         pfnalign = mmu_btop(align);
1166 
1167                 /*
1168                  * in order to satisfy the request, must minimally
1169                  * acquire minctg contiguous pages
1170                  */
1171                 minctg = howmany(*pgcnt, sgllen);
1172 
1173                 ASSERT(hi >= lo);
1174 
1175                 /*
1176                  * start from where last searched if the minctg >= lastctgcnt
1177                  */
1178                 if (minctg < lastctgcnt || startpfn < lo || startpfn > hi)
1179                         startpfn = lo;
1180         } else {
1181                 hi = physmax - 1;
1182                 lo = 0;
1183                 sgllen = 1;
1184                 pfnseg = mmu.highest_pfn;
1185                 minctg = *pgcnt;
1186 
1187                 if (minctg < lastctgcnt)
1188                         startpfn = lo;
1189         }
1190         lastctgcnt = minctg;
1191 
1192         ASSERT(pfnseg + 1 >= (uint64_t)minctg);
1193 
1194         /* conserve 16m memory - start search above 16m when possible */
1195         if (hi > PFN_16M && startpfn < PFN_16M)
1196                 startpfn = PFN_16M;
1197 
1198         pfn = startpfn;
1199         if (pfnalign)
1200                 pfn = P2ROUNDUP(pfn, pfnalign);
1201 
1202         while (pfn + minctg - 1 <= hi) {
1203 
1204                 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1205                 if (plist) {
1206                         page_list_concat(&pplist, &plist);
1207                         sgllen--;
1208                         /*
1209                          * return when contig pages no longer needed
1210                          */
1211                         if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1212                                 startpfn = pfn;
1213                                 CONTIG_UNLOCK();
1214                                 check_dma(mattr, pplist, *pgcnt);
1215                                 return (pplist);
1216                         }
1217                         minctg = howmany(*pgcnt, sgllen);
1218                 }
1219                 if (pfnalign)
1220                         pfn = P2ROUNDUP(pfn, pfnalign);
1221         }
1222 
1223         /* cannot find contig pages in specified range */
1224         if (startpfn == lo) {
1225                 CONTIG_UNLOCK();
1226                 return (NULL);
1227         }
1228 
1229         /* did not start with lo previously */
1230         pfn = lo;
1231         if (pfnalign)
1232                 pfn = P2ROUNDUP(pfn, pfnalign);
1233 
1234         /* allow search to go above startpfn */
1235         while (pfn < startpfn) {
1236 
1237                 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1238                 if (plist != NULL) {
1239 
1240                         page_list_concat(&pplist, &plist);
1241                         sgllen--;
1242 
1243                         /*
1244                          * return when contig pages no longer needed
1245                          */
1246                         if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1247                                 startpfn = pfn;
1248                                 CONTIG_UNLOCK();
1249                                 check_dma(mattr, pplist, *pgcnt);
1250                                 return (pplist);
1251                         }
1252                         minctg = howmany(*pgcnt, sgllen);
1253                 }
1254                 if (pfnalign)
1255                         pfn = P2ROUNDUP(pfn, pfnalign);
1256         }
1257         CONTIG_UNLOCK();
1258         return (NULL);
1259 }
1260 #endif  /* !__xpv */
1261 
1262 /*
1263  * mnode_range_cnt() calculates the number of memory ranges for mnode and
1264  * memranges[]. Used to determine the size of page lists and mnoderanges.
1265  */
1266 int
1267 mnode_range_cnt(int mnode)
1268 {
1269 #if defined(__xpv)
1270         ASSERT(mnode == 0);
1271         return (1);
1272 #else   /* __xpv */
1273         int     mri;
1274         int     mnrcnt = 0;
1275 
1276         if (mem_node_config[mnode].exists != 0) {
1277                 mri = nranges - 1;
1278 
1279                 /* find the memranges index below contained in mnode range */
1280 
1281                 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1282                         mri--;
1283 
1284                 /*
1285                  * increment mnode range counter when memranges or mnode
1286                  * boundary is reached.
1287                  */
1288                 while (mri >= 0 &&
1289                     mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1290                         mnrcnt++;
1291                         if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1292                                 mri--;
1293                         else
1294                                 break;
1295                 }
1296         }
1297         ASSERT(mnrcnt <= MAX_MNODE_MRANGES);
1298         return (mnrcnt);
1299 #endif  /* __xpv */
1300 }
1301 
1302 /*
1303  * mnode_range_setup() initializes mnoderanges.
1304  */
1305 void
1306 mnode_range_setup(mnoderange_t *mnoderanges)
1307 {
1308         mnoderange_t *mp = mnoderanges;
1309         int     mnode, mri;
1310         int     mindex = 0;     /* current index into mnoderanges array */
1311         int     i, j;
1312         pfn_t   hipfn;
1313         int     last, hi;
1314 
1315         for (mnode = 0; mnode < max_mem_nodes; mnode++) {
1316                 if (mem_node_config[mnode].exists == 0)
1317                         continue;
1318 
1319                 mri = nranges - 1;
1320 
1321                 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1322                         mri--;
1323 
1324                 while (mri >= 0 && mem_node_config[mnode].physmax >=
1325                     MEMRANGELO(mri)) {
1326                         mnoderanges->mnr_pfnlo = MAX(MEMRANGELO(mri),
1327                             mem_node_config[mnode].physbase);
1328                         mnoderanges->mnr_pfnhi = MIN(MEMRANGEHI(mri),
1329                             mem_node_config[mnode].physmax);
1330                         mnoderanges->mnr_mnode = mnode;
1331                         mnoderanges->mnr_memrange = mri;
1332                         mnoderanges->mnr_exists = 1;
1333                         mnoderanges++;
1334                         mindex++;
1335                         if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1336                                 mri--;
1337                         else
1338                                 break;
1339                 }
1340         }
1341 
1342         /*
1343          * For now do a simple sort of the mnoderanges array to fill in
1344          * the mnr_next fields.  Since mindex is expected to be relatively
1345          * small, using a simple O(N^2) algorithm.
1346          */
1347         for (i = 0; i < mindex; i++) {
1348                 if (mp[i].mnr_pfnlo == 0)       /* find lowest */
1349                         break;
1350         }
1351         ASSERT(i < mindex);
1352         last = i;
1353         mtype16m = last;
1354         mp[last].mnr_next = -1;
1355         for (i = 0; i < mindex - 1; i++) {
1356                 hipfn = (pfn_t)(-1);
1357                 hi = -1;
1358                 /* find next highest mnode range */
1359                 for (j = 0; j < mindex; j++) {
1360                         if (mp[j].mnr_pfnlo > mp[last].mnr_pfnlo &&
1361                             mp[j].mnr_pfnlo < hipfn) {
1362                                 hipfn = mp[j].mnr_pfnlo;
1363                                 hi = j;
1364                         }
1365                 }
1366                 mp[hi].mnr_next = last;
1367                 last = hi;
1368         }
1369         mtypetop = last;
1370 }
1371 
1372 #ifndef __xpv
1373 /*
1374  * Update mnoderanges for memory hot-add DR operations.
1375  */
1376 static void
1377 mnode_range_add(int mnode)
1378 {
1379         int     *prev;
1380         int     n, mri;
1381         pfn_t   start, end;
1382         extern  void membar_sync(void);
1383 
1384         ASSERT(0 <= mnode && mnode < max_mem_nodes);
1385         ASSERT(mem_node_config[mnode].exists);
1386         start = mem_node_config[mnode].physbase;
1387         end = mem_node_config[mnode].physmax;
1388         ASSERT(start <= end);
1389         mutex_enter(&mnoderange_lock);
1390 
1391 #ifdef  DEBUG
1392         /* Check whether it interleaves with other memory nodes. */
1393         for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1394                 ASSERT(mnoderanges[n].mnr_exists);
1395                 if (mnoderanges[n].mnr_mnode == mnode)
1396                         continue;
1397                 ASSERT(start > mnoderanges[n].mnr_pfnhi ||
1398                     end < mnoderanges[n].mnr_pfnlo);
1399         }
1400 #endif  /* DEBUG */
1401 
1402         mri = nranges - 1;
1403         while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1404                 mri--;
1405         while (mri >= 0 && mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1406                 /* Check whether mtype already exists. */
1407                 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1408                         if (mnoderanges[n].mnr_mnode == mnode &&
1409                             mnoderanges[n].mnr_memrange == mri) {
1410                                 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri),
1411                                     start);
1412                                 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri),
1413                                     end);
1414                                 break;
1415                         }
1416                 }
1417 
1418                 /* Add a new entry if it doesn't exist yet. */
1419                 if (n == -1) {
1420                         /* Try to find an unused entry in mnoderanges array. */
1421                         for (n = 0; n < mnoderangecnt; n++) {
1422                                 if (mnoderanges[n].mnr_exists == 0)
1423                                         break;
1424                         }
1425                         ASSERT(n < mnoderangecnt);
1426                         mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), start);
1427                         mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), end);
1428                         mnoderanges[n].mnr_mnode = mnode;
1429                         mnoderanges[n].mnr_memrange = mri;
1430                         mnoderanges[n].mnr_exists = 1;
1431                         /* Page 0 should always be present. */
1432                         for (prev = &mtypetop;
1433                             mnoderanges[*prev].mnr_pfnlo > start;
1434                             prev = &mnoderanges[*prev].mnr_next) {
1435                                 ASSERT(mnoderanges[*prev].mnr_next >= 0);
1436                                 ASSERT(mnoderanges[*prev].mnr_pfnlo > end);
1437                         }
1438                         mnoderanges[n].mnr_next = *prev;
1439                         membar_sync();
1440                         *prev = n;
1441                 }
1442 
1443                 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1444                         mri--;
1445                 else
1446                         break;
1447         }
1448 
1449         mutex_exit(&mnoderange_lock);
1450 }
1451 
1452 /*
1453  * Update mnoderanges for memory hot-removal DR operations.
1454  */
1455 static void
1456 mnode_range_del(int mnode)
1457 {
1458         _NOTE(ARGUNUSED(mnode));
1459         ASSERT(0 <= mnode && mnode < max_mem_nodes);
1460         /* TODO: support deletion operation. */
1461         ASSERT(0);
1462 }
1463 
1464 void
1465 plat_slice_add(pfn_t start, pfn_t end)
1466 {
1467         mem_node_add_slice(start, end);
1468         if (plat_dr_enabled()) {
1469                 mnode_range_add(PFN_2_MEM_NODE(start));
1470         }
1471 }
1472 
1473 void
1474 plat_slice_del(pfn_t start, pfn_t end)
1475 {
1476         ASSERT(PFN_2_MEM_NODE(start) == PFN_2_MEM_NODE(end));
1477         ASSERT(plat_dr_enabled());
1478         mnode_range_del(PFN_2_MEM_NODE(start));
1479         mem_node_del_slice(start, end);
1480 }
1481 #endif  /* __xpv */
1482 
1483 /*ARGSUSED*/
1484 int
1485 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz)
1486 {
1487         int mtype = mtypetop;
1488 
1489 #if !defined(__xpv)
1490 #if defined(__i386)
1491         /*
1492          * set the mtype range
1493          * - kmem requests need to be below 4g if restricted_kmemalloc is set.
1494          * - for non kmem requests, set range to above 4g if memory below 4g
1495          * runs low.
1496          */
1497         if (restricted_kmemalloc && VN_ISKAS(vp) &&
1498             (caddr_t)(vaddr) >= kernelheap &&
1499             (caddr_t)(vaddr) < ekernelheap) {
1500                 ASSERT(physmax4g);
1501                 mtype = mtype4g;
1502                 if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz),
1503                     btop(pgsz), *flags)) {
1504                         *flags |= PGI_MT_RANGE16M;
1505                 } else {
1506                         VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1507                         VM_STAT_COND_ADD((*flags & PG_PANIC),
1508                             vmm_vmstats.pgpanicalloc);
1509                         *flags |= PGI_MT_RANGE0;
1510                 }
1511                 return (mtype);
1512         }
1513 #endif  /* __i386 */
1514 
1515         if (RESTRICT4G_ALLOC) {
1516                 VM_STAT_ADD(vmm_vmstats.restrict4gcnt);
1517                 /* here only for > 4g systems */
1518                 *flags |= PGI_MT_RANGE4G;
1519         } else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) {
1520                 *flags |= PGI_MT_RANGE16M;
1521         } else {
1522                 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1523                 VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc);
1524                 *flags |= PGI_MT_RANGE0;
1525         }
1526 #endif /* !__xpv */
1527         return (mtype);
1528 }
1529 
1530 
1531 /* mtype init for page_get_replacement_page */
1532 /*ARGSUSED*/
1533 int
1534 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt)
1535 {
1536         int mtype = mtypetop;
1537 #if !defined(__xpv)
1538         if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) {
1539                 *flags |= PGI_MT_RANGE16M;
1540         } else {
1541                 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1542                 *flags |= PGI_MT_RANGE0;
1543         }
1544 #endif
1545         return (mtype);
1546 }
1547 
1548 /*
1549  * Determine if the mnode range specified in mtype contains memory belonging
1550  * to memory node mnode.  If flags & PGI_MT_RANGE is set then mtype contains
1551  * the range from high pfn to 0, 16m or 4g.
1552  *
1553  * Return first mnode range type index found otherwise return -1 if none found.
1554  */
1555 int
1556 mtype_func(int mnode, int mtype, uint_t flags)
1557 {
1558         if (flags & PGI_MT_RANGE) {
1559                 int     mnr_lim = MRI_0;
1560 
1561                 if (flags & PGI_MT_NEXT) {
1562                         mtype = mnoderanges[mtype].mnr_next;
1563                 }
1564                 if (flags & PGI_MT_RANGE4G)
1565                         mnr_lim = MRI_4G;       /* exclude 0-4g range */
1566                 else if (flags & PGI_MT_RANGE16M)
1567                         mnr_lim = MRI_16M;      /* exclude 0-16m range */
1568                 while (mtype != -1 &&
1569                     mnoderanges[mtype].mnr_memrange <= mnr_lim) {
1570                         if (mnoderanges[mtype].mnr_mnode == mnode)
1571                                 return (mtype);
1572                         mtype = mnoderanges[mtype].mnr_next;
1573                 }
1574         } else if (mnoderanges[mtype].mnr_mnode == mnode) {
1575                 return (mtype);
1576         }
1577         return (-1);
1578 }
1579 
1580 /*
1581  * Update the page list max counts with the pfn range specified by the
1582  * input parameters.
1583  */
1584 void
1585 mtype_modify_max(pfn_t startpfn, long cnt)
1586 {
1587         int             mtype;
1588         pgcnt_t         inc;
1589         spgcnt_t        scnt = (spgcnt_t)(cnt);
1590         pgcnt_t         acnt = ABS(scnt);
1591         pfn_t           endpfn = startpfn + acnt;
1592         pfn_t           pfn, lo;
1593 
1594         if (!physmax4g)
1595                 return;
1596 
1597         mtype = mtypetop;
1598         for (pfn = endpfn; pfn > startpfn; ) {
1599                 ASSERT(mtype != -1);
1600                 lo = mnoderanges[mtype].mnr_pfnlo;
1601                 if (pfn > lo) {
1602                         if (startpfn >= lo) {
1603                                 inc = pfn - startpfn;
1604                         } else {
1605                                 inc = pfn - lo;
1606                         }
1607                         if (mnoderanges[mtype].mnr_memrange != MRI_4G) {
1608                                 if (scnt > 0)
1609                                         maxmem4g += inc;
1610                                 else
1611                                         maxmem4g -= inc;
1612                         }
1613                         pfn -= inc;
1614                 }
1615                 mtype = mnoderanges[mtype].mnr_next;
1616         }
1617 }
1618 
1619 int
1620 mtype_2_mrange(int mtype)
1621 {
1622         return (mnoderanges[mtype].mnr_memrange);
1623 }
1624 
1625 void
1626 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi)
1627 {
1628         _NOTE(ARGUNUSED(mnode));
1629         ASSERT(mnoderanges[mtype].mnr_mnode == mnode);
1630         *pfnlo = mnoderanges[mtype].mnr_pfnlo;
1631         *pfnhi = mnoderanges[mtype].mnr_pfnhi;
1632 }
1633 
1634 size_t
1635 plcnt_sz(size_t ctrs_sz)
1636 {
1637 #ifdef DEBUG
1638         int     szc, colors;
1639 
1640         ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes;
1641         for (szc = 0; szc < mmu_page_sizes; szc++) {
1642                 colors = page_get_pagecolors(szc);
1643                 ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors;
1644         }
1645 #endif
1646         return (ctrs_sz);
1647 }
1648 
1649 caddr_t
1650 plcnt_init(caddr_t addr)
1651 {
1652 #ifdef DEBUG
1653         int     mt, szc, colors;
1654 
1655         for (mt = 0; mt < mnoderangecnt; mt++) {
1656                 mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr;
1657                 addr += (sizeof (struct mnr_mts) * mmu_page_sizes);
1658                 for (szc = 0; szc < mmu_page_sizes; szc++) {
1659                         colors = page_get_pagecolors(szc);
1660                         mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors;
1661                         mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt =
1662                             (pgcnt_t *)addr;
1663                         addr += (sizeof (pgcnt_t) * colors);
1664                 }
1665         }
1666 #endif
1667         return (addr);
1668 }
1669 
1670 void
1671 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags)
1672 {
1673         _NOTE(ARGUNUSED(pp));
1674 #ifdef DEBUG
1675         int     bin = PP_2_BIN(pp);
1676 
1677         atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt);
1678         atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin],
1679             cnt);
1680 #endif
1681         ASSERT(mtype == PP_2_MTYPE(pp));
1682         if (physmax4g && mnoderanges[mtype].mnr_memrange != MRI_4G)
1683                 atomic_add_long(&freemem4g, cnt);
1684         if (flags & PG_CACHE_LIST)
1685                 atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt);
1686         else
1687                 atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt);
1688         atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt);
1689 }
1690 
1691 /*
1692  * Returns the free page count for mnode
1693  */
1694 int
1695 mnode_pgcnt(int mnode)
1696 {
1697         int     mtype = mtypetop;
1698         int     flags = PGI_MT_RANGE0;
1699         pgcnt_t pgcnt = 0;
1700 
1701         mtype = mtype_func(mnode, mtype, flags);
1702 
1703         while (mtype != -1) {
1704                 pgcnt += MTYPE_FREEMEM(mtype);
1705                 mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT);
1706         }
1707         return (pgcnt);
1708 }
1709 
1710 /*
1711  * Initialize page coloring variables based on the l2 cache parameters.
1712  * Calculate and return memory needed for page coloring data structures.
1713  */
1714 size_t
1715 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc)
1716 {
1717         _NOTE(ARGUNUSED(l2_linesz));
1718         size_t  colorsz = 0;
1719         int     i;
1720         int     colors;
1721 
1722 #if defined(__xpv)
1723         /*
1724          * Hypervisor domains currently don't have any concept of NUMA.
1725          * Hence we'll act like there is only 1 memrange.
1726          */
1727         i = memrange_num(1);
1728 #else /* !__xpv */
1729         /*
1730          * Reduce the memory ranges lists if we don't have large amounts
1731          * of memory. This avoids searching known empty free lists.
1732          * To support memory DR operations, we need to keep memory ranges
1733          * for possible memory hot-add operations.
1734          */
1735         if (plat_dr_physmax > physmax)
1736                 i = memrange_num(plat_dr_physmax);
1737         else
1738                 i = memrange_num(physmax);
1739 #if defined(__i386)
1740         if (i > MRI_4G)
1741                 restricted_kmemalloc = 0;
1742 #endif
1743         /* physmax greater than 4g */
1744         if (i == MRI_4G)
1745                 physmax4g = 1;
1746 #endif /* !__xpv */
1747         memranges += i;
1748         nranges -= i;
1749 
1750         ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES);
1751 
1752         ASSERT(ISP2(l2_linesz));
1753         ASSERT(l2_sz > MMU_PAGESIZE);
1754 
1755         /* l2_assoc is 0 for fully associative l2 cache */
1756         if (l2_assoc)
1757                 l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE));
1758         else
1759                 l2_colors = 1;
1760 
1761         ASSERT(ISP2(l2_colors));
1762 
1763         /* for scalability, configure at least PAGE_COLORS_MIN color bins */
1764         page_colors = MAX(l2_colors, PAGE_COLORS_MIN);
1765 
1766         /*
1767          * cpu_page_colors is non-zero when a page color may be spread across
1768          * multiple bins.
1769          */
1770         if (l2_colors < page_colors)
1771                 cpu_page_colors = l2_colors;
1772 
1773         ASSERT(ISP2(page_colors));
1774 
1775         page_colors_mask = page_colors - 1;
1776 
1777         ASSERT(ISP2(CPUSETSIZE()));
1778         page_coloring_shift = lowbit(CPUSETSIZE());
1779 
1780         /* initialize number of colors per page size */
1781         for (i = 0; i <= mmu.max_page_level; i++) {
1782                 hw_page_array[i].hp_size = LEVEL_SIZE(i);
1783                 hw_page_array[i].hp_shift = LEVEL_SHIFT(i);
1784                 hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0);
1785                 hw_page_array[i].hp_colors = (page_colors_mask >>
1786                     (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift))
1787                     + 1;
1788                 colorequivszc[i] = 0;
1789         }
1790 
1791         /*
1792          * The value of cpu_page_colors determines if additional color bins
1793          * need to be checked for a particular color in the page_get routines.
1794          */
1795         if (cpu_page_colors != 0) {
1796 
1797                 int a = lowbit(page_colors) - lowbit(cpu_page_colors);
1798                 ASSERT(a > 0);
1799                 ASSERT(a < 16);
1800 
1801                 for (i = 0; i <= mmu.max_page_level; i++) {
1802                         if ((colors = hw_page_array[i].hp_colors) <= 1) {
1803                                 colorequivszc[i] = 0;
1804                                 continue;
1805                         }
1806                         while ((colors >> a) == 0)
1807                                 a--;
1808                         ASSERT(a >= 0);
1809 
1810                         /* higher 4 bits encodes color equiv mask */
1811                         colorequivszc[i] = (a << 4);
1812                 }
1813         }
1814 
1815         /* factor in colorequiv to check additional 'equivalent' bins. */
1816         if (colorequiv > 1) {
1817 
1818                 int a = lowbit(colorequiv) - 1;
1819                 if (a > 15)
1820                         a = 15;
1821 
1822                 for (i = 0; i <= mmu.max_page_level; i++) {
1823                         if ((colors = hw_page_array[i].hp_colors) <= 1) {
1824                                 continue;
1825                         }
1826                         while ((colors >> a) == 0)
1827                                 a--;
1828                         if ((a << 4) > colorequivszc[i]) {
1829                                 colorequivszc[i] = (a << 4);
1830                         }
1831                 }
1832         }
1833 
1834         /* size for mnoderanges */
1835         for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++)
1836                 mnoderangecnt += mnode_range_cnt(i);
1837         if (plat_dr_support_memory()) {
1838                 /*
1839                  * Reserve enough space for memory DR operations.
1840                  * Two extra mnoderanges for possbile fragmentations,
1841                  * one for the 2G boundary and the other for the 4G boundary.
1842                  * We don't expect a memory board crossing the 16M boundary
1843                  * for memory hot-add operations on x86 platforms.
1844                  */
1845                 mnoderangecnt += 2 + max_mem_nodes - lgrp_plat_node_cnt;
1846         }
1847         colorsz = mnoderangecnt * sizeof (mnoderange_t);
1848 
1849         /* size for fpc_mutex and cpc_mutex */
1850         colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX);
1851 
1852         /* size of page_freelists */
1853         colorsz += mnoderangecnt * sizeof (page_t ***);
1854         colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **);
1855 
1856         for (i = 0; i < mmu_page_sizes; i++) {
1857                 colors = page_get_pagecolors(i);
1858                 colorsz += mnoderangecnt * colors * sizeof (page_t *);
1859         }
1860 
1861         /* size of page_cachelists */
1862         colorsz += mnoderangecnt * sizeof (page_t **);
1863         colorsz += mnoderangecnt * page_colors * sizeof (page_t *);
1864 
1865         return (colorsz);
1866 }
1867 
1868 /*
1869  * Called once at startup to configure page_coloring data structures and
1870  * does the 1st page_free()/page_freelist_add().
1871  */
1872 void
1873 page_coloring_setup(caddr_t pcmemaddr)
1874 {
1875         int     i;
1876         int     j;
1877         int     k;
1878         caddr_t addr;
1879         int     colors;
1880 
1881         /*
1882          * do page coloring setup
1883          */
1884         addr = pcmemaddr;
1885 
1886         mnoderanges = (mnoderange_t *)addr;
1887         addr += (mnoderangecnt * sizeof (mnoderange_t));
1888 
1889         mnode_range_setup(mnoderanges);
1890 
1891         if (physmax4g)
1892                 mtype4g = pfn_2_mtype(0xfffff);
1893 
1894         for (k = 0; k < NPC_MUTEX; k++) {
1895                 fpc_mutex[k] = (kmutex_t *)addr;
1896                 addr += (max_mem_nodes * sizeof (kmutex_t));
1897         }
1898         for (k = 0; k < NPC_MUTEX; k++) {
1899                 cpc_mutex[k] = (kmutex_t *)addr;
1900                 addr += (max_mem_nodes * sizeof (kmutex_t));
1901         }
1902         page_freelists = (page_t ****)addr;
1903         addr += (mnoderangecnt * sizeof (page_t ***));
1904 
1905         page_cachelists = (page_t ***)addr;
1906         addr += (mnoderangecnt * sizeof (page_t **));
1907 
1908         for (i = 0; i < mnoderangecnt; i++) {
1909                 page_freelists[i] = (page_t ***)addr;
1910                 addr += (mmu_page_sizes * sizeof (page_t **));
1911 
1912                 for (j = 0; j < mmu_page_sizes; j++) {
1913                         colors = page_get_pagecolors(j);
1914                         page_freelists[i][j] = (page_t **)addr;
1915                         addr += (colors * sizeof (page_t *));
1916                 }
1917                 page_cachelists[i] = (page_t **)addr;
1918                 addr += (page_colors * sizeof (page_t *));
1919         }
1920 }
1921 
1922 #if defined(__xpv)
1923 /*
1924  * Give back 10% of the io_pool pages to the free list.
1925  * Don't shrink the pool below some absolute minimum.
1926  */
1927 static void
1928 page_io_pool_shrink()
1929 {
1930         int retcnt;
1931         page_t *pp, *pp_first, *pp_last, **curpool;
1932         mfn_t mfn;
1933         int bothpools = 0;
1934 
1935         mutex_enter(&io_pool_lock);
1936         io_pool_shrink_attempts++;      /* should be a kstat? */
1937         retcnt = io_pool_cnt / 10;
1938         if (io_pool_cnt - retcnt < io_pool_cnt_min)
1939                 retcnt = io_pool_cnt - io_pool_cnt_min;
1940         if (retcnt <= 0)
1941                 goto done;
1942         io_pool_shrinks++;      /* should be a kstat? */
1943         curpool = &io_pool_4g;
1944 domore:
1945         /*
1946          * Loop through taking pages from the end of the list
1947          * (highest mfns) till amount to return reached.
1948          */
1949         for (pp = *curpool; pp && retcnt > 0; ) {
1950                 pp_first = pp_last = pp->p_prev;
1951                 if (pp_first == *curpool)
1952                         break;
1953                 retcnt--;
1954                 io_pool_cnt--;
1955                 page_io_pool_sub(curpool, pp_first, pp_last);
1956                 if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn)
1957                         start_mfn = mfn;
1958                 page_free(pp_first, 1);
1959                 pp = *curpool;
1960         }
1961         if (retcnt != 0 && !bothpools) {
1962                 /*
1963                  * If not enough found in less constrained pool try the
1964                  * more constrained one.
1965                  */
1966                 curpool = &io_pool_16m;
1967                 bothpools = 1;
1968                 goto domore;
1969         }
1970 done:
1971         mutex_exit(&io_pool_lock);
1972 }
1973 
1974 #endif  /* __xpv */
1975 
1976 uint_t
1977 page_create_update_flags_x86(uint_t flags)
1978 {
1979 #if defined(__xpv)
1980         /*
1981          * Check this is an urgent allocation and free pages are depleted.
1982          */
1983         if (!(flags & PG_WAIT) && freemem < desfree)
1984                 page_io_pool_shrink();
1985 #else /* !__xpv */
1986         /*
1987          * page_create_get_something may call this because 4g memory may be
1988          * depleted. Set flags to allow for relocation of base page below
1989          * 4g if necessary.
1990          */
1991         if (physmax4g)
1992                 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
1993 #endif /* __xpv */
1994         return (flags);
1995 }
1996 
1997 /*ARGSUSED*/
1998 int
1999 bp_color(struct buf *bp)
2000 {
2001         return (0);
2002 }
2003 
2004 #if defined(__xpv)
2005 
2006 /*
2007  * Take pages out of an io_pool
2008  */
2009 static void
2010 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last)
2011 {
2012         if (*poolp == pp_first) {
2013                 *poolp = pp_last->p_next;
2014                 if (*poolp == pp_first)
2015                         *poolp = NULL;
2016         }
2017         pp_first->p_prev->p_next = pp_last->p_next;
2018         pp_last->p_next->p_prev = pp_first->p_prev;
2019         pp_first->p_prev = pp_last;
2020         pp_last->p_next = pp_first;
2021 }
2022 
2023 /*
2024  * Put a page on the io_pool list. The list is ordered by increasing MFN.
2025  */
2026 static void
2027 page_io_pool_add(page_t **poolp, page_t *pp)
2028 {
2029         page_t  *look;
2030         mfn_t   mfn = mfn_list[pp->p_pagenum];
2031 
2032         if (*poolp == NULL) {
2033                 *poolp = pp;
2034                 pp->p_next = pp;
2035                 pp->p_prev = pp;
2036                 return;
2037         }
2038 
2039         /*
2040          * Since we try to take pages from the high end of the pool
2041          * chances are good that the pages to be put on the list will
2042          * go at or near the end of the list. so start at the end and
2043          * work backwards.
2044          */
2045         look = (*poolp)->p_prev;
2046         while (mfn < mfn_list[look->p_pagenum]) {
2047                 look = look->p_prev;
2048                 if (look == (*poolp)->p_prev)
2049                         break; /* backed all the way to front of list */
2050         }
2051 
2052         /* insert after look */
2053         pp->p_prev = look;
2054         pp->p_next = look->p_next;
2055         pp->p_next->p_prev = pp;
2056         look->p_next = pp;
2057         if (mfn < mfn_list[(*poolp)->p_pagenum]) {
2058                 /*
2059                  * we inserted a new first list element
2060                  * adjust pool pointer to newly inserted element
2061                  */
2062                 *poolp = pp;
2063         }
2064 }
2065 
2066 /*
2067  * Add a page to the io_pool.  Setting the force flag will force the page
2068  * into the io_pool no matter what.
2069  */
2070 static void
2071 add_page_to_pool(page_t *pp, int force)
2072 {
2073         page_t *highest;
2074         page_t *freep = NULL;
2075 
2076         mutex_enter(&io_pool_lock);
2077         /*
2078          * Always keep the scarce low memory pages
2079          */
2080         if (mfn_list[pp->p_pagenum] < PFN_16MEG) {
2081                 ++io_pool_cnt;
2082                 page_io_pool_add(&io_pool_16m, pp);
2083                 goto done;
2084         }
2085         if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) {
2086                 ++io_pool_cnt;
2087                 page_io_pool_add(&io_pool_4g, pp);
2088         } else {
2089                 highest = io_pool_4g->p_prev;
2090                 if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) {
2091                         page_io_pool_sub(&io_pool_4g, highest, highest);
2092                         page_io_pool_add(&io_pool_4g, pp);
2093                         freep = highest;
2094                 } else {
2095                         freep = pp;
2096                 }
2097         }
2098 done:
2099         mutex_exit(&io_pool_lock);
2100         if (freep)
2101                 page_free(freep, 1);
2102 }
2103 
2104 
2105 int contig_pfn_cnt;     /* no of pfns in the contig pfn list */
2106 int contig_pfn_max;     /* capacity of the contig pfn list */
2107 int next_alloc_pfn;     /* next position in list to start a contig search */
2108 int contig_pfnlist_updates;     /* pfn list update count */
2109 int contig_pfnlist_builds;      /* how many times have we (re)built list */
2110 int contig_pfnlist_buildfailed; /* how many times has list build failed */
2111 int create_contig_pending;      /* nonzero means taskq creating contig list */
2112 pfn_t *contig_pfn_list = NULL;  /* list of contig pfns in ascending mfn order */
2113 
2114 /*
2115  * Function to use in sorting a list of pfns by their underlying mfns.
2116  */
2117 static int
2118 mfn_compare(const void *pfnp1, const void *pfnp2)
2119 {
2120         mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1];
2121         mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2];
2122 
2123         if (mfn1 > mfn2)
2124                 return (1);
2125         if (mfn1 < mfn2)
2126                 return (-1);
2127         return (0);
2128 }
2129 
2130 /*
2131  * Compact the contig_pfn_list by tossing all the non-contiguous
2132  * elements from the list.
2133  */
2134 static void
2135 compact_contig_pfn_list(void)
2136 {
2137         pfn_t pfn, lapfn, prev_lapfn;
2138         mfn_t mfn;
2139         int i, newcnt = 0;
2140 
2141         prev_lapfn = 0;
2142         for (i = 0; i < contig_pfn_cnt - 1; i++) {
2143                 pfn = contig_pfn_list[i];
2144                 lapfn = contig_pfn_list[i + 1];
2145                 mfn = mfn_list[pfn];
2146                 /*
2147                  * See if next pfn is for a contig mfn
2148                  */
2149                 if (mfn_list[lapfn] != mfn + 1)
2150                         continue;
2151                 /*
2152                  * pfn and lookahead are both put in list
2153                  * unless pfn is the previous lookahead.
2154                  */
2155                 if (pfn != prev_lapfn)
2156                         contig_pfn_list[newcnt++] = pfn;
2157                 contig_pfn_list[newcnt++] = lapfn;
2158                 prev_lapfn = lapfn;
2159         }
2160         for (i = newcnt; i < contig_pfn_cnt; i++)
2161                 contig_pfn_list[i] = 0;
2162         contig_pfn_cnt = newcnt;
2163 }
2164 
2165 /*ARGSUSED*/
2166 static void
2167 call_create_contiglist(void *arg)
2168 {
2169         (void) create_contig_pfnlist(PG_WAIT);
2170 }
2171 
2172 /*
2173  * Create list of freelist pfns that have underlying
2174  * contiguous mfns.  The list is kept in ascending mfn order.
2175  * returns 1 if list created else 0.
2176  */
2177 static int
2178 create_contig_pfnlist(uint_t flags)
2179 {
2180         pfn_t pfn;
2181         page_t *pp;
2182         int ret = 1;
2183 
2184         mutex_enter(&contig_list_lock);
2185         if (contig_pfn_list != NULL)
2186                 goto out;
2187         contig_pfn_max = freemem + (freemem / 10);
2188         contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t),
2189             (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP);
2190         if (contig_pfn_list == NULL) {
2191                 /*
2192                  * If we could not create the contig list (because
2193                  * we could not sleep for memory).  Dispatch a taskq that can
2194                  * sleep to get the memory.
2195                  */
2196                 if (!create_contig_pending) {
2197                         if (taskq_dispatch(system_taskq, call_create_contiglist,
2198                             NULL, TQ_NOSLEEP) != NULL)
2199                                 create_contig_pending = 1;
2200                 }
2201                 contig_pfnlist_buildfailed++;   /* count list build failures */
2202                 ret = 0;
2203                 goto out;
2204         }
2205         create_contig_pending = 0;
2206         ASSERT(contig_pfn_cnt == 0);
2207         for (pfn = 0; pfn < mfn_count; pfn++) {
2208                 pp = page_numtopp_nolock(pfn);
2209                 if (pp == NULL || !PP_ISFREE(pp))
2210                         continue;
2211                 contig_pfn_list[contig_pfn_cnt] = pfn;
2212                 if (++contig_pfn_cnt == contig_pfn_max)
2213                         break;
2214         }
2215         /*
2216          * Sanity check the new list.
2217          */
2218         if (contig_pfn_cnt < 2) { /* no contig pfns */
2219                 contig_pfn_cnt = 0;
2220                 contig_pfnlist_buildfailed++;
2221                 kmem_free(contig_pfn_list, contig_pfn_max * sizeof (pfn_t));
2222                 contig_pfn_list = NULL;
2223                 contig_pfn_max = 0;
2224                 ret = 0;
2225                 goto out;
2226         }
2227         qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare);
2228         compact_contig_pfn_list();
2229         /*
2230          * Make sure next search of the newly created contiguous pfn
2231          * list starts at the beginning of the list.
2232          */
2233         next_alloc_pfn = 0;
2234         contig_pfnlist_builds++;        /* count list builds */
2235 out:
2236         mutex_exit(&contig_list_lock);
2237         return (ret);
2238 }
2239 
2240 
2241 /*
2242  * Toss the current contig pfnlist.  Someone is about to do a massive
2243  * update to pfn<->mfn mappings.  So we have them destroy the list and lock
2244  * it till they are done with their update.
2245  */
2246 void
2247 clear_and_lock_contig_pfnlist()
2248 {
2249         pfn_t *listp = NULL;
2250         size_t listsize;
2251 
2252         mutex_enter(&contig_list_lock);
2253         if (contig_pfn_list != NULL) {
2254                 listp = contig_pfn_list;
2255                 listsize = contig_pfn_max * sizeof (pfn_t);
2256                 contig_pfn_list = NULL;
2257                 contig_pfn_max = contig_pfn_cnt = 0;
2258         }
2259         if (listp != NULL)
2260                 kmem_free(listp, listsize);
2261 }
2262 
2263 /*
2264  * Unlock the contig_pfn_list.  The next attempted use of it will cause
2265  * it to be re-created.
2266  */
2267 void
2268 unlock_contig_pfnlist()
2269 {
2270         mutex_exit(&contig_list_lock);
2271 }
2272 
2273 /*
2274  * Update the contiguous pfn list in response to a pfn <-> mfn reassignment
2275  */
2276 void
2277 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn)
2278 {
2279         int probe_hi, probe_lo, probe_pos, insert_after, insert_point;
2280         pfn_t probe_pfn;
2281         mfn_t probe_mfn;
2282         int drop_lock = 0;
2283 
2284         if (mutex_owner(&contig_list_lock) != curthread) {
2285                 drop_lock = 1;
2286                 mutex_enter(&contig_list_lock);
2287         }
2288         if (contig_pfn_list == NULL)
2289                 goto done;
2290         contig_pfnlist_updates++;
2291         /*
2292          * Find the pfn in the current list.  Use a binary chop to locate it.
2293          */
2294         probe_hi = contig_pfn_cnt - 1;
2295         probe_lo = 0;
2296         probe_pos = (probe_hi + probe_lo) / 2;
2297         while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) {
2298                 if (probe_pos == probe_lo) { /* pfn not in list */
2299                         probe_pos = -1;
2300                         break;
2301                 }
2302                 if (pfn_to_mfn(probe_pfn) <= oldmfn)
2303                         probe_lo = probe_pos;
2304                 else
2305                         probe_hi = probe_pos;
2306                 probe_pos = (probe_hi + probe_lo) / 2;
2307         }
2308         if (probe_pos >= 0) {
2309                 /*
2310                  * Remove pfn from list and ensure next alloc
2311                  * position stays in bounds.
2312                  */
2313                 if (--contig_pfn_cnt <= next_alloc_pfn)
2314                         next_alloc_pfn = 0;
2315                 if (contig_pfn_cnt < 2) { /* no contig pfns */
2316                         contig_pfn_cnt = 0;
2317                         kmem_free(contig_pfn_list,
2318                             contig_pfn_max * sizeof (pfn_t));
2319                         contig_pfn_list = NULL;
2320                         contig_pfn_max = 0;
2321                         goto done;
2322                 }
2323                 ovbcopy(&contig_pfn_list[probe_pos + 1],
2324                     &contig_pfn_list[probe_pos],
2325                     (contig_pfn_cnt - probe_pos) * sizeof (pfn_t));
2326         }
2327         if (newmfn == MFN_INVALID)
2328                 goto done;
2329         /*
2330          * Check if new mfn has adjacent mfns in the list
2331          */
2332         probe_hi = contig_pfn_cnt - 1;
2333         probe_lo = 0;
2334         insert_after = -2;
2335         do {
2336                 probe_pos = (probe_hi + probe_lo) / 2;
2337                 probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]);
2338                 if (newmfn == probe_mfn + 1)
2339                         insert_after = probe_pos;
2340                 else if (newmfn == probe_mfn - 1)
2341                         insert_after = probe_pos - 1;
2342                 if (probe_pos == probe_lo)
2343                         break;
2344                 if (probe_mfn <= newmfn)
2345                         probe_lo = probe_pos;
2346                 else
2347                         probe_hi = probe_pos;
2348         } while (insert_after == -2);
2349         /*
2350          * If there is space in the list and there are adjacent mfns
2351          * insert the pfn in to its proper place in the list.
2352          */
2353         if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) {
2354                 insert_point = insert_after + 1;
2355                 ovbcopy(&contig_pfn_list[insert_point],
2356                     &contig_pfn_list[insert_point + 1],
2357                     (contig_pfn_cnt - insert_point) * sizeof (pfn_t));
2358                 contig_pfn_list[insert_point] = pfn;
2359                 contig_pfn_cnt++;
2360         }
2361 done:
2362         if (drop_lock)
2363                 mutex_exit(&contig_list_lock);
2364 }
2365 
2366 /*
2367  * Called to (re-)populate the io_pool from the free page lists.
2368  */
2369 long
2370 populate_io_pool(void)
2371 {
2372         pfn_t pfn;
2373         mfn_t mfn, max_mfn;
2374         page_t *pp;
2375 
2376         /*
2377          * Figure out the bounds of the pool on first invocation.
2378          * We use a percentage of memory for the io pool size.
2379          * we allow that to shrink, but not to less than a fixed minimum
2380          */
2381         if (io_pool_cnt_max == 0) {
2382                 io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct);
2383                 io_pool_cnt_lowater = io_pool_cnt_max;
2384                 /*
2385                  * This is the first time in populate_io_pool, grab a va to use
2386                  * when we need to allocate pages.
2387                  */
2388                 io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2389         }
2390         /*
2391          * If we are out of pages in the pool, then grow the size of the pool
2392          */
2393         if (io_pool_cnt == 0) {
2394                 /*
2395                  * Grow the max size of the io pool by 5%, but never more than
2396                  * 25% of physical memory.
2397                  */
2398                 if (io_pool_cnt_max < physmem / 4)
2399                         io_pool_cnt_max += io_pool_cnt_max / 20;
2400         }
2401         io_pool_grows++;        /* should be a kstat? */
2402 
2403         /*
2404          * Get highest mfn on this platform, but limit to the 32 bit DMA max.
2405          */
2406         (void) mfn_to_pfn(start_mfn);
2407         max_mfn = MIN(cached_max_mfn, PFN_4GIG);
2408         for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) {
2409                 pfn = mfn_to_pfn(mfn);
2410                 if (pfn & PFN_IS_FOREIGN_MFN)
2411                         continue;
2412                 /*
2413                  * try to allocate it from free pages
2414                  */
2415                 pp = page_numtopp_alloc(pfn);
2416                 if (pp == NULL)
2417                         continue;
2418                 PP_CLRFREE(pp);
2419                 add_page_to_pool(pp, 1);
2420                 if (io_pool_cnt >= io_pool_cnt_max)
2421                         break;
2422         }
2423 
2424         return (io_pool_cnt);
2425 }
2426 
2427 /*
2428  * Destroy a page that was being used for DMA I/O. It may or
2429  * may not actually go back to the io_pool.
2430  */
2431 void
2432 page_destroy_io(page_t *pp)
2433 {
2434         mfn_t mfn = mfn_list[pp->p_pagenum];
2435 
2436         /*
2437          * When the page was alloc'd a reservation was made, release it now
2438          */
2439         page_unresv(1);
2440         /*
2441          * Unload translations, if any, then hash out the
2442          * page to erase its identity.
2443          */
2444         (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2445         page_hashout(pp, NULL);
2446 
2447         /*
2448          * If the page came from the free lists, just put it back to them.
2449          * DomU pages always go on the free lists as well.
2450          */
2451         if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) {
2452                 page_free(pp, 1);
2453                 return;
2454         }
2455 
2456         add_page_to_pool(pp, 0);
2457 }
2458 
2459 
2460 long contig_searches;           /* count of times contig pages requested */
2461 long contig_search_restarts;    /* count of contig ranges tried */
2462 long contig_search_failed;      /* count of contig alloc failures */
2463 
2464 /*
2465  * Free partial page list
2466  */
2467 static void
2468 free_partial_list(page_t **pplist)
2469 {
2470         page_t *pp;
2471 
2472         while (*pplist != NULL) {
2473                 pp = *pplist;
2474                 page_io_pool_sub(pplist, pp, pp);
2475                 page_free(pp, 1);
2476         }
2477 }
2478 
2479 /*
2480  * Look thru the contiguous pfns that are not part of the io_pool for
2481  * contiguous free pages.  Return a list of the found pages or NULL.
2482  */
2483 page_t *
2484 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg,
2485     pgcnt_t pfnalign)
2486 {
2487         page_t *pp, *plist = NULL;
2488         mfn_t mfn, prev_mfn, start_mfn;
2489         pfn_t pfn;
2490         int pages_needed, pages_requested;
2491         int search_start;
2492 
2493         /*
2494          * create the contig pfn list if not already done
2495          */
2496 retry:
2497         mutex_enter(&contig_list_lock);
2498         if (contig_pfn_list == NULL) {
2499                 mutex_exit(&contig_list_lock);
2500                 if (!create_contig_pfnlist(flags)) {
2501                         return (NULL);
2502                 }
2503                 goto retry;
2504         }
2505         contig_searches++;
2506         /*
2507          * Search contiguous pfn list for physically contiguous pages not in
2508          * the io_pool.  Start the search where the last search left off.
2509          */
2510         pages_requested = pages_needed = npages;
2511         search_start = next_alloc_pfn;
2512         start_mfn = prev_mfn = 0;
2513         while (pages_needed) {
2514                 pfn = contig_pfn_list[next_alloc_pfn];
2515                 mfn = pfn_to_mfn(pfn);
2516                 /*
2517                  * Check if mfn is first one or contig to previous one and
2518                  * if page corresponding to mfn is free and that mfn
2519                  * range is not crossing a segment boundary.
2520                  */
2521                 if ((prev_mfn == 0 || mfn == prev_mfn + 1) &&
2522                     (pp = page_numtopp_alloc(pfn)) != NULL &&
2523                     !((mfn & pfnseg) < (start_mfn & pfnseg))) {
2524                         PP_CLRFREE(pp);
2525                         page_io_pool_add(&plist, pp);
2526                         pages_needed--;
2527                         if (prev_mfn == 0) {
2528                                 if (pfnalign &&
2529                                     mfn != P2ROUNDUP(mfn, pfnalign)) {
2530                                         /*
2531                                          * not properly aligned
2532                                          */
2533                                         contig_search_restarts++;
2534                                         free_partial_list(&plist);
2535                                         pages_needed = pages_requested;
2536                                         start_mfn = prev_mfn = 0;
2537                                         goto skip;
2538                                 }
2539                                 start_mfn = mfn;
2540                         }
2541                         prev_mfn = mfn;
2542                 } else {
2543                         contig_search_restarts++;
2544                         free_partial_list(&plist);
2545                         pages_needed = pages_requested;
2546                         start_mfn = prev_mfn = 0;
2547                 }
2548 skip:
2549                 if (++next_alloc_pfn == contig_pfn_cnt)
2550                         next_alloc_pfn = 0;
2551                 if (next_alloc_pfn == search_start)
2552                         break; /* all pfns searched */
2553         }
2554         mutex_exit(&contig_list_lock);
2555         if (pages_needed) {
2556                 contig_search_failed++;
2557                 /*
2558                  * Failed to find enough contig pages.
2559                  * free partial page list
2560                  */
2561                 free_partial_list(&plist);
2562         }
2563         return (plist);
2564 }
2565 
2566 /*
2567  * Search the reserved io pool pages for a page range with the
2568  * desired characteristics.
2569  */
2570 page_t *
2571 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg)
2572 {
2573         page_t *pp_first, *pp_last;
2574         page_t *pp, **poolp;
2575         pgcnt_t nwanted, pfnalign;
2576         uint64_t pfnseg;
2577         mfn_t mfn, tmfn, hi_mfn, lo_mfn;
2578         int align, attempt = 0;
2579 
2580         if (minctg == 1)
2581                 contig = 0;
2582         lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2583         hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2584         pfnseg = mmu_btop(mattr->dma_attr_seg);
2585         align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2586         if (align > MMU_PAGESIZE)
2587                 pfnalign = mmu_btop(align);
2588         else
2589                 pfnalign = 0;
2590 
2591 try_again:
2592         /*
2593          * See if we want pages for a legacy device
2594          */
2595         if (hi_mfn < PFN_16MEG)
2596                 poolp = &io_pool_16m;
2597         else
2598                 poolp = &io_pool_4g;
2599 try_smaller:
2600         /*
2601          * Take pages from I/O pool. We'll use pages from the highest
2602          * MFN range possible.
2603          */
2604         pp_first = pp_last = NULL;
2605         mutex_enter(&io_pool_lock);
2606         nwanted = minctg;
2607         for (pp = *poolp; pp && nwanted > 0; ) {
2608                 pp = pp->p_prev;
2609 
2610                 /*
2611                  * skip pages above allowable range
2612                  */
2613                 mfn = mfn_list[pp->p_pagenum];
2614                 if (hi_mfn < mfn)
2615                         goto skip;
2616 
2617                 /*
2618                  * stop at pages below allowable range
2619                  */
2620                 if (lo_mfn > mfn)
2621                         break;
2622 restart:
2623                 if (pp_last == NULL) {
2624                         /*
2625                          * Check alignment
2626                          */
2627                         tmfn = mfn - (minctg - 1);
2628                         if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign))
2629                                 goto skip; /* not properly aligned */
2630                         /*
2631                          * Check segment
2632                          */
2633                         if ((mfn & pfnseg) < (tmfn & pfnseg))
2634                                 goto skip; /* crosses seg boundary */
2635                         /*
2636                          * Start building page list
2637                          */
2638                         pp_first = pp_last = pp;
2639                         nwanted--;
2640                 } else {
2641                         /*
2642                          * check physical contiguity if required
2643                          */
2644                         if (contig &&
2645                             mfn_list[pp_first->p_pagenum] != mfn + 1) {
2646                                 /*
2647                                  * not a contiguous page, restart list.
2648                                  */
2649                                 pp_last = NULL;
2650                                 nwanted = minctg;
2651                                 goto restart;
2652                         } else { /* add page to list */
2653                                 pp_first = pp;
2654                                 nwanted--;
2655                         }
2656                 }
2657 skip:
2658                 if (pp == *poolp)
2659                         break;
2660         }
2661 
2662         /*
2663          * If we didn't find memory. Try the more constrained pool, then
2664          * sweep free pages into the DMA pool and try again.
2665          */
2666         if (nwanted != 0) {
2667                 mutex_exit(&io_pool_lock);
2668                 /*
2669                  * If we were looking in the less constrained pool and
2670                  * didn't find pages, try the more constrained pool.
2671                  */
2672                 if (poolp == &io_pool_4g) {
2673                         poolp = &io_pool_16m;
2674                         goto try_smaller;
2675                 }
2676                 kmem_reap();
2677                 if (++attempt < 4) {
2678                         /*
2679                          * Grab some more io_pool pages
2680                          */
2681                         (void) populate_io_pool();
2682                         goto try_again; /* go around and retry */
2683                 }
2684                 return (NULL);
2685         }
2686         /*
2687          * Found the pages, now snip them from the list
2688          */
2689         page_io_pool_sub(poolp, pp_first, pp_last);
2690         io_pool_cnt -= minctg;
2691         /*
2692          * reset low water mark
2693          */
2694         if (io_pool_cnt < io_pool_cnt_lowater)
2695                 io_pool_cnt_lowater = io_pool_cnt;
2696         mutex_exit(&io_pool_lock);
2697         return (pp_first);
2698 }
2699 
2700 page_t *
2701 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr,
2702     ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg)
2703 {
2704         uint_t kflags;
2705         int order, extra, extpages, i, contig, nbits, extents;
2706         page_t *pp, *expp, *pp_first, **pplist = NULL;
2707         mfn_t *mfnlist = NULL;
2708 
2709         contig = flags & PG_PHYSCONTIG;
2710         if (minctg == 1)
2711                 contig = 0;
2712         flags &= ~PG_PHYSCONTIG;
2713         kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP;
2714         /*
2715          * Hypervisor will allocate extents, if we want contig
2716          * pages extent must be >= minctg
2717          */
2718         if (contig) {
2719                 order = highbit(minctg) - 1;
2720                 if (minctg & ((1 << order) - 1))
2721                         order++;
2722                 extpages = 1 << order;
2723         } else {
2724                 order = 0;
2725                 extpages = minctg;
2726         }
2727         if (extpages > minctg) {
2728                 extra = extpages - minctg;
2729                 if (!page_resv(extra, kflags))
2730                         return (NULL);
2731         }
2732         pp_first = NULL;
2733         pplist = kmem_alloc(extpages * sizeof (page_t *), kflags);
2734         if (pplist == NULL)
2735                 goto balloon_fail;
2736         mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags);
2737         if (mfnlist == NULL)
2738                 goto balloon_fail;
2739         pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr);
2740         if (pp == NULL)
2741                 goto balloon_fail;
2742         pp_first = pp;
2743         if (extpages > minctg) {
2744                 /*
2745                  * fill out the rest of extent pages to swap
2746                  * with the hypervisor
2747                  */
2748                 for (i = 0; i < extra; i++) {
2749                         expp = page_create_va(vp,
2750                             (u_offset_t)(uintptr_t)io_pool_kva,
2751                             PAGESIZE, flags, &kvseg, io_pool_kva);
2752                         if (expp == NULL)
2753                                 goto balloon_fail;
2754                         (void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD);
2755                         page_io_unlock(expp);
2756                         page_hashout(expp, NULL);
2757                         page_io_lock(expp);
2758                         /*
2759                          * add page to end of list
2760                          */
2761                         expp->p_prev = pp_first->p_prev;
2762                         expp->p_next = pp_first;
2763                         expp->p_prev->p_next = expp;
2764                         pp_first->p_prev = expp;
2765                 }
2766 
2767         }
2768         for (i = 0; i < extpages; i++) {
2769                 pplist[i] = pp;
2770                 pp = pp->p_next;
2771         }
2772         nbits = highbit(mattr->dma_attr_addr_hi);
2773         extents = contig ? 1 : minctg;
2774         if (balloon_replace_pages(extents, pplist, nbits, order,
2775             mfnlist) != extents) {
2776                 if (ioalloc_dbg)
2777                         cmn_err(CE_NOTE, "request to hypervisor"
2778                             " for %d pages, maxaddr %" PRIx64 " failed",
2779                             extpages, mattr->dma_attr_addr_hi);
2780                 goto balloon_fail;
2781         }
2782 
2783         kmem_free(pplist, extpages * sizeof (page_t *));
2784         kmem_free(mfnlist, extpages * sizeof (mfn_t));
2785         /*
2786          * Return any excess pages to free list
2787          */
2788         if (extpages > minctg) {
2789                 for (i = 0; i < extra; i++) {
2790                         pp = pp_first->p_prev;
2791                         page_sub(&pp_first, pp);
2792                         page_io_unlock(pp);
2793                         page_unresv(1);
2794                         page_free(pp, 1);
2795                 }
2796         }
2797         return (pp_first);
2798 balloon_fail:
2799         /*
2800          * Return pages to free list and return failure
2801          */
2802         while (pp_first != NULL) {
2803                 pp = pp_first;
2804                 page_sub(&pp_first, pp);
2805                 page_io_unlock(pp);
2806                 if (pp->p_vnode != NULL)
2807                         page_hashout(pp, NULL);
2808                 page_free(pp, 1);
2809         }
2810         if (pplist)
2811                 kmem_free(pplist, extpages * sizeof (page_t *));
2812         if (mfnlist)
2813                 kmem_free(mfnlist, extpages * sizeof (mfn_t));
2814         page_unresv(extpages - minctg);
2815         return (NULL);
2816 }
2817 
2818 static void
2819 return_partial_alloc(page_t *plist)
2820 {
2821         page_t *pp;
2822 
2823         while (plist != NULL) {
2824                 pp = plist;
2825                 page_sub(&plist, pp);
2826                 page_io_unlock(pp);
2827                 page_destroy_io(pp);
2828         }
2829 }
2830 
2831 static page_t *
2832 page_get_contigpages(
2833         struct vnode    *vp,
2834         u_offset_t      off,
2835         int             *npagesp,
2836         uint_t          flags,
2837         caddr_t         vaddr,
2838         ddi_dma_attr_t  *mattr)
2839 {
2840         mfn_t   max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2841         page_t  *plist; /* list to return */
2842         page_t  *pp, *mcpl;
2843         int     contig, anyaddr, npages, getone = 0;
2844         mfn_t   lo_mfn;
2845         mfn_t   hi_mfn;
2846         pgcnt_t pfnalign = 0;
2847         int     align, sgllen;
2848         uint64_t pfnseg;
2849         pgcnt_t minctg;
2850 
2851         npages = *npagesp;
2852         ASSERT(mattr != NULL);
2853         lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2854         hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2855         sgllen = mattr->dma_attr_sgllen;
2856         pfnseg = mmu_btop(mattr->dma_attr_seg);
2857         align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2858         if (align > MMU_PAGESIZE)
2859                 pfnalign = mmu_btop(align);
2860 
2861         contig = flags & PG_PHYSCONTIG;
2862         if (npages == -1) {
2863                 npages = 1;
2864                 pfnalign = 0;
2865         }
2866         /*
2867          * Clear the contig flag if only one page is needed.
2868          */
2869         if (npages == 1) {
2870                 getone = 1;
2871                 contig = 0;
2872         }
2873 
2874         /*
2875          * Check if any page in the system is fine.
2876          */
2877         anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn;
2878         if (!contig && anyaddr && !pfnalign) {
2879                 flags &= ~PG_PHYSCONTIG;
2880                 plist = page_create_va(vp, off, npages * MMU_PAGESIZE,
2881                     flags, &kvseg, vaddr);
2882                 if (plist != NULL) {
2883                         *npagesp = 0;
2884                         return (plist);
2885                 }
2886         }
2887         plist = NULL;
2888         minctg = howmany(npages, sgllen);
2889         while (npages > sgllen || getone) {
2890                 if (minctg > npages)
2891                         minctg = npages;
2892                 mcpl = NULL;
2893                 /*
2894                  * We could want contig pages with no address range limits.
2895                  */
2896                 if (anyaddr && contig) {
2897                         /*
2898                          * Look for free contig pages to satisfy the request.
2899                          */
2900                         mcpl = find_contig_free(minctg, flags, pfnseg,
2901                             pfnalign);
2902                 }
2903                 /*
2904                  * Try the reserved io pools next
2905                  */
2906                 if (mcpl == NULL)
2907                         mcpl = page_io_pool_alloc(mattr, contig, minctg);
2908                 if (mcpl != NULL) {
2909                         pp = mcpl;
2910                         do {
2911                                 if (!page_hashin(pp, vp, off, NULL)) {
2912                                         panic("page_get_contigpages:"
2913                                             " hashin failed"
2914                                             " pp %p, vp %p, off %llx",
2915                                             (void *)pp, (void *)vp, off);
2916                                 }
2917                                 off += MMU_PAGESIZE;
2918                                 PP_CLRFREE(pp);
2919                                 PP_CLRAGED(pp);
2920                                 page_set_props(pp, P_REF);
2921                                 page_io_lock(pp);
2922                                 pp = pp->p_next;
2923                         } while (pp != mcpl);
2924                 } else {
2925                         /*
2926                          * Hypervisor exchange doesn't handle segment or
2927                          * alignment constraints
2928                          */
2929                         if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi ||
2930                             pfnalign)
2931                                 goto fail;
2932                         /*
2933                          * Try exchanging pages with the hypervisor
2934                          */
2935                         mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr,
2936                             flags, minctg);
2937                         if (mcpl == NULL)
2938                                 goto fail;
2939                         off += minctg * MMU_PAGESIZE;
2940                 }
2941                 check_dma(mattr, mcpl, minctg);
2942                 /*
2943                  * Here with a minctg run of contiguous pages, add them to the
2944                  * list we will return for this request.
2945                  */
2946                 page_list_concat(&plist, &mcpl);
2947                 npages -= minctg;
2948                 *npagesp = npages;
2949                 sgllen--;
2950                 if (getone)
2951                         break;
2952         }
2953         return (plist);
2954 fail:
2955         return_partial_alloc(plist);
2956         return (NULL);
2957 }
2958 
2959 /*
2960  * Allocator for domain 0 I/O pages. We match the required
2961  * DMA attributes and contiguity constraints.
2962  */
2963 /*ARGSUSED*/
2964 page_t *
2965 page_create_io(
2966         struct vnode    *vp,
2967         u_offset_t      off,
2968         uint_t          bytes,
2969         uint_t          flags,
2970         struct as       *as,
2971         caddr_t         vaddr,
2972         ddi_dma_attr_t  *mattr)
2973 {
2974         page_t  *plist = NULL, *pp;
2975         int     npages = 0, contig, anyaddr, pages_req;
2976         mfn_t   lo_mfn;
2977         mfn_t   hi_mfn;
2978         pgcnt_t pfnalign = 0;
2979         int     align;
2980         int     is_domu = 0;
2981         int     dummy, bytes_got;
2982         mfn_t   max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2983 
2984         ASSERT(mattr != NULL);
2985         lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2986         hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2987         align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2988         if (align > MMU_PAGESIZE)
2989                 pfnalign = mmu_btop(align);
2990 
2991         /*
2992          * Clear the contig flag if only one page is needed or the scatter
2993          * gather list length is >= npages.
2994          */
2995         pages_req = npages = mmu_btopr(bytes);
2996         contig = (flags & PG_PHYSCONTIG);
2997         bytes = P2ROUNDUP(bytes, MMU_PAGESIZE);
2998         if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages)
2999                 contig = 0;
3000 
3001         /*
3002          * Check if any old page in the system is fine.
3003          * DomU should always go down this path.
3004          */
3005         is_domu = !DOMAIN_IS_INITDOMAIN(xen_info);
3006         anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
3007         if ((!contig && anyaddr) || is_domu) {
3008                 flags &= ~PG_PHYSCONTIG;
3009                 plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr);
3010                 if (plist != NULL)
3011                         return (plist);
3012                 else if (is_domu)
3013                         return (NULL); /* no memory available */
3014         }
3015         /*
3016          * DomU should never reach here
3017          */
3018         if (contig) {
3019                 plist = page_get_contigpages(vp, off, &npages, flags, vaddr,
3020                     mattr);
3021                 if (plist == NULL)
3022                         goto fail;
3023                 bytes_got = (pages_req - npages) << MMU_PAGESHIFT;
3024                 vaddr += bytes_got;
3025                 off += bytes_got;
3026                 /*
3027                  * We now have all the contiguous pages we need, but
3028                  * we may still need additional non-contiguous pages.
3029                  */
3030         }
3031         /*
3032          * now loop collecting the requested number of pages, these do
3033          * not have to be contiguous pages but we will use the contig
3034          * page alloc code to get the pages since it will honor any
3035          * other constraints the pages may have.
3036          */
3037         while (npages--) {
3038                 dummy = -1;
3039                 pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr);
3040                 if (pp == NULL)
3041                         goto fail;
3042                 page_add(&plist, pp);
3043                 vaddr += MMU_PAGESIZE;
3044                 off += MMU_PAGESIZE;
3045         }
3046         return (plist);
3047 fail:
3048         /*
3049          * Failed to get enough pages, return ones we did get
3050          */
3051         return_partial_alloc(plist);
3052         return (NULL);
3053 }
3054 
3055 /*
3056  * Lock and return the page with the highest mfn that we can find.  last_mfn
3057  * holds the last one found, so the next search can start from there.  We
3058  * also keep a counter so that we don't loop forever if the machine has no
3059  * free pages.
3060  *
3061  * This is called from the balloon thread to find pages to give away.  new_high
3062  * is used when new mfn's have been added to the system - we will reset our
3063  * search if the new mfn's are higher than our current search position.
3064  */
3065 page_t *
3066 page_get_high_mfn(mfn_t new_high)
3067 {
3068         static mfn_t last_mfn = 0;
3069         pfn_t pfn;
3070         page_t *pp;
3071         ulong_t loop_count = 0;
3072 
3073         if (new_high > last_mfn)
3074                 last_mfn = new_high;
3075 
3076         for (; loop_count < mfn_count; loop_count++, last_mfn--) {
3077                 if (last_mfn == 0) {
3078                         last_mfn = cached_max_mfn;
3079                 }
3080 
3081                 pfn = mfn_to_pfn(last_mfn);
3082                 if (pfn & PFN_IS_FOREIGN_MFN)
3083                         continue;
3084 
3085                 /* See if the page is free.  If so, lock it. */
3086                 pp = page_numtopp_alloc(pfn);
3087                 if (pp == NULL)
3088                         continue;
3089                 PP_CLRFREE(pp);
3090 
3091                 ASSERT(PAGE_EXCL(pp));
3092                 ASSERT(pp->p_vnode == NULL);
3093                 ASSERT(!hat_page_is_mapped(pp));
3094                 last_mfn--;
3095                 return (pp);
3096         }
3097         return (NULL);
3098 }
3099 
3100 #else /* !__xpv */
3101 
3102 /*
3103  * get a page from any list with the given mnode
3104  */
3105 static page_t *
3106 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags,
3107     int mnode, int mtype, ddi_dma_attr_t *dma_attr)
3108 {
3109         kmutex_t                *pcm;
3110         int                     i;
3111         page_t                  *pp;
3112         page_t                  *first_pp;
3113         uint64_t                pgaddr;
3114         ulong_t                 bin;
3115         int                     mtypestart;
3116         int                     plw_initialized;
3117         page_list_walker_t      plw;
3118 
3119         VM_STAT_ADD(pga_vmstats.pgma_alloc);
3120 
3121         ASSERT((flags & PG_MATCH_COLOR) == 0);
3122         ASSERT(szc == 0);
3123         ASSERT(dma_attr != NULL);
3124 
3125         MTYPE_START(mnode, mtype, flags);
3126         if (mtype < 0) {
3127                 VM_STAT_ADD(pga_vmstats.pgma_allocempty);
3128                 return (NULL);
3129         }
3130 
3131         mtypestart = mtype;
3132 
3133         bin = origbin;
3134 
3135         /*
3136          * check up to page_colors + 1 bins - origbin may be checked twice
3137          * because of BIN_STEP skip
3138          */
3139         do {
3140                 plw_initialized = 0;
3141 
3142                 for (plw.plw_count = 0;
3143                     plw.plw_count < page_colors; plw.plw_count++) {
3144 
3145                         if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL)
3146                                 goto nextfreebin;
3147 
3148                         pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST);
3149                         mutex_enter(pcm);
3150                         pp = PAGE_FREELISTS(mnode, szc, bin, mtype);
3151                         first_pp = pp;
3152                         while (pp != NULL) {
3153                                 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3154                                     SE_EXCL) == 0) {
3155                                         pp = pp->p_next;
3156                                         if (pp == first_pp) {
3157                                                 pp = NULL;
3158                                         }
3159                                         continue;
3160                                 }
3161 
3162                                 ASSERT(PP_ISFREE(pp));
3163                                 ASSERT(PP_ISAGED(pp));
3164                                 ASSERT(pp->p_vnode == NULL);
3165                                 ASSERT(pp->p_hash == NULL);
3166                                 ASSERT(pp->p_offset == (u_offset_t)-1);
3167                                 ASSERT(pp->p_szc == szc);
3168                                 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3169                                 /* check if page within DMA attributes */
3170                                 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3171                                 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3172                                     (pgaddr + MMU_PAGESIZE - 1 <=
3173                                     dma_attr->dma_attr_addr_hi)) {
3174                                         break;
3175                                 }
3176 
3177                                 /* continue looking */
3178                                 page_unlock(pp);
3179                                 pp = pp->p_next;
3180                                 if (pp == first_pp)
3181                                         pp = NULL;
3182 
3183                         }
3184                         if (pp != NULL) {
3185                                 ASSERT(mtype == PP_2_MTYPE(pp));
3186                                 ASSERT(pp->p_szc == 0);
3187 
3188                                 /* found a page with specified DMA attributes */
3189                                 page_sub(&PAGE_FREELISTS(mnode, szc, bin,
3190                                     mtype), pp);
3191                                 page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST);
3192 
3193                                 if ((PP_ISFREE(pp) == 0) ||
3194                                     (PP_ISAGED(pp) == 0)) {
3195                                         cmn_err(CE_PANIC, "page %p is not free",
3196                                             (void *)pp);
3197                                 }
3198 
3199                                 mutex_exit(pcm);
3200                                 check_dma(dma_attr, pp, 1);
3201                                 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3202                                 return (pp);
3203                         }
3204                         mutex_exit(pcm);
3205 nextfreebin:
3206                         if (plw_initialized == 0) {
3207                                 page_list_walk_init(szc, 0, bin, 1, 0, &plw);
3208                                 ASSERT(plw.plw_ceq_dif == page_colors);
3209                                 plw_initialized = 1;
3210                         }
3211 
3212                         if (plw.plw_do_split) {
3213                                 pp = page_freelist_split(szc, bin, mnode,
3214                                     mtype,
3215                                     mmu_btop(dma_attr->dma_attr_addr_lo),
3216                                     mmu_btop(dma_attr->dma_attr_addr_hi + 1),
3217                                     &plw);
3218                                 if (pp != NULL) {
3219                                         check_dma(dma_attr, pp, 1);
3220                                         return (pp);
3221                                 }
3222                         }
3223 
3224                         bin = page_list_walk_next_bin(szc, bin, &plw);
3225                 }
3226 
3227                 MTYPE_NEXT(mnode, mtype, flags);
3228         } while (mtype >= 0);
3229 
3230         /* failed to find a page in the freelist; try it in the cachelist */
3231 
3232         /* reset mtype start for cachelist search */
3233         mtype = mtypestart;
3234         ASSERT(mtype >= 0);
3235 
3236         /* start with the bin of matching color */
3237         bin = origbin;
3238 
3239         do {
3240                 for (i = 0; i <= page_colors; i++) {
3241                         if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL)
3242                                 goto nextcachebin;
3243                         pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST);
3244                         mutex_enter(pcm);
3245                         pp = PAGE_CACHELISTS(mnode, bin, mtype);
3246                         first_pp = pp;
3247                         while (pp != NULL) {
3248                                 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3249                                     SE_EXCL) == 0) {
3250                                         pp = pp->p_next;
3251                                         if (pp == first_pp)
3252                                                 pp = NULL;
3253                                         continue;
3254                                 }
3255                                 ASSERT(pp->p_vnode);
3256                                 ASSERT(PP_ISAGED(pp) == 0);
3257                                 ASSERT(pp->p_szc == 0);
3258                                 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3259 
3260                                 /* check if page within DMA attributes */
3261 
3262                                 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3263                                 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3264                                     (pgaddr + MMU_PAGESIZE - 1 <=
3265                                     dma_attr->dma_attr_addr_hi)) {
3266                                         break;
3267                                 }
3268 
3269                                 /* continue looking */
3270                                 page_unlock(pp);
3271                                 pp = pp->p_next;
3272                                 if (pp == first_pp)
3273                                         pp = NULL;
3274                         }
3275 
3276                         if (pp != NULL) {
3277                                 ASSERT(mtype == PP_2_MTYPE(pp));
3278                                 ASSERT(pp->p_szc == 0);
3279 
3280                                 /* found a page with specified DMA attributes */
3281                                 page_sub(&PAGE_CACHELISTS(mnode, bin,
3282                                     mtype), pp);
3283                                 page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST);
3284 
3285                                 mutex_exit(pcm);
3286                                 ASSERT(pp->p_vnode);
3287                                 ASSERT(PP_ISAGED(pp) == 0);
3288                                 check_dma(dma_attr, pp, 1);
3289                                 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3290                                 return (pp);
3291                         }
3292                         mutex_exit(pcm);
3293 nextcachebin:
3294                         bin += (i == 0) ? BIN_STEP : 1;
3295                         bin &= page_colors_mask;
3296                 }
3297                 MTYPE_NEXT(mnode, mtype, flags);
3298         } while (mtype >= 0);
3299 
3300         VM_STAT_ADD(pga_vmstats.pgma_allocfailed);
3301         return (NULL);
3302 }
3303 
3304 /*
3305  * This function is similar to page_get_freelist()/page_get_cachelist()
3306  * but it searches both the lists to find a page with the specified
3307  * color (or no color) and DMA attributes. The search is done in the
3308  * freelist first and then in the cache list within the highest memory
3309  * range (based on DMA attributes) before searching in the lower
3310  * memory ranges.
3311  *
3312  * Note: This function is called only by page_create_io().
3313  */
3314 /*ARGSUSED*/
3315 static page_t *
3316 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr,
3317     size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t *lgrp)
3318 {
3319         uint_t          bin;
3320         int             mtype;
3321         page_t          *pp;
3322         int             n;
3323         int             m;
3324         int             szc;
3325         int             fullrange;
3326         int             mnode;
3327         int             local_failed_stat = 0;
3328         lgrp_mnode_cookie_t     lgrp_cookie;
3329 
3330         VM_STAT_ADD(pga_vmstats.pga_alloc);
3331 
3332         /* only base pagesize currently supported */
3333         if (size != MMU_PAGESIZE)
3334                 return (NULL);
3335 
3336         /*
3337          * If we're passed a specific lgroup, we use it.  Otherwise,
3338          * assume first-touch placement is desired.
3339          */
3340         if (!LGRP_EXISTS(lgrp))
3341                 lgrp = lgrp_home_lgrp();
3342 
3343         /* LINTED */
3344         AS_2_BIN(as, seg, vp, vaddr, bin, 0);
3345 
3346         /*
3347          * Only hold one freelist or cachelist lock at a time, that way we
3348          * can start anywhere and not have to worry about lock
3349          * ordering.
3350          */
3351         if (dma_attr == NULL) {
3352                 n = mtype16m;
3353                 m = mtypetop;
3354                 fullrange = 1;
3355                 VM_STAT_ADD(pga_vmstats.pga_nulldmaattr);
3356         } else {
3357                 pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo);
3358                 pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi);
3359 
3360                 /*
3361                  * We can guarantee alignment only for page boundary.
3362                  */
3363                 if (dma_attr->dma_attr_align > MMU_PAGESIZE)
3364                         return (NULL);
3365 
3366                 /* Sanity check the dma_attr */
3367                 if (pfnlo > pfnhi)
3368                         return (NULL);
3369 
3370                 n = pfn_2_mtype(pfnlo);
3371                 m = pfn_2_mtype(pfnhi);
3372 
3373                 fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) &&
3374                     (pfnhi >= mnoderanges[m].mnr_pfnhi));
3375         }
3376         VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange);
3377 
3378         szc = 0;
3379 
3380         /* cylcing thru mtype handled by RANGE0 if n == mtype16m */
3381         if (n == mtype16m) {
3382                 flags |= PGI_MT_RANGE0;
3383                 n = m;
3384         }
3385 
3386         /*
3387          * Try local memory node first, but try remote if we can't
3388          * get a page of the right color.
3389          */
3390         LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER);
3391         while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) {
3392                 /*
3393                  * allocate pages from high pfn to low.
3394                  */
3395                 mtype = m;
3396                 do {
3397                         if (fullrange != 0) {
3398                                 pp = page_get_mnode_freelist(mnode,
3399                                     bin, mtype, szc, flags);
3400                                 if (pp == NULL) {
3401                                         pp = page_get_mnode_cachelist(
3402                                             bin, flags, mnode, mtype);
3403                                 }
3404                         } else {
3405                                 pp = page_get_mnode_anylist(bin, szc,
3406                                     flags, mnode, mtype, dma_attr);
3407                         }
3408                         if (pp != NULL) {
3409                                 VM_STAT_ADD(pga_vmstats.pga_allocok);
3410                                 check_dma(dma_attr, pp, 1);
3411                                 return (pp);
3412                         }
3413                 } while (mtype != n &&
3414                     (mtype = mnoderanges[mtype].mnr_next) != -1);
3415                 if (!local_failed_stat) {
3416                         lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1);
3417                         local_failed_stat = 1;
3418                 }
3419         }
3420         VM_STAT_ADD(pga_vmstats.pga_allocfailed);
3421 
3422         return (NULL);
3423 }
3424 
3425 /*
3426  * page_create_io()
3427  *
3428  * This function is a copy of page_create_va() with an additional
3429  * argument 'mattr' that specifies DMA memory requirements to
3430  * the page list functions. This function is used by the segkmem
3431  * allocator so it is only to create new pages (i.e PG_EXCL is
3432  * set).
3433  *
3434  * Note: This interface is currently used by x86 PSM only and is
3435  *       not fully specified so the commitment level is only for
3436  *       private interface specific to x86. This interface uses PSM
3437  *       specific page_get_anylist() interface.
3438  */
3439 
3440 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \
3441         for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
3442                 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
3443                         break; \
3444         } \
3445 }
3446 
3447 
3448 page_t *
3449 page_create_io(
3450         struct vnode    *vp,
3451         u_offset_t      off,
3452         uint_t          bytes,
3453         uint_t          flags,
3454         struct as       *as,
3455         caddr_t         vaddr,
3456         ddi_dma_attr_t  *mattr) /* DMA memory attributes if any */
3457 {
3458         page_t          *plist = NULL;
3459         uint_t          plist_len = 0;
3460         pgcnt_t         npages;
3461         page_t          *npp = NULL;
3462         uint_t          pages_req;
3463         page_t          *pp;
3464         kmutex_t        *phm = NULL;
3465         uint_t          index;
3466 
3467         TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
3468             "page_create_start:vp %p off %llx bytes %u flags %x",
3469             vp, off, bytes, flags);
3470 
3471         ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0);
3472 
3473         pages_req = npages = mmu_btopr(bytes);
3474 
3475         /*
3476          * Do the freemem and pcf accounting.
3477          */
3478         if (!page_create_wait(npages, flags)) {
3479                 return (NULL);
3480         }
3481 
3482         TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
3483             "page_create_success:vp %p off %llx", vp, off);
3484 
3485         /*
3486          * If satisfying this request has left us with too little
3487          * memory, start the wheels turning to get some back.  The
3488          * first clause of the test prevents waking up the pageout
3489          * daemon in situations where it would decide that there's
3490          * nothing to do.
3491          */
3492         if (nscan < desscan && freemem < minfree) {
3493                 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
3494                     "pageout_cv_signal:freemem %ld", freemem);
3495                 cv_signal(&proc_pageout->p_cv);
3496         }
3497 
3498         if (flags & PG_PHYSCONTIG) {
3499 
3500                 plist = page_get_contigpage(&npages, mattr, 1);
3501                 if (plist == NULL) {
3502                         page_create_putback(npages);
3503                         return (NULL);
3504                 }
3505 
3506                 pp = plist;
3507 
3508                 do {
3509                         if (!page_hashin(pp, vp, off, NULL)) {
3510                                 panic("pg_creat_io: hashin failed %p %p %llx",
3511                                     (void *)pp, (void *)vp, off);
3512                         }
3513                         VM_STAT_ADD(page_create_new);
3514                         off += MMU_PAGESIZE;
3515                         PP_CLRFREE(pp);
3516                         PP_CLRAGED(pp);
3517                         page_set_props(pp, P_REF);
3518                         pp = pp->p_next;
3519                 } while (pp != plist);
3520 
3521                 if (!npages) {
3522                         check_dma(mattr, plist, pages_req);
3523                         return (plist);
3524                 } else {
3525                         vaddr += (pages_req - npages) << MMU_PAGESHIFT;
3526                 }
3527 
3528                 /*
3529                  * fall-thru:
3530                  *
3531                  * page_get_contigpage returns when npages <= sgllen.
3532                  * Grab the rest of the non-contig pages below from anylist.
3533                  */
3534         }
3535 
3536         /*
3537          * Loop around collecting the requested number of pages.
3538          * Most of the time, we have to `create' a new page. With
3539          * this in mind, pull the page off the free list before
3540          * getting the hash lock.  This will minimize the hash
3541          * lock hold time, nesting, and the like.  If it turns
3542          * out we don't need the page, we put it back at the end.
3543          */
3544         while (npages--) {
3545                 phm = NULL;
3546 
3547                 index = PAGE_HASH_FUNC(vp, off);
3548 top:
3549                 ASSERT(phm == NULL);
3550                 ASSERT(index == PAGE_HASH_FUNC(vp, off));
3551                 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3552 
3553                 if (npp == NULL) {
3554                         /*
3555                          * Try to get the page of any color either from
3556                          * the freelist or from the cache list.
3557                          */
3558                         npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE,
3559                             flags & ~PG_MATCH_COLOR, mattr, NULL);
3560                         if (npp == NULL) {
3561                                 if (mattr == NULL) {
3562                                         /*
3563                                          * Not looking for a special page;
3564                                          * panic!
3565                                          */
3566                                         panic("no page found %d", (int)npages);
3567                                 }
3568                                 /*
3569                                  * No page found! This can happen
3570                                  * if we are looking for a page
3571                                  * within a specific memory range
3572                                  * for DMA purposes. If PG_WAIT is
3573                                  * specified then we wait for a
3574                                  * while and then try again. The
3575                                  * wait could be forever if we
3576                                  * don't get the page(s) we need.
3577                                  *
3578                                  * Note: XXX We really need a mechanism
3579                                  * to wait for pages in the desired
3580                                  * range. For now, we wait for any
3581                                  * pages and see if we can use it.
3582                                  */
3583 
3584                                 if ((mattr != NULL) && (flags & PG_WAIT)) {
3585                                         delay(10);
3586                                         goto top;
3587                                 }
3588                                 goto fail; /* undo accounting stuff */
3589                         }
3590 
3591                         if (PP_ISAGED(npp) == 0) {
3592                                 /*
3593                                  * Since this page came from the
3594                                  * cachelist, we must destroy the
3595                                  * old vnode association.
3596                                  */
3597                                 page_hashout(npp, (kmutex_t *)NULL);
3598                         }
3599                 }
3600 
3601                 /*
3602                  * We own this page!
3603                  */
3604                 ASSERT(PAGE_EXCL(npp));
3605                 ASSERT(npp->p_vnode == NULL);
3606                 ASSERT(!hat_page_is_mapped(npp));
3607                 PP_CLRFREE(npp);
3608                 PP_CLRAGED(npp);
3609 
3610                 /*
3611                  * Here we have a page in our hot little mits and are
3612                  * just waiting to stuff it on the appropriate lists.
3613                  * Get the mutex and check to see if it really does
3614                  * not exist.
3615                  */
3616                 phm = PAGE_HASH_MUTEX(index);
3617                 mutex_enter(phm);
3618                 PAGE_HASH_SEARCH(index, pp, vp, off);
3619                 if (pp == NULL) {
3620                         VM_STAT_ADD(page_create_new);
3621                         pp = npp;
3622                         npp = NULL;
3623                         if (!page_hashin(pp, vp, off, phm)) {
3624                                 /*
3625                                  * Since we hold the page hash mutex and
3626                                  * just searched for this page, page_hashin
3627                                  * had better not fail.  If it does, that
3628                                  * means somethread did not follow the
3629                                  * page hash mutex rules.  Panic now and
3630                                  * get it over with.  As usual, go down
3631                                  * holding all the locks.
3632                                  */
3633                                 ASSERT(MUTEX_HELD(phm));
3634                                 panic("page_create: hashin fail %p %p %llx %p",
3635                                     (void *)pp, (void *)vp, off, (void *)phm);
3636 
3637                         }
3638                         ASSERT(MUTEX_HELD(phm));
3639                         mutex_exit(phm);
3640                         phm = NULL;
3641 
3642                         /*
3643                          * Hat layer locking need not be done to set
3644                          * the following bits since the page is not hashed
3645                          * and was on the free list (i.e., had no mappings).
3646                          *
3647                          * Set the reference bit to protect
3648                          * against immediate pageout
3649                          *
3650                          * XXXmh modify freelist code to set reference
3651                          * bit so we don't have to do it here.
3652                          */
3653                         page_set_props(pp, P_REF);
3654                 } else {
3655                         ASSERT(MUTEX_HELD(phm));
3656                         mutex_exit(phm);
3657                         phm = NULL;
3658                         /*
3659                          * NOTE: This should not happen for pages associated
3660                          *       with kernel vnode 'kvp'.
3661                          */
3662                         /* XX64 - to debug why this happens! */
3663                         ASSERT(!VN_ISKAS(vp));
3664                         if (VN_ISKAS(vp))
3665                                 cmn_err(CE_NOTE,
3666                                     "page_create: page not expected "
3667                                     "in hash list for kernel vnode - pp 0x%p",
3668                                     (void *)pp);
3669                         VM_STAT_ADD(page_create_exists);
3670                         goto fail;
3671                 }
3672 
3673                 /*
3674                  * Got a page!  It is locked.  Acquire the i/o
3675                  * lock since we are going to use the p_next and
3676                  * p_prev fields to link the requested pages together.
3677                  */
3678                 page_io_lock(pp);
3679                 page_add(&plist, pp);
3680                 plist = plist->p_next;
3681                 off += MMU_PAGESIZE;
3682                 vaddr += MMU_PAGESIZE;
3683         }
3684 
3685         check_dma(mattr, plist, pages_req);
3686         return (plist);
3687 
3688 fail:
3689         if (npp != NULL) {
3690                 /*
3691                  * Did not need this page after all.
3692                  * Put it back on the free list.
3693                  */
3694                 VM_STAT_ADD(page_create_putbacks);
3695                 PP_SETFREE(npp);
3696                 PP_SETAGED(npp);
3697                 npp->p_offset = (u_offset_t)-1;
3698                 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
3699                 page_unlock(npp);
3700         }
3701 
3702         /*
3703          * Give up the pages we already got.
3704          */
3705         while (plist != NULL) {
3706                 pp = plist;
3707                 page_sub(&plist, pp);
3708                 page_io_unlock(pp);
3709                 plist_len++;
3710                 /*LINTED: constant in conditional ctx*/
3711                 VN_DISPOSE(pp, B_INVAL, 0, kcred);
3712         }
3713 
3714         /*
3715          * VN_DISPOSE does freemem accounting for the pages in plist
3716          * by calling page_free. So, we need to undo the pcf accounting
3717          * for only the remaining pages.
3718          */
3719         VM_STAT_ADD(page_create_putbacks);
3720         page_create_putback(pages_req - plist_len);
3721 
3722         return (NULL);
3723 }
3724 #endif /* !__xpv */
3725 
3726 
3727 /*
3728  * Copy the data from the physical page represented by "frompp" to
3729  * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and
3730  * CPU->cpu_caddr2.  It assumes that no one uses either map at interrupt
3731  * level and no one sleeps with an active mapping there.
3732  *
3733  * Note that the ref/mod bits in the page_t's are not affected by
3734  * this operation, hence it is up to the caller to update them appropriately.
3735  */
3736 int
3737 ppcopy(page_t *frompp, page_t *topp)
3738 {
3739         caddr_t         pp_addr1;
3740         caddr_t         pp_addr2;
3741         hat_mempte_t    pte1;
3742         hat_mempte_t    pte2;
3743         kmutex_t        *ppaddr_mutex;
3744         label_t         ljb;
3745         int             ret = 1;
3746 
3747         ASSERT_STACK_ALIGNED();
3748         ASSERT(PAGE_LOCKED(frompp));
3749         ASSERT(PAGE_LOCKED(topp));
3750 
3751         if (kpm_enable) {
3752                 pp_addr1 = hat_kpm_page2va(frompp, 0);
3753                 pp_addr2 = hat_kpm_page2va(topp, 0);
3754                 kpreempt_disable();
3755         } else {
3756                 /*
3757                  * disable pre-emption so that CPU can't change
3758                  */
3759                 kpreempt_disable();
3760 
3761                 pp_addr1 = CPU->cpu_caddr1;
3762                 pp_addr2 = CPU->cpu_caddr2;
3763                 pte1 = CPU->cpu_caddr1pte;
3764                 pte2 = CPU->cpu_caddr2pte;
3765 
3766                 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3767                 mutex_enter(ppaddr_mutex);
3768 
3769                 hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1,
3770                     PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST);
3771                 hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2,
3772                     PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3773                     HAT_LOAD_NOCONSIST);
3774         }
3775 
3776         if (on_fault(&ljb)) {
3777                 ret = 0;
3778                 goto faulted;
3779         }
3780         if (use_sse_pagecopy)
3781 #ifdef __xpv
3782                 page_copy_no_xmm(pp_addr2, pp_addr1);
3783 #else
3784                 hwblkpagecopy(pp_addr1, pp_addr2);
3785 #endif
3786         else
3787                 bcopy(pp_addr1, pp_addr2, PAGESIZE);
3788 
3789         no_fault();
3790 faulted:
3791         if (!kpm_enable) {
3792 #ifdef __xpv
3793                 /*
3794                  * We can't leave unused mappings laying about under the
3795                  * hypervisor, so blow them away.
3796                  */
3797                 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0,
3798                     UVMF_INVLPG | UVMF_LOCAL) < 0)
3799                         panic("HYPERVISOR_update_va_mapping() failed");
3800                 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3801                     UVMF_INVLPG | UVMF_LOCAL) < 0)
3802                         panic("HYPERVISOR_update_va_mapping() failed");
3803 #endif
3804                 mutex_exit(ppaddr_mutex);
3805         }
3806         kpreempt_enable();
3807         return (ret);
3808 }
3809 
3810 void
3811 pagezero(page_t *pp, uint_t off, uint_t len)
3812 {
3813         ASSERT(PAGE_LOCKED(pp));
3814         pfnzero(page_pptonum(pp), off, len);
3815 }
3816 
3817 /*
3818  * Zero the physical page from off to off + len given by pfn
3819  * without changing the reference and modified bits of page.
3820  *
3821  * We use this using CPU private page address #2, see ppcopy() for more info.
3822  * pfnzero() must not be called at interrupt level.
3823  */
3824 void
3825 pfnzero(pfn_t pfn, uint_t off, uint_t len)
3826 {
3827         caddr_t         pp_addr2;
3828         hat_mempte_t    pte2;
3829         kmutex_t        *ppaddr_mutex = NULL;
3830 
3831         ASSERT_STACK_ALIGNED();
3832         ASSERT(len <= MMU_PAGESIZE);
3833         ASSERT(off <= MMU_PAGESIZE);
3834         ASSERT(off + len <= MMU_PAGESIZE);
3835 
3836         if (kpm_enable && !pfn_is_foreign(pfn)) {
3837                 pp_addr2 = hat_kpm_pfn2va(pfn);
3838                 kpreempt_disable();
3839         } else {
3840                 kpreempt_disable();
3841 
3842                 pp_addr2 = CPU->cpu_caddr2;
3843                 pte2 = CPU->cpu_caddr2pte;
3844 
3845                 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3846                 mutex_enter(ppaddr_mutex);
3847 
3848                 hat_mempte_remap(pfn, pp_addr2, pte2,
3849                     PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3850                     HAT_LOAD_NOCONSIST);
3851         }
3852 
3853         if (use_sse_pagezero) {
3854 #ifdef __xpv
3855                 uint_t rem;
3856 
3857                 /*
3858                  * zero a byte at a time until properly aligned for
3859                  * block_zero_no_xmm().
3860                  */
3861                 while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0)
3862                         pp_addr2[off++] = 0;
3863 
3864                 /*
3865                  * Now use faster block_zero_no_xmm() for any range
3866                  * that is properly aligned and sized.
3867                  */
3868                 rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN));
3869                 len -= rem;
3870                 if (len != 0) {
3871                         block_zero_no_xmm(pp_addr2 + off, len);
3872                         off += len;
3873                 }
3874 
3875                 /*
3876                  * zero remainder with byte stores.
3877                  */
3878                 while (rem-- > 0)
3879                         pp_addr2[off++] = 0;
3880 #else
3881                 hwblkclr(pp_addr2 + off, len);
3882 #endif
3883         } else {
3884                 bzero(pp_addr2 + off, len);
3885         }
3886 
3887         if (!kpm_enable || pfn_is_foreign(pfn)) {
3888 #ifdef __xpv
3889                 /*
3890                  * On the hypervisor this page might get used for a page
3891                  * table before any intervening change to this mapping,
3892                  * so blow it away.
3893                  */
3894                 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3895                     UVMF_INVLPG) < 0)
3896                         panic("HYPERVISOR_update_va_mapping() failed");
3897 #endif
3898                 mutex_exit(ppaddr_mutex);
3899         }
3900 
3901         kpreempt_enable();
3902 }
3903 
3904 /*
3905  * Platform-dependent page scrub call.
3906  */
3907 void
3908 pagescrub(page_t *pp, uint_t off, uint_t len)
3909 {
3910         /*
3911          * For now, we rely on the fact that pagezero() will
3912          * always clear UEs.
3913          */
3914         pagezero(pp, off, len);
3915 }
3916 
3917 /*
3918  * set up two private addresses for use on a given CPU for use in ppcopy()
3919  */
3920 void
3921 setup_vaddr_for_ppcopy(struct cpu *cpup)
3922 {
3923         void *addr;
3924         hat_mempte_t pte_pa;
3925 
3926         addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3927         pte_pa = hat_mempte_setup(addr);
3928         cpup->cpu_caddr1 = addr;
3929         cpup->cpu_caddr1pte = pte_pa;
3930 
3931         addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3932         pte_pa = hat_mempte_setup(addr);
3933         cpup->cpu_caddr2 = addr;
3934         cpup->cpu_caddr2pte = pte_pa;
3935 
3936         mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL);
3937 }
3938 
3939 /*
3940  * Undo setup_vaddr_for_ppcopy
3941  */
3942 void
3943 teardown_vaddr_for_ppcopy(struct cpu *cpup)
3944 {
3945         mutex_destroy(&cpup->cpu_ppaddr_mutex);
3946 
3947         hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte);
3948         cpup->cpu_caddr2pte = 0;
3949         vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1));
3950         cpup->cpu_caddr2 = 0;
3951 
3952         hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte);
3953         cpup->cpu_caddr1pte = 0;
3954         vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1));
3955         cpup->cpu_caddr1 = 0;
3956 }
3957 
3958 /*
3959  * Function for flushing D-cache when performing module relocations
3960  * to an alternate mapping.  Unnecessary on Intel / AMD platforms.
3961  */
3962 void
3963 dcache_flushall()
3964 {}
3965 
3966 /*
3967  * Allocate a memory page.  The argument 'seed' can be any pseudo-random
3968  * number to vary where the pages come from.  This is quite a hacked up
3969  * method -- it works for now, but really needs to be fixed up a bit.
3970  *
3971  * We currently use page_create_va() on the kvp with fake offsets,
3972  * segments and virt address.  This is pretty bogus, but was copied from the
3973  * old hat_i86.c code.  A better approach would be to specify either mnode
3974  * random or mnode local and takes a page from whatever color has the MOST
3975  * available - this would have a minimal impact on page coloring.
3976  */
3977 page_t *
3978 page_get_physical(uintptr_t seed)
3979 {
3980         page_t *pp;
3981         u_offset_t offset;
3982         static struct seg tmpseg;
3983         static uintptr_t ctr = 0;
3984 
3985         /*
3986          * This code is gross, we really need a simpler page allocator.
3987          *
3988          * We need to assign an offset for the page to call page_create_va()
3989          * To avoid conflicts with other pages, we get creative with the offset.
3990          * For 32 bits, we need an offset > 4Gig
3991          * For 64 bits, need an offset somewhere in the VA hole.
3992          */
3993         offset = seed;
3994         if (offset > kernelbase)
3995                 offset -= kernelbase;
3996         offset <<= MMU_PAGESHIFT;
3997 #if defined(__amd64)
3998         offset += mmu.hole_start;       /* something in VA hole */
3999 #else
4000         offset += 1ULL << 40;     /* something > 4 Gig */
4001 #endif
4002 
4003         if (page_resv(1, KM_NOSLEEP) == 0)
4004                 return (NULL);
4005 
4006 #ifdef  DEBUG
4007         pp = page_exists(&kvp, offset);
4008         if (pp != NULL)
4009                 panic("page already exists %p", (void *)pp);
4010 #endif
4011 
4012         pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL,
4013             &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE));       /* changing VA usage */
4014         if (pp != NULL) {
4015                 page_io_unlock(pp);
4016                 page_downgrade(pp);
4017         }
4018         return (pp);
4019 }