1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 /*
  25  * Copyright (c) 2010, Intel Corporation.
  26  * All rights reserved.
  27  */
  28 
  29 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
  30 /*      All Rights Reserved   */
  31 
  32 /*
  33  * Portions of this source code were derived from Berkeley 4.3 BSD
  34  * under license from the Regents of the University of California.
  35  */
  36 
  37 /*
  38  * UNIX machine dependent virtual memory support.
  39  */
  40 
  41 #include <sys/types.h>
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/user.h>
  45 #include <sys/proc.h>
  46 #include <sys/kmem.h>
  47 #include <sys/vmem.h>
  48 #include <sys/buf.h>
  49 #include <sys/cpuvar.h>
  50 #include <sys/lgrp.h>
  51 #include <sys/disp.h>
  52 #include <sys/vm.h>
  53 #include <sys/mman.h>
  54 #include <sys/vnode.h>
  55 #include <sys/cred.h>
  56 #include <sys/exec.h>
  57 #include <sys/exechdr.h>
  58 #include <sys/debug.h>
  59 #include <sys/vmsystm.h>
  60 #include <sys/swap.h>
  61 #include <sys/dumphdr.h>
  62 #include <sys/random.h>
  63 
  64 #include <vm/hat.h>
  65 #include <vm/as.h>
  66 #include <vm/seg.h>
  67 #include <vm/seg_kp.h>
  68 #include <vm/seg_vn.h>
  69 #include <vm/page.h>
  70 #include <vm/seg_kmem.h>
  71 #include <vm/seg_kpm.h>
  72 #include <vm/vm_dep.h>
  73 
  74 #include <sys/cpu.h>
  75 #include <sys/vm_machparam.h>
  76 #include <sys/memlist.h>
  77 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */
  78 #include <vm/hat_i86.h>
  79 #include <sys/x86_archext.h>
  80 #include <sys/elf_386.h>
  81 #include <sys/cmn_err.h>
  82 #include <sys/archsystm.h>
  83 #include <sys/machsystm.h>
  84 #include <sys/secflags.h>
  85 
  86 #include <sys/vtrace.h>
  87 #include <sys/ddidmareq.h>
  88 #include <sys/promif.h>
  89 #include <sys/memnode.h>
  90 #include <sys/stack.h>
  91 #include <util/qsort.h>
  92 #include <sys/taskq.h>
  93 
  94 #ifdef __xpv
  95 
  96 #include <sys/hypervisor.h>
  97 #include <sys/xen_mmu.h>
  98 #include <sys/balloon_impl.h>
  99 
 100 /*
 101  * domain 0 pages usable for DMA are kept pre-allocated and kept in
 102  * distinct lists, ordered by increasing mfn.
 103  */
 104 static kmutex_t io_pool_lock;
 105 static kmutex_t contig_list_lock;
 106 static page_t *io_pool_4g;      /* pool for 32 bit dma limited devices */
 107 static page_t *io_pool_16m;     /* pool for 24 bit dma limited legacy devices */
 108 static long io_pool_cnt;
 109 static long io_pool_cnt_max = 0;
 110 #define DEFAULT_IO_POOL_MIN     128
 111 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN;
 112 static long io_pool_cnt_lowater = 0;
 113 static long io_pool_shrink_attempts; /* how many times did we try to shrink */
 114 static long io_pool_shrinks;    /* how many times did we really shrink */
 115 static long io_pool_grows;      /* how many times did we grow */
 116 static mfn_t start_mfn = 1;
 117 static caddr_t io_pool_kva;     /* use to alloc pages when needed */
 118 
 119 static int create_contig_pfnlist(uint_t);
 120 
 121 /*
 122  * percentage of phys mem to hold in the i/o pool
 123  */
 124 #define DEFAULT_IO_POOL_PCT     2
 125 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT;
 126 static void page_io_pool_sub(page_t **, page_t *, page_t *);
 127 int ioalloc_dbg = 0;
 128 
 129 #endif /* __xpv */
 130 
 131 uint_t vac_colors = 1;
 132 
 133 int largepagesupport = 0;
 134 extern uint_t page_create_new;
 135 extern uint_t page_create_exists;
 136 extern uint_t page_create_putbacks;
 137 /*
 138  * Allow users to disable the kernel's use of SSE.
 139  */
 140 extern int use_sse_pagecopy, use_sse_pagezero;
 141 
 142 /*
 143  * combined memory ranges from mnode and memranges[] to manage single
 144  * mnode/mtype dimension in the page lists.
 145  */
 146 typedef struct {
 147         pfn_t   mnr_pfnlo;
 148         pfn_t   mnr_pfnhi;
 149         int     mnr_mnode;
 150         int     mnr_memrange;           /* index into memranges[] */
 151         int     mnr_next;               /* next lower PA mnoderange */
 152         int     mnr_exists;
 153         /* maintain page list stats */
 154         pgcnt_t mnr_mt_clpgcnt;         /* cache list cnt */
 155         pgcnt_t mnr_mt_flpgcnt[MMU_PAGE_SIZES]; /* free list cnt per szc */
 156         pgcnt_t mnr_mt_totcnt;          /* sum of cache and free lists */
 157 #ifdef DEBUG
 158         struct mnr_mts {                /* mnode/mtype szc stats */
 159                 pgcnt_t mnr_mts_pgcnt;
 160                 int     mnr_mts_colors;
 161                 pgcnt_t *mnr_mtsc_pgcnt;
 162         }       *mnr_mts;
 163 #endif
 164 } mnoderange_t;
 165 
 166 #define MEMRANGEHI(mtype)                                               \
 167         ((mtype > 0) ? memranges[mtype - 1] - 1: physmax)
 168 #define MEMRANGELO(mtype)       (memranges[mtype])
 169 
 170 #define MTYPE_FREEMEM(mt)       (mnoderanges[mt].mnr_mt_totcnt)
 171 
 172 /*
 173  * As the PC architecture evolved memory up was clumped into several
 174  * ranges for various historical I/O devices to do DMA.
 175  * < 16Meg - ISA bus
 176  * < 2Gig - ???
 177  * < 4Gig - PCI bus or drivers that don't understand PAE mode
 178  *
 179  * These are listed in reverse order, so that we can skip over unused
 180  * ranges on machines with small memories.
 181  *
 182  * For now under the Hypervisor, we'll only ever have one memrange.
 183  */
 184 #define PFN_4GIG        0x100000
 185 #define PFN_16MEG       0x1000
 186 /* Indices into the memory range (arch_memranges) array. */
 187 #define MRI_4G          0
 188 #define MRI_2G          1
 189 #define MRI_16M         2
 190 #define MRI_0           3
 191 static pfn_t arch_memranges[NUM_MEM_RANGES] = {
 192     PFN_4GIG,   /* pfn range for 4G and above */
 193     0x80000,    /* pfn range for 2G-4G */
 194     PFN_16MEG,  /* pfn range for 16M-2G */
 195     0x00000,    /* pfn range for 0-16M */
 196 };
 197 pfn_t *memranges = &arch_memranges[0];
 198 int nranges = NUM_MEM_RANGES;
 199 
 200 /*
 201  * This combines mem_node_config and memranges into one data
 202  * structure to be used for page list management.
 203  */
 204 mnoderange_t    *mnoderanges;
 205 int             mnoderangecnt;
 206 int             mtype4g;
 207 int             mtype16m;
 208 int             mtypetop;       /* index of highest pfn'ed mnoderange */
 209 
 210 /*
 211  * 4g memory management variables for systems with more than 4g of memory:
 212  *
 213  * physical memory below 4g is required for 32bit dma devices and, currently,
 214  * for kmem memory. On systems with more than 4g of memory, the pool of memory
 215  * below 4g can be depleted without any paging activity given that there is
 216  * likely to be sufficient memory above 4g.
 217  *
 218  * physmax4g is set true if the largest pfn is over 4g. The rest of the
 219  * 4g memory management code is enabled only when physmax4g is true.
 220  *
 221  * maxmem4g is the count of the maximum number of pages on the page lists
 222  * with physical addresses below 4g. It can be a lot less then 4g given that
 223  * BIOS may reserve large chunks of space below 4g for hot plug pci devices,
 224  * agp aperture etc.
 225  *
 226  * freemem4g maintains the count of the number of available pages on the
 227  * page lists with physical addresses below 4g.
 228  *
 229  * DESFREE4G specifies the desired amount of below 4g memory. It defaults to
 230  * 6% (desfree4gshift = 4) of maxmem4g.
 231  *
 232  * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G
 233  * and the amount of physical memory above 4g is greater than freemem4g.
 234  * In this case, page_get_* routines will restrict below 4g allocations
 235  * for requests that don't specifically require it.
 236  */
 237 
 238 #define DESFREE4G       (maxmem4g >> desfree4gshift)
 239 
 240 #define RESTRICT4G_ALLOC                                        \
 241         (physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem))
 242 
 243 static pgcnt_t  maxmem4g;
 244 static pgcnt_t  freemem4g;
 245 static int      physmax4g;
 246 static int      desfree4gshift = 4;     /* maxmem4g shift to derive DESFREE4G */
 247 
 248 /*
 249  * 16m memory management:
 250  *
 251  * reserve some amount of physical memory below 16m for legacy devices.
 252  *
 253  * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above
 254  * 16m or if the 16m pool drops below DESFREE16M.
 255  *
 256  * In this case, general page allocations via page_get_{free,cache}list
 257  * routines will be restricted from allocating from the 16m pool. Allocations
 258  * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations
 259  * are not restricted.
 260  */
 261 
 262 #define FREEMEM16M      MTYPE_FREEMEM(mtype16m)
 263 #define DESFREE16M      desfree16m
 264 #define RESTRICT16M_ALLOC(freemem, pgcnt, flags)                \
 265         ((freemem != 0) && ((flags & PG_PANIC) == 0) &&             \
 266             ((freemem >= (FREEMEM16M)) ||                    \
 267             (FREEMEM16M  < (DESFREE16M + pgcnt))))
 268 
 269 static pgcnt_t  desfree16m = 0x380;
 270 
 271 /*
 272  * This can be patched via /etc/system to allow old non-PAE aware device
 273  * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM.
 274  */
 275 int restricted_kmemalloc = 0;
 276 
 277 #ifdef VM_STATS
 278 struct {
 279         ulong_t pga_alloc;
 280         ulong_t pga_notfullrange;
 281         ulong_t pga_nulldmaattr;
 282         ulong_t pga_allocok;
 283         ulong_t pga_allocfailed;
 284         ulong_t pgma_alloc;
 285         ulong_t pgma_allocok;
 286         ulong_t pgma_allocfailed;
 287         ulong_t pgma_allocempty;
 288 } pga_vmstats;
 289 #endif
 290 
 291 uint_t mmu_page_sizes;
 292 
 293 /* How many page sizes the users can see */
 294 uint_t mmu_exported_page_sizes;
 295 
 296 /* page sizes that legacy applications can see */
 297 uint_t mmu_legacy_page_sizes;
 298 
 299 /*
 300  * Number of pages in 1 GB.  Don't enable automatic large pages if we have
 301  * fewer than this many pages.
 302  */
 303 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
 304 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT);
 305 
 306 /*
 307  * Maximum and default segment size tunables for user private
 308  * and shared anon memory, and user text and initialized data.
 309  * These can be patched via /etc/system to allow large pages
 310  * to be used for mapping application private and shared anon memory.
 311  */
 312 size_t mcntl0_lpsize = MMU_PAGESIZE;
 313 size_t max_uheap_lpsize = MMU_PAGESIZE;
 314 size_t default_uheap_lpsize = MMU_PAGESIZE;
 315 size_t max_ustack_lpsize = MMU_PAGESIZE;
 316 size_t default_ustack_lpsize = MMU_PAGESIZE;
 317 size_t max_privmap_lpsize = MMU_PAGESIZE;
 318 size_t max_uidata_lpsize = MMU_PAGESIZE;
 319 size_t max_utext_lpsize = MMU_PAGESIZE;
 320 size_t max_shm_lpsize = MMU_PAGESIZE;
 321 
 322 
 323 /*
 324  * initialized by page_coloring_init().
 325  */
 326 uint_t  page_colors;
 327 uint_t  page_colors_mask;
 328 uint_t  page_coloring_shift;
 329 int     cpu_page_colors;
 330 static uint_t   l2_colors;
 331 
 332 /*
 333  * Page freelists and cachelists are dynamically allocated once mnoderangecnt
 334  * and page_colors are calculated from the l2 cache n-way set size.  Within a
 335  * mnode range, the page freelist and cachelist are hashed into bins based on
 336  * color. This makes it easier to search for a page within a specific memory
 337  * range.
 338  */
 339 #define PAGE_COLORS_MIN 16
 340 
 341 page_t ****page_freelists;
 342 page_t ***page_cachelists;
 343 
 344 
 345 /*
 346  * Used by page layer to know about page sizes
 347  */
 348 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1];
 349 
 350 kmutex_t        *fpc_mutex[NPC_MUTEX];
 351 kmutex_t        *cpc_mutex[NPC_MUTEX];
 352 
 353 /* Lock to protect mnoderanges array for memory DR operations. */
 354 static kmutex_t mnoderange_lock;
 355 
 356 /*
 357  * Only let one thread at a time try to coalesce large pages, to
 358  * prevent them from working against each other.
 359  */
 360 static kmutex_t contig_lock;
 361 #define CONTIG_LOCK()   mutex_enter(&contig_lock);
 362 #define CONTIG_UNLOCK() mutex_exit(&contig_lock);
 363 
 364 #define PFN_16M         (mmu_btop((uint64_t)0x1000000))
 365 
 366 /*
 367  * Return the optimum page size for a given mapping
 368  */
 369 /*ARGSUSED*/
 370 size_t
 371 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl)
 372 {
 373         level_t l = 0;
 374         size_t pgsz = MMU_PAGESIZE;
 375         size_t max_lpsize;
 376         uint_t mszc;
 377 
 378         ASSERT(maptype != MAPPGSZ_VA);
 379 
 380         if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) {
 381                 return (MMU_PAGESIZE);
 382         }
 383 
 384         switch (maptype) {
 385         case MAPPGSZ_HEAP:
 386         case MAPPGSZ_STK:
 387                 max_lpsize = memcntl ? mcntl0_lpsize : (maptype ==
 388                     MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize);
 389                 if (max_lpsize == MMU_PAGESIZE) {
 390                         return (MMU_PAGESIZE);
 391                 }
 392                 if (len == 0) {
 393                         len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase +
 394                             p->p_brksize - p->p_bssbase : p->p_stksize;
 395                 }
 396                 len = (maptype == MAPPGSZ_HEAP) ? MAX(len,
 397                     default_uheap_lpsize) : MAX(len, default_ustack_lpsize);
 398 
 399                 /*
 400                  * use the pages size that best fits len
 401                  */
 402                 for (l = mmu.umax_page_level; l > 0; --l) {
 403                         if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) {
 404                                 continue;
 405                         } else {
 406                                 pgsz = LEVEL_SIZE(l);
 407                         }
 408                         break;
 409                 }
 410 
 411                 mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc :
 412                     p->p_stkpageszc);
 413                 if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) {
 414                         pgsz = hw_page_array[mszc].hp_size;
 415                 }
 416                 return (pgsz);
 417 
 418         case MAPPGSZ_ISM:
 419                 for (l = mmu.umax_page_level; l > 0; --l) {
 420                         if (len >= LEVEL_SIZE(l))
 421                                 return (LEVEL_SIZE(l));
 422                 }
 423                 return (LEVEL_SIZE(0));
 424         }
 425         return (pgsz);
 426 }
 427 
 428 static uint_t
 429 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize,
 430     size_t min_physmem)
 431 {
 432         caddr_t eaddr = addr + size;
 433         uint_t szcvec = 0;
 434         caddr_t raddr;
 435         caddr_t readdr;
 436         size_t  pgsz;
 437         int i;
 438 
 439         if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) {
 440                 return (0);
 441         }
 442 
 443         for (i = mmu_exported_page_sizes - 1; i > 0; i--) {
 444                 pgsz = page_get_pagesize(i);
 445                 if (pgsz > max_lpsize) {
 446                         continue;
 447                 }
 448                 raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
 449                 readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
 450                 if (raddr < addr || raddr >= readdr) {
 451                         continue;
 452                 }
 453                 if (P2PHASE((uintptr_t)addr ^ off, pgsz)) {
 454                         continue;
 455                 }
 456                 /*
 457                  * Set szcvec to the remaining page sizes.
 458                  */
 459                 szcvec = ((1 << (i + 1)) - 1) & ~1;
 460                 break;
 461         }
 462         return (szcvec);
 463 }
 464 
 465 /*
 466  * Return a bit vector of large page size codes that
 467  * can be used to map [addr, addr + len) region.
 468  */
 469 /*ARGSUSED*/
 470 uint_t
 471 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type,
 472     int memcntl)
 473 {
 474         size_t max_lpsize = mcntl0_lpsize;
 475 
 476         if (mmu.max_page_level == 0)
 477                 return (0);
 478 
 479         if (flags & MAP_TEXT) {
 480                 if (!memcntl)
 481                         max_lpsize = max_utext_lpsize;
 482                 return (map_szcvec(addr, size, off, max_lpsize,
 483                     shm_lpg_min_physmem));
 484 
 485         } else if (flags & MAP_INITDATA) {
 486                 if (!memcntl)
 487                         max_lpsize = max_uidata_lpsize;
 488                 return (map_szcvec(addr, size, off, max_lpsize,
 489                     privm_lpg_min_physmem));
 490 
 491         } else if (type == MAPPGSZC_SHM) {
 492                 if (!memcntl)
 493                         max_lpsize = max_shm_lpsize;
 494                 return (map_szcvec(addr, size, off, max_lpsize,
 495                     shm_lpg_min_physmem));
 496 
 497         } else if (type == MAPPGSZC_HEAP) {
 498                 if (!memcntl)
 499                         max_lpsize = max_uheap_lpsize;
 500                 return (map_szcvec(addr, size, off, max_lpsize,
 501                     privm_lpg_min_physmem));
 502 
 503         } else if (type == MAPPGSZC_STACK) {
 504                 if (!memcntl)
 505                         max_lpsize = max_ustack_lpsize;
 506                 return (map_szcvec(addr, size, off, max_lpsize,
 507                     privm_lpg_min_physmem));
 508 
 509         } else {
 510                 if (!memcntl)
 511                         max_lpsize = max_privmap_lpsize;
 512                 return (map_szcvec(addr, size, off, max_lpsize,
 513                     privm_lpg_min_physmem));
 514         }
 515 }
 516 
 517 /*
 518  * Handle a pagefault.
 519  */
 520 faultcode_t
 521 pagefault(
 522         caddr_t addr,
 523         enum fault_type type,
 524         enum seg_rw rw,
 525         int iskernel)
 526 {
 527         struct as *as;
 528         struct hat *hat;
 529         struct proc *p;
 530         kthread_t *t;
 531         faultcode_t res;
 532         caddr_t base;
 533         size_t len;
 534         int err;
 535         int mapped_red;
 536         uintptr_t ea;
 537 
 538         ASSERT_STACK_ALIGNED();
 539 
 540         if (INVALID_VADDR(addr))
 541                 return (FC_NOMAP);
 542 
 543         mapped_red = segkp_map_red();
 544 
 545         if (iskernel) {
 546                 as = &kas;
 547                 hat = as->a_hat;
 548         } else {
 549                 t = curthread;
 550                 p = ttoproc(t);
 551                 as = p->p_as;
 552                 hat = as->a_hat;
 553         }
 554 
 555         /*
 556          * Dispatch pagefault.
 557          */
 558         res = as_fault(hat, as, addr, 1, type, rw);
 559 
 560         /*
 561          * If this isn't a potential unmapped hole in the user's
 562          * UNIX data or stack segments, just return status info.
 563          */
 564         if (res != FC_NOMAP || iskernel)
 565                 goto out;
 566 
 567         /*
 568          * Check to see if we happened to faulted on a currently unmapped
 569          * part of the UNIX data or stack segments.  If so, create a zfod
 570          * mapping there and then try calling the fault routine again.
 571          */
 572         base = p->p_brkbase;
 573         len = p->p_brksize;
 574 
 575         if (addr < base || addr >= base + len) {          /* data seg? */
 576                 base = (caddr_t)p->p_usrstack - p->p_stksize;
 577                 len = p->p_stksize;
 578                 if (addr < base || addr >= p->p_usrstack) {    /* stack seg? */
 579                         /* not in either UNIX data or stack segments */
 580                         res = FC_NOMAP;
 581                         goto out;
 582                 }
 583         }
 584 
 585         /*
 586          * the rest of this function implements a 3.X 4.X 5.X compatibility
 587          * This code is probably not needed anymore
 588          */
 589         if (p->p_model == DATAMODEL_ILP32) {
 590 
 591                 /* expand the gap to the page boundaries on each side */
 592                 ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE);
 593                 base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE);
 594                 len = ea - (uintptr_t)base;
 595 
 596                 as_rangelock(as);
 597                 if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) ==
 598                     0) {
 599                         err = as_map(as, base, len, segvn_create, zfod_argsp);
 600                         as_rangeunlock(as);
 601                         if (err) {
 602                                 res = FC_MAKE_ERR(err);
 603                                 goto out;
 604                         }
 605                 } else {
 606                         /*
 607                          * This page is already mapped by another thread after
 608                          * we returned from as_fault() above.  We just fall
 609                          * through as_fault() below.
 610                          */
 611                         as_rangeunlock(as);
 612                 }
 613 
 614                 res = as_fault(hat, as, addr, 1, F_INVAL, rw);
 615         }
 616 
 617 out:
 618         if (mapped_red)
 619                 segkp_unmap_red();
 620 
 621         return (res);
 622 }
 623 
 624 void
 625 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags)
 626 {
 627         struct proc *p = curproc;
 628         caddr_t userlimit = (flags & _MAP_LOW32) ?
 629             (caddr_t)_userlimit32 : p->p_as->a_userlimit;
 630 
 631         map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags);
 632 }
 633 
 634 /*ARGSUSED*/
 635 int
 636 map_addr_vacalign_check(caddr_t addr, u_offset_t off)
 637 {
 638         return (0);
 639 }
 640 
 641 /*
 642  * The maximum amount a randomized mapping will be slewed.  We should perhaps
 643  * arrange things so these tunables can be separate for mmap, mmapobj, and
 644  * ld.so
 645  */
 646 size_t aslr_max_map_skew = 256 * 1024 * 1024; /* 256MB */
 647 
 648 /*
 649  * map_addr_proc() is the routine called when the system is to
 650  * choose an address for the user.  We will pick an address
 651  * range which is the highest available below userlimit.
 652  *
 653  * Every mapping will have a redzone of a single page on either side of
 654  * the request. This is done to leave one page unmapped between segments.
 655  * This is not required, but it's useful for the user because if their
 656  * program strays across a segment boundary, it will catch a fault
 657  * immediately making debugging a little easier.  Currently the redzone
 658  * is mandatory.
 659  *
 660  * addrp is a value/result parameter.
 661  *      On input it is a hint from the user to be used in a completely
 662  *      machine dependent fashion.  We decide to completely ignore this hint.
 663  *      If MAP_ALIGN was specified, addrp contains the minimal alignment, which
 664  *      must be some "power of two" multiple of pagesize.
 665  *
 666  *      On output it is NULL if no address can be found in the current
 667  *      processes address space or else an address that is currently
 668  *      not mapped for len bytes with a page of red zone on either side.
 669  *
 670  *      vacalign is not needed on x86 (it's for viturally addressed caches)
 671  */
 672 /*ARGSUSED*/
 673 void
 674 map_addr_proc(
 675         caddr_t *addrp,
 676         size_t len,
 677         offset_t off,
 678         int vacalign,
 679         caddr_t userlimit,
 680         struct proc *p,
 681         uint_t flags)
 682 {
 683         struct as *as = p->p_as;
 684         caddr_t addr;
 685         caddr_t base;
 686         size_t slen;
 687         size_t align_amount;
 688 
 689         ASSERT32(userlimit == as->a_userlimit);
 690 
 691         base = p->p_brkbase;
 692 #if defined(__amd64)
 693         /*
 694          * XX64 Yes, this needs more work.
 695          */
 696         if (p->p_model == DATAMODEL_NATIVE) {
 697                 if (userlimit < as->a_userlimit) {
 698                         /*
 699                          * This happens when a program wants to map
 700                          * something in a range that's accessible to a
 701                          * program in a smaller address space.  For example,
 702                          * a 64-bit program calling mmap32(2) to guarantee
 703                          * that the returned address is below 4Gbytes.
 704                          */
 705                         ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff));
 706 
 707                         if (userlimit > base)
 708                                 slen = userlimit - base;
 709                         else {
 710                                 *addrp = NULL;
 711                                 return;
 712                         }
 713                 } else {
 714                         /*
 715                          * XX64 This layout is probably wrong .. but in
 716                          * the event we make the amd64 address space look
 717                          * like sparcv9 i.e. with the stack -above- the
 718                          * heap, this bit of code might even be correct.
 719                          */
 720                         slen = p->p_usrstack - base -
 721                             ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK);
 722                 }
 723         } else
 724 #endif
 725                 slen = userlimit - base;
 726 
 727         /* Make len be a multiple of PAGESIZE */
 728         len = (len + PAGEOFFSET) & PAGEMASK;
 729 
 730         /*
 731          * figure out what the alignment should be
 732          *
 733          * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same????
 734          */
 735         if (len <= ELF_386_MAXPGSZ) {
 736                 /*
 737                  * Align virtual addresses to ensure that ELF shared libraries
 738                  * are mapped with the appropriate alignment constraints by
 739                  * the run-time linker.
 740                  */
 741                 align_amount = ELF_386_MAXPGSZ;
 742         } else {
 743                 /*
 744                  * For 32-bit processes, only those which have specified
 745                  * MAP_ALIGN and an addr will be aligned on a larger page size.
 746                  * Not doing so can potentially waste up to 1G of process
 747                  * address space.
 748                  */
 749                 int lvl = (p->p_model == DATAMODEL_ILP32) ? 1 :
 750                     mmu.umax_page_level;
 751 
 752                 while (lvl && len < LEVEL_SIZE(lvl))
 753                         --lvl;
 754 
 755                 align_amount = LEVEL_SIZE(lvl);
 756         }
 757         if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount))
 758                 align_amount = (uintptr_t)*addrp;
 759 
 760         ASSERT(ISP2(align_amount));
 761         ASSERT(align_amount == 0 || align_amount >= PAGESIZE);
 762 
 763         off = off & (align_amount - 1);
 764 
 765         /*
 766          * Look for a large enough hole starting below userlimit.
 767          * After finding it, use the upper part.
 768          */
 769         if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount,
 770             PAGESIZE, off) == 0) {
 771                 caddr_t as_addr;
 772 
 773                 /*
 774                  * addr is the highest possible address to use since we have
 775                  * a PAGESIZE redzone at the beginning and end.
 776                  */
 777                 addr = base + slen - (PAGESIZE + len);
 778                 as_addr = addr;
 779                 /*
 780                  * Round address DOWN to the alignment amount and
 781                  * add the offset in.
 782                  * If addr is greater than as_addr, len would not be large
 783                  * enough to include the redzone, so we must adjust down
 784                  * by the alignment amount.
 785                  */
 786                 addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1)));
 787                 addr += (uintptr_t)off;
 788                 if (addr > as_addr) {
 789                         addr -= align_amount;
 790                 }
 791 
 792                 /*
 793                  * If randomization is requested, slew the allocation
 794                  * backwards, within the same gap, by a random amount.
 795                  */
 796                 if (flags & _MAP_RANDOMIZE) {
 797                         uint32_t slew;
 798 
 799                         (void) random_get_pseudo_bytes((uint8_t *)&slew,
 800                             sizeof (slew));
 801 
 802                         slew = slew % MIN(aslr_max_map_skew, (addr - base));
 803                         addr -= P2ALIGN(slew, align_amount);
 804                 }
 805 
 806                 ASSERT(addr > base);
 807                 ASSERT(addr + len < base + slen);
 808                 ASSERT(((uintptr_t)addr & (align_amount - 1)) ==
 809                     ((uintptr_t)(off)));
 810                 *addrp = addr;
 811         } else {
 812                 *addrp = NULL;  /* no more virtual space */
 813         }
 814 }
 815 
 816 int valid_va_range_aligned_wraparound;
 817 
 818 /*
 819  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
 820  * addresses at least "minlen" long, where the base of the range is at "off"
 821  * phase from an "align" boundary and there is space for a "redzone"-sized
 822  * redzone on either side of the range.  On success, 1 is returned and *basep
 823  * and *lenp are adjusted to describe the acceptable range (including
 824  * the redzone).  On failure, 0 is returned.
 825  */
 826 /*ARGSUSED3*/
 827 int
 828 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir,
 829     size_t align, size_t redzone, size_t off)
 830 {
 831         uintptr_t hi, lo;
 832         size_t tot_len;
 833 
 834         ASSERT(align == 0 ? off == 0 : off < align);
 835         ASSERT(ISP2(align));
 836         ASSERT(align == 0 || align >= PAGESIZE);
 837 
 838         lo = (uintptr_t)*basep;
 839         hi = lo + *lenp;
 840         tot_len = minlen + 2 * redzone; /* need at least this much space */
 841 
 842         /*
 843          * If hi rolled over the top, try cutting back.
 844          */
 845         if (hi < lo) {
 846                 *lenp = 0UL - lo - 1UL;
 847                 /* See if this really happens. If so, then we figure out why */
 848                 valid_va_range_aligned_wraparound++;
 849                 hi = lo + *lenp;
 850         }
 851         if (*lenp < tot_len) {
 852                 return (0);
 853         }
 854 
 855 #if defined(__amd64)
 856         /*
 857          * Deal with a possible hole in the address range between
 858          * hole_start and hole_end that should never be mapped.
 859          */
 860         if (lo < hole_start) {
 861                 if (hi > hole_start) {
 862                         if (hi < hole_end) {
 863                                 hi = hole_start;
 864                         } else {
 865                                 /* lo < hole_start && hi >= hole_end */
 866                                 if (dir == AH_LO) {
 867                                         /*
 868                                          * prefer lowest range
 869                                          */
 870                                         if (hole_start - lo >= tot_len)
 871                                                 hi = hole_start;
 872                                         else if (hi - hole_end >= tot_len)
 873                                                 lo = hole_end;
 874                                         else
 875                                                 return (0);
 876                                 } else {
 877                                         /*
 878                                          * prefer highest range
 879                                          */
 880                                         if (hi - hole_end >= tot_len)
 881                                                 lo = hole_end;
 882                                         else if (hole_start - lo >= tot_len)
 883                                                 hi = hole_start;
 884                                         else
 885                                                 return (0);
 886                                 }
 887                         }
 888                 }
 889         } else {
 890                 /* lo >= hole_start */
 891                 if (hi < hole_end)
 892                         return (0);
 893                 if (lo < hole_end)
 894                         lo = hole_end;
 895         }
 896 #endif
 897 
 898         if (hi - lo < tot_len)
 899                 return (0);
 900 
 901         if (align > 1) {
 902                 uintptr_t tlo = lo + redzone;
 903                 uintptr_t thi = hi - redzone;
 904                 tlo = (uintptr_t)P2PHASEUP(tlo, align, off);
 905                 if (tlo < lo + redzone) {
 906                         return (0);
 907                 }
 908                 if (thi < tlo || thi - tlo < minlen) {
 909                         return (0);
 910                 }
 911         }
 912 
 913         *basep = (caddr_t)lo;
 914         *lenp = hi - lo;
 915         return (1);
 916 }
 917 
 918 /*
 919  * Determine whether [*basep, *basep + *lenp) contains a mappable range of
 920  * addresses at least "minlen" long.  On success, 1 is returned and *basep
 921  * and *lenp are adjusted to describe the acceptable range.  On failure, 0
 922  * is returned.
 923  */
 924 int
 925 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir)
 926 {
 927         return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0));
 928 }
 929 
 930 /*
 931  * Default to forbidding the first 64k of address space.  This protects most
 932  * reasonably sized structures from dereferences through NULL:
 933  *     ((foo_t *)0)->bar
 934  */
 935 uintptr_t forbidden_null_mapping_sz = 0x10000;
 936 
 937 /*
 938  * Determine whether [addr, addr+len] are valid user addresses.
 939  */
 940 /*ARGSUSED*/
 941 int
 942 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as,
 943     caddr_t userlimit)
 944 {
 945         caddr_t eaddr = addr + len;
 946 
 947         if (eaddr <= addr || addr >= userlimit || eaddr > userlimit)
 948                 return (RANGE_BADADDR);
 949 
 950         if ((addr <= (caddr_t)forbidden_null_mapping_sz) &&
 951             secflag_enabled(as->a_proc, PROC_SEC_FORBIDNULLMAP))
 952                 return (RANGE_BADADDR);
 953 
 954 #if defined(__amd64)
 955         /*
 956          * Check for the VA hole
 957          */
 958         if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end)
 959                 return (RANGE_BADADDR);
 960 #endif
 961 
 962         return (RANGE_OKAY);
 963 }
 964 
 965 /*
 966  * Return 1 if the page frame is onboard memory, else 0.
 967  */
 968 int
 969 pf_is_memory(pfn_t pf)
 970 {
 971         if (pfn_is_foreign(pf))
 972                 return (0);
 973         return (address_in_memlist(phys_install, pfn_to_pa(pf), 1));
 974 }
 975 
 976 /*
 977  * return the memrange containing pfn
 978  */
 979 int
 980 memrange_num(pfn_t pfn)
 981 {
 982         int n;
 983 
 984         for (n = 0; n < nranges - 1; ++n) {
 985                 if (pfn >= memranges[n])
 986                         break;
 987         }
 988         return (n);
 989 }
 990 
 991 /*
 992  * return the mnoderange containing pfn
 993  */
 994 /*ARGSUSED*/
 995 int
 996 pfn_2_mtype(pfn_t pfn)
 997 {
 998 #if defined(__xpv)
 999         return (0);
1000 #else
1001         int     n;
1002 
1003         /* Always start from highest pfn and work our way down */
1004         for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1005                 if (pfn >= mnoderanges[n].mnr_pfnlo) {
1006                         break;
1007                 }
1008         }
1009         return (n);
1010 #endif
1011 }
1012 
1013 #if !defined(__xpv)
1014 /*
1015  * is_contigpage_free:
1016  *      returns a page list of contiguous pages. It minimally has to return
1017  *      minctg pages. Caller determines minctg based on the scatter-gather
1018  *      list length.
1019  *
1020  *      pfnp is set to the next page frame to search on return.
1021  */
1022 static page_t *
1023 is_contigpage_free(
1024         pfn_t *pfnp,
1025         pgcnt_t *pgcnt,
1026         pgcnt_t minctg,
1027         uint64_t pfnseg,
1028         int iolock)
1029 {
1030         int     i = 0;
1031         pfn_t   pfn = *pfnp;
1032         page_t  *pp;
1033         page_t  *plist = NULL;
1034 
1035         /*
1036          * fail if pfn + minctg crosses a segment boundary.
1037          * Adjust for next starting pfn to begin at segment boundary.
1038          */
1039 
1040         if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) {
1041                 *pfnp = roundup(*pfnp, pfnseg + 1);
1042                 return (NULL);
1043         }
1044 
1045         do {
1046 retry:
1047                 pp = page_numtopp_nolock(pfn + i);
1048                 if ((pp == NULL) || IS_DUMP_PAGE(pp) ||
1049                     (page_trylock(pp, SE_EXCL) == 0)) {
1050                         (*pfnp)++;
1051                         break;
1052                 }
1053                 if (page_pptonum(pp) != pfn + i) {
1054                         page_unlock(pp);
1055                         goto retry;
1056                 }
1057 
1058                 if (!(PP_ISFREE(pp))) {
1059                         page_unlock(pp);
1060                         (*pfnp)++;
1061                         break;
1062                 }
1063 
1064                 if (!PP_ISAGED(pp)) {
1065                         page_list_sub(pp, PG_CACHE_LIST);
1066                         page_hashout(pp, (kmutex_t *)NULL);
1067                 } else {
1068                         page_list_sub(pp, PG_FREE_LIST);
1069                 }
1070 
1071                 if (iolock)
1072                         page_io_lock(pp);
1073                 page_list_concat(&plist, &pp);
1074 
1075                 /*
1076                  * exit loop when pgcnt satisfied or segment boundary reached.
1077                  */
1078 
1079         } while ((++i < *pgcnt) && ((pfn + i) & pfnseg));
1080 
1081         *pfnp += i;             /* set to next pfn to search */
1082 
1083         if (i >= minctg) {
1084                 *pgcnt -= i;
1085                 return (plist);
1086         }
1087 
1088         /*
1089          * failure: minctg not satisfied.
1090          *
1091          * if next request crosses segment boundary, set next pfn
1092          * to search from the segment boundary.
1093          */
1094         if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg))
1095                 *pfnp = roundup(*pfnp, pfnseg + 1);
1096 
1097         /* clean up any pages already allocated */
1098 
1099         while (plist) {
1100                 pp = plist;
1101                 page_sub(&plist, pp);
1102                 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
1103                 if (iolock)
1104                         page_io_unlock(pp);
1105                 page_unlock(pp);
1106         }
1107 
1108         return (NULL);
1109 }
1110 #endif  /* !__xpv */
1111 
1112 /*
1113  * verify that pages being returned from allocator have correct DMA attribute
1114  */
1115 #ifndef DEBUG
1116 #define check_dma(a, b, c) (void)(0)
1117 #else
1118 static void
1119 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt)
1120 {
1121         if (dma_attr == NULL)
1122                 return;
1123 
1124         while (cnt-- > 0) {
1125                 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) <
1126                     dma_attr->dma_attr_addr_lo)
1127                         panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp);
1128                 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >=
1129                     dma_attr->dma_attr_addr_hi)
1130                         panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp);
1131                 pp = pp->p_next;
1132         }
1133 }
1134 #endif
1135 
1136 #if !defined(__xpv)
1137 static page_t *
1138 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock)
1139 {
1140         pfn_t           pfn;
1141         int             sgllen;
1142         uint64_t        pfnseg;
1143         pgcnt_t         minctg;
1144         page_t          *pplist = NULL, *plist;
1145         uint64_t        lo, hi;
1146         pgcnt_t         pfnalign = 0;
1147         static pfn_t    startpfn;
1148         static pgcnt_t  lastctgcnt;
1149         uintptr_t       align;
1150 
1151         CONTIG_LOCK();
1152 
1153         if (mattr) {
1154                 lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET));
1155                 hi = mmu_btop(mattr->dma_attr_addr_hi);
1156                 if (hi >= physmax)
1157                         hi = physmax - 1;
1158                 sgllen = mattr->dma_attr_sgllen;
1159                 pfnseg = mmu_btop(mattr->dma_attr_seg);
1160 
1161                 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
1162                 if (align > MMU_PAGESIZE)
1163                         pfnalign = mmu_btop(align);
1164 
1165                 /*
1166                  * in order to satisfy the request, must minimally
1167                  * acquire minctg contiguous pages
1168                  */
1169                 minctg = howmany(*pgcnt, sgllen);
1170 
1171                 ASSERT(hi >= lo);
1172 
1173                 /*
1174                  * start from where last searched if the minctg >= lastctgcnt
1175                  */
1176                 if (minctg < lastctgcnt || startpfn < lo || startpfn > hi)
1177                         startpfn = lo;
1178         } else {
1179                 hi = physmax - 1;
1180                 lo = 0;
1181                 sgllen = 1;
1182                 pfnseg = mmu.highest_pfn;
1183                 minctg = *pgcnt;
1184 
1185                 if (minctg < lastctgcnt)
1186                         startpfn = lo;
1187         }
1188         lastctgcnt = minctg;
1189 
1190         ASSERT(pfnseg + 1 >= (uint64_t)minctg);
1191 
1192         /* conserve 16m memory - start search above 16m when possible */
1193         if (hi > PFN_16M && startpfn < PFN_16M)
1194                 startpfn = PFN_16M;
1195 
1196         pfn = startpfn;
1197         if (pfnalign)
1198                 pfn = P2ROUNDUP(pfn, pfnalign);
1199 
1200         while (pfn + minctg - 1 <= hi) {
1201 
1202                 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1203                 if (plist) {
1204                         page_list_concat(&pplist, &plist);
1205                         sgllen--;
1206                         /*
1207                          * return when contig pages no longer needed
1208                          */
1209                         if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1210                                 startpfn = pfn;
1211                                 CONTIG_UNLOCK();
1212                                 check_dma(mattr, pplist, *pgcnt);
1213                                 return (pplist);
1214                         }
1215                         minctg = howmany(*pgcnt, sgllen);
1216                 }
1217                 if (pfnalign)
1218                         pfn = P2ROUNDUP(pfn, pfnalign);
1219         }
1220 
1221         /* cannot find contig pages in specified range */
1222         if (startpfn == lo) {
1223                 CONTIG_UNLOCK();
1224                 return (NULL);
1225         }
1226 
1227         /* did not start with lo previously */
1228         pfn = lo;
1229         if (pfnalign)
1230                 pfn = P2ROUNDUP(pfn, pfnalign);
1231 
1232         /* allow search to go above startpfn */
1233         while (pfn < startpfn) {
1234 
1235                 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock);
1236                 if (plist != NULL) {
1237 
1238                         page_list_concat(&pplist, &plist);
1239                         sgllen--;
1240 
1241                         /*
1242                          * return when contig pages no longer needed
1243                          */
1244                         if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) {
1245                                 startpfn = pfn;
1246                                 CONTIG_UNLOCK();
1247                                 check_dma(mattr, pplist, *pgcnt);
1248                                 return (pplist);
1249                         }
1250                         minctg = howmany(*pgcnt, sgllen);
1251                 }
1252                 if (pfnalign)
1253                         pfn = P2ROUNDUP(pfn, pfnalign);
1254         }
1255         CONTIG_UNLOCK();
1256         return (NULL);
1257 }
1258 #endif  /* !__xpv */
1259 
1260 /*
1261  * mnode_range_cnt() calculates the number of memory ranges for mnode and
1262  * memranges[]. Used to determine the size of page lists and mnoderanges.
1263  */
1264 int
1265 mnode_range_cnt(int mnode)
1266 {
1267 #if defined(__xpv)
1268         ASSERT(mnode == 0);
1269         return (1);
1270 #else   /* __xpv */
1271         int     mri;
1272         int     mnrcnt = 0;
1273 
1274         if (mem_node_config[mnode].exists != 0) {
1275                 mri = nranges - 1;
1276 
1277                 /* find the memranges index below contained in mnode range */
1278 
1279                 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1280                         mri--;
1281 
1282                 /*
1283                  * increment mnode range counter when memranges or mnode
1284                  * boundary is reached.
1285                  */
1286                 while (mri >= 0 &&
1287                     mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1288                         mnrcnt++;
1289                         if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1290                                 mri--;
1291                         else
1292                                 break;
1293                 }
1294         }
1295         ASSERT(mnrcnt <= MAX_MNODE_MRANGES);
1296         return (mnrcnt);
1297 #endif  /* __xpv */
1298 }
1299 
1300 /*
1301  * mnode_range_setup() initializes mnoderanges.
1302  */
1303 void
1304 mnode_range_setup(mnoderange_t *mnoderanges)
1305 {
1306         mnoderange_t *mp = mnoderanges;
1307         int     mnode, mri;
1308         int     mindex = 0;     /* current index into mnoderanges array */
1309         int     i, j;
1310         pfn_t   hipfn;
1311         int     last, hi;
1312 
1313         for (mnode = 0; mnode < max_mem_nodes; mnode++) {
1314                 if (mem_node_config[mnode].exists == 0)
1315                         continue;
1316 
1317                 mri = nranges - 1;
1318 
1319                 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1320                         mri--;
1321 
1322                 while (mri >= 0 && mem_node_config[mnode].physmax >=
1323                     MEMRANGELO(mri)) {
1324                         mnoderanges->mnr_pfnlo = MAX(MEMRANGELO(mri),
1325                             mem_node_config[mnode].physbase);
1326                         mnoderanges->mnr_pfnhi = MIN(MEMRANGEHI(mri),
1327                             mem_node_config[mnode].physmax);
1328                         mnoderanges->mnr_mnode = mnode;
1329                         mnoderanges->mnr_memrange = mri;
1330                         mnoderanges->mnr_exists = 1;
1331                         mnoderanges++;
1332                         mindex++;
1333                         if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1334                                 mri--;
1335                         else
1336                                 break;
1337                 }
1338         }
1339 
1340         /*
1341          * For now do a simple sort of the mnoderanges array to fill in
1342          * the mnr_next fields.  Since mindex is expected to be relatively
1343          * small, using a simple O(N^2) algorithm.
1344          */
1345         for (i = 0; i < mindex; i++) {
1346                 if (mp[i].mnr_pfnlo == 0)       /* find lowest */
1347                         break;
1348         }
1349         ASSERT(i < mindex);
1350         last = i;
1351         mtype16m = last;
1352         mp[last].mnr_next = -1;
1353         for (i = 0; i < mindex - 1; i++) {
1354                 hipfn = (pfn_t)(-1);
1355                 hi = -1;
1356                 /* find next highest mnode range */
1357                 for (j = 0; j < mindex; j++) {
1358                         if (mp[j].mnr_pfnlo > mp[last].mnr_pfnlo &&
1359                             mp[j].mnr_pfnlo < hipfn) {
1360                                 hipfn = mp[j].mnr_pfnlo;
1361                                 hi = j;
1362                         }
1363                 }
1364                 mp[hi].mnr_next = last;
1365                 last = hi;
1366         }
1367         mtypetop = last;
1368 }
1369 
1370 #ifndef __xpv
1371 /*
1372  * Update mnoderanges for memory hot-add DR operations.
1373  */
1374 static void
1375 mnode_range_add(int mnode)
1376 {
1377         int     *prev;
1378         int     n, mri;
1379         pfn_t   start, end;
1380         extern  void membar_sync(void);
1381 
1382         ASSERT(0 <= mnode && mnode < max_mem_nodes);
1383         ASSERT(mem_node_config[mnode].exists);
1384         start = mem_node_config[mnode].physbase;
1385         end = mem_node_config[mnode].physmax;
1386         ASSERT(start <= end);
1387         mutex_enter(&mnoderange_lock);
1388 
1389 #ifdef  DEBUG
1390         /* Check whether it interleaves with other memory nodes. */
1391         for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1392                 ASSERT(mnoderanges[n].mnr_exists);
1393                 if (mnoderanges[n].mnr_mnode == mnode)
1394                         continue;
1395                 ASSERT(start > mnoderanges[n].mnr_pfnhi ||
1396                     end < mnoderanges[n].mnr_pfnlo);
1397         }
1398 #endif  /* DEBUG */
1399 
1400         mri = nranges - 1;
1401         while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase)
1402                 mri--;
1403         while (mri >= 0 && mem_node_config[mnode].physmax >= MEMRANGELO(mri)) {
1404                 /* Check whether mtype already exists. */
1405                 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) {
1406                         if (mnoderanges[n].mnr_mnode == mnode &&
1407                             mnoderanges[n].mnr_memrange == mri) {
1408                                 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri),
1409                                     start);
1410                                 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri),
1411                                     end);
1412                                 break;
1413                         }
1414                 }
1415 
1416                 /* Add a new entry if it doesn't exist yet. */
1417                 if (n == -1) {
1418                         /* Try to find an unused entry in mnoderanges array. */
1419                         for (n = 0; n < mnoderangecnt; n++) {
1420                                 if (mnoderanges[n].mnr_exists == 0)
1421                                         break;
1422                         }
1423                         ASSERT(n < mnoderangecnt);
1424                         mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), start);
1425                         mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), end);
1426                         mnoderanges[n].mnr_mnode = mnode;
1427                         mnoderanges[n].mnr_memrange = mri;
1428                         mnoderanges[n].mnr_exists = 1;
1429                         /* Page 0 should always be present. */
1430                         for (prev = &mtypetop;
1431                             mnoderanges[*prev].mnr_pfnlo > start;
1432                             prev = &mnoderanges[*prev].mnr_next) {
1433                                 ASSERT(mnoderanges[*prev].mnr_next >= 0);
1434                                 ASSERT(mnoderanges[*prev].mnr_pfnlo > end);
1435                         }
1436                         mnoderanges[n].mnr_next = *prev;
1437                         membar_sync();
1438                         *prev = n;
1439                 }
1440 
1441                 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri))
1442                         mri--;
1443                 else
1444                         break;
1445         }
1446 
1447         mutex_exit(&mnoderange_lock);
1448 }
1449 
1450 /*
1451  * Update mnoderanges for memory hot-removal DR operations.
1452  */
1453 static void
1454 mnode_range_del(int mnode)
1455 {
1456         _NOTE(ARGUNUSED(mnode));
1457         ASSERT(0 <= mnode && mnode < max_mem_nodes);
1458         /* TODO: support deletion operation. */
1459         ASSERT(0);
1460 }
1461 
1462 void
1463 plat_slice_add(pfn_t start, pfn_t end)
1464 {
1465         mem_node_add_slice(start, end);
1466         if (plat_dr_enabled()) {
1467                 mnode_range_add(PFN_2_MEM_NODE(start));
1468         }
1469 }
1470 
1471 void
1472 plat_slice_del(pfn_t start, pfn_t end)
1473 {
1474         ASSERT(PFN_2_MEM_NODE(start) == PFN_2_MEM_NODE(end));
1475         ASSERT(plat_dr_enabled());
1476         mnode_range_del(PFN_2_MEM_NODE(start));
1477         mem_node_del_slice(start, end);
1478 }
1479 #endif  /* __xpv */
1480 
1481 /*ARGSUSED*/
1482 int
1483 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz)
1484 {
1485         int mtype = mtypetop;
1486 
1487 #if !defined(__xpv)
1488 #if defined(__i386)
1489         /*
1490          * set the mtype range
1491          * - kmem requests need to be below 4g if restricted_kmemalloc is set.
1492          * - for non kmem requests, set range to above 4g if memory below 4g
1493          * runs low.
1494          */
1495         if (restricted_kmemalloc && VN_ISKAS(vp) &&
1496             (caddr_t)(vaddr) >= kernelheap &&
1497             (caddr_t)(vaddr) < ekernelheap) {
1498                 ASSERT(physmax4g);
1499                 mtype = mtype4g;
1500                 if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz),
1501                     btop(pgsz), *flags)) {
1502                         *flags |= PGI_MT_RANGE16M;
1503                 } else {
1504                         VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1505                         VM_STAT_COND_ADD((*flags & PG_PANIC),
1506                             vmm_vmstats.pgpanicalloc);
1507                         *flags |= PGI_MT_RANGE0;
1508                 }
1509                 return (mtype);
1510         }
1511 #endif  /* __i386 */
1512 
1513         if (RESTRICT4G_ALLOC) {
1514                 VM_STAT_ADD(vmm_vmstats.restrict4gcnt);
1515                 /* here only for > 4g systems */
1516                 *flags |= PGI_MT_RANGE4G;
1517         } else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) {
1518                 *flags |= PGI_MT_RANGE16M;
1519         } else {
1520                 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1521                 VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc);
1522                 *flags |= PGI_MT_RANGE0;
1523         }
1524 #endif /* !__xpv */
1525         return (mtype);
1526 }
1527 
1528 
1529 /* mtype init for page_get_replacement_page */
1530 /*ARGSUSED*/
1531 int
1532 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt)
1533 {
1534         int mtype = mtypetop;
1535 #if !defined(__xpv)
1536         if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) {
1537                 *flags |= PGI_MT_RANGE16M;
1538         } else {
1539                 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt);
1540                 *flags |= PGI_MT_RANGE0;
1541         }
1542 #endif
1543         return (mtype);
1544 }
1545 
1546 /*
1547  * Determine if the mnode range specified in mtype contains memory belonging
1548  * to memory node mnode.  If flags & PGI_MT_RANGE is set then mtype contains
1549  * the range from high pfn to 0, 16m or 4g.
1550  *
1551  * Return first mnode range type index found otherwise return -1 if none found.
1552  */
1553 int
1554 mtype_func(int mnode, int mtype, uint_t flags)
1555 {
1556         if (flags & PGI_MT_RANGE) {
1557                 int     mnr_lim = MRI_0;
1558 
1559                 if (flags & PGI_MT_NEXT) {
1560                         mtype = mnoderanges[mtype].mnr_next;
1561                 }
1562                 if (flags & PGI_MT_RANGE4G)
1563                         mnr_lim = MRI_4G;       /* exclude 0-4g range */
1564                 else if (flags & PGI_MT_RANGE16M)
1565                         mnr_lim = MRI_16M;      /* exclude 0-16m range */
1566                 while (mtype != -1 &&
1567                     mnoderanges[mtype].mnr_memrange <= mnr_lim) {
1568                         if (mnoderanges[mtype].mnr_mnode == mnode)
1569                                 return (mtype);
1570                         mtype = mnoderanges[mtype].mnr_next;
1571                 }
1572         } else if (mnoderanges[mtype].mnr_mnode == mnode) {
1573                 return (mtype);
1574         }
1575         return (-1);
1576 }
1577 
1578 /*
1579  * Update the page list max counts with the pfn range specified by the
1580  * input parameters.
1581  */
1582 void
1583 mtype_modify_max(pfn_t startpfn, long cnt)
1584 {
1585         int             mtype;
1586         pgcnt_t         inc;
1587         spgcnt_t        scnt = (spgcnt_t)(cnt);
1588         pgcnt_t         acnt = ABS(scnt);
1589         pfn_t           endpfn = startpfn + acnt;
1590         pfn_t           pfn, lo;
1591 
1592         if (!physmax4g)
1593                 return;
1594 
1595         mtype = mtypetop;
1596         for (pfn = endpfn; pfn > startpfn; ) {
1597                 ASSERT(mtype != -1);
1598                 lo = mnoderanges[mtype].mnr_pfnlo;
1599                 if (pfn > lo) {
1600                         if (startpfn >= lo) {
1601                                 inc = pfn - startpfn;
1602                         } else {
1603                                 inc = pfn - lo;
1604                         }
1605                         if (mnoderanges[mtype].mnr_memrange != MRI_4G) {
1606                                 if (scnt > 0)
1607                                         maxmem4g += inc;
1608                                 else
1609                                         maxmem4g -= inc;
1610                         }
1611                         pfn -= inc;
1612                 }
1613                 mtype = mnoderanges[mtype].mnr_next;
1614         }
1615 }
1616 
1617 int
1618 mtype_2_mrange(int mtype)
1619 {
1620         return (mnoderanges[mtype].mnr_memrange);
1621 }
1622 
1623 void
1624 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi)
1625 {
1626         _NOTE(ARGUNUSED(mnode));
1627         ASSERT(mnoderanges[mtype].mnr_mnode == mnode);
1628         *pfnlo = mnoderanges[mtype].mnr_pfnlo;
1629         *pfnhi = mnoderanges[mtype].mnr_pfnhi;
1630 }
1631 
1632 size_t
1633 plcnt_sz(size_t ctrs_sz)
1634 {
1635 #ifdef DEBUG
1636         int     szc, colors;
1637 
1638         ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes;
1639         for (szc = 0; szc < mmu_page_sizes; szc++) {
1640                 colors = page_get_pagecolors(szc);
1641                 ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors;
1642         }
1643 #endif
1644         return (ctrs_sz);
1645 }
1646 
1647 caddr_t
1648 plcnt_init(caddr_t addr)
1649 {
1650 #ifdef DEBUG
1651         int     mt, szc, colors;
1652 
1653         for (mt = 0; mt < mnoderangecnt; mt++) {
1654                 mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr;
1655                 addr += (sizeof (struct mnr_mts) * mmu_page_sizes);
1656                 for (szc = 0; szc < mmu_page_sizes; szc++) {
1657                         colors = page_get_pagecolors(szc);
1658                         mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors;
1659                         mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt =
1660                             (pgcnt_t *)addr;
1661                         addr += (sizeof (pgcnt_t) * colors);
1662                 }
1663         }
1664 #endif
1665         return (addr);
1666 }
1667 
1668 void
1669 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags)
1670 {
1671         _NOTE(ARGUNUSED(pp));
1672 #ifdef DEBUG
1673         int     bin = PP_2_BIN(pp);
1674 
1675         atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt);
1676         atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin],
1677             cnt);
1678 #endif
1679         ASSERT(mtype == PP_2_MTYPE(pp));
1680         if (physmax4g && mnoderanges[mtype].mnr_memrange != MRI_4G)
1681                 atomic_add_long(&freemem4g, cnt);
1682         if (flags & PG_CACHE_LIST)
1683                 atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt);
1684         else
1685                 atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt);
1686         atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt);
1687 }
1688 
1689 /*
1690  * Returns the free page count for mnode
1691  */
1692 int
1693 mnode_pgcnt(int mnode)
1694 {
1695         int     mtype = mtypetop;
1696         int     flags = PGI_MT_RANGE0;
1697         pgcnt_t pgcnt = 0;
1698 
1699         mtype = mtype_func(mnode, mtype, flags);
1700 
1701         while (mtype != -1) {
1702                 pgcnt += MTYPE_FREEMEM(mtype);
1703                 mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT);
1704         }
1705         return (pgcnt);
1706 }
1707 
1708 /*
1709  * Initialize page coloring variables based on the l2 cache parameters.
1710  * Calculate and return memory needed for page coloring data structures.
1711  */
1712 size_t
1713 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc)
1714 {
1715         _NOTE(ARGUNUSED(l2_linesz));
1716         size_t  colorsz = 0;
1717         int     i;
1718         int     colors;
1719 
1720 #if defined(__xpv)
1721         /*
1722          * Hypervisor domains currently don't have any concept of NUMA.
1723          * Hence we'll act like there is only 1 memrange.
1724          */
1725         i = memrange_num(1);
1726 #else /* !__xpv */
1727         /*
1728          * Reduce the memory ranges lists if we don't have large amounts
1729          * of memory. This avoids searching known empty free lists.
1730          * To support memory DR operations, we need to keep memory ranges
1731          * for possible memory hot-add operations.
1732          */
1733         if (plat_dr_physmax > physmax)
1734                 i = memrange_num(plat_dr_physmax);
1735         else
1736                 i = memrange_num(physmax);
1737 #if defined(__i386)
1738         if (i > MRI_4G)
1739                 restricted_kmemalloc = 0;
1740 #endif
1741         /* physmax greater than 4g */
1742         if (i == MRI_4G)
1743                 physmax4g = 1;
1744 #endif /* !__xpv */
1745         memranges += i;
1746         nranges -= i;
1747 
1748         ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES);
1749 
1750         ASSERT(ISP2(l2_linesz));
1751         ASSERT(l2_sz > MMU_PAGESIZE);
1752 
1753         /* l2_assoc is 0 for fully associative l2 cache */
1754         if (l2_assoc)
1755                 l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE));
1756         else
1757                 l2_colors = 1;
1758 
1759         ASSERT(ISP2(l2_colors));
1760 
1761         /* for scalability, configure at least PAGE_COLORS_MIN color bins */
1762         page_colors = MAX(l2_colors, PAGE_COLORS_MIN);
1763 
1764         /*
1765          * cpu_page_colors is non-zero when a page color may be spread across
1766          * multiple bins.
1767          */
1768         if (l2_colors < page_colors)
1769                 cpu_page_colors = l2_colors;
1770 
1771         ASSERT(ISP2(page_colors));
1772 
1773         page_colors_mask = page_colors - 1;
1774 
1775         ASSERT(ISP2(CPUSETSIZE()));
1776         page_coloring_shift = lowbit(CPUSETSIZE());
1777 
1778         /* initialize number of colors per page size */
1779         for (i = 0; i <= mmu.max_page_level; i++) {
1780                 hw_page_array[i].hp_size = LEVEL_SIZE(i);
1781                 hw_page_array[i].hp_shift = LEVEL_SHIFT(i);
1782                 hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0);
1783                 hw_page_array[i].hp_colors = (page_colors_mask >>
1784                     (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift))
1785                     + 1;
1786                 colorequivszc[i] = 0;
1787         }
1788 
1789         /*
1790          * The value of cpu_page_colors determines if additional color bins
1791          * need to be checked for a particular color in the page_get routines.
1792          */
1793         if (cpu_page_colors != 0) {
1794 
1795                 int a = lowbit(page_colors) - lowbit(cpu_page_colors);
1796                 ASSERT(a > 0);
1797                 ASSERT(a < 16);
1798 
1799                 for (i = 0; i <= mmu.max_page_level; i++) {
1800                         if ((colors = hw_page_array[i].hp_colors) <= 1) {
1801                                 colorequivszc[i] = 0;
1802                                 continue;
1803                         }
1804                         while ((colors >> a) == 0)
1805                                 a--;
1806                         ASSERT(a >= 0);
1807 
1808                         /* higher 4 bits encodes color equiv mask */
1809                         colorequivszc[i] = (a << 4);
1810                 }
1811         }
1812 
1813         /* factor in colorequiv to check additional 'equivalent' bins. */
1814         if (colorequiv > 1) {
1815 
1816                 int a = lowbit(colorequiv) - 1;
1817                 if (a > 15)
1818                         a = 15;
1819 
1820                 for (i = 0; i <= mmu.max_page_level; i++) {
1821                         if ((colors = hw_page_array[i].hp_colors) <= 1) {
1822                                 continue;
1823                         }
1824                         while ((colors >> a) == 0)
1825                                 a--;
1826                         if ((a << 4) > colorequivszc[i]) {
1827                                 colorequivszc[i] = (a << 4);
1828                         }
1829                 }
1830         }
1831 
1832         /* size for mnoderanges */
1833         for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++)
1834                 mnoderangecnt += mnode_range_cnt(i);
1835         if (plat_dr_support_memory()) {
1836                 /*
1837                  * Reserve enough space for memory DR operations.
1838                  * Two extra mnoderanges for possbile fragmentations,
1839                  * one for the 2G boundary and the other for the 4G boundary.
1840                  * We don't expect a memory board crossing the 16M boundary
1841                  * for memory hot-add operations on x86 platforms.
1842                  */
1843                 mnoderangecnt += 2 + max_mem_nodes - lgrp_plat_node_cnt;
1844         }
1845         colorsz = mnoderangecnt * sizeof (mnoderange_t);
1846 
1847         /* size for fpc_mutex and cpc_mutex */
1848         colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX);
1849 
1850         /* size of page_freelists */
1851         colorsz += mnoderangecnt * sizeof (page_t ***);
1852         colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **);
1853 
1854         for (i = 0; i < mmu_page_sizes; i++) {
1855                 colors = page_get_pagecolors(i);
1856                 colorsz += mnoderangecnt * colors * sizeof (page_t *);
1857         }
1858 
1859         /* size of page_cachelists */
1860         colorsz += mnoderangecnt * sizeof (page_t **);
1861         colorsz += mnoderangecnt * page_colors * sizeof (page_t *);
1862 
1863         return (colorsz);
1864 }
1865 
1866 /*
1867  * Called once at startup to configure page_coloring data structures and
1868  * does the 1st page_free()/page_freelist_add().
1869  */
1870 void
1871 page_coloring_setup(caddr_t pcmemaddr)
1872 {
1873         int     i;
1874         int     j;
1875         int     k;
1876         caddr_t addr;
1877         int     colors;
1878 
1879         /*
1880          * do page coloring setup
1881          */
1882         addr = pcmemaddr;
1883 
1884         mnoderanges = (mnoderange_t *)addr;
1885         addr += (mnoderangecnt * sizeof (mnoderange_t));
1886 
1887         mnode_range_setup(mnoderanges);
1888 
1889         if (physmax4g)
1890                 mtype4g = pfn_2_mtype(0xfffff);
1891 
1892         for (k = 0; k < NPC_MUTEX; k++) {
1893                 fpc_mutex[k] = (kmutex_t *)addr;
1894                 addr += (max_mem_nodes * sizeof (kmutex_t));
1895         }
1896         for (k = 0; k < NPC_MUTEX; k++) {
1897                 cpc_mutex[k] = (kmutex_t *)addr;
1898                 addr += (max_mem_nodes * sizeof (kmutex_t));
1899         }
1900         page_freelists = (page_t ****)addr;
1901         addr += (mnoderangecnt * sizeof (page_t ***));
1902 
1903         page_cachelists = (page_t ***)addr;
1904         addr += (mnoderangecnt * sizeof (page_t **));
1905 
1906         for (i = 0; i < mnoderangecnt; i++) {
1907                 page_freelists[i] = (page_t ***)addr;
1908                 addr += (mmu_page_sizes * sizeof (page_t **));
1909 
1910                 for (j = 0; j < mmu_page_sizes; j++) {
1911                         colors = page_get_pagecolors(j);
1912                         page_freelists[i][j] = (page_t **)addr;
1913                         addr += (colors * sizeof (page_t *));
1914                 }
1915                 page_cachelists[i] = (page_t **)addr;
1916                 addr += (page_colors * sizeof (page_t *));
1917         }
1918 }
1919 
1920 #if defined(__xpv)
1921 /*
1922  * Give back 10% of the io_pool pages to the free list.
1923  * Don't shrink the pool below some absolute minimum.
1924  */
1925 static void
1926 page_io_pool_shrink()
1927 {
1928         int retcnt;
1929         page_t *pp, *pp_first, *pp_last, **curpool;
1930         mfn_t mfn;
1931         int bothpools = 0;
1932 
1933         mutex_enter(&io_pool_lock);
1934         io_pool_shrink_attempts++;      /* should be a kstat? */
1935         retcnt = io_pool_cnt / 10;
1936         if (io_pool_cnt - retcnt < io_pool_cnt_min)
1937                 retcnt = io_pool_cnt - io_pool_cnt_min;
1938         if (retcnt <= 0)
1939                 goto done;
1940         io_pool_shrinks++;      /* should be a kstat? */
1941         curpool = &io_pool_4g;
1942 domore:
1943         /*
1944          * Loop through taking pages from the end of the list
1945          * (highest mfns) till amount to return reached.
1946          */
1947         for (pp = *curpool; pp && retcnt > 0; ) {
1948                 pp_first = pp_last = pp->p_prev;
1949                 if (pp_first == *curpool)
1950                         break;
1951                 retcnt--;
1952                 io_pool_cnt--;
1953                 page_io_pool_sub(curpool, pp_first, pp_last);
1954                 if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn)
1955                         start_mfn = mfn;
1956                 page_free(pp_first, 1);
1957                 pp = *curpool;
1958         }
1959         if (retcnt != 0 && !bothpools) {
1960                 /*
1961                  * If not enough found in less constrained pool try the
1962                  * more constrained one.
1963                  */
1964                 curpool = &io_pool_16m;
1965                 bothpools = 1;
1966                 goto domore;
1967         }
1968 done:
1969         mutex_exit(&io_pool_lock);
1970 }
1971 
1972 #endif  /* __xpv */
1973 
1974 uint_t
1975 page_create_update_flags_x86(uint_t flags)
1976 {
1977 #if defined(__xpv)
1978         /*
1979          * Check this is an urgent allocation and free pages are depleted.
1980          */
1981         if (!(flags & PG_WAIT) && freemem < desfree)
1982                 page_io_pool_shrink();
1983 #else /* !__xpv */
1984         /*
1985          * page_create_get_something may call this because 4g memory may be
1986          * depleted. Set flags to allow for relocation of base page below
1987          * 4g if necessary.
1988          */
1989         if (physmax4g)
1990                 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
1991 #endif /* __xpv */
1992         return (flags);
1993 }
1994 
1995 /*ARGSUSED*/
1996 int
1997 bp_color(struct buf *bp)
1998 {
1999         return (0);
2000 }
2001 
2002 #if defined(__xpv)
2003 
2004 /*
2005  * Take pages out of an io_pool
2006  */
2007 static void
2008 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last)
2009 {
2010         if (*poolp == pp_first) {
2011                 *poolp = pp_last->p_next;
2012                 if (*poolp == pp_first)
2013                         *poolp = NULL;
2014         }
2015         pp_first->p_prev->p_next = pp_last->p_next;
2016         pp_last->p_next->p_prev = pp_first->p_prev;
2017         pp_first->p_prev = pp_last;
2018         pp_last->p_next = pp_first;
2019 }
2020 
2021 /*
2022  * Put a page on the io_pool list. The list is ordered by increasing MFN.
2023  */
2024 static void
2025 page_io_pool_add(page_t **poolp, page_t *pp)
2026 {
2027         page_t  *look;
2028         mfn_t   mfn = mfn_list[pp->p_pagenum];
2029 
2030         if (*poolp == NULL) {
2031                 *poolp = pp;
2032                 pp->p_next = pp;
2033                 pp->p_prev = pp;
2034                 return;
2035         }
2036 
2037         /*
2038          * Since we try to take pages from the high end of the pool
2039          * chances are good that the pages to be put on the list will
2040          * go at or near the end of the list. so start at the end and
2041          * work backwards.
2042          */
2043         look = (*poolp)->p_prev;
2044         while (mfn < mfn_list[look->p_pagenum]) {
2045                 look = look->p_prev;
2046                 if (look == (*poolp)->p_prev)
2047                         break; /* backed all the way to front of list */
2048         }
2049 
2050         /* insert after look */
2051         pp->p_prev = look;
2052         pp->p_next = look->p_next;
2053         pp->p_next->p_prev = pp;
2054         look->p_next = pp;
2055         if (mfn < mfn_list[(*poolp)->p_pagenum]) {
2056                 /*
2057                  * we inserted a new first list element
2058                  * adjust pool pointer to newly inserted element
2059                  */
2060                 *poolp = pp;
2061         }
2062 }
2063 
2064 /*
2065  * Add a page to the io_pool.  Setting the force flag will force the page
2066  * into the io_pool no matter what.
2067  */
2068 static void
2069 add_page_to_pool(page_t *pp, int force)
2070 {
2071         page_t *highest;
2072         page_t *freep = NULL;
2073 
2074         mutex_enter(&io_pool_lock);
2075         /*
2076          * Always keep the scarce low memory pages
2077          */
2078         if (mfn_list[pp->p_pagenum] < PFN_16MEG) {
2079                 ++io_pool_cnt;
2080                 page_io_pool_add(&io_pool_16m, pp);
2081                 goto done;
2082         }
2083         if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) {
2084                 ++io_pool_cnt;
2085                 page_io_pool_add(&io_pool_4g, pp);
2086         } else {
2087                 highest = io_pool_4g->p_prev;
2088                 if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) {
2089                         page_io_pool_sub(&io_pool_4g, highest, highest);
2090                         page_io_pool_add(&io_pool_4g, pp);
2091                         freep = highest;
2092                 } else {
2093                         freep = pp;
2094                 }
2095         }
2096 done:
2097         mutex_exit(&io_pool_lock);
2098         if (freep)
2099                 page_free(freep, 1);
2100 }
2101 
2102 
2103 int contig_pfn_cnt;     /* no of pfns in the contig pfn list */
2104 int contig_pfn_max;     /* capacity of the contig pfn list */
2105 int next_alloc_pfn;     /* next position in list to start a contig search */
2106 int contig_pfnlist_updates;     /* pfn list update count */
2107 int contig_pfnlist_builds;      /* how many times have we (re)built list */
2108 int contig_pfnlist_buildfailed; /* how many times has list build failed */
2109 int create_contig_pending;      /* nonzero means taskq creating contig list */
2110 pfn_t *contig_pfn_list = NULL;  /* list of contig pfns in ascending mfn order */
2111 
2112 /*
2113  * Function to use in sorting a list of pfns by their underlying mfns.
2114  */
2115 static int
2116 mfn_compare(const void *pfnp1, const void *pfnp2)
2117 {
2118         mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1];
2119         mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2];
2120 
2121         if (mfn1 > mfn2)
2122                 return (1);
2123         if (mfn1 < mfn2)
2124                 return (-1);
2125         return (0);
2126 }
2127 
2128 /*
2129  * Compact the contig_pfn_list by tossing all the non-contiguous
2130  * elements from the list.
2131  */
2132 static void
2133 compact_contig_pfn_list(void)
2134 {
2135         pfn_t pfn, lapfn, prev_lapfn;
2136         mfn_t mfn;
2137         int i, newcnt = 0;
2138 
2139         prev_lapfn = 0;
2140         for (i = 0; i < contig_pfn_cnt - 1; i++) {
2141                 pfn = contig_pfn_list[i];
2142                 lapfn = contig_pfn_list[i + 1];
2143                 mfn = mfn_list[pfn];
2144                 /*
2145                  * See if next pfn is for a contig mfn
2146                  */
2147                 if (mfn_list[lapfn] != mfn + 1)
2148                         continue;
2149                 /*
2150                  * pfn and lookahead are both put in list
2151                  * unless pfn is the previous lookahead.
2152                  */
2153                 if (pfn != prev_lapfn)
2154                         contig_pfn_list[newcnt++] = pfn;
2155                 contig_pfn_list[newcnt++] = lapfn;
2156                 prev_lapfn = lapfn;
2157         }
2158         for (i = newcnt; i < contig_pfn_cnt; i++)
2159                 contig_pfn_list[i] = 0;
2160         contig_pfn_cnt = newcnt;
2161 }
2162 
2163 /*ARGSUSED*/
2164 static void
2165 call_create_contiglist(void *arg)
2166 {
2167         (void) create_contig_pfnlist(PG_WAIT);
2168 }
2169 
2170 /*
2171  * Create list of freelist pfns that have underlying
2172  * contiguous mfns.  The list is kept in ascending mfn order.
2173  * returns 1 if list created else 0.
2174  */
2175 static int
2176 create_contig_pfnlist(uint_t flags)
2177 {
2178         pfn_t pfn;
2179         page_t *pp;
2180         int ret = 1;
2181 
2182         mutex_enter(&contig_list_lock);
2183         if (contig_pfn_list != NULL)
2184                 goto out;
2185         contig_pfn_max = freemem + (freemem / 10);
2186         contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t),
2187             (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP);
2188         if (contig_pfn_list == NULL) {
2189                 /*
2190                  * If we could not create the contig list (because
2191                  * we could not sleep for memory).  Dispatch a taskq that can
2192                  * sleep to get the memory.
2193                  */
2194                 if (!create_contig_pending) {
2195                         if (taskq_dispatch(system_taskq, call_create_contiglist,
2196                             NULL, TQ_NOSLEEP) != NULL)
2197                                 create_contig_pending = 1;
2198                 }
2199                 contig_pfnlist_buildfailed++;   /* count list build failures */
2200                 ret = 0;
2201                 goto out;
2202         }
2203         create_contig_pending = 0;
2204         ASSERT(contig_pfn_cnt == 0);
2205         for (pfn = 0; pfn < mfn_count; pfn++) {
2206                 pp = page_numtopp_nolock(pfn);
2207                 if (pp == NULL || !PP_ISFREE(pp))
2208                         continue;
2209                 contig_pfn_list[contig_pfn_cnt] = pfn;
2210                 if (++contig_pfn_cnt == contig_pfn_max)
2211                         break;
2212         }
2213         /*
2214          * Sanity check the new list.
2215          */
2216         if (contig_pfn_cnt < 2) { /* no contig pfns */
2217                 contig_pfn_cnt = 0;
2218                 contig_pfnlist_buildfailed++;
2219                 kmem_free(contig_pfn_list, contig_pfn_max * sizeof (pfn_t));
2220                 contig_pfn_list = NULL;
2221                 contig_pfn_max = 0;
2222                 ret = 0;
2223                 goto out;
2224         }
2225         qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare);
2226         compact_contig_pfn_list();
2227         /*
2228          * Make sure next search of the newly created contiguous pfn
2229          * list starts at the beginning of the list.
2230          */
2231         next_alloc_pfn = 0;
2232         contig_pfnlist_builds++;        /* count list builds */
2233 out:
2234         mutex_exit(&contig_list_lock);
2235         return (ret);
2236 }
2237 
2238 
2239 /*
2240  * Toss the current contig pfnlist.  Someone is about to do a massive
2241  * update to pfn<->mfn mappings.  So we have them destroy the list and lock
2242  * it till they are done with their update.
2243  */
2244 void
2245 clear_and_lock_contig_pfnlist()
2246 {
2247         pfn_t *listp = NULL;
2248         size_t listsize;
2249 
2250         mutex_enter(&contig_list_lock);
2251         if (contig_pfn_list != NULL) {
2252                 listp = contig_pfn_list;
2253                 listsize = contig_pfn_max * sizeof (pfn_t);
2254                 contig_pfn_list = NULL;
2255                 contig_pfn_max = contig_pfn_cnt = 0;
2256         }
2257         if (listp != NULL)
2258                 kmem_free(listp, listsize);
2259 }
2260 
2261 /*
2262  * Unlock the contig_pfn_list.  The next attempted use of it will cause
2263  * it to be re-created.
2264  */
2265 void
2266 unlock_contig_pfnlist()
2267 {
2268         mutex_exit(&contig_list_lock);
2269 }
2270 
2271 /*
2272  * Update the contiguous pfn list in response to a pfn <-> mfn reassignment
2273  */
2274 void
2275 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn)
2276 {
2277         int probe_hi, probe_lo, probe_pos, insert_after, insert_point;
2278         pfn_t probe_pfn;
2279         mfn_t probe_mfn;
2280         int drop_lock = 0;
2281 
2282         if (mutex_owner(&contig_list_lock) != curthread) {
2283                 drop_lock = 1;
2284                 mutex_enter(&contig_list_lock);
2285         }
2286         if (contig_pfn_list == NULL)
2287                 goto done;
2288         contig_pfnlist_updates++;
2289         /*
2290          * Find the pfn in the current list.  Use a binary chop to locate it.
2291          */
2292         probe_hi = contig_pfn_cnt - 1;
2293         probe_lo = 0;
2294         probe_pos = (probe_hi + probe_lo) / 2;
2295         while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) {
2296                 if (probe_pos == probe_lo) { /* pfn not in list */
2297                         probe_pos = -1;
2298                         break;
2299                 }
2300                 if (pfn_to_mfn(probe_pfn) <= oldmfn)
2301                         probe_lo = probe_pos;
2302                 else
2303                         probe_hi = probe_pos;
2304                 probe_pos = (probe_hi + probe_lo) / 2;
2305         }
2306         if (probe_pos >= 0) {
2307                 /*
2308                  * Remove pfn from list and ensure next alloc
2309                  * position stays in bounds.
2310                  */
2311                 if (--contig_pfn_cnt <= next_alloc_pfn)
2312                         next_alloc_pfn = 0;
2313                 if (contig_pfn_cnt < 2) { /* no contig pfns */
2314                         contig_pfn_cnt = 0;
2315                         kmem_free(contig_pfn_list,
2316                             contig_pfn_max * sizeof (pfn_t));
2317                         contig_pfn_list = NULL;
2318                         contig_pfn_max = 0;
2319                         goto done;
2320                 }
2321                 ovbcopy(&contig_pfn_list[probe_pos + 1],
2322                     &contig_pfn_list[probe_pos],
2323                     (contig_pfn_cnt - probe_pos) * sizeof (pfn_t));
2324         }
2325         if (newmfn == MFN_INVALID)
2326                 goto done;
2327         /*
2328          * Check if new mfn has adjacent mfns in the list
2329          */
2330         probe_hi = contig_pfn_cnt - 1;
2331         probe_lo = 0;
2332         insert_after = -2;
2333         do {
2334                 probe_pos = (probe_hi + probe_lo) / 2;
2335                 probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]);
2336                 if (newmfn == probe_mfn + 1)
2337                         insert_after = probe_pos;
2338                 else if (newmfn == probe_mfn - 1)
2339                         insert_after = probe_pos - 1;
2340                 if (probe_pos == probe_lo)
2341                         break;
2342                 if (probe_mfn <= newmfn)
2343                         probe_lo = probe_pos;
2344                 else
2345                         probe_hi = probe_pos;
2346         } while (insert_after == -2);
2347         /*
2348          * If there is space in the list and there are adjacent mfns
2349          * insert the pfn in to its proper place in the list.
2350          */
2351         if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) {
2352                 insert_point = insert_after + 1;
2353                 ovbcopy(&contig_pfn_list[insert_point],
2354                     &contig_pfn_list[insert_point + 1],
2355                     (contig_pfn_cnt - insert_point) * sizeof (pfn_t));
2356                 contig_pfn_list[insert_point] = pfn;
2357                 contig_pfn_cnt++;
2358         }
2359 done:
2360         if (drop_lock)
2361                 mutex_exit(&contig_list_lock);
2362 }
2363 
2364 /*
2365  * Called to (re-)populate the io_pool from the free page lists.
2366  */
2367 long
2368 populate_io_pool(void)
2369 {
2370         pfn_t pfn;
2371         mfn_t mfn, max_mfn;
2372         page_t *pp;
2373 
2374         /*
2375          * Figure out the bounds of the pool on first invocation.
2376          * We use a percentage of memory for the io pool size.
2377          * we allow that to shrink, but not to less than a fixed minimum
2378          */
2379         if (io_pool_cnt_max == 0) {
2380                 io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct);
2381                 io_pool_cnt_lowater = io_pool_cnt_max;
2382                 /*
2383                  * This is the first time in populate_io_pool, grab a va to use
2384                  * when we need to allocate pages.
2385                  */
2386                 io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
2387         }
2388         /*
2389          * If we are out of pages in the pool, then grow the size of the pool
2390          */
2391         if (io_pool_cnt == 0) {
2392                 /*
2393                  * Grow the max size of the io pool by 5%, but never more than
2394                  * 25% of physical memory.
2395                  */
2396                 if (io_pool_cnt_max < physmem / 4)
2397                         io_pool_cnt_max += io_pool_cnt_max / 20;
2398         }
2399         io_pool_grows++;        /* should be a kstat? */
2400 
2401         /*
2402          * Get highest mfn on this platform, but limit to the 32 bit DMA max.
2403          */
2404         (void) mfn_to_pfn(start_mfn);
2405         max_mfn = MIN(cached_max_mfn, PFN_4GIG);
2406         for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) {
2407                 pfn = mfn_to_pfn(mfn);
2408                 if (pfn & PFN_IS_FOREIGN_MFN)
2409                         continue;
2410                 /*
2411                  * try to allocate it from free pages
2412                  */
2413                 pp = page_numtopp_alloc(pfn);
2414                 if (pp == NULL)
2415                         continue;
2416                 PP_CLRFREE(pp);
2417                 add_page_to_pool(pp, 1);
2418                 if (io_pool_cnt >= io_pool_cnt_max)
2419                         break;
2420         }
2421 
2422         return (io_pool_cnt);
2423 }
2424 
2425 /*
2426  * Destroy a page that was being used for DMA I/O. It may or
2427  * may not actually go back to the io_pool.
2428  */
2429 void
2430 page_destroy_io(page_t *pp)
2431 {
2432         mfn_t mfn = mfn_list[pp->p_pagenum];
2433 
2434         /*
2435          * When the page was alloc'd a reservation was made, release it now
2436          */
2437         page_unresv(1);
2438         /*
2439          * Unload translations, if any, then hash out the
2440          * page to erase its identity.
2441          */
2442         (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2443         page_hashout(pp, NULL);
2444 
2445         /*
2446          * If the page came from the free lists, just put it back to them.
2447          * DomU pages always go on the free lists as well.
2448          */
2449         if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) {
2450                 page_free(pp, 1);
2451                 return;
2452         }
2453 
2454         add_page_to_pool(pp, 0);
2455 }
2456 
2457 
2458 long contig_searches;           /* count of times contig pages requested */
2459 long contig_search_restarts;    /* count of contig ranges tried */
2460 long contig_search_failed;      /* count of contig alloc failures */
2461 
2462 /*
2463  * Free partial page list
2464  */
2465 static void
2466 free_partial_list(page_t **pplist)
2467 {
2468         page_t *pp;
2469 
2470         while (*pplist != NULL) {
2471                 pp = *pplist;
2472                 page_io_pool_sub(pplist, pp, pp);
2473                 page_free(pp, 1);
2474         }
2475 }
2476 
2477 /*
2478  * Look thru the contiguous pfns that are not part of the io_pool for
2479  * contiguous free pages.  Return a list of the found pages or NULL.
2480  */
2481 page_t *
2482 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg,
2483     pgcnt_t pfnalign)
2484 {
2485         page_t *pp, *plist = NULL;
2486         mfn_t mfn, prev_mfn, start_mfn;
2487         pfn_t pfn;
2488         int pages_needed, pages_requested;
2489         int search_start;
2490 
2491         /*
2492          * create the contig pfn list if not already done
2493          */
2494 retry:
2495         mutex_enter(&contig_list_lock);
2496         if (contig_pfn_list == NULL) {
2497                 mutex_exit(&contig_list_lock);
2498                 if (!create_contig_pfnlist(flags)) {
2499                         return (NULL);
2500                 }
2501                 goto retry;
2502         }
2503         contig_searches++;
2504         /*
2505          * Search contiguous pfn list for physically contiguous pages not in
2506          * the io_pool.  Start the search where the last search left off.
2507          */
2508         pages_requested = pages_needed = npages;
2509         search_start = next_alloc_pfn;
2510         start_mfn = prev_mfn = 0;
2511         while (pages_needed) {
2512                 pfn = contig_pfn_list[next_alloc_pfn];
2513                 mfn = pfn_to_mfn(pfn);
2514                 /*
2515                  * Check if mfn is first one or contig to previous one and
2516                  * if page corresponding to mfn is free and that mfn
2517                  * range is not crossing a segment boundary.
2518                  */
2519                 if ((prev_mfn == 0 || mfn == prev_mfn + 1) &&
2520                     (pp = page_numtopp_alloc(pfn)) != NULL &&
2521                     !((mfn & pfnseg) < (start_mfn & pfnseg))) {
2522                         PP_CLRFREE(pp);
2523                         page_io_pool_add(&plist, pp);
2524                         pages_needed--;
2525                         if (prev_mfn == 0) {
2526                                 if (pfnalign &&
2527                                     mfn != P2ROUNDUP(mfn, pfnalign)) {
2528                                         /*
2529                                          * not properly aligned
2530                                          */
2531                                         contig_search_restarts++;
2532                                         free_partial_list(&plist);
2533                                         pages_needed = pages_requested;
2534                                         start_mfn = prev_mfn = 0;
2535                                         goto skip;
2536                                 }
2537                                 start_mfn = mfn;
2538                         }
2539                         prev_mfn = mfn;
2540                 } else {
2541                         contig_search_restarts++;
2542                         free_partial_list(&plist);
2543                         pages_needed = pages_requested;
2544                         start_mfn = prev_mfn = 0;
2545                 }
2546 skip:
2547                 if (++next_alloc_pfn == contig_pfn_cnt)
2548                         next_alloc_pfn = 0;
2549                 if (next_alloc_pfn == search_start)
2550                         break; /* all pfns searched */
2551         }
2552         mutex_exit(&contig_list_lock);
2553         if (pages_needed) {
2554                 contig_search_failed++;
2555                 /*
2556                  * Failed to find enough contig pages.
2557                  * free partial page list
2558                  */
2559                 free_partial_list(&plist);
2560         }
2561         return (plist);
2562 }
2563 
2564 /*
2565  * Search the reserved io pool pages for a page range with the
2566  * desired characteristics.
2567  */
2568 page_t *
2569 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg)
2570 {
2571         page_t *pp_first, *pp_last;
2572         page_t *pp, **poolp;
2573         pgcnt_t nwanted, pfnalign;
2574         uint64_t pfnseg;
2575         mfn_t mfn, tmfn, hi_mfn, lo_mfn;
2576         int align, attempt = 0;
2577 
2578         if (minctg == 1)
2579                 contig = 0;
2580         lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2581         hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2582         pfnseg = mmu_btop(mattr->dma_attr_seg);
2583         align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2584         if (align > MMU_PAGESIZE)
2585                 pfnalign = mmu_btop(align);
2586         else
2587                 pfnalign = 0;
2588 
2589 try_again:
2590         /*
2591          * See if we want pages for a legacy device
2592          */
2593         if (hi_mfn < PFN_16MEG)
2594                 poolp = &io_pool_16m;
2595         else
2596                 poolp = &io_pool_4g;
2597 try_smaller:
2598         /*
2599          * Take pages from I/O pool. We'll use pages from the highest
2600          * MFN range possible.
2601          */
2602         pp_first = pp_last = NULL;
2603         mutex_enter(&io_pool_lock);
2604         nwanted = minctg;
2605         for (pp = *poolp; pp && nwanted > 0; ) {
2606                 pp = pp->p_prev;
2607 
2608                 /*
2609                  * skip pages above allowable range
2610                  */
2611                 mfn = mfn_list[pp->p_pagenum];
2612                 if (hi_mfn < mfn)
2613                         goto skip;
2614 
2615                 /*
2616                  * stop at pages below allowable range
2617                  */
2618                 if (lo_mfn > mfn)
2619                         break;
2620 restart:
2621                 if (pp_last == NULL) {
2622                         /*
2623                          * Check alignment
2624                          */
2625                         tmfn = mfn - (minctg - 1);
2626                         if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign))
2627                                 goto skip; /* not properly aligned */
2628                         /*
2629                          * Check segment
2630                          */
2631                         if ((mfn & pfnseg) < (tmfn & pfnseg))
2632                                 goto skip; /* crosses seg boundary */
2633                         /*
2634                          * Start building page list
2635                          */
2636                         pp_first = pp_last = pp;
2637                         nwanted--;
2638                 } else {
2639                         /*
2640                          * check physical contiguity if required
2641                          */
2642                         if (contig &&
2643                             mfn_list[pp_first->p_pagenum] != mfn + 1) {
2644                                 /*
2645                                  * not a contiguous page, restart list.
2646                                  */
2647                                 pp_last = NULL;
2648                                 nwanted = minctg;
2649                                 goto restart;
2650                         } else { /* add page to list */
2651                                 pp_first = pp;
2652                                 nwanted--;
2653                         }
2654                 }
2655 skip:
2656                 if (pp == *poolp)
2657                         break;
2658         }
2659 
2660         /*
2661          * If we didn't find memory. Try the more constrained pool, then
2662          * sweep free pages into the DMA pool and try again.
2663          */
2664         if (nwanted != 0) {
2665                 mutex_exit(&io_pool_lock);
2666                 /*
2667                  * If we were looking in the less constrained pool and
2668                  * didn't find pages, try the more constrained pool.
2669                  */
2670                 if (poolp == &io_pool_4g) {
2671                         poolp = &io_pool_16m;
2672                         goto try_smaller;
2673                 }
2674                 kmem_reap();
2675                 if (++attempt < 4) {
2676                         /*
2677                          * Grab some more io_pool pages
2678                          */
2679                         (void) populate_io_pool();
2680                         goto try_again; /* go around and retry */
2681                 }
2682                 return (NULL);
2683         }
2684         /*
2685          * Found the pages, now snip them from the list
2686          */
2687         page_io_pool_sub(poolp, pp_first, pp_last);
2688         io_pool_cnt -= minctg;
2689         /*
2690          * reset low water mark
2691          */
2692         if (io_pool_cnt < io_pool_cnt_lowater)
2693                 io_pool_cnt_lowater = io_pool_cnt;
2694         mutex_exit(&io_pool_lock);
2695         return (pp_first);
2696 }
2697 
2698 page_t *
2699 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr,
2700     ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg)
2701 {
2702         uint_t kflags;
2703         int order, extra, extpages, i, contig, nbits, extents;
2704         page_t *pp, *expp, *pp_first, **pplist = NULL;
2705         mfn_t *mfnlist = NULL;
2706 
2707         contig = flags & PG_PHYSCONTIG;
2708         if (minctg == 1)
2709                 contig = 0;
2710         flags &= ~PG_PHYSCONTIG;
2711         kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP;
2712         /*
2713          * Hypervisor will allocate extents, if we want contig
2714          * pages extent must be >= minctg
2715          */
2716         if (contig) {
2717                 order = highbit(minctg) - 1;
2718                 if (minctg & ((1 << order) - 1))
2719                         order++;
2720                 extpages = 1 << order;
2721         } else {
2722                 order = 0;
2723                 extpages = minctg;
2724         }
2725         if (extpages > minctg) {
2726                 extra = extpages - minctg;
2727                 if (!page_resv(extra, kflags))
2728                         return (NULL);
2729         }
2730         pp_first = NULL;
2731         pplist = kmem_alloc(extpages * sizeof (page_t *), kflags);
2732         if (pplist == NULL)
2733                 goto balloon_fail;
2734         mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags);
2735         if (mfnlist == NULL)
2736                 goto balloon_fail;
2737         pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr);
2738         if (pp == NULL)
2739                 goto balloon_fail;
2740         pp_first = pp;
2741         if (extpages > minctg) {
2742                 /*
2743                  * fill out the rest of extent pages to swap
2744                  * with the hypervisor
2745                  */
2746                 for (i = 0; i < extra; i++) {
2747                         expp = page_create_va(vp,
2748                             (u_offset_t)(uintptr_t)io_pool_kva,
2749                             PAGESIZE, flags, &kvseg, io_pool_kva);
2750                         if (expp == NULL)
2751                                 goto balloon_fail;
2752                         (void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD);
2753                         page_io_unlock(expp);
2754                         page_hashout(expp, NULL);
2755                         page_io_lock(expp);
2756                         /*
2757                          * add page to end of list
2758                          */
2759                         expp->p_prev = pp_first->p_prev;
2760                         expp->p_next = pp_first;
2761                         expp->p_prev->p_next = expp;
2762                         pp_first->p_prev = expp;
2763                 }
2764 
2765         }
2766         for (i = 0; i < extpages; i++) {
2767                 pplist[i] = pp;
2768                 pp = pp->p_next;
2769         }
2770         nbits = highbit(mattr->dma_attr_addr_hi);
2771         extents = contig ? 1 : minctg;
2772         if (balloon_replace_pages(extents, pplist, nbits, order,
2773             mfnlist) != extents) {
2774                 if (ioalloc_dbg)
2775                         cmn_err(CE_NOTE, "request to hypervisor"
2776                             " for %d pages, maxaddr %" PRIx64 " failed",
2777                             extpages, mattr->dma_attr_addr_hi);
2778                 goto balloon_fail;
2779         }
2780 
2781         kmem_free(pplist, extpages * sizeof (page_t *));
2782         kmem_free(mfnlist, extpages * sizeof (mfn_t));
2783         /*
2784          * Return any excess pages to free list
2785          */
2786         if (extpages > minctg) {
2787                 for (i = 0; i < extra; i++) {
2788                         pp = pp_first->p_prev;
2789                         page_sub(&pp_first, pp);
2790                         page_io_unlock(pp);
2791                         page_unresv(1);
2792                         page_free(pp, 1);
2793                 }
2794         }
2795         return (pp_first);
2796 balloon_fail:
2797         /*
2798          * Return pages to free list and return failure
2799          */
2800         while (pp_first != NULL) {
2801                 pp = pp_first;
2802                 page_sub(&pp_first, pp);
2803                 page_io_unlock(pp);
2804                 if (pp->p_vnode != NULL)
2805                         page_hashout(pp, NULL);
2806                 page_free(pp, 1);
2807         }
2808         if (pplist)
2809                 kmem_free(pplist, extpages * sizeof (page_t *));
2810         if (mfnlist)
2811                 kmem_free(mfnlist, extpages * sizeof (mfn_t));
2812         page_unresv(extpages - minctg);
2813         return (NULL);
2814 }
2815 
2816 static void
2817 return_partial_alloc(page_t *plist)
2818 {
2819         page_t *pp;
2820 
2821         while (plist != NULL) {
2822                 pp = plist;
2823                 page_sub(&plist, pp);
2824                 page_io_unlock(pp);
2825                 page_destroy_io(pp);
2826         }
2827 }
2828 
2829 static page_t *
2830 page_get_contigpages(
2831         struct vnode    *vp,
2832         u_offset_t      off,
2833         int             *npagesp,
2834         uint_t          flags,
2835         caddr_t         vaddr,
2836         ddi_dma_attr_t  *mattr)
2837 {
2838         mfn_t   max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2839         page_t  *plist; /* list to return */
2840         page_t  *pp, *mcpl;
2841         int     contig, anyaddr, npages, getone = 0;
2842         mfn_t   lo_mfn;
2843         mfn_t   hi_mfn;
2844         pgcnt_t pfnalign = 0;
2845         int     align, sgllen;
2846         uint64_t pfnseg;
2847         pgcnt_t minctg;
2848 
2849         npages = *npagesp;
2850         ASSERT(mattr != NULL);
2851         lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2852         hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2853         sgllen = mattr->dma_attr_sgllen;
2854         pfnseg = mmu_btop(mattr->dma_attr_seg);
2855         align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2856         if (align > MMU_PAGESIZE)
2857                 pfnalign = mmu_btop(align);
2858 
2859         contig = flags & PG_PHYSCONTIG;
2860         if (npages == -1) {
2861                 npages = 1;
2862                 pfnalign = 0;
2863         }
2864         /*
2865          * Clear the contig flag if only one page is needed.
2866          */
2867         if (npages == 1) {
2868                 getone = 1;
2869                 contig = 0;
2870         }
2871 
2872         /*
2873          * Check if any page in the system is fine.
2874          */
2875         anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn;
2876         if (!contig && anyaddr && !pfnalign) {
2877                 flags &= ~PG_PHYSCONTIG;
2878                 plist = page_create_va(vp, off, npages * MMU_PAGESIZE,
2879                     flags, &kvseg, vaddr);
2880                 if (plist != NULL) {
2881                         *npagesp = 0;
2882                         return (plist);
2883                 }
2884         }
2885         plist = NULL;
2886         minctg = howmany(npages, sgllen);
2887         while (npages > sgllen || getone) {
2888                 if (minctg > npages)
2889                         minctg = npages;
2890                 mcpl = NULL;
2891                 /*
2892                  * We could want contig pages with no address range limits.
2893                  */
2894                 if (anyaddr && contig) {
2895                         /*
2896                          * Look for free contig pages to satisfy the request.
2897                          */
2898                         mcpl = find_contig_free(minctg, flags, pfnseg,
2899                             pfnalign);
2900                 }
2901                 /*
2902                  * Try the reserved io pools next
2903                  */
2904                 if (mcpl == NULL)
2905                         mcpl = page_io_pool_alloc(mattr, contig, minctg);
2906                 if (mcpl != NULL) {
2907                         pp = mcpl;
2908                         do {
2909                                 if (!page_hashin(pp, vp, off, NULL)) {
2910                                         panic("page_get_contigpages:"
2911                                             " hashin failed"
2912                                             " pp %p, vp %p, off %llx",
2913                                             (void *)pp, (void *)vp, off);
2914                                 }
2915                                 off += MMU_PAGESIZE;
2916                                 PP_CLRFREE(pp);
2917                                 PP_CLRAGED(pp);
2918                                 page_set_props(pp, P_REF);
2919                                 page_io_lock(pp);
2920                                 pp = pp->p_next;
2921                         } while (pp != mcpl);
2922                 } else {
2923                         /*
2924                          * Hypervisor exchange doesn't handle segment or
2925                          * alignment constraints
2926                          */
2927                         if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi ||
2928                             pfnalign)
2929                                 goto fail;
2930                         /*
2931                          * Try exchanging pages with the hypervisor
2932                          */
2933                         mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr,
2934                             flags, minctg);
2935                         if (mcpl == NULL)
2936                                 goto fail;
2937                         off += minctg * MMU_PAGESIZE;
2938                 }
2939                 check_dma(mattr, mcpl, minctg);
2940                 /*
2941                  * Here with a minctg run of contiguous pages, add them to the
2942                  * list we will return for this request.
2943                  */
2944                 page_list_concat(&plist, &mcpl);
2945                 npages -= minctg;
2946                 *npagesp = npages;
2947                 sgllen--;
2948                 if (getone)
2949                         break;
2950         }
2951         return (plist);
2952 fail:
2953         return_partial_alloc(plist);
2954         return (NULL);
2955 }
2956 
2957 /*
2958  * Allocator for domain 0 I/O pages. We match the required
2959  * DMA attributes and contiguity constraints.
2960  */
2961 /*ARGSUSED*/
2962 page_t *
2963 page_create_io(
2964         struct vnode    *vp,
2965         u_offset_t      off,
2966         uint_t          bytes,
2967         uint_t          flags,
2968         struct as       *as,
2969         caddr_t         vaddr,
2970         ddi_dma_attr_t  *mattr)
2971 {
2972         page_t  *plist = NULL, *pp;
2973         int     npages = 0, contig, anyaddr, pages_req;
2974         mfn_t   lo_mfn;
2975         mfn_t   hi_mfn;
2976         pgcnt_t pfnalign = 0;
2977         int     align;
2978         int     is_domu = 0;
2979         int     dummy, bytes_got;
2980         mfn_t   max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
2981 
2982         ASSERT(mattr != NULL);
2983         lo_mfn = mmu_btop(mattr->dma_attr_addr_lo);
2984         hi_mfn = mmu_btop(mattr->dma_attr_addr_hi);
2985         align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer);
2986         if (align > MMU_PAGESIZE)
2987                 pfnalign = mmu_btop(align);
2988 
2989         /*
2990          * Clear the contig flag if only one page is needed or the scatter
2991          * gather list length is >= npages.
2992          */
2993         pages_req = npages = mmu_btopr(bytes);
2994         contig = (flags & PG_PHYSCONTIG);
2995         bytes = P2ROUNDUP(bytes, MMU_PAGESIZE);
2996         if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages)
2997                 contig = 0;
2998 
2999         /*
3000          * Check if any old page in the system is fine.
3001          * DomU should always go down this path.
3002          */
3003         is_domu = !DOMAIN_IS_INITDOMAIN(xen_info);
3004         anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign;
3005         if ((!contig && anyaddr) || is_domu) {
3006                 flags &= ~PG_PHYSCONTIG;
3007                 plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr);
3008                 if (plist != NULL)
3009                         return (plist);
3010                 else if (is_domu)
3011                         return (NULL); /* no memory available */
3012         }
3013         /*
3014          * DomU should never reach here
3015          */
3016         if (contig) {
3017                 plist = page_get_contigpages(vp, off, &npages, flags, vaddr,
3018                     mattr);
3019                 if (plist == NULL)
3020                         goto fail;
3021                 bytes_got = (pages_req - npages) << MMU_PAGESHIFT;
3022                 vaddr += bytes_got;
3023                 off += bytes_got;
3024                 /*
3025                  * We now have all the contiguous pages we need, but
3026                  * we may still need additional non-contiguous pages.
3027                  */
3028         }
3029         /*
3030          * now loop collecting the requested number of pages, these do
3031          * not have to be contiguous pages but we will use the contig
3032          * page alloc code to get the pages since it will honor any
3033          * other constraints the pages may have.
3034          */
3035         while (npages--) {
3036                 dummy = -1;
3037                 pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr);
3038                 if (pp == NULL)
3039                         goto fail;
3040                 page_add(&plist, pp);
3041                 vaddr += MMU_PAGESIZE;
3042                 off += MMU_PAGESIZE;
3043         }
3044         return (plist);
3045 fail:
3046         /*
3047          * Failed to get enough pages, return ones we did get
3048          */
3049         return_partial_alloc(plist);
3050         return (NULL);
3051 }
3052 
3053 /*
3054  * Lock and return the page with the highest mfn that we can find.  last_mfn
3055  * holds the last one found, so the next search can start from there.  We
3056  * also keep a counter so that we don't loop forever if the machine has no
3057  * free pages.
3058  *
3059  * This is called from the balloon thread to find pages to give away.  new_high
3060  * is used when new mfn's have been added to the system - we will reset our
3061  * search if the new mfn's are higher than our current search position.
3062  */
3063 page_t *
3064 page_get_high_mfn(mfn_t new_high)
3065 {
3066         static mfn_t last_mfn = 0;
3067         pfn_t pfn;
3068         page_t *pp;
3069         ulong_t loop_count = 0;
3070 
3071         if (new_high > last_mfn)
3072                 last_mfn = new_high;
3073 
3074         for (; loop_count < mfn_count; loop_count++, last_mfn--) {
3075                 if (last_mfn == 0) {
3076                         last_mfn = cached_max_mfn;
3077                 }
3078 
3079                 pfn = mfn_to_pfn(last_mfn);
3080                 if (pfn & PFN_IS_FOREIGN_MFN)
3081                         continue;
3082 
3083                 /* See if the page is free.  If so, lock it. */
3084                 pp = page_numtopp_alloc(pfn);
3085                 if (pp == NULL)
3086                         continue;
3087                 PP_CLRFREE(pp);
3088 
3089                 ASSERT(PAGE_EXCL(pp));
3090                 ASSERT(pp->p_vnode == NULL);
3091                 ASSERT(!hat_page_is_mapped(pp));
3092                 last_mfn--;
3093                 return (pp);
3094         }
3095         return (NULL);
3096 }
3097 
3098 #else /* !__xpv */
3099 
3100 /*
3101  * get a page from any list with the given mnode
3102  */
3103 static page_t *
3104 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags,
3105     int mnode, int mtype, ddi_dma_attr_t *dma_attr)
3106 {
3107         kmutex_t                *pcm;
3108         int                     i;
3109         page_t                  *pp;
3110         page_t                  *first_pp;
3111         uint64_t                pgaddr;
3112         ulong_t                 bin;
3113         int                     mtypestart;
3114         int                     plw_initialized;
3115         page_list_walker_t      plw;
3116 
3117         VM_STAT_ADD(pga_vmstats.pgma_alloc);
3118 
3119         ASSERT((flags & PG_MATCH_COLOR) == 0);
3120         ASSERT(szc == 0);
3121         ASSERT(dma_attr != NULL);
3122 
3123         MTYPE_START(mnode, mtype, flags);
3124         if (mtype < 0) {
3125                 VM_STAT_ADD(pga_vmstats.pgma_allocempty);
3126                 return (NULL);
3127         }
3128 
3129         mtypestart = mtype;
3130 
3131         bin = origbin;
3132 
3133         /*
3134          * check up to page_colors + 1 bins - origbin may be checked twice
3135          * because of BIN_STEP skip
3136          */
3137         do {
3138                 plw_initialized = 0;
3139 
3140                 for (plw.plw_count = 0;
3141                     plw.plw_count < page_colors; plw.plw_count++) {
3142 
3143                         if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL)
3144                                 goto nextfreebin;
3145 
3146                         pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST);
3147                         mutex_enter(pcm);
3148                         pp = PAGE_FREELISTS(mnode, szc, bin, mtype);
3149                         first_pp = pp;
3150                         while (pp != NULL) {
3151                                 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3152                                     SE_EXCL) == 0) {
3153                                         pp = pp->p_next;
3154                                         if (pp == first_pp) {
3155                                                 pp = NULL;
3156                                         }
3157                                         continue;
3158                                 }
3159 
3160                                 ASSERT(PP_ISFREE(pp));
3161                                 ASSERT(PP_ISAGED(pp));
3162                                 ASSERT(pp->p_vnode == NULL);
3163                                 ASSERT(pp->p_hash == NULL);
3164                                 ASSERT(pp->p_offset == (u_offset_t)-1);
3165                                 ASSERT(pp->p_szc == szc);
3166                                 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3167                                 /* check if page within DMA attributes */
3168                                 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3169                                 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3170                                     (pgaddr + MMU_PAGESIZE - 1 <=
3171                                     dma_attr->dma_attr_addr_hi)) {
3172                                         break;
3173                                 }
3174 
3175                                 /* continue looking */
3176                                 page_unlock(pp);
3177                                 pp = pp->p_next;
3178                                 if (pp == first_pp)
3179                                         pp = NULL;
3180 
3181                         }
3182                         if (pp != NULL) {
3183                                 ASSERT(mtype == PP_2_MTYPE(pp));
3184                                 ASSERT(pp->p_szc == 0);
3185 
3186                                 /* found a page with specified DMA attributes */
3187                                 page_sub(&PAGE_FREELISTS(mnode, szc, bin,
3188                                     mtype), pp);
3189                                 page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST);
3190 
3191                                 if ((PP_ISFREE(pp) == 0) ||
3192                                     (PP_ISAGED(pp) == 0)) {
3193                                         cmn_err(CE_PANIC, "page %p is not free",
3194                                             (void *)pp);
3195                                 }
3196 
3197                                 mutex_exit(pcm);
3198                                 check_dma(dma_attr, pp, 1);
3199                                 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3200                                 return (pp);
3201                         }
3202                         mutex_exit(pcm);
3203 nextfreebin:
3204                         if (plw_initialized == 0) {
3205                                 page_list_walk_init(szc, 0, bin, 1, 0, &plw);
3206                                 ASSERT(plw.plw_ceq_dif == page_colors);
3207                                 plw_initialized = 1;
3208                         }
3209 
3210                         if (plw.plw_do_split) {
3211                                 pp = page_freelist_split(szc, bin, mnode,
3212                                     mtype,
3213                                     mmu_btop(dma_attr->dma_attr_addr_lo),
3214                                     mmu_btop(dma_attr->dma_attr_addr_hi + 1),
3215                                     &plw);
3216                                 if (pp != NULL) {
3217                                         check_dma(dma_attr, pp, 1);
3218                                         return (pp);
3219                                 }
3220                         }
3221 
3222                         bin = page_list_walk_next_bin(szc, bin, &plw);
3223                 }
3224 
3225                 MTYPE_NEXT(mnode, mtype, flags);
3226         } while (mtype >= 0);
3227 
3228         /* failed to find a page in the freelist; try it in the cachelist */
3229 
3230         /* reset mtype start for cachelist search */
3231         mtype = mtypestart;
3232         ASSERT(mtype >= 0);
3233 
3234         /* start with the bin of matching color */
3235         bin = origbin;
3236 
3237         do {
3238                 for (i = 0; i <= page_colors; i++) {
3239                         if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL)
3240                                 goto nextcachebin;
3241                         pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST);
3242                         mutex_enter(pcm);
3243                         pp = PAGE_CACHELISTS(mnode, bin, mtype);
3244                         first_pp = pp;
3245                         while (pp != NULL) {
3246                                 if (IS_DUMP_PAGE(pp) || page_trylock(pp,
3247                                     SE_EXCL) == 0) {
3248                                         pp = pp->p_next;
3249                                         if (pp == first_pp)
3250                                                 pp = NULL;
3251                                         continue;
3252                                 }
3253                                 ASSERT(pp->p_vnode);
3254                                 ASSERT(PP_ISAGED(pp) == 0);
3255                                 ASSERT(pp->p_szc == 0);
3256                                 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode);
3257 
3258                                 /* check if page within DMA attributes */
3259 
3260                                 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum));
3261                                 if ((pgaddr >= dma_attr->dma_attr_addr_lo) &&
3262                                     (pgaddr + MMU_PAGESIZE - 1 <=
3263                                     dma_attr->dma_attr_addr_hi)) {
3264                                         break;
3265                                 }
3266 
3267                                 /* continue looking */
3268                                 page_unlock(pp);
3269                                 pp = pp->p_next;
3270                                 if (pp == first_pp)
3271                                         pp = NULL;
3272                         }
3273 
3274                         if (pp != NULL) {
3275                                 ASSERT(mtype == PP_2_MTYPE(pp));
3276                                 ASSERT(pp->p_szc == 0);
3277 
3278                                 /* found a page with specified DMA attributes */
3279                                 page_sub(&PAGE_CACHELISTS(mnode, bin,
3280                                     mtype), pp);
3281                                 page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST);
3282 
3283                                 mutex_exit(pcm);
3284                                 ASSERT(pp->p_vnode);
3285                                 ASSERT(PP_ISAGED(pp) == 0);
3286                                 check_dma(dma_attr, pp, 1);
3287                                 VM_STAT_ADD(pga_vmstats.pgma_allocok);
3288                                 return (pp);
3289                         }
3290                         mutex_exit(pcm);
3291 nextcachebin:
3292                         bin += (i == 0) ? BIN_STEP : 1;
3293                         bin &= page_colors_mask;
3294                 }
3295                 MTYPE_NEXT(mnode, mtype, flags);
3296         } while (mtype >= 0);
3297 
3298         VM_STAT_ADD(pga_vmstats.pgma_allocfailed);
3299         return (NULL);
3300 }
3301 
3302 /*
3303  * This function is similar to page_get_freelist()/page_get_cachelist()
3304  * but it searches both the lists to find a page with the specified
3305  * color (or no color) and DMA attributes. The search is done in the
3306  * freelist first and then in the cache list within the highest memory
3307  * range (based on DMA attributes) before searching in the lower
3308  * memory ranges.
3309  *
3310  * Note: This function is called only by page_create_io().
3311  */
3312 /*ARGSUSED*/
3313 static page_t *
3314 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr,
3315     size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t *lgrp)
3316 {
3317         uint_t          bin;
3318         int             mtype;
3319         page_t          *pp;
3320         int             n;
3321         int             m;
3322         int             szc;
3323         int             fullrange;
3324         int             mnode;
3325         int             local_failed_stat = 0;
3326         lgrp_mnode_cookie_t     lgrp_cookie;
3327 
3328         VM_STAT_ADD(pga_vmstats.pga_alloc);
3329 
3330         /* only base pagesize currently supported */
3331         if (size != MMU_PAGESIZE)
3332                 return (NULL);
3333 
3334         /*
3335          * If we're passed a specific lgroup, we use it.  Otherwise,
3336          * assume first-touch placement is desired.
3337          */
3338         if (!LGRP_EXISTS(lgrp))
3339                 lgrp = lgrp_home_lgrp();
3340 
3341         /* LINTED */
3342         AS_2_BIN(as, seg, vp, vaddr, bin, 0);
3343 
3344         /*
3345          * Only hold one freelist or cachelist lock at a time, that way we
3346          * can start anywhere and not have to worry about lock
3347          * ordering.
3348          */
3349         if (dma_attr == NULL) {
3350                 n = mtype16m;
3351                 m = mtypetop;
3352                 fullrange = 1;
3353                 VM_STAT_ADD(pga_vmstats.pga_nulldmaattr);
3354         } else {
3355                 pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo);
3356                 pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi);
3357 
3358                 /*
3359                  * We can guarantee alignment only for page boundary.
3360                  */
3361                 if (dma_attr->dma_attr_align > MMU_PAGESIZE)
3362                         return (NULL);
3363 
3364                 /* Sanity check the dma_attr */
3365                 if (pfnlo > pfnhi)
3366                         return (NULL);
3367 
3368                 n = pfn_2_mtype(pfnlo);
3369                 m = pfn_2_mtype(pfnhi);
3370 
3371                 fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) &&
3372                     (pfnhi >= mnoderanges[m].mnr_pfnhi));
3373         }
3374         VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange);
3375 
3376         szc = 0;
3377 
3378         /* cylcing thru mtype handled by RANGE0 if n == mtype16m */
3379         if (n == mtype16m) {
3380                 flags |= PGI_MT_RANGE0;
3381                 n = m;
3382         }
3383 
3384         /*
3385          * Try local memory node first, but try remote if we can't
3386          * get a page of the right color.
3387          */
3388         LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER);
3389         while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) {
3390                 /*
3391                  * allocate pages from high pfn to low.
3392                  */
3393                 mtype = m;
3394                 do {
3395                         if (fullrange != 0) {
3396                                 pp = page_get_mnode_freelist(mnode,
3397                                     bin, mtype, szc, flags);
3398                                 if (pp == NULL) {
3399                                         pp = page_get_mnode_cachelist(
3400                                             bin, flags, mnode, mtype);
3401                                 }
3402                         } else {
3403                                 pp = page_get_mnode_anylist(bin, szc,
3404                                     flags, mnode, mtype, dma_attr);
3405                         }
3406                         if (pp != NULL) {
3407                                 VM_STAT_ADD(pga_vmstats.pga_allocok);
3408                                 check_dma(dma_attr, pp, 1);
3409                                 return (pp);
3410                         }
3411                 } while (mtype != n &&
3412                     (mtype = mnoderanges[mtype].mnr_next) != -1);
3413                 if (!local_failed_stat) {
3414                         lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1);
3415                         local_failed_stat = 1;
3416                 }
3417         }
3418         VM_STAT_ADD(pga_vmstats.pga_allocfailed);
3419 
3420         return (NULL);
3421 }
3422 
3423 /*
3424  * page_create_io()
3425  *
3426  * This function is a copy of page_create_va() with an additional
3427  * argument 'mattr' that specifies DMA memory requirements to
3428  * the page list functions. This function is used by the segkmem
3429  * allocator so it is only to create new pages (i.e PG_EXCL is
3430  * set).
3431  *
3432  * Note: This interface is currently used by x86 PSM only and is
3433  *       not fully specified so the commitment level is only for
3434  *       private interface specific to x86. This interface uses PSM
3435  *       specific page_get_anylist() interface.
3436  */
3437 
3438 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \
3439         for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
3440                 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
3441                         break; \
3442         } \
3443 }
3444 
3445 
3446 page_t *
3447 page_create_io(
3448         struct vnode    *vp,
3449         u_offset_t      off,
3450         uint_t          bytes,
3451         uint_t          flags,
3452         struct as       *as,
3453         caddr_t         vaddr,
3454         ddi_dma_attr_t  *mattr) /* DMA memory attributes if any */
3455 {
3456         page_t          *plist = NULL;
3457         uint_t          plist_len = 0;
3458         pgcnt_t         npages;
3459         page_t          *npp = NULL;
3460         uint_t          pages_req;
3461         page_t          *pp;
3462         kmutex_t        *phm = NULL;
3463         uint_t          index;
3464 
3465         TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
3466             "page_create_start:vp %p off %llx bytes %u flags %x",
3467             vp, off, bytes, flags);
3468 
3469         ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0);
3470 
3471         pages_req = npages = mmu_btopr(bytes);
3472 
3473         /*
3474          * Do the freemem and pcf accounting.
3475          */
3476         if (!page_create_wait(npages, flags)) {
3477                 return (NULL);
3478         }
3479 
3480         TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
3481             "page_create_success:vp %p off %llx", vp, off);
3482 
3483         /*
3484          * If satisfying this request has left us with too little
3485          * memory, start the wheels turning to get some back.  The
3486          * first clause of the test prevents waking up the pageout
3487          * daemon in situations where it would decide that there's
3488          * nothing to do.
3489          */
3490         if (nscan < desscan && freemem < minfree) {
3491                 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
3492                     "pageout_cv_signal:freemem %ld", freemem);
3493                 cv_signal(&proc_pageout->p_cv);
3494         }
3495 
3496         if (flags & PG_PHYSCONTIG) {
3497 
3498                 plist = page_get_contigpage(&npages, mattr, 1);
3499                 if (plist == NULL) {
3500                         page_create_putback(npages);
3501                         return (NULL);
3502                 }
3503 
3504                 pp = plist;
3505 
3506                 do {
3507                         if (!page_hashin(pp, vp, off, NULL)) {
3508                                 panic("pg_creat_io: hashin failed %p %p %llx",
3509                                     (void *)pp, (void *)vp, off);
3510                         }
3511                         VM_STAT_ADD(page_create_new);
3512                         off += MMU_PAGESIZE;
3513                         PP_CLRFREE(pp);
3514                         PP_CLRAGED(pp);
3515                         page_set_props(pp, P_REF);
3516                         pp = pp->p_next;
3517                 } while (pp != plist);
3518 
3519                 if (!npages) {
3520                         check_dma(mattr, plist, pages_req);
3521                         return (plist);
3522                 } else {
3523                         vaddr += (pages_req - npages) << MMU_PAGESHIFT;
3524                 }
3525 
3526                 /*
3527                  * fall-thru:
3528                  *
3529                  * page_get_contigpage returns when npages <= sgllen.
3530                  * Grab the rest of the non-contig pages below from anylist.
3531                  */
3532         }
3533 
3534         /*
3535          * Loop around collecting the requested number of pages.
3536          * Most of the time, we have to `create' a new page. With
3537          * this in mind, pull the page off the free list before
3538          * getting the hash lock.  This will minimize the hash
3539          * lock hold time, nesting, and the like.  If it turns
3540          * out we don't need the page, we put it back at the end.
3541          */
3542         while (npages--) {
3543                 phm = NULL;
3544 
3545                 index = PAGE_HASH_FUNC(vp, off);
3546 top:
3547                 ASSERT(phm == NULL);
3548                 ASSERT(index == PAGE_HASH_FUNC(vp, off));
3549                 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3550 
3551                 if (npp == NULL) {
3552                         /*
3553                          * Try to get the page of any color either from
3554                          * the freelist or from the cache list.
3555                          */
3556                         npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE,
3557                             flags & ~PG_MATCH_COLOR, mattr, NULL);
3558                         if (npp == NULL) {
3559                                 if (mattr == NULL) {
3560                                         /*
3561                                          * Not looking for a special page;
3562                                          * panic!
3563                                          */
3564                                         panic("no page found %d", (int)npages);
3565                                 }
3566                                 /*
3567                                  * No page found! This can happen
3568                                  * if we are looking for a page
3569                                  * within a specific memory range
3570                                  * for DMA purposes. If PG_WAIT is
3571                                  * specified then we wait for a
3572                                  * while and then try again. The
3573                                  * wait could be forever if we
3574                                  * don't get the page(s) we need.
3575                                  *
3576                                  * Note: XXX We really need a mechanism
3577                                  * to wait for pages in the desired
3578                                  * range. For now, we wait for any
3579                                  * pages and see if we can use it.
3580                                  */
3581 
3582                                 if ((mattr != NULL) && (flags & PG_WAIT)) {
3583                                         delay(10);
3584                                         goto top;
3585                                 }
3586                                 goto fail; /* undo accounting stuff */
3587                         }
3588 
3589                         if (PP_ISAGED(npp) == 0) {
3590                                 /*
3591                                  * Since this page came from the
3592                                  * cachelist, we must destroy the
3593                                  * old vnode association.
3594                                  */
3595                                 page_hashout(npp, (kmutex_t *)NULL);
3596                         }
3597                 }
3598 
3599                 /*
3600                  * We own this page!
3601                  */
3602                 ASSERT(PAGE_EXCL(npp));
3603                 ASSERT(npp->p_vnode == NULL);
3604                 ASSERT(!hat_page_is_mapped(npp));
3605                 PP_CLRFREE(npp);
3606                 PP_CLRAGED(npp);
3607 
3608                 /*
3609                  * Here we have a page in our hot little mits and are
3610                  * just waiting to stuff it on the appropriate lists.
3611                  * Get the mutex and check to see if it really does
3612                  * not exist.
3613                  */
3614                 phm = PAGE_HASH_MUTEX(index);
3615                 mutex_enter(phm);
3616                 PAGE_HASH_SEARCH(index, pp, vp, off);
3617                 if (pp == NULL) {
3618                         VM_STAT_ADD(page_create_new);
3619                         pp = npp;
3620                         npp = NULL;
3621                         if (!page_hashin(pp, vp, off, phm)) {
3622                                 /*
3623                                  * Since we hold the page hash mutex and
3624                                  * just searched for this page, page_hashin
3625                                  * had better not fail.  If it does, that
3626                                  * means somethread did not follow the
3627                                  * page hash mutex rules.  Panic now and
3628                                  * get it over with.  As usual, go down
3629                                  * holding all the locks.
3630                                  */
3631                                 ASSERT(MUTEX_HELD(phm));
3632                                 panic("page_create: hashin fail %p %p %llx %p",
3633                                     (void *)pp, (void *)vp, off, (void *)phm);
3634 
3635                         }
3636                         ASSERT(MUTEX_HELD(phm));
3637                         mutex_exit(phm);
3638                         phm = NULL;
3639 
3640                         /*
3641                          * Hat layer locking need not be done to set
3642                          * the following bits since the page is not hashed
3643                          * and was on the free list (i.e., had no mappings).
3644                          *
3645                          * Set the reference bit to protect
3646                          * against immediate pageout
3647                          *
3648                          * XXXmh modify freelist code to set reference
3649                          * bit so we don't have to do it here.
3650                          */
3651                         page_set_props(pp, P_REF);
3652                 } else {
3653                         ASSERT(MUTEX_HELD(phm));
3654                         mutex_exit(phm);
3655                         phm = NULL;
3656                         /*
3657                          * NOTE: This should not happen for pages associated
3658                          *       with kernel vnode 'kvp'.
3659                          */
3660                         /* XX64 - to debug why this happens! */
3661                         ASSERT(!VN_ISKAS(vp));
3662                         if (VN_ISKAS(vp))
3663                                 cmn_err(CE_NOTE,
3664                                     "page_create: page not expected "
3665                                     "in hash list for kernel vnode - pp 0x%p",
3666                                     (void *)pp);
3667                         VM_STAT_ADD(page_create_exists);
3668                         goto fail;
3669                 }
3670 
3671                 /*
3672                  * Got a page!  It is locked.  Acquire the i/o
3673                  * lock since we are going to use the p_next and
3674                  * p_prev fields to link the requested pages together.
3675                  */
3676                 page_io_lock(pp);
3677                 page_add(&plist, pp);
3678                 plist = plist->p_next;
3679                 off += MMU_PAGESIZE;
3680                 vaddr += MMU_PAGESIZE;
3681         }
3682 
3683         check_dma(mattr, plist, pages_req);
3684         return (plist);
3685 
3686 fail:
3687         if (npp != NULL) {
3688                 /*
3689                  * Did not need this page after all.
3690                  * Put it back on the free list.
3691                  */
3692                 VM_STAT_ADD(page_create_putbacks);
3693                 PP_SETFREE(npp);
3694                 PP_SETAGED(npp);
3695                 npp->p_offset = (u_offset_t)-1;
3696                 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
3697                 page_unlock(npp);
3698         }
3699 
3700         /*
3701          * Give up the pages we already got.
3702          */
3703         while (plist != NULL) {
3704                 pp = plist;
3705                 page_sub(&plist, pp);
3706                 page_io_unlock(pp);
3707                 plist_len++;
3708                 /*LINTED: constant in conditional ctx*/
3709                 VN_DISPOSE(pp, B_INVAL, 0, kcred);
3710         }
3711 
3712         /*
3713          * VN_DISPOSE does freemem accounting for the pages in plist
3714          * by calling page_free. So, we need to undo the pcf accounting
3715          * for only the remaining pages.
3716          */
3717         VM_STAT_ADD(page_create_putbacks);
3718         page_create_putback(pages_req - plist_len);
3719 
3720         return (NULL);
3721 }
3722 #endif /* !__xpv */
3723 
3724 
3725 /*
3726  * Copy the data from the physical page represented by "frompp" to
3727  * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and
3728  * CPU->cpu_caddr2.  It assumes that no one uses either map at interrupt
3729  * level and no one sleeps with an active mapping there.
3730  *
3731  * Note that the ref/mod bits in the page_t's are not affected by
3732  * this operation, hence it is up to the caller to update them appropriately.
3733  */
3734 int
3735 ppcopy(page_t *frompp, page_t *topp)
3736 {
3737         caddr_t         pp_addr1;
3738         caddr_t         pp_addr2;
3739         hat_mempte_t    pte1;
3740         hat_mempte_t    pte2;
3741         kmutex_t        *ppaddr_mutex;
3742         label_t         ljb;
3743         int             ret = 1;
3744 
3745         ASSERT_STACK_ALIGNED();
3746         ASSERT(PAGE_LOCKED(frompp));
3747         ASSERT(PAGE_LOCKED(topp));
3748 
3749         if (kpm_enable) {
3750                 pp_addr1 = hat_kpm_page2va(frompp, 0);
3751                 pp_addr2 = hat_kpm_page2va(topp, 0);
3752                 kpreempt_disable();
3753         } else {
3754                 /*
3755                  * disable pre-emption so that CPU can't change
3756                  */
3757                 kpreempt_disable();
3758 
3759                 pp_addr1 = CPU->cpu_caddr1;
3760                 pp_addr2 = CPU->cpu_caddr2;
3761                 pte1 = CPU->cpu_caddr1pte;
3762                 pte2 = CPU->cpu_caddr2pte;
3763 
3764                 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3765                 mutex_enter(ppaddr_mutex);
3766 
3767                 hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1,
3768                     PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST);
3769                 hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2,
3770                     PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3771                     HAT_LOAD_NOCONSIST);
3772         }
3773 
3774         if (on_fault(&ljb)) {
3775                 ret = 0;
3776                 goto faulted;
3777         }
3778         if (use_sse_pagecopy)
3779 #ifdef __xpv
3780                 page_copy_no_xmm(pp_addr2, pp_addr1);
3781 #else
3782                 hwblkpagecopy(pp_addr1, pp_addr2);
3783 #endif
3784         else
3785                 bcopy(pp_addr1, pp_addr2, PAGESIZE);
3786 
3787         no_fault();
3788 faulted:
3789         if (!kpm_enable) {
3790 #ifdef __xpv
3791                 /*
3792                  * We can't leave unused mappings laying about under the
3793                  * hypervisor, so blow them away.
3794                  */
3795                 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0,
3796                     UVMF_INVLPG | UVMF_LOCAL) < 0)
3797                         panic("HYPERVISOR_update_va_mapping() failed");
3798                 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3799                     UVMF_INVLPG | UVMF_LOCAL) < 0)
3800                         panic("HYPERVISOR_update_va_mapping() failed");
3801 #endif
3802                 mutex_exit(ppaddr_mutex);
3803         }
3804         kpreempt_enable();
3805         return (ret);
3806 }
3807 
3808 void
3809 pagezero(page_t *pp, uint_t off, uint_t len)
3810 {
3811         ASSERT(PAGE_LOCKED(pp));
3812         pfnzero(page_pptonum(pp), off, len);
3813 }
3814 
3815 /*
3816  * Zero the physical page from off to off + len given by pfn
3817  * without changing the reference and modified bits of page.
3818  *
3819  * We use this using CPU private page address #2, see ppcopy() for more info.
3820  * pfnzero() must not be called at interrupt level.
3821  */
3822 void
3823 pfnzero(pfn_t pfn, uint_t off, uint_t len)
3824 {
3825         caddr_t         pp_addr2;
3826         hat_mempte_t    pte2;
3827         kmutex_t        *ppaddr_mutex = NULL;
3828 
3829         ASSERT_STACK_ALIGNED();
3830         ASSERT(len <= MMU_PAGESIZE);
3831         ASSERT(off <= MMU_PAGESIZE);
3832         ASSERT(off + len <= MMU_PAGESIZE);
3833 
3834         if (kpm_enable && !pfn_is_foreign(pfn)) {
3835                 pp_addr2 = hat_kpm_pfn2va(pfn);
3836                 kpreempt_disable();
3837         } else {
3838                 kpreempt_disable();
3839 
3840                 pp_addr2 = CPU->cpu_caddr2;
3841                 pte2 = CPU->cpu_caddr2pte;
3842 
3843                 ppaddr_mutex = &CPU->cpu_ppaddr_mutex;
3844                 mutex_enter(ppaddr_mutex);
3845 
3846                 hat_mempte_remap(pfn, pp_addr2, pte2,
3847                     PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
3848                     HAT_LOAD_NOCONSIST);
3849         }
3850 
3851         if (use_sse_pagezero) {
3852 #ifdef __xpv
3853                 uint_t rem;
3854 
3855                 /*
3856                  * zero a byte at a time until properly aligned for
3857                  * block_zero_no_xmm().
3858                  */
3859                 while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0)
3860                         pp_addr2[off++] = 0;
3861 
3862                 /*
3863                  * Now use faster block_zero_no_xmm() for any range
3864                  * that is properly aligned and sized.
3865                  */
3866                 rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN));
3867                 len -= rem;
3868                 if (len != 0) {
3869                         block_zero_no_xmm(pp_addr2 + off, len);
3870                         off += len;
3871                 }
3872 
3873                 /*
3874                  * zero remainder with byte stores.
3875                  */
3876                 while (rem-- > 0)
3877                         pp_addr2[off++] = 0;
3878 #else
3879                 hwblkclr(pp_addr2 + off, len);
3880 #endif
3881         } else {
3882                 bzero(pp_addr2 + off, len);
3883         }
3884 
3885         if (!kpm_enable || pfn_is_foreign(pfn)) {
3886 #ifdef __xpv
3887                 /*
3888                  * On the hypervisor this page might get used for a page
3889                  * table before any intervening change to this mapping,
3890                  * so blow it away.
3891                  */
3892                 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0,
3893                     UVMF_INVLPG) < 0)
3894                         panic("HYPERVISOR_update_va_mapping() failed");
3895 #endif
3896                 mutex_exit(ppaddr_mutex);
3897         }
3898 
3899         kpreempt_enable();
3900 }
3901 
3902 /*
3903  * Platform-dependent page scrub call.
3904  */
3905 void
3906 pagescrub(page_t *pp, uint_t off, uint_t len)
3907 {
3908         /*
3909          * For now, we rely on the fact that pagezero() will
3910          * always clear UEs.
3911          */
3912         pagezero(pp, off, len);
3913 }
3914 
3915 /*
3916  * set up two private addresses for use on a given CPU for use in ppcopy()
3917  */
3918 void
3919 setup_vaddr_for_ppcopy(struct cpu *cpup)
3920 {
3921         void *addr;
3922         hat_mempte_t pte_pa;
3923 
3924         addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3925         pte_pa = hat_mempte_setup(addr);
3926         cpup->cpu_caddr1 = addr;
3927         cpup->cpu_caddr1pte = pte_pa;
3928 
3929         addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP);
3930         pte_pa = hat_mempte_setup(addr);
3931         cpup->cpu_caddr2 = addr;
3932         cpup->cpu_caddr2pte = pte_pa;
3933 
3934         mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL);
3935 }
3936 
3937 /*
3938  * Undo setup_vaddr_for_ppcopy
3939  */
3940 void
3941 teardown_vaddr_for_ppcopy(struct cpu *cpup)
3942 {
3943         mutex_destroy(&cpup->cpu_ppaddr_mutex);
3944 
3945         hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte);
3946         cpup->cpu_caddr2pte = 0;
3947         vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1));
3948         cpup->cpu_caddr2 = 0;
3949 
3950         hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte);
3951         cpup->cpu_caddr1pte = 0;
3952         vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1));
3953         cpup->cpu_caddr1 = 0;
3954 }
3955 
3956 /*
3957  * Function for flushing D-cache when performing module relocations
3958  * to an alternate mapping.  Unnecessary on Intel / AMD platforms.
3959  */
3960 void
3961 dcache_flushall()
3962 {}
3963 
3964 /*
3965  * Allocate a memory page.  The argument 'seed' can be any pseudo-random
3966  * number to vary where the pages come from.  This is quite a hacked up
3967  * method -- it works for now, but really needs to be fixed up a bit.
3968  *
3969  * We currently use page_create_va() on the kvp with fake offsets,
3970  * segments and virt address.  This is pretty bogus, but was copied from the
3971  * old hat_i86.c code.  A better approach would be to specify either mnode
3972  * random or mnode local and takes a page from whatever color has the MOST
3973  * available - this would have a minimal impact on page coloring.
3974  */
3975 page_t *
3976 page_get_physical(uintptr_t seed)
3977 {
3978         page_t *pp;
3979         u_offset_t offset;
3980         static struct seg tmpseg;
3981         static uintptr_t ctr = 0;
3982 
3983         /*
3984          * This code is gross, we really need a simpler page allocator.
3985          *
3986          * We need to assign an offset for the page to call page_create_va()
3987          * To avoid conflicts with other pages, we get creative with the offset.
3988          * For 32 bits, we need an offset > 4Gig
3989          * For 64 bits, need an offset somewhere in the VA hole.
3990          */
3991         offset = seed;
3992         if (offset > kernelbase)
3993                 offset -= kernelbase;
3994         offset <<= MMU_PAGESHIFT;
3995 #if defined(__amd64)
3996         offset += mmu.hole_start;       /* something in VA hole */
3997 #else
3998         offset += 1ULL << 40;     /* something > 4 Gig */
3999 #endif
4000 
4001         if (page_resv(1, KM_NOSLEEP) == 0)
4002                 return (NULL);
4003 
4004 #ifdef  DEBUG
4005         pp = page_exists(&kvp, offset);
4006         if (pp != NULL)
4007                 panic("page already exists %p", (void *)pp);
4008 #endif
4009 
4010         pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL,
4011             &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE));       /* changing VA usage */
4012         if (pp != NULL) {
4013                 page_io_unlock(pp);
4014                 page_downgrade(pp);
4015         }
4016         return (pp);
4017 }