1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2012 DEY Storage Systems, Inc.  All rights reserved.
  25  * Copyright 2017 Nexenta Systems, Inc.
  26  * Copyright (c) 2018 Joyent, Inc.
  27  * Copyright (c) 2015 by Delphix. All rights reserved.
  28  */
  29 /*
  30  * Copyright (c) 2010, Intel Corporation.
  31  * All rights reserved.
  32  */
  33 
  34 #include <sys/types.h>
  35 #include <sys/t_lock.h>
  36 #include <sys/param.h>
  37 #include <sys/sysmacros.h>
  38 #include <sys/signal.h>
  39 #include <sys/systm.h>
  40 #include <sys/user.h>
  41 #include <sys/mman.h>
  42 #include <sys/vm.h>
  43 #include <sys/conf.h>
  44 #include <sys/avintr.h>
  45 #include <sys/autoconf.h>
  46 #include <sys/disp.h>
  47 #include <sys/class.h>
  48 #include <sys/bitmap.h>
  49 
  50 #include <sys/privregs.h>
  51 
  52 #include <sys/proc.h>
  53 #include <sys/buf.h>
  54 #include <sys/kmem.h>
  55 #include <sys/mem.h>
  56 #include <sys/kstat.h>
  57 
  58 #include <sys/reboot.h>
  59 
  60 #include <sys/cred.h>
  61 #include <sys/vnode.h>
  62 #include <sys/file.h>
  63 
  64 #include <sys/procfs.h>
  65 
  66 #include <sys/vfs.h>
  67 #include <sys/cmn_err.h>
  68 #include <sys/utsname.h>
  69 #include <sys/debug.h>
  70 #include <sys/kdi.h>
  71 
  72 #include <sys/dumphdr.h>
  73 #include <sys/bootconf.h>
  74 #include <sys/memlist_plat.h>
  75 #include <sys/varargs.h>
  76 #include <sys/promif.h>
  77 #include <sys/modctl.h>
  78 
  79 #include <sys/sunddi.h>
  80 #include <sys/sunndi.h>
  81 #include <sys/ndi_impldefs.h>
  82 #include <sys/ddidmareq.h>
  83 #include <sys/psw.h>
  84 #include <sys/regset.h>
  85 #include <sys/clock.h>
  86 #include <sys/pte.h>
  87 #include <sys/tss.h>
  88 #include <sys/stack.h>
  89 #include <sys/trap.h>
  90 #include <sys/fp.h>
  91 #include <vm/kboot_mmu.h>
  92 #include <vm/anon.h>
  93 #include <vm/as.h>
  94 #include <vm/page.h>
  95 #include <vm/seg.h>
  96 #include <vm/seg_dev.h>
  97 #include <vm/seg_kmem.h>
  98 #include <vm/seg_kpm.h>
  99 #include <vm/seg_map.h>
 100 #include <vm/seg_vn.h>
 101 #include <vm/seg_kp.h>
 102 #include <sys/memnode.h>
 103 #include <vm/vm_dep.h>
 104 #include <sys/thread.h>
 105 #include <sys/sysconf.h>
 106 #include <sys/vm_machparam.h>
 107 #include <sys/archsystm.h>
 108 #include <sys/machsystm.h>
 109 #include <vm/hat.h>
 110 #include <vm/hat_i86.h>
 111 #include <sys/pmem.h>
 112 #include <sys/smp_impldefs.h>
 113 #include <sys/x86_archext.h>
 114 #include <sys/cpuvar.h>
 115 #include <sys/segments.h>
 116 #include <sys/clconf.h>
 117 #include <sys/kobj.h>
 118 #include <sys/kobj_lex.h>
 119 #include <sys/cpc_impl.h>
 120 #include <sys/cpu_module.h>
 121 #include <sys/smbios.h>
 122 #include <sys/debug_info.h>
 123 #include <sys/bootinfo.h>
 124 #include <sys/ddi_periodic.h>
 125 #include <sys/systeminfo.h>
 126 #include <sys/multiboot.h>
 127 #include <sys/ramdisk.h>
 128 
 129 #ifdef  __xpv
 130 
 131 #include <sys/hypervisor.h>
 132 #include <sys/xen_mmu.h>
 133 #include <sys/evtchn_impl.h>
 134 #include <sys/gnttab.h>
 135 #include <sys/xpv_panic.h>
 136 #include <xen/sys/xenbus_comms.h>
 137 #include <xen/public/physdev.h>
 138 
 139 extern void xen_late_startup(void);
 140 
 141 struct xen_evt_data cpu0_evt_data;
 142 
 143 #else   /* __xpv */
 144 #include <sys/memlist_impl.h>
 145 
 146 extern void mem_config_init(void);
 147 #endif /* __xpv */
 148 
 149 extern void progressbar_init(void);
 150 extern void brand_init(void);
 151 extern void pcf_init(void);
 152 extern void pg_init(void);
 153 extern void ssp_init(void);
 154 
 155 extern int size_pse_array(pgcnt_t, int);
 156 
 157 #if defined(_SOFT_HOSTID)
 158 
 159 #include <sys/rtc.h>
 160 
 161 static int32_t set_soft_hostid(void);
 162 static char hostid_file[] = "/etc/hostid";
 163 
 164 #endif
 165 
 166 void *gfx_devinfo_list;
 167 
 168 #if defined(__amd64) && !defined(__xpv)
 169 extern void immu_startup(void);
 170 #endif
 171 
 172 /*
 173  * XXX make declaration below "static" when drivers no longer use this
 174  * interface.
 175  */
 176 extern caddr_t p0_va;   /* Virtual address for accessing physical page 0 */
 177 
 178 /*
 179  * segkp
 180  */
 181 extern int segkp_fromheap;
 182 
 183 static void kvm_init(void);
 184 static void startup_init(void);
 185 static void startup_memlist(void);
 186 static void startup_kmem(void);
 187 static void startup_modules(void);
 188 static void startup_vm(void);
 189 static void startup_end(void);
 190 static void layout_kernel_va(void);
 191 
 192 /*
 193  * Declare these as initialized data so we can patch them.
 194  */
 195 #ifdef __i386
 196 
 197 /*
 198  * Due to virtual address space limitations running in 32 bit mode, restrict
 199  * the amount of physical memory configured to a max of PHYSMEM pages (16g).
 200  *
 201  * If the physical max memory size of 64g were allowed to be configured, the
 202  * size of user virtual address space will be less than 1g. A limited user
 203  * address space greatly reduces the range of applications that can run.
 204  *
 205  * If more physical memory than PHYSMEM is required, users should preferably
 206  * run in 64 bit mode which has far looser virtual address space limitations.
 207  *
 208  * If 64 bit mode is not available (as in IA32) and/or more physical memory
 209  * than PHYSMEM is required in 32 bit mode, physmem can be set to the desired
 210  * value or to 0 (to configure all available memory) via eeprom(1M). kernelbase
 211  * should also be carefully tuned to balance out the need of the user
 212  * application while minimizing the risk of kernel heap exhaustion due to
 213  * kernelbase being set too high.
 214  */
 215 #define PHYSMEM 0x400000
 216 
 217 #else /* __amd64 */
 218 
 219 /*
 220  * For now we can handle memory with physical addresses up to about
 221  * 64 Terabytes. This keeps the kernel above the VA hole, leaving roughly
 222  * half the VA space for seg_kpm. When systems get bigger than 64TB this
 223  * code will need revisiting. There is an implicit assumption that there
 224  * are no *huge* holes in the physical address space too.
 225  */
 226 #define TERABYTE                (1ul << 40)
 227 #define PHYSMEM_MAX64           mmu_btop(64 * TERABYTE)
 228 #define PHYSMEM                 PHYSMEM_MAX64
 229 #define AMD64_VA_HOLE_END       0xFFFF800000000000ul
 230 
 231 #endif /* __amd64 */
 232 
 233 pgcnt_t physmem = PHYSMEM;
 234 pgcnt_t obp_pages;      /* Memory used by PROM for its text and data */
 235 
 236 char *kobj_file_buf;
 237 int kobj_file_bufsize;  /* set in /etc/system */
 238 
 239 /* Global variables for MP support. Used in mp_startup */
 240 caddr_t rm_platter_va = 0;
 241 uint32_t rm_platter_pa;
 242 
 243 int     auto_lpg_disable = 1;
 244 
 245 /*
 246  * Some CPUs have holes in the middle of the 64-bit virtual address range.
 247  */
 248 uintptr_t hole_start, hole_end;
 249 
 250 /*
 251  * kpm mapping window
 252  */
 253 caddr_t kpm_vbase;
 254 size_t  kpm_size;
 255 static int kpm_desired;
 256 #ifdef __amd64
 257 static uintptr_t segkpm_base = (uintptr_t)SEGKPM_BASE;
 258 #endif
 259 
 260 /*
 261  * Configuration parameters set at boot time.
 262  */
 263 
 264 caddr_t econtig;                /* end of first block of contiguous kernel */
 265 
 266 struct bootops          *bootops = 0;   /* passed in from boot */
 267 struct bootops          **bootopsp;
 268 struct boot_syscalls    *sysp;          /* passed in from boot */
 269 
 270 char bootblock_fstype[16];
 271 
 272 char kern_bootargs[OBP_MAXPATHLEN];
 273 char kern_bootfile[OBP_MAXPATHLEN];
 274 
 275 /*
 276  * ZFS zio segment.  This allows us to exclude large portions of ZFS data that
 277  * gets cached in kmem caches on the heap.  If this is set to zero, we allocate
 278  * zio buffers from their own segment, otherwise they are allocated from the
 279  * heap.  The optimization of allocating zio buffers from their own segment is
 280  * only valid on 64-bit kernels.
 281  */
 282 #if defined(__amd64)
 283 int segzio_fromheap = 0;
 284 #else
 285 int segzio_fromheap = 1;
 286 #endif
 287 
 288 /*
 289  * Give folks an escape hatch for disabling SMAP via kmdb. Doesn't work
 290  * post-boot.
 291  */
 292 int disable_smap = 0;
 293 
 294 /*
 295  * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this
 296  * depends on number of BOP_ALLOC calls made and requested size, memory size
 297  * combination and whether boot.bin memory needs to be freed.
 298  */
 299 #define POSS_NEW_FRAGMENTS      12
 300 
 301 /*
 302  * VM data structures
 303  */
 304 long page_hashsz;               /* Size of page hash table (power of two) */
 305 unsigned int page_hashsz_shift; /* log2(page_hashsz) */
 306 struct page *pp_base;           /* Base of initial system page struct array */
 307 struct page **page_hash;        /* Page hash table */
 308 pad_mutex_t *pse_mutex;         /* Locks protecting pp->p_selock */
 309 size_t pse_table_size;          /* Number of mutexes in pse_mutex[] */
 310 int pse_shift;                  /* log2(pse_table_size) */
 311 struct seg ktextseg;            /* Segment used for kernel executable image */
 312 struct seg kvalloc;             /* Segment used for "valloc" mapping */
 313 struct seg kpseg;               /* Segment used for pageable kernel virt mem */
 314 struct seg kmapseg;             /* Segment used for generic kernel mappings */
 315 struct seg kdebugseg;           /* Segment used for the kernel debugger */
 316 
 317 struct seg *segkmap = &kmapseg;     /* Kernel generic mapping segment */
 318 static struct seg *segmap = &kmapseg;       /* easier to use name for in here */
 319 
 320 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */
 321 
 322 #if defined(__amd64)
 323 struct seg kvseg_core;          /* Segment used for the core heap */
 324 struct seg kpmseg;              /* Segment used for physical mapping */
 325 struct seg *segkpm = &kpmseg;       /* 64bit kernel physical mapping segment */
 326 #else
 327 struct seg *segkpm = NULL;      /* Unused on IA32 */
 328 #endif
 329 
 330 caddr_t segkp_base;             /* Base address of segkp */
 331 caddr_t segzio_base;            /* Base address of segzio */
 332 #if defined(__amd64)
 333 pgcnt_t segkpsize = btop(SEGKPDEFSIZE); /* size of segkp segment in pages */
 334 #else
 335 pgcnt_t segkpsize = 0;
 336 #endif
 337 pgcnt_t segziosize = 0;         /* size of zio segment in pages */
 338 
 339 /*
 340  * A static DR page_t VA map is reserved that can map the page structures
 341  * for a domain's entire RA space. The pages that back this space are
 342  * dynamically allocated and need not be physically contiguous.  The DR
 343  * map size is derived from KPM size.
 344  * This mechanism isn't used by x86 yet, so just stubs here.
 345  */
 346 int ppvm_enable = 0;            /* Static virtual map for page structs */
 347 page_t *ppvm_base = NULL;       /* Base of page struct map */
 348 pgcnt_t ppvm_size = 0;          /* Size of page struct map */
 349 
 350 /*
 351  * VA range available to the debugger
 352  */
 353 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
 354 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
 355 
 356 struct memseg *memseg_base;
 357 struct vnode unused_pages_vp;
 358 
 359 #define FOURGB  0x100000000LL
 360 
 361 struct memlist *memlist;
 362 
 363 caddr_t s_text;         /* start of kernel text segment */
 364 caddr_t e_text;         /* end of kernel text segment */
 365 caddr_t s_data;         /* start of kernel data segment */
 366 caddr_t e_data;         /* end of kernel data segment */
 367 caddr_t modtext;        /* start of loadable module text reserved */
 368 caddr_t e_modtext;      /* end of loadable module text reserved */
 369 caddr_t moddata;        /* start of loadable module data reserved */
 370 caddr_t e_moddata;      /* end of loadable module data reserved */
 371 
 372 struct memlist *phys_install;   /* Total installed physical memory */
 373 struct memlist *phys_avail;     /* Total available physical memory */
 374 struct memlist *bios_rsvd;      /* Bios reserved memory */
 375 
 376 /*
 377  * kphysm_init returns the number of pages that were processed
 378  */
 379 static pgcnt_t kphysm_init(page_t *, pgcnt_t);
 380 
 381 #define IO_PROP_SIZE    64      /* device property size */
 382 
 383 /*
 384  * a couple useful roundup macros
 385  */
 386 #define ROUND_UP_PAGE(x)        \
 387         ((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)MMU_PAGESIZE))
 388 #define ROUND_UP_LPAGE(x)       \
 389         ((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[1]))
 390 #define ROUND_UP_4MEG(x)        \
 391         ((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)FOUR_MEG))
 392 #define ROUND_UP_TOPLEVEL(x)    \
 393         ((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[mmu.max_level]))
 394 
 395 /*
 396  *      32-bit Kernel's Virtual memory layout.
 397  *              +-----------------------+
 398  *              |                       |
 399  * 0xFFC00000  -|-----------------------|- ARGSBASE
 400  *              |       debugger        |
 401  * 0xFF800000  -|-----------------------|- SEGDEBUGBASE
 402  *              |      Kernel Data      |
 403  * 0xFEC00000  -|-----------------------|
 404  *              |      Kernel Text      |
 405  * 0xFE800000  -|-----------------------|- KERNEL_TEXT (0xFB400000 on Xen)
 406  *              |---       GDT       ---|- GDT page (GDT_VA)
 407  *              |---    debug info   ---|- debug info (DEBUG_INFO_VA)
 408  *              |                       |
 409  *              |   page_t structures   |
 410  *              |   memsegs, memlists,  |
 411  *              |   page hash, etc.     |
 412  * ---         -|-----------------------|- ekernelheap, valloc_base (floating)
 413  *              |                       |  (segkp is just an arena in the heap)
 414  *              |                       |
 415  *              |       kvseg           |
 416  *              |                       |
 417  *              |                       |
 418  * ---         -|-----------------------|- kernelheap (floating)
 419  *              |        Segkmap        |
 420  * 0xC3002000  -|-----------------------|- segmap_start (floating)
 421  *              |       Red Zone        |
 422  * 0xC3000000  -|-----------------------|- kernelbase / userlimit (floating)
 423  *              |                       |                       ||
 424  *              |     Shared objects    |                       \/
 425  *              |                       |
 426  *              :                       :
 427  *              |       user data       |
 428  *              |-----------------------|
 429  *              |       user text       |
 430  * 0x08048000  -|-----------------------|
 431  *              |       user stack      |
 432  *              :                       :
 433  *              |       invalid         |
 434  * 0x00000000   +-----------------------+
 435  *
 436  *
 437  *              64-bit Kernel's Virtual memory layout. (assuming 64 bit app)
 438  *                      +-----------------------+
 439  *                      |                       |
 440  * 0xFFFFFFFF.FFC00000  |-----------------------|- ARGSBASE
 441  *                      |       debugger (?)    |
 442  * 0xFFFFFFFF.FF800000  |-----------------------|- SEGDEBUGBASE
 443  *                      |      unused           |
 444  *                      +-----------------------+
 445  *                      |      Kernel Data      |
 446  * 0xFFFFFFFF.FBC00000  |-----------------------|
 447  *                      |      Kernel Text      |
 448  * 0xFFFFFFFF.FB800000  |-----------------------|- KERNEL_TEXT
 449  *                      |---    debug info   ---|- debug info (DEBUG_INFO_VA)
 450  *                      |---       GDT       ---|- GDT page (GDT_VA)
 451  *                      |---       IDT       ---|- IDT page (IDT_VA)
 452  *                      |---       LDT       ---|- LDT pages (LDT_VA)
 453  *                      |                       |
 454  *                      |      Core heap        | (used for loadable modules)
 455  * 0xFFFFFFFF.C0000000  |-----------------------|- core_base / ekernelheap
 456  *                      |        Kernel         |
 457  *                      |         heap          |
 458  * 0xFFFFFXXX.XXX00000  |-----------------------|- kernelheap (floating)
 459  *                      |        segmap         |
 460  * 0xFFFFFXXX.XXX00000  |-----------------------|- segmap_start (floating)
 461  *                      |    device mappings    |
 462  * 0xFFFFFXXX.XXX00000  |-----------------------|- toxic_addr (floating)
 463  *                      |         segzio        |
 464  * 0xFFFFFXXX.XXX00000  |-----------------------|- segzio_base (floating)
 465  *                      |         segkp         |
 466  * ---                  |-----------------------|- segkp_base (floating)
 467  *                      |   page_t structures   |  valloc_base + valloc_sz
 468  *                      |   memsegs, memlists,  |
 469  *                      |   page hash, etc.     |
 470  * 0xFFFFFF00.00000000  |-----------------------|- valloc_base (lower if >256GB)
 471  *                      |        segkpm         |
 472  * 0xFFFFFE00.00000000  |-----------------------|
 473  *                      |       Red Zone        |
 474  * 0xFFFFFD80.00000000  |-----------------------|- KERNELBASE (lower if >256GB)
 475  *                      |     User stack        |- User space memory
 476  *                      |                       |
 477  *                      | shared objects, etc   |       (grows downwards)
 478  *                      :                       :
 479  *                      |                       |
 480  * 0xFFFF8000.00000000  |-----------------------|
 481  *                      |                       |
 482  *                      | VA Hole / unused      |
 483  *                      |                       |
 484  * 0x00008000.00000000  |-----------------------|
 485  *                      |                       |
 486  *                      |                       |
 487  *                      :                       :
 488  *                      |       user heap       |       (grows upwards)
 489  *                      |                       |
 490  *                      |       user data       |
 491  *                      |-----------------------|
 492  *                      |       user text       |
 493  * 0x00000000.04000000  |-----------------------|
 494  *                      |       invalid         |
 495  * 0x00000000.00000000  +-----------------------+
 496  *
 497  * A 32 bit app on the 64 bit kernel sees the same layout as on the 32 bit
 498  * kernel, except that userlimit is raised to 0xfe000000
 499  *
 500  * Floating values:
 501  *
 502  * valloc_base: start of the kernel's memory management/tracking data
 503  * structures.  This region contains page_t structures for
 504  * physical memory, memsegs, memlists, and the page hash.
 505  *
 506  * core_base: start of the kernel's "core" heap area on 64-bit systems.
 507  * This area is intended to be used for global data as well as for module
 508  * text/data that does not fit into the nucleus pages.  The core heap is
 509  * restricted to a 2GB range, allowing every address within it to be
 510  * accessed using rip-relative addressing
 511  *
 512  * ekernelheap: end of kernelheap and start of segmap.
 513  *
 514  * kernelheap: start of kernel heap.  On 32-bit systems, this starts right
 515  * above a red zone that separates the user's address space from the
 516  * kernel's.  On 64-bit systems, it sits above segkp and segkpm.
 517  *
 518  * segmap_start: start of segmap. The length of segmap can be modified
 519  * through eeprom. The default length is 16MB on 32-bit systems and 64MB
 520  * on 64-bit systems.
 521  *
 522  * kernelbase: On a 32-bit kernel the default value of 0xd4000000 will be
 523  * decreased by 2X the size required for page_t.  This allows the kernel
 524  * heap to grow in size with physical memory.  With sizeof(page_t) == 80
 525  * bytes, the following shows the values of kernelbase and kernel heap
 526  * sizes for different memory configurations (assuming default segmap and
 527  * segkp sizes).
 528  *
 529  *      mem     size for        kernelbase      kernel heap
 530  *      size    page_t's                        size
 531  *      ----    ---------       ----------      -----------
 532  *      1gb     0x01400000      0xd1800000      684MB
 533  *      2gb     0x02800000      0xcf000000      704MB
 534  *      4gb     0x05000000      0xca000000      744MB
 535  *      6gb     0x07800000      0xc5000000      784MB
 536  *      8gb     0x0a000000      0xc0000000      824MB
 537  *      16gb    0x14000000      0xac000000      984MB
 538  *      32gb    0x28000000      0x84000000      1304MB
 539  *      64gb    0x50000000      0x34000000      1944MB (*)
 540  *
 541  * kernelbase is less than the abi minimum of 0xc0000000 for memory
 542  * configurations above 8gb.
 543  *
 544  * (*) support for memory configurations above 32gb will require manual tuning
 545  * of kernelbase to balance out the need of user applications.
 546  */
 547 
 548 /* real-time-clock initialization parameters */
 549 extern time_t process_rtc_config_file(void);
 550 
 551 uintptr_t       kernelbase;
 552 uintptr_t       postbootkernelbase;     /* not set till boot loader is gone */
 553 uintptr_t       eprom_kernelbase;
 554 size_t          segmapsize;
 555 uintptr_t       segmap_start;
 556 int             segmapfreelists;
 557 pgcnt_t         npages;
 558 pgcnt_t         orig_npages;
 559 size_t          core_size;              /* size of "core" heap */
 560 uintptr_t       core_base;              /* base address of "core" heap */
 561 
 562 /*
 563  * List of bootstrap pages. We mark these as allocated in startup.
 564  * release_bootstrap() will free them when we're completely done with
 565  * the bootstrap.
 566  */
 567 static page_t *bootpages;
 568 
 569 /*
 570  * boot time pages that have a vnode from the ramdisk will keep that forever.
 571  */
 572 static page_t *rd_pages;
 573 
 574 /*
 575  * Lower 64K
 576  */
 577 static page_t *lower_pages = NULL;
 578 static int lower_pages_count = 0;
 579 
 580 struct system_hardware system_hardware;
 581 
 582 /*
 583  * Enable some debugging messages concerning memory usage...
 584  */
 585 static void
 586 print_memlist(char *title, struct memlist *mp)
 587 {
 588         prom_printf("MEMLIST: %s:\n", title);
 589         while (mp != NULL)  {
 590                 prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n",
 591                     mp->ml_address, mp->ml_size);
 592                 mp = mp->ml_next;
 593         }
 594 }
 595 
 596 /*
 597  * XX64 need a comment here.. are these just default values, surely
 598  * we read the "cpuid" type information to figure this out.
 599  */
 600 int     l2cache_sz = 0x80000;
 601 int     l2cache_linesz = 0x40;
 602 int     l2cache_assoc = 1;
 603 
 604 static size_t   textrepl_min_gb = 10;
 605 
 606 /*
 607  * on 64 bit we use a predifined VA range for mapping devices in the kernel
 608  * on 32 bit the mappings are intermixed in the heap, so we use a bit map
 609  */
 610 #ifdef __amd64
 611 
 612 vmem_t          *device_arena;
 613 uintptr_t       toxic_addr = (uintptr_t)NULL;
 614 size_t          toxic_size = 1024 * 1024 * 1024; /* Sparc uses 1 gig too */
 615 
 616 #else   /* __i386 */
 617 
 618 ulong_t         *toxic_bit_map; /* one bit for each 4k of VA in heap_arena */
 619 size_t          toxic_bit_map_len = 0;  /* in bits */
 620 
 621 #endif  /* __i386 */
 622 
 623 /*
 624  * Simple boot time debug facilities
 625  */
 626 static char *prm_dbg_str[] = {
 627         "%s:%d: '%s' is 0x%x\n",
 628         "%s:%d: '%s' is 0x%llx\n"
 629 };
 630 
 631 int prom_debug;
 632 
 633 #define PRM_DEBUG(q)    if (prom_debug)         \
 634         prom_printf(prm_dbg_str[sizeof (q) >> 3], "startup.c", __LINE__, #q, q);
 635 #define PRM_POINT(q)    if (prom_debug)         \
 636         prom_printf("%s:%d: %s\n", "startup.c", __LINE__, q);
 637 
 638 /*
 639  * This structure is used to keep track of the intial allocations
 640  * done in startup_memlist(). The value of NUM_ALLOCATIONS needs to
 641  * be >= the number of ADD_TO_ALLOCATIONS() executed in the code.
 642  */
 643 #define NUM_ALLOCATIONS 8
 644 int num_allocations = 0;
 645 struct {
 646         void **al_ptr;
 647         size_t al_size;
 648 } allocations[NUM_ALLOCATIONS];
 649 size_t valloc_sz = 0;
 650 uintptr_t valloc_base;
 651 
 652 #define ADD_TO_ALLOCATIONS(ptr, size) {                                 \
 653                 size = ROUND_UP_PAGE(size);                             \
 654                 if (num_allocations == NUM_ALLOCATIONS)                 \
 655                         panic("too many ADD_TO_ALLOCATIONS()");         \
 656                 allocations[num_allocations].al_ptr = (void**)&ptr; \
 657                 allocations[num_allocations].al_size = size;            \
 658                 valloc_sz += size;                                      \
 659                 ++num_allocations;                                      \
 660         }
 661 
 662 /*
 663  * Allocate all the initial memory needed by the page allocator.
 664  */
 665 static void
 666 perform_allocations(void)
 667 {
 668         caddr_t mem;
 669         int i;
 670         int valloc_align;
 671 
 672         PRM_DEBUG(valloc_base);
 673         PRM_DEBUG(valloc_sz);
 674         valloc_align = mmu.level_size[mmu.max_page_level > 0];
 675         mem = BOP_ALLOC(bootops, (caddr_t)valloc_base, valloc_sz, valloc_align);
 676         if (mem != (caddr_t)valloc_base)
 677                 panic("BOP_ALLOC() failed");
 678         bzero(mem, valloc_sz);
 679         for (i = 0; i < num_allocations; ++i) {
 680                 *allocations[i].al_ptr = (void *)mem;
 681                 mem += allocations[i].al_size;
 682         }
 683 }
 684 
 685 /*
 686  * Set up and enable SMAP now before we start other CPUs, but after the kernel's
 687  * VM has been set up so we can use hot_patch_kernel_text().
 688  *
 689  * We can only patch 1, 2, or 4 bytes, but not three bytes. So instead, we
 690  * replace the four byte word at the patch point. See uts/intel/ia32/ml/copy.s
 691  * for more information on what's going on here.
 692  */
 693 static void
 694 startup_smap(void)
 695 {
 696         int i;
 697         uint32_t inst;
 698         uint8_t *instp;
 699         char sym[128];
 700         struct modctl *modp;
 701 
 702         extern int _smap_enable_patch_count;
 703         extern int _smap_disable_patch_count;
 704 
 705         if (disable_smap != 0)
 706                 remove_x86_feature(x86_featureset, X86FSET_SMAP);
 707 
 708         if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE)
 709                 return;
 710 
 711         for (i = 0; i < _smap_enable_patch_count; i++) {
 712                 int sizep;
 713 
 714                 VERIFY3U(i, <, _smap_enable_patch_count);
 715                 VERIFY(snprintf(sym, sizeof (sym), "_smap_enable_patch_%d", i) <
 716                     sizeof (sym));
 717                 instp = (uint8_t *)(void *)kobj_getelfsym(sym, NULL, &sizep);
 718                 VERIFY(instp != 0);
 719                 inst = (instp[3] << 24) | (SMAP_CLAC_INSTR & 0x00ffffff);
 720                 hot_patch_kernel_text((caddr_t)instp, inst, 4);
 721         }
 722 
 723         for (i = 0; i < _smap_disable_patch_count; i++) {
 724                 int sizep;
 725 
 726                 VERIFY(snprintf(sym, sizeof (sym), "_smap_disable_patch_%d",
 727                     i) < sizeof (sym));
 728                 instp = (uint8_t *)(void *)kobj_getelfsym(sym, NULL, &sizep);
 729                 VERIFY(instp != 0);
 730                 inst = (instp[3] << 24) | (SMAP_STAC_INSTR & 0x00ffffff);
 731                 hot_patch_kernel_text((caddr_t)instp, inst, 4);
 732         }
 733 
 734         /*
 735          * Hotinline calls to smap_enable and smap_disable within
 736          * unix module. Hotinlines in other modules are done on
 737          * mod_load().
 738          */
 739         modp = mod_hold_by_name("unix");
 740         do_hotinlines(modp->mod_mp);
 741         mod_release_mod(modp);
 742 
 743         setcr4(getcr4() | CR4_SMAP);
 744         smap_enable();
 745 }
 746 
 747 /*
 748  * Our world looks like this at startup time.
 749  *
 750  * In a 32-bit OS, boot loads the kernel text at 0xfe800000 and kernel data
 751  * at 0xfec00000.  On a 64-bit OS, kernel text and data are loaded at
 752  * 0xffffffff.fe800000 and 0xffffffff.fec00000 respectively.  Those
 753  * addresses are fixed in the binary at link time.
 754  *
 755  * On the text page:
 756  * unix/genunix/krtld/module text loads.
 757  *
 758  * On the data page:
 759  * unix/genunix/krtld/module data loads.
 760  *
 761  * Machine-dependent startup code
 762  */
 763 void
 764 startup(void)
 765 {
 766 #if !defined(__xpv)
 767         extern void startup_pci_bios(void);
 768 #endif
 769         extern cpuset_t cpu_ready_set;
 770 
 771         /*
 772          * Make sure that nobody tries to use sekpm until we have
 773          * initialized it properly.
 774          */
 775 #if defined(__amd64)
 776         kpm_desired = 1;
 777 #endif
 778         kpm_enable = 0;
 779         CPUSET_ONLY(cpu_ready_set, 0);  /* cpu 0 is boot cpu */
 780 
 781 #if defined(__xpv)      /* XXPV fix me! */
 782         {
 783                 extern int segvn_use_regions;
 784                 segvn_use_regions = 0;
 785         }
 786 #endif
 787         ssp_init();
 788         progressbar_init();
 789         startup_init();
 790 #if defined(__xpv)
 791         startup_xen_version();
 792 #endif
 793         startup_memlist();
 794         startup_kmem();
 795         startup_vm();
 796 #if !defined(__xpv)
 797         /*
 798          * Note we need to do this even on fast reboot in order to access
 799          * the irq routing table (used for pci labels).
 800          */
 801         startup_pci_bios();
 802         startup_smap();
 803 #endif
 804 #if defined(__xpv)
 805         startup_xen_mca();
 806 #endif
 807         startup_modules();
 808 
 809         startup_end();
 810 }
 811 
 812 static void
 813 startup_init()
 814 {
 815         PRM_POINT("startup_init() starting...");
 816 
 817         /*
 818          * Complete the extraction of cpuid data
 819          */
 820         cpuid_pass2(CPU);
 821 
 822         (void) check_boot_version(BOP_GETVERSION(bootops));
 823 
 824         /*
 825          * Check for prom_debug in boot environment
 826          */
 827         if (BOP_GETPROPLEN(bootops, "prom_debug") >= 0) {
 828                 ++prom_debug;
 829                 PRM_POINT("prom_debug found in boot enviroment");
 830         }
 831 
 832         /*
 833          * Collect node, cpu and memory configuration information.
 834          */
 835         get_system_configuration();
 836 
 837         /*
 838          * Halt if this is an unsupported processor.
 839          */
 840         if (x86_type == X86_TYPE_486 || x86_type == X86_TYPE_CYRIX_486) {
 841                 printf("\n486 processor (\"%s\") detected.\n",
 842                     CPU->cpu_brandstr);
 843                 halt("This processor is not supported by this release "
 844                     "of Solaris.");
 845         }
 846 
 847         PRM_POINT("startup_init() done");
 848 }
 849 
 850 /*
 851  * Callback for copy_memlist_filter() to filter nucleus, kadb/kmdb, (ie.
 852  * everything mapped above KERNEL_TEXT) pages from phys_avail. Note it
 853  * also filters out physical page zero.  There is some reliance on the
 854  * boot loader allocating only a few contiguous physical memory chunks.
 855  */
 856 static void
 857 avail_filter(uint64_t *addr, uint64_t *size)
 858 {
 859         uintptr_t va;
 860         uintptr_t next_va;
 861         pfn_t pfn;
 862         uint64_t pfn_addr;
 863         uint64_t pfn_eaddr;
 864         uint_t prot;
 865         size_t len;
 866         uint_t change;
 867 
 868         if (prom_debug)
 869                 prom_printf("\tFilter: in: a=%" PRIx64 ", s=%" PRIx64 "\n",
 870                     *addr, *size);
 871 
 872         /*
 873          * page zero is required for BIOS.. never make it available
 874          */
 875         if (*addr == 0) {
 876                 *addr += MMU_PAGESIZE;
 877                 *size -= MMU_PAGESIZE;
 878         }
 879 
 880         /*
 881          * First we trim from the front of the range. Since kbm_probe()
 882          * walks ranges in virtual order, but addr/size are physical, we need
 883          * to the list until no changes are seen.  This deals with the case
 884          * where page "p" is mapped at v, page "p + PAGESIZE" is mapped at w
 885          * but w < v.
 886          */
 887         do {
 888                 change = 0;
 889                 for (va = KERNEL_TEXT;
 890                     *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0;
 891                     va = next_va) {
 892 
 893                         next_va = va + len;
 894                         pfn_addr = pfn_to_pa(pfn);
 895                         pfn_eaddr = pfn_addr + len;
 896 
 897                         if (pfn_addr <= *addr && pfn_eaddr > *addr) {
 898                                 change = 1;
 899                                 while (*size > 0 && len > 0) {
 900                                         *addr += MMU_PAGESIZE;
 901                                         *size -= MMU_PAGESIZE;
 902                                         len -= MMU_PAGESIZE;
 903                                 }
 904                         }
 905                 }
 906                 if (change && prom_debug)
 907                         prom_printf("\t\ttrim: a=%" PRIx64 ", s=%" PRIx64 "\n",
 908                             *addr, *size);
 909         } while (change);
 910 
 911         /*
 912          * Trim pages from the end of the range.
 913          */
 914         for (va = KERNEL_TEXT;
 915             *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0;
 916             va = next_va) {
 917 
 918                 next_va = va + len;
 919                 pfn_addr = pfn_to_pa(pfn);
 920 
 921                 if (pfn_addr >= *addr && pfn_addr < *addr + *size)
 922                         *size = pfn_addr - *addr;
 923         }
 924 
 925         if (prom_debug)
 926                 prom_printf("\tFilter out: a=%" PRIx64 ", s=%" PRIx64 "\n",
 927                     *addr, *size);
 928 }
 929 
 930 static void
 931 kpm_init()
 932 {
 933         struct segkpm_crargs b;
 934 
 935         /*
 936          * These variables were all designed for sfmmu in which segkpm is
 937          * mapped using a single pagesize - either 8KB or 4MB.  On x86, we
 938          * might use 2+ page sizes on a single machine, so none of these
 939          * variables have a single correct value.  They are set up as if we
 940          * always use a 4KB pagesize, which should do no harm.  In the long
 941          * run, we should get rid of KPM's assumption that only a single
 942          * pagesize is used.
 943          */
 944         kpm_pgshft = MMU_PAGESHIFT;
 945         kpm_pgsz =  MMU_PAGESIZE;
 946         kpm_pgoff = MMU_PAGEOFFSET;
 947         kpmp2pshft = 0;
 948         kpmpnpgs = 1;
 949         ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0);
 950 
 951         PRM_POINT("about to create segkpm");
 952         rw_enter(&kas.a_lock, RW_WRITER);
 953 
 954         if (seg_attach(&kas, kpm_vbase, kpm_size, segkpm) < 0)
 955                 panic("cannot attach segkpm");
 956 
 957         b.prot = PROT_READ | PROT_WRITE;
 958         b.nvcolors = 1;
 959 
 960         if (segkpm_create(segkpm, (caddr_t)&b) != 0)
 961                 panic("segkpm_create segkpm");
 962 
 963         rw_exit(&kas.a_lock);
 964 
 965         kpm_enable = 1;
 966 
 967         /*
 968          * As the KPM was disabled while setting up the system, go back and fix
 969          * CPU zero's access to its user page table. This is a bit gross, but
 970          * we have a chicken and egg problem otherwise.
 971          */
 972         ASSERT(CPU->cpu_hat_info->hci_user_l3ptes == NULL);
 973         CPU->cpu_hat_info->hci_user_l3ptes =
 974             (x86pte_t *)hat_kpm_mapin_pfn(CPU->cpu_hat_info->hci_user_l3pfn);
 975 }
 976 
 977 /*
 978  * The debug info page provides enough information to allow external
 979  * inspectors (e.g. when running under a hypervisor) to bootstrap
 980  * themselves into allowing full-blown kernel debugging.
 981  */
 982 static void
 983 init_debug_info(void)
 984 {
 985         caddr_t mem;
 986         debug_info_t *di;
 987 
 988 #ifndef __lint
 989         ASSERT(sizeof (debug_info_t) < MMU_PAGESIZE);
 990 #endif
 991 
 992         mem = BOP_ALLOC(bootops, (caddr_t)DEBUG_INFO_VA, MMU_PAGESIZE,
 993             MMU_PAGESIZE);
 994 
 995         if (mem != (caddr_t)DEBUG_INFO_VA)
 996                 panic("BOP_ALLOC() failed");
 997         bzero(mem, MMU_PAGESIZE);
 998 
 999         di = (debug_info_t *)mem;
1000 
1001         di->di_magic = DEBUG_INFO_MAGIC;
1002         di->di_version = DEBUG_INFO_VERSION;
1003         di->di_modules = (uintptr_t)&modules;
1004         di->di_s_text = (uintptr_t)s_text;
1005         di->di_e_text = (uintptr_t)e_text;
1006         di->di_s_data = (uintptr_t)s_data;
1007         di->di_e_data = (uintptr_t)e_data;
1008         di->di_hat_htable_off = offsetof(hat_t, hat_htable);
1009         di->di_ht_pfn_off = offsetof(htable_t, ht_pfn);
1010 }
1011 
1012 /*
1013  * Build the memlists and other kernel essential memory system data structures.
1014  * This is everything at valloc_base.
1015  */
1016 static void
1017 startup_memlist(void)
1018 {
1019         size_t memlist_sz;
1020         size_t memseg_sz;
1021         size_t pagehash_sz;
1022         size_t pp_sz;
1023         uintptr_t va;
1024         size_t len;
1025         uint_t prot;
1026         pfn_t pfn;
1027         int memblocks;
1028         pfn_t rsvd_high_pfn;
1029         pgcnt_t rsvd_pgcnt;
1030         size_t rsvdmemlist_sz;
1031         int rsvdmemblocks;
1032         caddr_t pagecolor_mem;
1033         size_t pagecolor_memsz;
1034         caddr_t page_ctrs_mem;
1035         size_t page_ctrs_size;
1036         size_t pse_table_alloc_size;
1037         struct memlist *current;
1038         extern void startup_build_mem_nodes(struct memlist *);
1039 
1040         /* XX64 fix these - they should be in include files */
1041         extern size_t page_coloring_init(uint_t, int, int);
1042         extern void page_coloring_setup(caddr_t);
1043 
1044         PRM_POINT("startup_memlist() starting...");
1045 
1046         /*
1047          * Use leftover large page nucleus text/data space for loadable modules.
1048          * Use at most MODTEXT/MODDATA.
1049          */
1050         len = kbm_nucleus_size;
1051         ASSERT(len > MMU_PAGESIZE);
1052 
1053         moddata = (caddr_t)ROUND_UP_PAGE(e_data);
1054         e_moddata = (caddr_t)P2ROUNDUP((uintptr_t)e_data, (uintptr_t)len);
1055         if (e_moddata - moddata > MODDATA)
1056                 e_moddata = moddata + MODDATA;
1057 
1058         modtext = (caddr_t)ROUND_UP_PAGE(e_text);
1059         e_modtext = (caddr_t)P2ROUNDUP((uintptr_t)e_text, (uintptr_t)len);
1060         if (e_modtext - modtext > MODTEXT)
1061                 e_modtext = modtext + MODTEXT;
1062 
1063         econtig = e_moddata;
1064 
1065         PRM_DEBUG(modtext);
1066         PRM_DEBUG(e_modtext);
1067         PRM_DEBUG(moddata);
1068         PRM_DEBUG(e_moddata);
1069         PRM_DEBUG(econtig);
1070 
1071         /*
1072          * Examine the boot loader physical memory map to find out:
1073          * - total memory in system - physinstalled
1074          * - the max physical address - physmax
1075          * - the number of discontiguous segments of memory.
1076          */
1077         if (prom_debug)
1078                 print_memlist("boot physinstalled",
1079                     bootops->boot_mem->physinstalled);
1080         installed_top_size_ex(bootops->boot_mem->physinstalled, &physmax,
1081             &physinstalled, &memblocks);
1082         PRM_DEBUG(physmax);
1083         PRM_DEBUG(physinstalled);
1084         PRM_DEBUG(memblocks);
1085 
1086         /*
1087          * Compute maximum physical address for memory DR operations.
1088          * Memory DR operations are unsupported on xpv or 32bit OSes.
1089          */
1090 #ifdef  __amd64
1091         if (plat_dr_support_memory()) {
1092                 if (plat_dr_physmax == 0) {
1093                         uint_t pabits = UINT_MAX;
1094 
1095                         cpuid_get_addrsize(CPU, &pabits, NULL);
1096                         plat_dr_physmax = btop(1ULL << pabits);
1097                 }
1098                 if (plat_dr_physmax > PHYSMEM_MAX64)
1099                         plat_dr_physmax = PHYSMEM_MAX64;
1100         } else
1101 #endif
1102                 plat_dr_physmax = 0;
1103 
1104         /*
1105          * Examine the bios reserved memory to find out:
1106          * - the number of discontiguous segments of memory.
1107          */
1108         if (prom_debug)
1109                 print_memlist("boot reserved mem",
1110                     bootops->boot_mem->rsvdmem);
1111         installed_top_size_ex(bootops->boot_mem->rsvdmem, &rsvd_high_pfn,
1112             &rsvd_pgcnt, &rsvdmemblocks);
1113         PRM_DEBUG(rsvd_high_pfn);
1114         PRM_DEBUG(rsvd_pgcnt);
1115         PRM_DEBUG(rsvdmemblocks);
1116 
1117         /*
1118          * Initialize hat's mmu parameters.
1119          * Check for enforce-prot-exec in boot environment. It's used to
1120          * enable/disable support for the page table entry NX bit.
1121          * The default is to enforce PROT_EXEC on processors that support NX.
1122          * Boot seems to round up the "len", but 8 seems to be big enough.
1123          */
1124         mmu_init();
1125 
1126 #ifdef  __i386
1127         /*
1128          * physmax is lowered if there is more memory than can be
1129          * physically addressed in 32 bit (PAE/non-PAE) modes.
1130          */
1131         if (mmu.pae_hat) {
1132                 if (PFN_ABOVE64G(physmax)) {
1133                         physinstalled -= (physmax - (PFN_64G - 1));
1134                         physmax = PFN_64G - 1;
1135                 }
1136         } else {
1137                 if (PFN_ABOVE4G(physmax)) {
1138                         physinstalled -= (physmax - (PFN_4G - 1));
1139                         physmax = PFN_4G - 1;
1140                 }
1141         }
1142 #endif
1143 
1144         startup_build_mem_nodes(bootops->boot_mem->physinstalled);
1145 
1146         if (BOP_GETPROPLEN(bootops, "enforce-prot-exec") >= 0) {
1147                 int len = BOP_GETPROPLEN(bootops, "enforce-prot-exec");
1148                 char value[8];
1149 
1150                 if (len < 8)
1151                         (void) BOP_GETPROP(bootops, "enforce-prot-exec", value);
1152                 else
1153                         (void) strcpy(value, "");
1154                 if (strcmp(value, "off") == 0)
1155                         mmu.pt_nx = 0;
1156         }
1157         PRM_DEBUG(mmu.pt_nx);
1158 
1159         /*
1160          * We will need page_t's for every page in the system, except for
1161          * memory mapped at or above above the start of the kernel text segment.
1162          *
1163          * pages above e_modtext are attributed to kernel debugger (obp_pages)
1164          */
1165         npages = physinstalled - 1; /* avail_filter() skips page 0, so "- 1" */
1166         obp_pages = 0;
1167         va = KERNEL_TEXT;
1168         while (kbm_probe(&va, &len, &pfn, &prot) != 0) {
1169                 npages -= len >> MMU_PAGESHIFT;
1170                 if (va >= (uintptr_t)e_moddata)
1171                         obp_pages += len >> MMU_PAGESHIFT;
1172                 va += len;
1173         }
1174         PRM_DEBUG(npages);
1175         PRM_DEBUG(obp_pages);
1176 
1177         /*
1178          * If physmem is patched to be non-zero, use it instead of the computed
1179          * value unless it is larger than the actual amount of memory on hand.
1180          */
1181         if (physmem == 0 || physmem > npages) {
1182                 physmem = npages;
1183         } else if (physmem < npages) {
1184                 orig_npages = npages;
1185                 npages = physmem;
1186         }
1187         PRM_DEBUG(physmem);
1188 
1189         /*
1190          * We now compute the sizes of all the  initial allocations for
1191          * structures the kernel needs in order do kmem_alloc(). These
1192          * include:
1193          *      memsegs
1194          *      memlists
1195          *      page hash table
1196          *      page_t's
1197          *      page coloring data structs
1198          */
1199         memseg_sz = sizeof (struct memseg) * (memblocks + POSS_NEW_FRAGMENTS);
1200         ADD_TO_ALLOCATIONS(memseg_base, memseg_sz);
1201         PRM_DEBUG(memseg_sz);
1202 
1203         /*
1204          * Reserve space for memlists. There's no real good way to know exactly
1205          * how much room we'll need, but this should be a good upper bound.
1206          */
1207         memlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) *
1208             (memblocks + POSS_NEW_FRAGMENTS));
1209         ADD_TO_ALLOCATIONS(memlist, memlist_sz);
1210         PRM_DEBUG(memlist_sz);
1211 
1212         /*
1213          * Reserve space for bios reserved memlists.
1214          */
1215         rsvdmemlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) *
1216             (rsvdmemblocks + POSS_NEW_FRAGMENTS));
1217         ADD_TO_ALLOCATIONS(bios_rsvd, rsvdmemlist_sz);
1218         PRM_DEBUG(rsvdmemlist_sz);
1219 
1220         /* LINTED */
1221         ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), sizeof (struct page)));
1222         /*
1223          * The page structure hash table size is a power of 2
1224          * such that the average hash chain length is PAGE_HASHAVELEN.
1225          */
1226         page_hashsz = npages / PAGE_HASHAVELEN;
1227         page_hashsz_shift = highbit(page_hashsz);
1228         page_hashsz = 1 << page_hashsz_shift;
1229         pagehash_sz = sizeof (struct page *) * page_hashsz;
1230         ADD_TO_ALLOCATIONS(page_hash, pagehash_sz);
1231         PRM_DEBUG(pagehash_sz);
1232 
1233         /*
1234          * Set aside room for the page structures themselves.
1235          */
1236         PRM_DEBUG(npages);
1237         pp_sz = sizeof (struct page) * npages;
1238         ADD_TO_ALLOCATIONS(pp_base, pp_sz);
1239         PRM_DEBUG(pp_sz);
1240 
1241         /*
1242          * determine l2 cache info and memory size for page coloring
1243          */
1244         (void) getl2cacheinfo(CPU,
1245             &l2cache_sz, &l2cache_linesz, &l2cache_assoc);
1246         pagecolor_memsz =
1247             page_coloring_init(l2cache_sz, l2cache_linesz, l2cache_assoc);
1248         ADD_TO_ALLOCATIONS(pagecolor_mem, pagecolor_memsz);
1249         PRM_DEBUG(pagecolor_memsz);
1250 
1251         page_ctrs_size = page_ctrs_sz();
1252         ADD_TO_ALLOCATIONS(page_ctrs_mem, page_ctrs_size);
1253         PRM_DEBUG(page_ctrs_size);
1254 
1255         /*
1256          * Allocate the array that protects pp->p_selock.
1257          */
1258         pse_shift = size_pse_array(physmem, max_ncpus);
1259         pse_table_size = 1 << pse_shift;
1260         pse_table_alloc_size = pse_table_size * sizeof (pad_mutex_t);
1261         ADD_TO_ALLOCATIONS(pse_mutex, pse_table_alloc_size);
1262 
1263 #if defined(__amd64)
1264         valloc_sz = ROUND_UP_LPAGE(valloc_sz);
1265         valloc_base = VALLOC_BASE;
1266 
1267         /*
1268          * The default values of VALLOC_BASE and SEGKPM_BASE should work
1269          * for values of physmax up to 256GB (1/4 TB). They need adjusting when
1270          * memory is at addresses above 256GB. When adjusted, segkpm_base must
1271          * be aligned on KERNEL_REDZONE_SIZE boundary (span of top level pte).
1272          *
1273          * In the general case (>256GB), we use (4 * physmem) for the
1274          * kernel's virtual addresses, which is divided approximately
1275          * as follows:
1276          *  - 1 * physmem for segkpm
1277          *  - 1.5 * physmem for segzio
1278          *  - 1.5 * physmem for heap
1279          * Total: 4.0 * physmem
1280          *
1281          * Note that the segzio and heap sizes are more than physmem so that
1282          * VA fragmentation does not prevent either of them from being
1283          * able to use nearly all of physmem.  The value of 1.5x is determined
1284          * experimentally and may need to change if the workload changes.
1285          */
1286         if (physmax + 1 > mmu_btop(TERABYTE / 4) ||
1287             plat_dr_physmax > mmu_btop(TERABYTE / 4)) {
1288                 uint64_t kpm_resv_amount = mmu_ptob(physmax + 1);
1289 
1290                 if (kpm_resv_amount < mmu_ptob(plat_dr_physmax)) {
1291                         kpm_resv_amount = mmu_ptob(plat_dr_physmax);
1292                 }
1293 
1294                 /*
1295                  * This is what actually controls the KVA : UVA split.
1296                  * The kernel uses high VA, and this is lowering the
1297                  * boundary, thus increasing the amount of VA for the kernel.
1298                  * This gives the kernel 4 * (amount of physical memory) VA.
1299                  *
1300                  * The maximum VA is UINT64_MAX and we are using
1301                  * 64-bit 2's complement math, so e.g. if you have 512GB
1302                  * of memory, segkpm_base = -(4 * 512GB) == -2TB ==
1303                  * UINT64_MAX - 2TB (approximately).  So the kernel's
1304                  * VA is [UINT64_MAX-2TB to UINT64_MAX].
1305                  */
1306                 segkpm_base = -(P2ROUNDUP((4 * kpm_resv_amount),
1307                     KERNEL_REDZONE_SIZE));
1308 
1309                 /* make sure we leave some space for user apps above hole */
1310                 segkpm_base = MAX(segkpm_base, AMD64_VA_HOLE_END + TERABYTE);
1311                 if (segkpm_base > SEGKPM_BASE)
1312                         segkpm_base = SEGKPM_BASE;
1313                 PRM_DEBUG(segkpm_base);
1314 
1315                 valloc_base = segkpm_base + P2ROUNDUP(kpm_resv_amount, ONE_GIG);
1316                 if (valloc_base < segkpm_base)
1317                         panic("not enough kernel VA to support memory size");
1318                 PRM_DEBUG(valloc_base);
1319         }
1320 #else   /* __i386 */
1321         valloc_base = (uintptr_t)(MISC_VA_BASE - valloc_sz);
1322         valloc_base = P2ALIGN(valloc_base, mmu.level_size[1]);
1323         PRM_DEBUG(valloc_base);
1324 #endif  /* __i386 */
1325 
1326         /*
1327          * do all the initial allocations
1328          */
1329         perform_allocations();
1330 
1331         /*
1332          * Build phys_install and phys_avail in kernel memspace.
1333          * - phys_install should be all memory in the system.
1334          * - phys_avail is phys_install minus any memory mapped before this
1335          *    point above KERNEL_TEXT.
1336          */
1337         current = phys_install = memlist;
1338         copy_memlist_filter(bootops->boot_mem->physinstalled, &current, NULL);
1339         if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1340                 panic("physinstalled was too big!");
1341         if (prom_debug)
1342                 print_memlist("phys_install", phys_install);
1343 
1344         phys_avail = current;
1345         PRM_POINT("Building phys_avail:\n");
1346         copy_memlist_filter(bootops->boot_mem->physinstalled, &current,
1347             avail_filter);
1348         if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1349                 panic("physavail was too big!");
1350         if (prom_debug)
1351                 print_memlist("phys_avail", phys_avail);
1352 #ifndef __xpv
1353         /*
1354          * Free unused memlist items, which may be used by memory DR driver
1355          * at runtime.
1356          */
1357         if ((caddr_t)current < (caddr_t)memlist + memlist_sz) {
1358                 memlist_free_block((caddr_t)current,
1359                     (caddr_t)memlist + memlist_sz - (caddr_t)current);
1360         }
1361 #endif
1362 
1363         /*
1364          * Build bios reserved memspace
1365          */
1366         current = bios_rsvd;
1367         copy_memlist_filter(bootops->boot_mem->rsvdmem, &current, NULL);
1368         if ((caddr_t)current > (caddr_t)bios_rsvd + rsvdmemlist_sz)
1369                 panic("bios_rsvd was too big!");
1370         if (prom_debug)
1371                 print_memlist("bios_rsvd", bios_rsvd);
1372 #ifndef __xpv
1373         /*
1374          * Free unused memlist items, which may be used by memory DR driver
1375          * at runtime.
1376          */
1377         if ((caddr_t)current < (caddr_t)bios_rsvd + rsvdmemlist_sz) {
1378                 memlist_free_block((caddr_t)current,
1379                     (caddr_t)bios_rsvd + rsvdmemlist_sz - (caddr_t)current);
1380         }
1381 #endif
1382 
1383         /*
1384          * setup page coloring
1385          */
1386         page_coloring_setup(pagecolor_mem);
1387         page_lock_init();       /* currently a no-op */
1388 
1389         /*
1390          * free page list counters
1391          */
1392         (void) page_ctrs_alloc(page_ctrs_mem);
1393 
1394         /*
1395          * Size the pcf array based on the number of cpus in the box at
1396          * boot time.
1397          */
1398 
1399         pcf_init();
1400 
1401         /*
1402          * Initialize the page structures from the memory lists.
1403          */
1404         availrmem_initial = availrmem = freemem = 0;
1405         PRM_POINT("Calling kphysm_init()...");
1406         npages = kphysm_init(pp_base, npages);
1407         PRM_POINT("kphysm_init() done");
1408         PRM_DEBUG(npages);
1409 
1410         init_debug_info();
1411 
1412         /*
1413          * Now that page_t's have been initialized, remove all the
1414          * initial allocation pages from the kernel free page lists.
1415          */
1416         boot_mapin((caddr_t)valloc_base, valloc_sz);
1417         boot_mapin((caddr_t)MISC_VA_BASE, MISC_VA_SIZE);
1418         PRM_POINT("startup_memlist() done");
1419 
1420         PRM_DEBUG(valloc_sz);
1421 
1422 #if defined(__amd64)
1423         if ((availrmem >> (30 - MMU_PAGESHIFT)) >=
1424             textrepl_min_gb && l2cache_sz <= 2 << 20) {
1425                 extern size_t textrepl_size_thresh;
1426                 textrepl_size_thresh = (16 << 20) - 1;
1427         }
1428 #endif
1429 }
1430 
1431 /*
1432  * Layout the kernel's part of address space and initialize kmem allocator.
1433  */
1434 static void
1435 startup_kmem(void)
1436 {
1437         extern void page_set_colorequiv_arr(void);
1438 #if !defined(__xpv)
1439         extern uint64_t kpti_kbase;
1440 #endif
1441 
1442         PRM_POINT("startup_kmem() starting...");
1443 
1444 #if defined(__amd64)
1445         if (eprom_kernelbase && eprom_kernelbase != KERNELBASE)
1446                 cmn_err(CE_NOTE, "!kernelbase cannot be changed on 64-bit "
1447                     "systems.");
1448         kernelbase = segkpm_base - KERNEL_REDZONE_SIZE;
1449         core_base = (uintptr_t)COREHEAP_BASE;
1450         core_size = (size_t)MISC_VA_BASE - COREHEAP_BASE;
1451 #else   /* __i386 */
1452         /*
1453          * We configure kernelbase based on:
1454          *
1455          * 1. user specified kernelbase via eeprom command. Value cannot exceed
1456          *    KERNELBASE_MAX. we large page align eprom_kernelbase
1457          *
1458          * 2. Default to KERNELBASE and adjust to 2X less the size for page_t.
1459          *    On large memory systems we must lower kernelbase to allow
1460          *    enough room for page_t's for all of memory.
1461          *
1462          * The value set here, might be changed a little later.
1463          */
1464         if (eprom_kernelbase) {
1465                 kernelbase = eprom_kernelbase & mmu.level_mask[1];
1466                 if (kernelbase > KERNELBASE_MAX)
1467                         kernelbase = KERNELBASE_MAX;
1468         } else {
1469                 kernelbase = (uintptr_t)KERNELBASE;
1470                 kernelbase -= ROUND_UP_4MEG(2 * valloc_sz);
1471         }
1472         ASSERT((kernelbase & mmu.level_offset[1]) == 0);
1473         core_base = valloc_base;
1474         core_size = 0;
1475 #endif  /* __i386 */
1476 
1477         PRM_DEBUG(core_base);
1478         PRM_DEBUG(core_size);
1479         PRM_DEBUG(kernelbase);
1480 
1481 #if defined(__i386)
1482         segkp_fromheap = 1;
1483 #endif  /* __i386 */
1484 
1485         ekernelheap = (char *)core_base;
1486         PRM_DEBUG(ekernelheap);
1487 
1488         /*
1489          * Now that we know the real value of kernelbase,
1490          * update variables that were initialized with a value of
1491          * KERNELBASE (in common/conf/param.c).
1492          *
1493          * XXX  The problem with this sort of hackery is that the
1494          *      compiler just may feel like putting the const declarations
1495          *      (in param.c) into the .text section.  Perhaps they should
1496          *      just be declared as variables there?
1497          */
1498 
1499         *(uintptr_t *)&_kernelbase = kernelbase;
1500         *(uintptr_t *)&_userlimit = kernelbase;
1501 #if defined(__amd64)
1502         *(uintptr_t *)&_userlimit -= KERNELBASE - USERLIMIT;
1503 #if !defined(__xpv)
1504         kpti_kbase = kernelbase;
1505 #endif
1506 #else
1507         *(uintptr_t *)&_userlimit32 = _userlimit;
1508 #endif
1509         PRM_DEBUG(_kernelbase);
1510         PRM_DEBUG(_userlimit);
1511         PRM_DEBUG(_userlimit32);
1512 
1513         /* We have to re-do this now that we've modified _userlimit. */
1514         mmu_calc_user_slots();
1515 
1516         layout_kernel_va();
1517 
1518 #if defined(__i386)
1519         /*
1520          * If segmap is too large we can push the bottom of the kernel heap
1521          * higher than the base.  Or worse, it could exceed the top of the
1522          * VA space entirely, causing it to wrap around.
1523          */
1524         if (kernelheap >= ekernelheap || (uintptr_t)kernelheap < kernelbase)
1525                 panic("too little address space available for kernelheap,"
1526                     " use eeprom for lower kernelbase or smaller segmapsize");
1527 #endif  /* __i386 */
1528 
1529         /*
1530          * Initialize the kernel heap. Note 3rd argument must be > 1st.
1531          */
1532         kernelheap_init(kernelheap, ekernelheap,
1533             kernelheap + MMU_PAGESIZE,
1534             (void *)core_base, (void *)(core_base + core_size));
1535 
1536 #if defined(__xpv)
1537         /*
1538          * Link pending events struct into cpu struct
1539          */
1540         CPU->cpu_m.mcpu_evt_pend = &cpu0_evt_data;
1541 #endif
1542         /*
1543          * Initialize kernel memory allocator.
1544          */
1545         kmem_init();
1546 
1547         /*
1548          * Factor in colorequiv to check additional 'equivalent' bins
1549          */
1550         page_set_colorequiv_arr();
1551 
1552         /*
1553          * print this out early so that we know what's going on
1554          */
1555         print_x86_featureset(x86_featureset);
1556 
1557         /*
1558          * Initialize bp_mapin().
1559          */
1560         bp_init(MMU_PAGESIZE, HAT_STORECACHING_OK);
1561 
1562         /*
1563          * orig_npages is non-zero if physmem has been configured for less
1564          * than the available memory.
1565          */
1566         if (orig_npages) {
1567                 cmn_err(CE_WARN, "!%slimiting physmem to 0x%lx of 0x%lx pages",
1568                     (npages == PHYSMEM ? "Due to virtual address space " : ""),
1569                     npages, orig_npages);
1570         }
1571 #if defined(__i386)
1572         if (eprom_kernelbase && (eprom_kernelbase != kernelbase))
1573                 cmn_err(CE_WARN, "kernelbase value, User specified 0x%lx, "
1574                     "System using 0x%lx",
1575                     (uintptr_t)eprom_kernelbase, (uintptr_t)kernelbase);
1576 #endif
1577 
1578 #ifdef  KERNELBASE_ABI_MIN
1579         if (kernelbase < (uintptr_t)KERNELBASE_ABI_MIN) {
1580                 cmn_err(CE_NOTE, "!kernelbase set to 0x%lx, system is not "
1581                     "i386 ABI compliant.", (uintptr_t)kernelbase);
1582         }
1583 #endif
1584 
1585 #ifndef __xpv
1586         if (plat_dr_support_memory()) {
1587                 mem_config_init();
1588         }
1589 #else   /* __xpv */
1590         /*
1591          * Some of the xen start information has to be relocated up
1592          * into the kernel's permanent address space.
1593          */
1594         PRM_POINT("calling xen_relocate_start_info()");
1595         xen_relocate_start_info();
1596         PRM_POINT("xen_relocate_start_info() done");
1597 
1598         /*
1599          * (Update the vcpu pointer in our cpu structure to point into
1600          * the relocated shared info.)
1601          */
1602         CPU->cpu_m.mcpu_vcpu_info =
1603             &HYPERVISOR_shared_info->vcpu_info[CPU->cpu_id];
1604 #endif  /* __xpv */
1605 
1606         PRM_POINT("startup_kmem() done");
1607 }
1608 
1609 #ifndef __xpv
1610 /*
1611  * If we have detected that we are running in an HVM environment, we need
1612  * to prepend the PV driver directory to the module search path.
1613  */
1614 #define HVM_MOD_DIR "/platform/i86hvm/kernel"
1615 static void
1616 update_default_path()
1617 {
1618         char *current, *newpath;
1619         int newlen;
1620 
1621         /*
1622          * We are about to resync with krtld.  krtld will reset its
1623          * internal module search path iff Solaris has set default_path.
1624          * We want to be sure we're prepending this new directory to the
1625          * right search path.
1626          */
1627         current = (default_path == NULL) ? kobj_module_path : default_path;
1628 
1629         newlen = strlen(HVM_MOD_DIR) + strlen(current) + 2;
1630         newpath = kmem_alloc(newlen, KM_SLEEP);
1631         (void) strcpy(newpath, HVM_MOD_DIR);
1632         (void) strcat(newpath, " ");
1633         (void) strcat(newpath, current);
1634 
1635         default_path = newpath;
1636 }
1637 #endif
1638 
1639 static void
1640 startup_modules(void)
1641 {
1642         int cnt;
1643         extern void prom_setup(void);
1644         int32_t v, h;
1645         char d[11];
1646         char *cp;
1647         cmi_hdl_t hdl;
1648 
1649         PRM_POINT("startup_modules() starting...");
1650 
1651 #ifndef __xpv
1652         /*
1653          * Initialize ten-micro second timer so that drivers will
1654          * not get short changed in their init phase. This was
1655          * not getting called until clkinit which, on fast cpu's
1656          * caused the drv_usecwait to be way too short.
1657          */
1658         microfind();
1659 
1660         if ((get_hwenv() & HW_XEN_HVM) != 0)
1661                 update_default_path();
1662 #endif
1663 
1664         /*
1665          * Read the GMT lag from /etc/rtc_config.
1666          */
1667         sgmtl(process_rtc_config_file());
1668 
1669         /*
1670          * Calculate default settings of system parameters based upon
1671          * maxusers, yet allow to be overridden via the /etc/system file.
1672          */
1673         param_calc(0);
1674 
1675         mod_setup();
1676 
1677         /*
1678          * Initialize system parameters.
1679          */
1680         param_init();
1681 
1682         /*
1683          * Initialize the default brands
1684          */
1685         brand_init();
1686 
1687         /*
1688          * maxmem is the amount of physical memory we're playing with.
1689          */
1690         maxmem = physmem;
1691 
1692         /*
1693          * Initialize segment management stuff.
1694          */
1695         seg_init();
1696 
1697         if (modload("fs", "specfs") == -1)
1698                 halt("Can't load specfs");
1699 
1700         if (modload("fs", "devfs") == -1)
1701                 halt("Can't load devfs");
1702 
1703         if (modload("fs", "dev") == -1)
1704                 halt("Can't load dev");
1705 
1706         if (modload("fs", "procfs") == -1)
1707                 halt("Can't load procfs");
1708 
1709         (void) modloadonly("sys", "lbl_edition");
1710 
1711         dispinit();
1712 
1713         /* Read cluster configuration data. */
1714         clconf_init();
1715 
1716 #if defined(__xpv)
1717         (void) ec_init();
1718         gnttab_init();
1719         (void) xs_early_init();
1720 #endif /* __xpv */
1721 
1722         /*
1723          * Create a kernel device tree. First, create rootnex and
1724          * then invoke bus specific code to probe devices.
1725          */
1726         setup_ddi();
1727 
1728 #ifdef __xpv
1729         if (DOMAIN_IS_INITDOMAIN(xen_info))
1730 #endif
1731         {
1732                 id_t smid;
1733                 smbios_system_t smsys;
1734                 smbios_info_t sminfo;
1735                 char *mfg;
1736                 /*
1737                  * Load the System Management BIOS into the global ksmbios
1738                  * handle, if an SMBIOS is present on this system.
1739                  * Also set "si-hw-provider" property, if not already set.
1740                  */
1741                 ksmbios = smbios_open(NULL, SMB_VERSION, ksmbios_flags, NULL);
1742                 if (ksmbios != NULL &&
1743                     ((smid = smbios_info_system(ksmbios, &smsys)) != SMB_ERR) &&
1744                     (smbios_info_common(ksmbios, smid, &sminfo)) != SMB_ERR) {
1745                         mfg = (char *)sminfo.smbi_manufacturer;
1746                         if (BOP_GETPROPLEN(bootops, "si-hw-provider") < 0) {
1747                                 extern char hw_provider[];
1748                                 int i;
1749                                 for (i = 0; i < SYS_NMLN; i++) {
1750                                         if (isprint(mfg[i]))
1751                                                 hw_provider[i] = mfg[i];
1752                                         else {
1753                                                 hw_provider[i] = '\0';
1754                                                 break;
1755                                         }
1756                                 }
1757                                 hw_provider[SYS_NMLN - 1] = '\0';
1758                         }
1759                 }
1760         }
1761 
1762 
1763         /*
1764          * Originally clconf_init() apparently needed the hostid.  But
1765          * this no longer appears to be true - it uses its own nodeid.
1766          * By placing the hostid logic here, we are able to make use of
1767          * the SMBIOS UUID.
1768          */
1769         if ((h = set_soft_hostid()) == HW_INVALID_HOSTID) {
1770                 cmn_err(CE_WARN, "Unable to set hostid");
1771         } else {
1772                 for (v = h, cnt = 0; cnt < 10; cnt++) {
1773                         d[cnt] = (char)(v % 10);
1774                         v /= 10;
1775                         if (v == 0)
1776                                 break;
1777                 }
1778                 for (cp = hw_serial; cnt >= 0; cnt--)
1779                         *cp++ = d[cnt] + '0';
1780                 *cp = 0;
1781         }
1782 
1783         /*
1784          * Set up the CPU module subsystem for the boot cpu in the native
1785          * case, and all physical cpu resource in the xpv dom0 case.
1786          * Modifies the device tree, so this must be done after
1787          * setup_ddi().
1788          */
1789 #ifdef __xpv
1790         /*
1791          * If paravirtualized and on dom0 then we initialize all physical
1792          * cpu handles now;  if paravirtualized on a domU then do not
1793          * initialize.
1794          */
1795         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1796                 xen_mc_lcpu_cookie_t cpi;
1797 
1798                 for (cpi = xen_physcpu_next(NULL); cpi != NULL;
1799                     cpi = xen_physcpu_next(cpi)) {
1800                         if ((hdl = cmi_init(CMI_HDL_SOLARIS_xVM_MCA,
1801                             xen_physcpu_chipid(cpi), xen_physcpu_coreid(cpi),
1802                             xen_physcpu_strandid(cpi))) != NULL &&
1803                             is_x86_feature(x86_featureset, X86FSET_MCA))
1804                                 cmi_mca_init(hdl);
1805                 }
1806         }
1807 #else
1808         /*
1809          * Initialize a handle for the boot cpu - others will initialize
1810          * as they startup.
1811          */
1812         if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
1813             cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) {
1814                 if (is_x86_feature(x86_featureset, X86FSET_MCA))
1815                         cmi_mca_init(hdl);
1816                 CPU->cpu_m.mcpu_cmi_hdl = hdl;
1817         }
1818 #endif  /* __xpv */
1819 
1820         /*
1821          * Fake a prom tree such that /dev/openprom continues to work
1822          */
1823         PRM_POINT("startup_modules: calling prom_setup...");
1824         prom_setup();
1825         PRM_POINT("startup_modules: done");
1826 
1827         /*
1828          * Load all platform specific modules
1829          */
1830         PRM_POINT("startup_modules: calling psm_modload...");
1831         psm_modload();
1832 
1833         PRM_POINT("startup_modules() done");
1834 }
1835 
1836 /*
1837  * claim a "setaside" boot page for use in the kernel
1838  */
1839 page_t *
1840 boot_claim_page(pfn_t pfn)
1841 {
1842         page_t *pp;
1843 
1844         pp = page_numtopp_nolock(pfn);
1845         ASSERT(pp != NULL);
1846 
1847         if (PP_ISBOOTPAGES(pp)) {
1848                 if (pp->p_next != NULL)
1849                         pp->p_next->p_prev = pp->p_prev;
1850                 if (pp->p_prev == NULL)
1851                         bootpages = pp->p_next;
1852                 else
1853                         pp->p_prev->p_next = pp->p_next;
1854         } else {
1855                 /*
1856                  * htable_attach() expects a base pagesize page
1857                  */
1858                 if (pp->p_szc != 0)
1859                         page_boot_demote(pp);
1860                 pp = page_numtopp(pfn, SE_EXCL);
1861         }
1862         return (pp);
1863 }
1864 
1865 /*
1866  * Walk through the pagetables looking for pages mapped in by boot.  If the
1867  * setaside flag is set the pages are expected to be returned to the
1868  * kernel later in boot, so we add them to the bootpages list.
1869  */
1870 static void
1871 protect_boot_range(uintptr_t low, uintptr_t high, int setaside)
1872 {
1873         uintptr_t va = low;
1874         size_t len;
1875         uint_t prot;
1876         pfn_t pfn;
1877         page_t *pp;
1878         pgcnt_t boot_protect_cnt = 0;
1879 
1880         while (kbm_probe(&va, &len, &pfn, &prot) != 0 && va < high) {
1881                 if (va + len >= high)
1882                         panic("0x%lx byte mapping at 0x%p exceeds boot's "
1883                             "legal range.", len, (void *)va);
1884 
1885                 while (len > 0) {
1886                         pp = page_numtopp_alloc(pfn);
1887                         if (pp != NULL) {
1888                                 if (setaside == 0)
1889                                         panic("Unexpected mapping by boot.  "
1890                                             "addr=%p pfn=%lx\n",
1891                                             (void *)va, pfn);
1892 
1893                                 pp->p_next = bootpages;
1894                                 pp->p_prev = NULL;
1895                                 PP_SETBOOTPAGES(pp);
1896                                 if (bootpages != NULL) {
1897                                         bootpages->p_prev = pp;
1898                                 }
1899                                 bootpages = pp;
1900                                 ++boot_protect_cnt;
1901                         }
1902 
1903                         ++pfn;
1904                         len -= MMU_PAGESIZE;
1905                         va += MMU_PAGESIZE;
1906                 }
1907         }
1908         PRM_DEBUG(boot_protect_cnt);
1909 }
1910 
1911 /*
1912  *
1913  */
1914 static void
1915 layout_kernel_va(void)
1916 {
1917         PRM_POINT("layout_kernel_va() starting...");
1918         /*
1919          * Establish the final size of the kernel's heap, size of segmap,
1920          * segkp, etc.
1921          */
1922 
1923 #if defined(__amd64)
1924 
1925         kpm_vbase = (caddr_t)segkpm_base;
1926         if (physmax + 1 < plat_dr_physmax) {
1927                 kpm_size = ROUND_UP_LPAGE(mmu_ptob(plat_dr_physmax));
1928         } else {
1929                 kpm_size = ROUND_UP_LPAGE(mmu_ptob(physmax + 1));
1930         }
1931         if ((uintptr_t)kpm_vbase + kpm_size > (uintptr_t)valloc_base)
1932                 panic("not enough room for kpm!");
1933         PRM_DEBUG(kpm_size);
1934         PRM_DEBUG(kpm_vbase);
1935 
1936         /*
1937          * By default we create a seg_kp in 64 bit kernels, it's a little
1938          * faster to access than embedding it in the heap.
1939          */
1940         segkp_base = (caddr_t)valloc_base + valloc_sz;
1941         if (!segkp_fromheap) {
1942                 size_t sz = mmu_ptob(segkpsize);
1943 
1944                 /*
1945                  * determine size of segkp
1946                  */
1947                 if (sz < SEGKPMINSIZE || sz > SEGKPMAXSIZE) {
1948                         sz = SEGKPDEFSIZE;
1949                         cmn_err(CE_WARN, "!Illegal value for segkpsize. "
1950                             "segkpsize has been reset to %ld pages",
1951                             mmu_btop(sz));
1952                 }
1953                 sz = MIN(sz, MAX(SEGKPMINSIZE, mmu_ptob(physmem)));
1954 
1955                 segkpsize = mmu_btop(ROUND_UP_LPAGE(sz));
1956         }
1957         PRM_DEBUG(segkp_base);
1958         PRM_DEBUG(segkpsize);
1959 
1960         /*
1961          * segzio is used for ZFS cached data. It uses a distinct VA
1962          * segment (from kernel heap) so that we can easily tell not to
1963          * include it in kernel crash dumps on 64 bit kernels. The trick is
1964          * to give it lots of VA, but not constrain the kernel heap.
1965          * We can use 1.5x physmem for segzio, leaving approximately
1966          * another 1.5x physmem for heap.  See also the comment in
1967          * startup_memlist().
1968          */
1969         segzio_base = segkp_base + mmu_ptob(segkpsize);
1970         if (segzio_fromheap) {
1971                 segziosize = 0;
1972         } else {
1973                 size_t physmem_size = mmu_ptob(physmem);
1974                 size_t size = (segziosize == 0) ?
1975                     physmem_size * 3 / 2 : mmu_ptob(segziosize);
1976 
1977                 if (size < SEGZIOMINSIZE)
1978                         size = SEGZIOMINSIZE;
1979                 segziosize = mmu_btop(ROUND_UP_LPAGE(size));
1980         }
1981         PRM_DEBUG(segziosize);
1982         PRM_DEBUG(segzio_base);
1983 
1984         /*
1985          * Put the range of VA for device mappings next, kmdb knows to not
1986          * grep in this range of addresses.
1987          */
1988         toxic_addr =
1989             ROUND_UP_LPAGE((uintptr_t)segzio_base + mmu_ptob(segziosize));
1990         PRM_DEBUG(toxic_addr);
1991         segmap_start = ROUND_UP_LPAGE(toxic_addr + toxic_size);
1992 #else /* __i386 */
1993         segmap_start = ROUND_UP_LPAGE(kernelbase);
1994 #endif /* __i386 */
1995         PRM_DEBUG(segmap_start);
1996 
1997         /*
1998          * Users can change segmapsize through eeprom. If the variable
1999          * is tuned through eeprom, there is no upper bound on the
2000          * size of segmap.
2001          */
2002         segmapsize = MAX(ROUND_UP_LPAGE(segmapsize), SEGMAPDEFAULT);
2003 
2004 #if defined(__i386)
2005         /*
2006          * 32-bit systems don't have segkpm or segkp, so segmap appears at
2007          * the bottom of the kernel's address range.  Set aside space for a
2008          * small red zone just below the start of segmap.
2009          */
2010         segmap_start += KERNEL_REDZONE_SIZE;
2011         segmapsize -= KERNEL_REDZONE_SIZE;
2012 #endif
2013 
2014         PRM_DEBUG(segmap_start);
2015         PRM_DEBUG(segmapsize);
2016         kernelheap = (caddr_t)ROUND_UP_LPAGE(segmap_start + segmapsize);
2017         PRM_DEBUG(kernelheap);
2018         PRM_POINT("layout_kernel_va() done...");
2019 }
2020 
2021 /*
2022  * Finish initializing the VM system, now that we are no longer
2023  * relying on the boot time memory allocators.
2024  */
2025 static void
2026 startup_vm(void)
2027 {
2028         struct segmap_crargs a;
2029 
2030         extern int use_brk_lpg, use_stk_lpg;
2031 
2032         PRM_POINT("startup_vm() starting...");
2033 
2034         /*
2035          * Initialize the hat layer.
2036          */
2037         hat_init();
2038 
2039         /*
2040          * Do final allocations of HAT data structures that need to
2041          * be allocated before quiescing the boot loader.
2042          */
2043         PRM_POINT("Calling hat_kern_alloc()...");
2044         hat_kern_alloc((caddr_t)segmap_start, segmapsize, ekernelheap);
2045         PRM_POINT("hat_kern_alloc() done");
2046 
2047 #ifndef __xpv
2048         /*
2049          * Setup Page Attribute Table
2050          */
2051         pat_sync();
2052 #endif
2053 
2054         /*
2055          * The next two loops are done in distinct steps in order
2056          * to be sure that any page that is doubly mapped (both above
2057          * KERNEL_TEXT and below kernelbase) is dealt with correctly.
2058          * Note this may never happen, but it might someday.
2059          */
2060         bootpages = NULL;
2061         PRM_POINT("Protecting boot pages");
2062 
2063         /*
2064          * Protect any pages mapped above KERNEL_TEXT that somehow have
2065          * page_t's. This can only happen if something weird allocated
2066          * in this range (like kadb/kmdb).
2067          */
2068         protect_boot_range(KERNEL_TEXT, (uintptr_t)-1, 0);
2069 
2070         /*
2071          * Before we can take over memory allocation/mapping from the boot
2072          * loader we must remove from our free page lists any boot allocated
2073          * pages that stay mapped until release_bootstrap().
2074          */
2075         protect_boot_range(0, kernelbase, 1);
2076 
2077 
2078         /*
2079          * Switch to running on regular HAT (not boot_mmu)
2080          */
2081         PRM_POINT("Calling hat_kern_setup()...");
2082         hat_kern_setup();
2083 
2084         /*
2085          * It is no longer safe to call BOP_ALLOC(), so make sure we don't.
2086          */
2087         bop_no_more_mem();
2088 
2089         PRM_POINT("hat_kern_setup() done");
2090 
2091         hat_cpu_online(CPU);
2092 
2093         /*
2094          * Initialize VM system
2095          */
2096         PRM_POINT("Calling kvm_init()...");
2097         kvm_init();
2098         PRM_POINT("kvm_init() done");
2099 
2100         /*
2101          * Tell kmdb that the VM system is now working
2102          */
2103         if (boothowto & RB_DEBUG)
2104                 kdi_dvec_vmready();
2105 
2106 #if defined(__xpv)
2107         /*
2108          * Populate the I/O pool on domain 0
2109          */
2110         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
2111                 extern long populate_io_pool(void);
2112                 long init_io_pool_cnt;
2113 
2114                 PRM_POINT("Populating reserve I/O page pool");
2115                 init_io_pool_cnt = populate_io_pool();
2116                 PRM_DEBUG(init_io_pool_cnt);
2117         }
2118 #endif
2119         /*
2120          * Mangle the brand string etc.
2121          */
2122         cpuid_pass3(CPU);
2123 
2124 #if defined(__amd64)
2125 
2126         /*
2127          * Create the device arena for toxic (to dtrace/kmdb) mappings.
2128          */
2129         device_arena = vmem_create("device", (void *)toxic_addr,
2130             toxic_size, MMU_PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
2131 
2132 #else   /* __i386 */
2133 
2134         /*
2135          * allocate the bit map that tracks toxic pages
2136          */
2137         toxic_bit_map_len = btop((ulong_t)(valloc_base - kernelbase));
2138         PRM_DEBUG(toxic_bit_map_len);
2139         toxic_bit_map =
2140             kmem_zalloc(BT_SIZEOFMAP(toxic_bit_map_len), KM_NOSLEEP);
2141         ASSERT(toxic_bit_map != NULL);
2142         PRM_DEBUG(toxic_bit_map);
2143 
2144 #endif  /* __i386 */
2145 
2146 
2147         /*
2148          * Now that we've got more VA, as well as the ability to allocate from
2149          * it, tell the debugger.
2150          */
2151         if (boothowto & RB_DEBUG)
2152                 kdi_dvec_memavail();
2153 
2154 #if !defined(__xpv)
2155         /*
2156          * Map page pfn=0 for drivers, such as kd, that need to pick up
2157          * parameters left there by controllers/BIOS.
2158          */
2159         PRM_POINT("setup up p0_va");
2160         p0_va = i86devmap(0, 1, PROT_READ);
2161         PRM_DEBUG(p0_va);
2162 #endif
2163 
2164         cmn_err(CE_CONT, "?mem = %luK (0x%lx)\n",
2165             physinstalled << (MMU_PAGESHIFT - 10), ptob(physinstalled));
2166 
2167         /*
2168          * disable automatic large pages for small memory systems or
2169          * when the disable flag is set.
2170          *
2171          * Do not yet consider page sizes larger than 2m/4m.
2172          */
2173         if (!auto_lpg_disable && mmu.max_page_level > 0) {
2174                 max_uheap_lpsize = LEVEL_SIZE(1);
2175                 max_ustack_lpsize = LEVEL_SIZE(1);
2176                 max_privmap_lpsize = LEVEL_SIZE(1);
2177                 max_uidata_lpsize = LEVEL_SIZE(1);
2178                 max_utext_lpsize = LEVEL_SIZE(1);
2179                 max_shm_lpsize = LEVEL_SIZE(1);
2180         }
2181         if (physmem < privm_lpg_min_physmem || mmu.max_page_level == 0 ||
2182             auto_lpg_disable) {
2183                 use_brk_lpg = 0;
2184                 use_stk_lpg = 0;
2185         }
2186         mcntl0_lpsize = LEVEL_SIZE(mmu.umax_page_level);
2187 
2188         PRM_POINT("Calling hat_init_finish()...");
2189         hat_init_finish();
2190         PRM_POINT("hat_init_finish() done");
2191 
2192         /*
2193          * Initialize the segkp segment type.
2194          */
2195         rw_enter(&kas.a_lock, RW_WRITER);
2196         PRM_POINT("Attaching segkp");
2197         if (segkp_fromheap) {
2198                 segkp->s_as = &kas;
2199         } else if (seg_attach(&kas, (caddr_t)segkp_base, mmu_ptob(segkpsize),
2200             segkp) < 0) {
2201                 panic("startup: cannot attach segkp");
2202                 /*NOTREACHED*/
2203         }
2204         PRM_POINT("Doing segkp_create()");
2205         if (segkp_create(segkp) != 0) {
2206                 panic("startup: segkp_create failed");
2207                 /*NOTREACHED*/
2208         }
2209         PRM_DEBUG(segkp);
2210         rw_exit(&kas.a_lock);
2211 
2212         /*
2213          * kpm segment
2214          */
2215         segmap_kpm = 0;
2216         if (kpm_desired)
2217                 kpm_init();
2218 
2219         /*
2220          * Now create segmap segment.
2221          */
2222         rw_enter(&kas.a_lock, RW_WRITER);
2223         if (seg_attach(&kas, (caddr_t)segmap_start, segmapsize, segmap) < 0) {
2224                 panic("cannot attach segmap");
2225                 /*NOTREACHED*/
2226         }
2227         PRM_DEBUG(segmap);
2228 
2229         a.prot = PROT_READ | PROT_WRITE;
2230         a.shmsize = 0;
2231         a.nfreelist = segmapfreelists;
2232 
2233         if (segmap_create(segmap, (caddr_t)&a) != 0)
2234                 panic("segmap_create segmap");
2235         rw_exit(&kas.a_lock);
2236 
2237         setup_vaddr_for_ppcopy(CPU);
2238 
2239         segdev_init();
2240 #if defined(__xpv)
2241         if (DOMAIN_IS_INITDOMAIN(xen_info))
2242 #endif
2243                 pmem_init();
2244 
2245         PRM_POINT("startup_vm() done");
2246 }
2247 
2248 /*
2249  * Load a tod module for the non-standard tod part found on this system.
2250  */
2251 static void
2252 load_tod_module(char *todmod)
2253 {
2254         if (modload("tod", todmod) == -1)
2255                 halt("Can't load TOD module");
2256 }
2257 
2258 static void
2259 startup_end(void)
2260 {
2261         int i;
2262         extern void setx86isalist(void);
2263         extern void cpu_event_init(void);
2264 
2265         PRM_POINT("startup_end() starting...");
2266 
2267         /*
2268          * Perform tasks that get done after most of the VM
2269          * initialization has been done but before the clock
2270          * and other devices get started.
2271          */
2272         kern_setup1();
2273 
2274         /*
2275          * Perform CPC initialization for this CPU.
2276          */
2277         kcpc_hw_init(CPU);
2278 
2279         /*
2280          * Initialize cpu event framework.
2281          */
2282         cpu_event_init();
2283 
2284 #if defined(OPTERON_WORKAROUND_6323525)
2285         if (opteron_workaround_6323525)
2286                 patch_workaround_6323525();
2287 #endif
2288         /*
2289          * If needed, load TOD module now so that ddi_get_time(9F) etc. work
2290          * (For now, "needed" is defined as set tod_module_name in /etc/system)
2291          */
2292         if (tod_module_name != NULL) {
2293                 PRM_POINT("load_tod_module()");
2294                 load_tod_module(tod_module_name);
2295         }
2296 
2297 #if defined(__xpv)
2298         /*
2299          * Forceload interposing TOD module for the hypervisor.
2300          */
2301         PRM_POINT("load_tod_module()");
2302         load_tod_module("xpvtod");
2303 #endif
2304 
2305         /*
2306          * Configure the system.
2307          */
2308         PRM_POINT("Calling configure()...");
2309         configure();            /* set up devices */
2310         PRM_POINT("configure() done");
2311 
2312         /*
2313          * We can now setup for XSAVE because fpu_probe is done in configure().
2314          */
2315         if (fp_save_mech == FP_XSAVE) {
2316                 xsave_setup_msr(CPU);
2317         }
2318 
2319         /*
2320          * Set the isa_list string to the defined instruction sets we
2321          * support.
2322          */
2323         setx86isalist();
2324         cpu_intr_alloc(CPU, NINTR_THREADS);
2325         psm_install();
2326 
2327         /*
2328          * We're done with bootops.  We don't unmap the bootstrap yet because
2329          * we're still using bootsvcs.
2330          */
2331         PRM_POINT("NULLing out bootops");
2332         *bootopsp = (struct bootops *)NULL;
2333         bootops = (struct bootops *)NULL;
2334 
2335 #if defined(__xpv)
2336         ec_init_debug_irq();
2337         xs_domu_init();
2338 #endif
2339 
2340 #if !defined(__xpv)
2341         /*
2342          * Intel IOMMU has been setup/initialized in ddi_impl.c
2343          * Start it up now.
2344          */
2345         immu_startup();
2346 
2347         /*
2348          * Now that we're no longer going to drop into real mode for a BIOS call
2349          * via bootops, we can enable PCID (which requires CR0.PG).
2350          */
2351         enable_pcid();
2352 #endif
2353 
2354         PRM_POINT("Enabling interrupts");
2355         (*picinitf)();
2356         sti();
2357 #if defined(__xpv)
2358         ASSERT(CPU->cpu_m.mcpu_vcpu_info->evtchn_upcall_mask == 0);
2359         xen_late_startup();
2360 #endif
2361 
2362         (void) add_avsoftintr((void *)&softlevel1_hdl, 1, softlevel1,
2363             "softlevel1", NULL, NULL); /* XXX to be moved later */
2364 
2365         /*
2366          * Register software interrupt handlers for ddi_periodic_add(9F).
2367          * Software interrupts up to the level 10 are supported.
2368          */
2369         for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) {
2370                 (void) add_avsoftintr((void *)&softlevel_hdl[i-1], i,
2371                     (avfunc)(uintptr_t)ddi_periodic_softintr, "ddi_periodic",
2372                     (caddr_t)(uintptr_t)i, NULL);
2373         }
2374 
2375 #if !defined(__xpv)
2376         if (modload("drv", "amd_iommu") < 0) {
2377                 PRM_POINT("No AMD IOMMU present\n");
2378         } else if (ddi_hold_installed_driver(ddi_name_to_major(
2379             "amd_iommu")) == NULL) {
2380                 prom_printf("ERROR: failed to attach AMD IOMMU\n");
2381         }
2382 #endif
2383         post_startup_cpu_fixups();
2384 
2385         PRM_POINT("startup_end() done");
2386 }
2387 
2388 /*
2389  * Don't remove the following 2 variables.  They are necessary
2390  * for reading the hostid from the legacy file (/kernel/misc/sysinit).
2391  */
2392 char *_hs1107 = hw_serial;
2393 ulong_t  _bdhs34;
2394 
2395 void
2396 post_startup(void)
2397 {
2398         extern void cpupm_init(cpu_t *);
2399         extern void cpu_event_init_cpu(cpu_t *);
2400 
2401         /*
2402          * Set the system wide, processor-specific flags to be passed
2403          * to userland via the aux vector for performance hints and
2404          * instruction set extensions.
2405          */
2406         bind_hwcap();
2407 
2408 #ifdef __xpv
2409         if (DOMAIN_IS_INITDOMAIN(xen_info))
2410 #endif
2411         {
2412 #if defined(__xpv)
2413                 xpv_panic_init();
2414 #else
2415                 /*
2416                  * Startup the memory scrubber.
2417                  * XXPV This should be running somewhere ..
2418                  */
2419                 if ((get_hwenv() & HW_VIRTUAL) == 0)
2420                         memscrub_init();
2421 #endif
2422         }
2423 
2424         /*
2425          * Complete CPU module initialization
2426          */
2427         cmi_post_startup();
2428 
2429         /*
2430          * Perform forceloading tasks for /etc/system.
2431          */
2432         (void) mod_sysctl(SYS_FORCELOAD, NULL);
2433 
2434         /*
2435          * ON4.0: Force /proc module in until clock interrupt handle fixed
2436          * ON4.0: This must be fixed or restated in /etc/systems.
2437          */
2438         (void) modload("fs", "procfs");
2439 
2440         (void) i_ddi_attach_hw_nodes("pit_beep");
2441 
2442 #if defined(__i386)
2443         /*
2444          * Check for required functional Floating Point hardware,
2445          * unless FP hardware explicitly disabled.
2446          */
2447         if (fpu_exists && (fpu_pentium_fdivbug || fp_kind == FP_NO))
2448                 halt("No working FP hardware found");
2449 #endif
2450 
2451         maxmem = freemem;
2452 
2453         cpu_event_init_cpu(CPU);
2454         cpupm_init(CPU);
2455         (void) mach_cpu_create_device_node(CPU, NULL);
2456 
2457         pg_init();
2458 }
2459 
2460 static int
2461 pp_in_range(page_t *pp, uint64_t low_addr, uint64_t high_addr)
2462 {
2463         return ((pp->p_pagenum >= btop(low_addr)) &&
2464             (pp->p_pagenum < btopr(high_addr)));
2465 }
2466 
2467 static int
2468 pp_in_module(page_t *pp, const rd_existing_t *modranges)
2469 {
2470         uint_t i;
2471 
2472         for (i = 0; modranges[i].phys != 0; i++) {
2473                 if (pp_in_range(pp, modranges[i].phys,
2474                     modranges[i].phys + modranges[i].size))
2475                         return (1);
2476         }
2477 
2478         return (0);
2479 }
2480 
2481 void
2482 release_bootstrap(void)
2483 {
2484         int root_is_ramdisk;
2485         page_t *pp;
2486         extern void kobj_boot_unmountroot(void);
2487         extern dev_t rootdev;
2488         uint_t i;
2489         char propname[32];
2490         rd_existing_t *modranges;
2491 #if !defined(__xpv)
2492         pfn_t   pfn;
2493 #endif
2494 
2495         /*
2496          * Save the bootfs module ranges so that we can reserve them below
2497          * for the real bootfs.
2498          */
2499         modranges = kmem_alloc(sizeof (rd_existing_t) * MAX_BOOT_MODULES,
2500             KM_SLEEP);
2501         for (i = 0; ; i++) {
2502                 uint64_t start, size;
2503 
2504                 modranges[i].phys = 0;
2505 
2506                 (void) snprintf(propname, sizeof (propname),
2507                     "module-addr-%u", i);
2508                 if (do_bsys_getproplen(NULL, propname) <= 0)
2509                         break;
2510                 (void) do_bsys_getprop(NULL, propname, &start);
2511 
2512                 (void) snprintf(propname, sizeof (propname),
2513                     "module-size-%u", i);
2514                 if (do_bsys_getproplen(NULL, propname) <= 0)
2515                         break;
2516                 (void) do_bsys_getprop(NULL, propname, &size);
2517 
2518                 modranges[i].phys = start;
2519                 modranges[i].size = size;
2520         }
2521 
2522         /* unmount boot ramdisk and release kmem usage */
2523         kobj_boot_unmountroot();
2524 
2525         /*
2526          * We're finished using the boot loader so free its pages.
2527          */
2528         PRM_POINT("Unmapping lower boot pages");
2529 
2530         clear_boot_mappings(0, _userlimit);
2531 
2532         postbootkernelbase = kernelbase;
2533 
2534         /*
2535          * If root isn't on ramdisk, destroy the hardcoded
2536          * ramdisk node now and release the memory. Else,
2537          * ramdisk memory is kept in rd_pages.
2538          */
2539         root_is_ramdisk = (getmajor(rootdev) == ddi_name_to_major("ramdisk"));
2540         if (!root_is_ramdisk) {
2541                 dev_info_t *dip = ddi_find_devinfo("ramdisk", -1, 0);
2542                 ASSERT(dip && ddi_get_parent(dip) == ddi_root_node());
2543                 ndi_rele_devi(dip);     /* held from ddi_find_devinfo */
2544                 (void) ddi_remove_child(dip, 0);
2545         }
2546 
2547         PRM_POINT("Releasing boot pages");
2548         while (bootpages) {
2549                 extern uint64_t ramdisk_start, ramdisk_end;
2550                 pp = bootpages;
2551                 bootpages = pp->p_next;
2552 
2553 
2554                 /* Keep pages for the lower 64K */
2555                 if (pp_in_range(pp, 0, 0x40000)) {
2556                         pp->p_next = lower_pages;
2557                         lower_pages = pp;
2558                         lower_pages_count++;
2559                         continue;
2560                 }
2561 
2562                 if (root_is_ramdisk && pp_in_range(pp, ramdisk_start,
2563                     ramdisk_end) || pp_in_module(pp, modranges)) {
2564                         pp->p_next = rd_pages;
2565                         rd_pages = pp;
2566                         continue;
2567                 }
2568                 pp->p_next = (struct page *)0;
2569                 pp->p_prev = (struct page *)0;
2570                 PP_CLRBOOTPAGES(pp);
2571                 page_free(pp, 1);
2572         }
2573         PRM_POINT("Boot pages released");
2574 
2575         kmem_free(modranges, sizeof (rd_existing_t) * 99);
2576 
2577 #if !defined(__xpv)
2578 /* XXPV -- note this following bunch of code needs to be revisited in Xen 3.0 */
2579         /*
2580          * Find 1 page below 1 MB so that other processors can boot up or
2581          * so that any processor can resume.
2582          * Make sure it has a kernel VA as well as a 1:1 mapping.
2583          * We should have just free'd one up.
2584          */
2585 
2586         /*
2587          * 0x10 pages is 64K.  Leave the bottom 64K alone
2588          * for BIOS.
2589          */
2590         for (pfn = 0x10; pfn < btop(1*1024*1024); pfn++) {
2591                 if (page_numtopp_alloc(pfn) == NULL)
2592                         continue;
2593                 rm_platter_va = i86devmap(pfn, 1,
2594                     PROT_READ | PROT_WRITE | PROT_EXEC);
2595                 rm_platter_pa = ptob(pfn);
2596                 break;
2597         }
2598         if (pfn == btop(1*1024*1024) && use_mp)
2599                 panic("No page below 1M available for starting "
2600                     "other processors or for resuming from system-suspend");
2601 #endif  /* !__xpv */
2602 }
2603 
2604 /*
2605  * Initialize the platform-specific parts of a page_t.
2606  */
2607 void
2608 add_physmem_cb(page_t *pp, pfn_t pnum)
2609 {
2610         pp->p_pagenum = pnum;
2611         pp->p_mapping = NULL;
2612         pp->p_embed = 0;
2613         pp->p_share = 0;
2614         pp->p_mlentry = 0;
2615 }
2616 
2617 /*
2618  * kphysm_init() initializes physical memory.
2619  */
2620 static pgcnt_t
2621 kphysm_init(page_t *pp, pgcnt_t npages)
2622 {
2623         struct memlist  *pmem;
2624         struct memseg   *cur_memseg;
2625         pfn_t           base_pfn;
2626         pfn_t           end_pfn;
2627         pgcnt_t         num;
2628         pgcnt_t         pages_done = 0;
2629         uint64_t        addr;
2630         uint64_t        size;
2631         extern pfn_t    ddiphysmin;
2632         extern int      mnode_xwa;
2633         int             ms = 0, me = 0;
2634 
2635         ASSERT(page_hash != NULL && page_hashsz != 0);
2636 
2637         cur_memseg = memseg_base;
2638         for (pmem = phys_avail; pmem && npages; pmem = pmem->ml_next) {
2639                 /*
2640                  * In a 32 bit kernel can't use higher memory if we're
2641                  * not booting in PAE mode. This check takes care of that.
2642                  */
2643                 addr = pmem->ml_address;
2644                 size = pmem->ml_size;
2645                 if (btop(addr) > physmax)
2646                         continue;
2647 
2648                 /*
2649                  * align addr and size - they may not be at page boundaries
2650                  */
2651                 if ((addr & MMU_PAGEOFFSET) != 0) {
2652                         addr += MMU_PAGEOFFSET;
2653                         addr &= ~(uint64_t)MMU_PAGEOFFSET;
2654                         size -= addr - pmem->ml_address;
2655                 }
2656 
2657                 /* only process pages below or equal to physmax */
2658                 if ((btop(addr + size) - 1) > physmax)
2659                         size = ptob(physmax - btop(addr) + 1);
2660 
2661                 num = btop(size);
2662                 if (num == 0)
2663                         continue;
2664 
2665                 if (num > npages)
2666                         num = npages;
2667 
2668                 npages -= num;
2669                 pages_done += num;
2670                 base_pfn = btop(addr);
2671 
2672                 if (prom_debug)
2673                         prom_printf("MEMSEG addr=0x%" PRIx64
2674                             " pgs=0x%lx pfn 0x%lx-0x%lx\n",
2675                             addr, num, base_pfn, base_pfn + num);
2676 
2677                 /*
2678                  * Ignore pages below ddiphysmin to simplify ddi memory
2679                  * allocation with non-zero addr_lo requests.
2680                  */
2681                 if (base_pfn < ddiphysmin) {
2682                         if (base_pfn + num <= ddiphysmin)
2683                                 continue;
2684                         pp += (ddiphysmin - base_pfn);
2685                         num -= (ddiphysmin - base_pfn);
2686                         base_pfn = ddiphysmin;
2687                 }
2688 
2689                 /*
2690                  * mnode_xwa is greater than 1 when large pages regions can
2691                  * cross memory node boundaries. To prevent the formation
2692                  * of these large pages, configure the memsegs based on the
2693                  * memory node ranges which had been made non-contiguous.
2694                  */
2695                 end_pfn = base_pfn + num - 1;
2696                 if (mnode_xwa > 1) {
2697                         ms = PFN_2_MEM_NODE(base_pfn);
2698                         me = PFN_2_MEM_NODE(end_pfn);
2699 
2700                         if (ms != me) {
2701                                 /*
2702                                  * current range spans more than 1 memory node.
2703                                  * Set num to only the pfn range in the start
2704                                  * memory node.
2705                                  */
2706                                 num = mem_node_config[ms].physmax - base_pfn
2707                                     + 1;
2708                                 ASSERT(end_pfn > mem_node_config[ms].physmax);
2709                         }
2710                 }
2711 
2712                 for (;;) {
2713                         /*
2714                          * Build the memsegs entry
2715                          */
2716                         cur_memseg->pages = pp;
2717                         cur_memseg->epages = pp + num;
2718                         cur_memseg->pages_base = base_pfn;
2719                         cur_memseg->pages_end = base_pfn + num;
2720 
2721                         /*
2722                          * Insert into memseg list in decreasing pfn range
2723                          * order. Low memory is typically more fragmented such
2724                          * that this ordering keeps the larger ranges at the
2725                          * front of the list for code that searches memseg.
2726                          * This ASSERTS that the memsegs coming in from boot
2727                          * are in increasing physical address order and not
2728                          * contiguous.
2729                          */
2730                         if (memsegs != NULL) {
2731                                 ASSERT(cur_memseg->pages_base >=
2732                                     memsegs->pages_end);
2733                                 cur_memseg->next = memsegs;
2734                         }
2735                         memsegs = cur_memseg;
2736 
2737                         /*
2738                          * add_physmem() initializes the PSM part of the page
2739                          * struct by calling the PSM back with add_physmem_cb().
2740                          * In addition it coalesces pages into larger pages as
2741                          * it initializes them.
2742                          */
2743                         add_physmem(pp, num, base_pfn);
2744                         cur_memseg++;
2745                         availrmem_initial += num;
2746                         availrmem += num;
2747 
2748                         pp += num;
2749                         if (ms >= me)
2750                                 break;
2751 
2752                         /* process next memory node range */
2753                         ms++;
2754                         base_pfn = mem_node_config[ms].physbase;
2755 
2756                         if (mnode_xwa > 1) {
2757                                 num = MIN(mem_node_config[ms].physmax,
2758                                     end_pfn) - base_pfn + 1;
2759                         } else {
2760                                 num = mem_node_config[ms].physmax -
2761                                     base_pfn + 1;
2762                         }
2763                 }
2764         }
2765 
2766         PRM_DEBUG(availrmem_initial);
2767         PRM_DEBUG(availrmem);
2768         PRM_DEBUG(freemem);
2769         build_pfn_hash();
2770         return (pages_done);
2771 }
2772 
2773 /*
2774  * Kernel VM initialization.
2775  */
2776 static void
2777 kvm_init(void)
2778 {
2779         ASSERT((((uintptr_t)s_text) & MMU_PAGEOFFSET) == 0);
2780 
2781         /*
2782          * Put the kernel segments in kernel address space.
2783          */
2784         rw_enter(&kas.a_lock, RW_WRITER);
2785         as_avlinit(&kas);
2786 
2787         (void) seg_attach(&kas, s_text, e_moddata - s_text, &ktextseg);
2788         (void) segkmem_create(&ktextseg);
2789 
2790         (void) seg_attach(&kas, (caddr_t)valloc_base, valloc_sz, &kvalloc);
2791         (void) segkmem_create(&kvalloc);
2792 
2793         (void) seg_attach(&kas, kernelheap,
2794             ekernelheap - kernelheap, &kvseg);
2795         (void) segkmem_create(&kvseg);
2796 
2797         if (core_size > 0) {
2798                 PRM_POINT("attaching kvseg_core");
2799                 (void) seg_attach(&kas, (caddr_t)core_base, core_size,
2800                     &kvseg_core);
2801                 (void) segkmem_create(&kvseg_core);
2802         }
2803 
2804         if (segziosize > 0) {
2805                 PRM_POINT("attaching segzio");
2806                 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2807                     &kzioseg);
2808                 (void) segkmem_zio_create(&kzioseg);
2809 
2810                 /* create zio area covering new segment */
2811                 segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2812         }
2813 
2814         (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2815         (void) segkmem_create(&kdebugseg);
2816 
2817         rw_exit(&kas.a_lock);
2818 
2819         /*
2820          * Ensure that the red zone at kernelbase is never accessible.
2821          */
2822         PRM_POINT("protecting redzone");
2823         (void) as_setprot(&kas, (caddr_t)kernelbase, KERNEL_REDZONE_SIZE, 0);
2824 
2825         /*
2826          * Make the text writable so that it can be hot patched by DTrace.
2827          */
2828         (void) as_setprot(&kas, s_text, e_modtext - s_text,
2829             PROT_READ | PROT_WRITE | PROT_EXEC);
2830 
2831         /*
2832          * Make data writable until end.
2833          */
2834         (void) as_setprot(&kas, s_data, e_moddata - s_data,
2835             PROT_READ | PROT_WRITE | PROT_EXEC);
2836 }
2837 
2838 #ifndef __xpv
2839 /*
2840  * Solaris adds an entry for Write Combining caching to the PAT
2841  */
2842 static uint64_t pat_attr_reg = PAT_DEFAULT_ATTRIBUTE;
2843 
2844 void
2845 pat_sync(void)
2846 {
2847         ulong_t cr0, cr0_orig, cr4;
2848 
2849         if (!is_x86_feature(x86_featureset, X86FSET_PAT))
2850                 return;
2851         cr0_orig = cr0 = getcr0();
2852         cr4 = getcr4();
2853 
2854         /* disable caching and flush all caches and TLBs */
2855         cr0 |= CR0_CD;
2856         cr0 &= ~CR0_NW;
2857         setcr0(cr0);
2858         invalidate_cache();
2859         if (cr4 & CR4_PGE) {
2860                 setcr4(cr4 & ~(ulong_t)CR4_PGE);
2861                 setcr4(cr4);
2862         } else {
2863                 reload_cr3();
2864         }
2865 
2866         /* add our entry to the PAT */
2867         wrmsr(REG_PAT, pat_attr_reg);
2868 
2869         /* flush TLBs and cache again, then reenable cr0 caching */
2870         if (cr4 & CR4_PGE) {
2871                 setcr4(cr4 & ~(ulong_t)CR4_PGE);
2872                 setcr4(cr4);
2873         } else {
2874                 reload_cr3();
2875         }
2876         invalidate_cache();
2877         setcr0(cr0_orig);
2878 }
2879 
2880 #endif /* !__xpv */
2881 
2882 #if defined(_SOFT_HOSTID)
2883 /*
2884  * On platforms that do not have a hardware serial number, attempt
2885  * to set one based on the contents of /etc/hostid.  If this file does
2886  * not exist, assume that we are to generate a new hostid and set
2887  * it in the kernel, for subsequent saving by a userland process
2888  * once the system is up and the root filesystem is mounted r/w.
2889  *
2890  * In order to gracefully support upgrade on OpenSolaris, if
2891  * /etc/hostid does not exist, we will attempt to get a serial number
2892  * using the legacy method (/kernel/misc/sysinit).
2893  *
2894  * If that isn't present, we attempt to use an SMBIOS UUID, which is
2895  * a hardware serial number.  Note that we don't automatically trust
2896  * all SMBIOS UUIDs (some older platforms are defective and ship duplicate
2897  * UUIDs in violation of the standard), we check against a blacklist.
2898  *
2899  * In an attempt to make the hostid less prone to abuse
2900  * (for license circumvention, etc), we store it in /etc/hostid
2901  * in rot47 format.
2902  */
2903 extern volatile unsigned long tenmicrodata;
2904 static int atoi(char *);
2905 
2906 /*
2907  * Set this to non-zero in /etc/system if you think your SMBIOS returns a
2908  * UUID that is not unique. (Also report it so that the smbios_uuid_blacklist
2909  * array can be updated.)
2910  */
2911 int smbios_broken_uuid = 0;
2912 
2913 /*
2914  * List of known bad UUIDs.  This is just the lower 32-bit values, since
2915  * that's what we use for the host id.  If your hostid falls here, you need
2916  * to contact your hardware OEM for a fix for your BIOS.
2917  */
2918 static unsigned char
2919 smbios_uuid_blacklist[][16] = {
2920 
2921         {       /* Reported bad UUID (Google search) */
2922                 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05,
2923                 0x00, 0x06, 0x00, 0x07, 0x00, 0x08, 0x00, 0x09,
2924         },
2925         {       /* Known bad DELL UUID */
2926                 0x4C, 0x4C, 0x45, 0x44, 0x00, 0x00, 0x20, 0x10,
2927                 0x80, 0x20, 0x80, 0xC0, 0x4F, 0x20, 0x20, 0x20,
2928         },
2929         {       /* Uninitialized flash */
2930                 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
2931                 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
2932         },
2933         {       /* All zeros */
2934                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2935                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
2936         },
2937 };
2938 
2939 static int32_t
2940 uuid_to_hostid(const uint8_t *uuid)
2941 {
2942         /*
2943          * Although the UUIDs are 128-bits, they may not distribute entropy
2944          * evenly.  We would like to use SHA or MD5, but those are located
2945          * in loadable modules and not available this early in boot.  As we
2946          * don't need the values to be cryptographically strong, we just
2947          * generate 32-bit vaue by xor'ing the various sequences together,
2948          * which ensures that the entire UUID contributes to the hostid.
2949          */
2950         uint32_t        id = 0;
2951 
2952         /* first check against the blacklist */
2953         for (int i = 0; i < (sizeof (smbios_uuid_blacklist) / 16); i++) {
2954                 if (bcmp(smbios_uuid_blacklist[0], uuid, 16) == 0) {
2955                         cmn_err(CE_CONT, "?Broken SMBIOS UUID. "
2956                             "Contact BIOS manufacturer for repair.\n");
2957                         return ((int32_t)HW_INVALID_HOSTID);
2958                 }
2959         }
2960 
2961         for (int i = 0; i < 16; i++)
2962                 id ^= ((uuid[i]) << (8 * (i % sizeof (id))));
2963 
2964         /* Make sure return value is positive */
2965         return (id & 0x7fffffff);
2966 }
2967 
2968 static int32_t
2969 set_soft_hostid(void)
2970 {
2971         struct _buf *file;
2972         char tokbuf[MAXNAMELEN];
2973         token_t token;
2974         int done = 0;
2975         u_longlong_t tmp;
2976         int i;
2977         int32_t hostid = (int32_t)HW_INVALID_HOSTID;
2978         unsigned char *c;
2979         hrtime_t tsc;
2980         smbios_system_t smsys;
2981 
2982         /*
2983          * If /etc/hostid file not found, we'd like to get a pseudo
2984          * random number to use at the hostid.  A nice way to do this
2985          * is to read the real time clock.  To remain xen-compatible,
2986          * we can't poke the real hardware, so we use tsc_read() to
2987          * read the real time clock.  However, there is an ominous
2988          * warning in tsc_read that says it can return zero, so we
2989          * deal with that possibility by falling back to using the
2990          * (hopefully random enough) value in tenmicrodata.
2991          */
2992 
2993         if ((file = kobj_open_file(hostid_file)) == (struct _buf *)-1) {
2994                 /*
2995                  * hostid file not found - try to load sysinit module
2996                  * and see if it has a nonzero hostid value...use that
2997                  * instead of generating a new hostid here if so.
2998                  */
2999                 if ((i = modload("misc", "sysinit")) != -1) {
3000                         if (strlen(hw_serial) > 0)
3001                                 hostid = (int32_t)atoi(hw_serial);
3002                         (void) modunload(i);
3003                 }
3004 
3005                 /*
3006                  * We try to use the SMBIOS UUID. But not if it is blacklisted
3007                  * in /etc/system.
3008                  */
3009                 if ((hostid == HW_INVALID_HOSTID) &&
3010                     (smbios_broken_uuid == 0) &&
3011                     (ksmbios != NULL) &&
3012                     (smbios_info_system(ksmbios, &smsys) != SMB_ERR) &&
3013                     (smsys.smbs_uuidlen >= 16)) {
3014                         hostid = uuid_to_hostid(smsys.smbs_uuid);
3015                 }
3016 
3017                 /*
3018                  * Generate a "random" hostid using the clock.  These
3019                  * hostids will change on each boot if the value is not
3020                  * saved to a persistent /etc/hostid file.
3021                  */
3022                 if (hostid == HW_INVALID_HOSTID) {
3023                         tsc = tsc_read();
3024                         if (tsc == 0)   /* tsc_read can return zero sometimes */
3025                                 hostid = (int32_t)tenmicrodata & 0x0CFFFFF;
3026                         else
3027                                 hostid = (int32_t)tsc & 0x0CFFFFF;
3028                 }
3029         } else {
3030                 /* hostid file found */
3031                 while (!done) {
3032                         token = kobj_lex(file, tokbuf, sizeof (tokbuf));
3033 
3034                         switch (token) {
3035                         case POUND:
3036                                 /*
3037                                  * skip comments
3038                                  */
3039                                 kobj_find_eol(file);
3040                                 break;
3041                         case STRING:
3042                                 /*
3043                                  * un-rot47 - obviously this
3044                                  * nonsense is ascii-specific
3045                                  */
3046                                 for (c = (unsigned char *)tokbuf;
3047                                     *c != '\0'; c++) {
3048                                         *c += 47;
3049                                         if (*c > '~')
3050                                                 *c -= 94;
3051                                         else if (*c < '!')
3052                                                 *c += 94;
3053                                 }
3054                                 /*
3055                                  * now we should have a real number
3056                                  */
3057 
3058                                 if (kobj_getvalue(tokbuf, &tmp) != 0)
3059                                         kobj_file_err(CE_WARN, file,
3060                                             "Bad value %s for hostid",
3061                                             tokbuf);
3062                                 else
3063                                         hostid = (int32_t)tmp;
3064 
3065                                 break;
3066                         case EOF:
3067                                 done = 1;
3068                                 /* FALLTHROUGH */
3069                         case NEWLINE:
3070                                 kobj_newline(file);
3071                                 break;
3072                         default:
3073                                 break;
3074 
3075                         }
3076                 }
3077                 if (hostid == HW_INVALID_HOSTID) /* didn't find a hostid */
3078                         kobj_file_err(CE_WARN, file,
3079                             "hostid missing or corrupt");
3080 
3081                 kobj_close_file(file);
3082         }
3083         /*
3084          * hostid is now the value read from /etc/hostid, or the
3085          * new hostid we generated in this routine or HW_INVALID_HOSTID if not
3086          * set.
3087          */
3088         return (hostid);
3089 }
3090 
3091 static int
3092 atoi(char *p)
3093 {
3094         int i = 0;
3095 
3096         while (*p != '\0')
3097                 i = 10 * i + (*p++ - '0');
3098 
3099         return (i);
3100 }
3101 
3102 #endif /* _SOFT_HOSTID */
3103 
3104 void
3105 get_system_configuration(void)
3106 {
3107         char    prop[32];
3108         u_longlong_t nodes_ll, cpus_pernode_ll, lvalue;
3109 
3110         if (BOP_GETPROPLEN(bootops, "nodes") > sizeof (prop) ||
3111             BOP_GETPROP(bootops, "nodes", prop) < 0 ||
3112             kobj_getvalue(prop, &nodes_ll) == -1 ||
3113             nodes_ll > MAXNODES ||
3114             BOP_GETPROPLEN(bootops, "cpus_pernode") > sizeof (prop) ||
3115             BOP_GETPROP(bootops, "cpus_pernode", prop) < 0 ||
3116             kobj_getvalue(prop, &cpus_pernode_ll) == -1) {
3117                 system_hardware.hd_nodes = 1;
3118                 system_hardware.hd_cpus_per_node = 0;
3119         } else {
3120                 system_hardware.hd_nodes = (int)nodes_ll;
3121                 system_hardware.hd_cpus_per_node = (int)cpus_pernode_ll;
3122         }
3123 
3124         if (BOP_GETPROPLEN(bootops, "kernelbase") > sizeof (prop) ||
3125             BOP_GETPROP(bootops, "kernelbase", prop) < 0 ||
3126             kobj_getvalue(prop, &lvalue) == -1)
3127                 eprom_kernelbase = 0;
3128         else
3129                 eprom_kernelbase = (uintptr_t)lvalue;
3130 
3131         if (BOP_GETPROPLEN(bootops, "segmapsize") > sizeof (prop) ||
3132             BOP_GETPROP(bootops, "segmapsize", prop) < 0 ||
3133             kobj_getvalue(prop, &lvalue) == -1)
3134                 segmapsize = SEGMAPDEFAULT;
3135         else
3136                 segmapsize = (uintptr_t)lvalue;
3137 
3138         if (BOP_GETPROPLEN(bootops, "segmapfreelists") > sizeof (prop) ||
3139             BOP_GETPROP(bootops, "segmapfreelists", prop) < 0 ||
3140             kobj_getvalue(prop, &lvalue) == -1)
3141                 segmapfreelists = 0;    /* use segmap driver default */
3142         else
3143                 segmapfreelists = (int)lvalue;
3144 
3145         /* physmem used to be here, but moved much earlier to fakebop.c */
3146 }
3147 
3148 /*
3149  * Add to a memory list.
3150  * start = start of new memory segment
3151  * len = length of new memory segment in bytes
3152  * new = pointer to a new struct memlist
3153  * memlistp = memory list to which to add segment.
3154  */
3155 void
3156 memlist_add(
3157         uint64_t start,
3158         uint64_t len,
3159         struct memlist *new,
3160         struct memlist **memlistp)
3161 {
3162         struct memlist *cur;
3163         uint64_t end = start + len;
3164 
3165         new->ml_address = start;
3166         new->ml_size = len;
3167 
3168         cur = *memlistp;
3169 
3170         while (cur) {
3171                 if (cur->ml_address >= end) {
3172                         new->ml_next = cur;
3173                         *memlistp = new;
3174                         new->ml_prev = cur->ml_prev;
3175                         cur->ml_prev = new;
3176                         return;
3177                 }
3178                 ASSERT(cur->ml_address + cur->ml_size <= start);
3179                 if (cur->ml_next == NULL) {
3180                         cur->ml_next = new;
3181                         new->ml_prev = cur;
3182                         new->ml_next = NULL;
3183                         return;
3184                 }
3185                 memlistp = &cur->ml_next;
3186                 cur = cur->ml_next;
3187         }
3188 }
3189 
3190 void
3191 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3192 {
3193         size_t tsize = e_modtext - modtext;
3194         size_t dsize = e_moddata - moddata;
3195 
3196         *text_arena = vmem_create("module_text", tsize ? modtext : NULL, tsize,
3197             1, segkmem_alloc, segkmem_free, heaptext_arena, 0, VM_SLEEP);
3198         *data_arena = vmem_create("module_data", dsize ? moddata : NULL, dsize,
3199             1, segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3200 }
3201 
3202 caddr_t
3203 kobj_text_alloc(vmem_t *arena, size_t size)
3204 {
3205         return (vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT));
3206 }
3207 
3208 /*ARGSUSED*/
3209 caddr_t
3210 kobj_texthole_alloc(caddr_t addr, size_t size)
3211 {
3212         panic("unexpected call to kobj_texthole_alloc()");
3213         /*NOTREACHED*/
3214         return (0);
3215 }
3216 
3217 /*ARGSUSED*/
3218 void
3219 kobj_texthole_free(caddr_t addr, size_t size)
3220 {
3221         panic("unexpected call to kobj_texthole_free()");
3222 }
3223 
3224 /*
3225  * This is called just after configure() in startup().
3226  *
3227  * The ISALIST concept is a bit hopeless on Intel, because
3228  * there's no guarantee of an ever-more-capable processor
3229  * given that various parts of the instruction set may appear
3230  * and disappear between different implementations.
3231  *
3232  * While it would be possible to correct it and even enhance
3233  * it somewhat, the explicit hardware capability bitmask allows
3234  * more flexibility.
3235  *
3236  * So, we just leave this alone.
3237  */
3238 void
3239 setx86isalist(void)
3240 {
3241         char *tp;
3242         size_t len;
3243         extern char *isa_list;
3244 
3245 #define TBUFSIZE        1024
3246 
3247         tp = kmem_alloc(TBUFSIZE, KM_SLEEP);
3248         *tp = '\0';
3249 
3250 #if defined(__amd64)
3251         (void) strcpy(tp, "amd64 ");
3252 #endif
3253 
3254         switch (x86_vendor) {
3255         case X86_VENDOR_Intel:
3256         case X86_VENDOR_AMD:
3257         case X86_VENDOR_TM:
3258                 if (is_x86_feature(x86_featureset, X86FSET_CMOV)) {
3259                         /*
3260                          * Pentium Pro or later
3261                          */
3262                         (void) strcat(tp, "pentium_pro");
3263                         (void) strcat(tp,
3264                             is_x86_feature(x86_featureset, X86FSET_MMX) ?
3265                             "+mmx pentium_pro " : " ");
3266                 }
3267                 /*FALLTHROUGH*/
3268         case X86_VENDOR_Cyrix:
3269                 /*
3270                  * The Cyrix 6x86 does not have any Pentium features
3271                  * accessible while not at privilege level 0.
3272                  */
3273                 if (is_x86_feature(x86_featureset, X86FSET_CPUID)) {
3274                         (void) strcat(tp, "pentium");
3275                         (void) strcat(tp,
3276                             is_x86_feature(x86_featureset, X86FSET_MMX) ?
3277                             "+mmx pentium " : " ");
3278                 }
3279                 break;
3280         default:
3281                 break;
3282         }
3283         (void) strcat(tp, "i486 i386 i86");
3284         len = strlen(tp) + 1;   /* account for NULL at end of string */
3285         isa_list = strcpy(kmem_alloc(len, KM_SLEEP), tp);
3286         kmem_free(tp, TBUFSIZE);
3287 
3288 #undef TBUFSIZE
3289 }
3290 
3291 
3292 #ifdef __amd64
3293 
3294 void *
3295 device_arena_alloc(size_t size, int vm_flag)
3296 {
3297         return (vmem_alloc(device_arena, size, vm_flag));
3298 }
3299 
3300 void
3301 device_arena_free(void *vaddr, size_t size)
3302 {
3303         vmem_free(device_arena, vaddr, size);
3304 }
3305 
3306 #else /* __i386 */
3307 
3308 void *
3309 device_arena_alloc(size_t size, int vm_flag)
3310 {
3311         caddr_t vaddr;
3312         uintptr_t v;
3313         size_t  start;
3314         size_t  end;
3315 
3316         vaddr = vmem_alloc(heap_arena, size, vm_flag);
3317         if (vaddr == NULL)
3318                 return (NULL);
3319 
3320         v = (uintptr_t)vaddr;
3321         ASSERT(v >= kernelbase);
3322         ASSERT(v + size <= valloc_base);
3323 
3324         start = btop(v - kernelbase);
3325         end = btop(v + size - 1 - kernelbase);
3326         ASSERT(start < toxic_bit_map_len);
3327         ASSERT(end < toxic_bit_map_len);
3328 
3329         while (start <= end) {
3330                 BT_ATOMIC_SET(toxic_bit_map, start);
3331                 ++start;
3332         }
3333         return (vaddr);
3334 }
3335 
3336 void
3337 device_arena_free(void *vaddr, size_t size)
3338 {
3339         uintptr_t v = (uintptr_t)vaddr;
3340         size_t  start;
3341         size_t  end;
3342 
3343         ASSERT(v >= kernelbase);
3344         ASSERT(v + size <= valloc_base);
3345 
3346         start = btop(v - kernelbase);
3347         end = btop(v + size - 1 - kernelbase);
3348         ASSERT(start < toxic_bit_map_len);
3349         ASSERT(end < toxic_bit_map_len);
3350 
3351         while (start <= end) {
3352                 ASSERT(BT_TEST(toxic_bit_map, start) != 0);
3353                 BT_ATOMIC_CLEAR(toxic_bit_map, start);
3354                 ++start;
3355         }
3356         vmem_free(heap_arena, vaddr, size);
3357 }
3358 
3359 /*
3360  * returns 1st address in range that is in device arena, or NULL
3361  * if len is not NULL it returns the length of the toxic range
3362  */
3363 void *
3364 device_arena_contains(void *vaddr, size_t size, size_t *len)
3365 {
3366         uintptr_t v = (uintptr_t)vaddr;
3367         uintptr_t eaddr = v + size;
3368         size_t start;
3369         size_t end;
3370 
3371         /*
3372          * if called very early by kmdb, just return NULL
3373          */
3374         if (toxic_bit_map == NULL)
3375                 return (NULL);
3376 
3377         /*
3378          * First check if we're completely outside the bitmap range.
3379          */
3380         if (v >= valloc_base || eaddr < kernelbase)
3381                 return (NULL);
3382 
3383         /*
3384          * Trim ends of search to look at only what the bitmap covers.
3385          */
3386         if (v < kernelbase)
3387                 v = kernelbase;
3388         start = btop(v - kernelbase);
3389         end = btop(eaddr - kernelbase);
3390         if (end >= toxic_bit_map_len)
3391                 end = toxic_bit_map_len;
3392 
3393         if (bt_range(toxic_bit_map, &start, &end, end) == 0)
3394                 return (NULL);
3395 
3396         v = kernelbase + ptob(start);
3397         if (len != NULL)
3398                 *len = ptob(end - start);
3399         return ((void *)v);
3400 }
3401 
3402 #endif  /* __i386 */