1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2012 DEY Storage Systems, Inc.  All rights reserved.
  25  * Copyright 2017 Nexenta Systems, Inc.
  26  * Copyright (c) 2018 Joyent, Inc.
  27  * Copyright (c) 2015 by Delphix. All rights reserved.
  28  */
  29 /*
  30  * Copyright (c) 2010, Intel Corporation.
  31  * All rights reserved.
  32  */
  33 
  34 #include <sys/types.h>
  35 #include <sys/t_lock.h>
  36 #include <sys/param.h>
  37 #include <sys/sysmacros.h>
  38 #include <sys/signal.h>
  39 #include <sys/systm.h>
  40 #include <sys/user.h>
  41 #include <sys/mman.h>
  42 #include <sys/vm.h>
  43 #include <sys/conf.h>
  44 #include <sys/avintr.h>
  45 #include <sys/autoconf.h>
  46 #include <sys/disp.h>
  47 #include <sys/class.h>
  48 #include <sys/bitmap.h>
  49 
  50 #include <sys/privregs.h>
  51 
  52 #include <sys/proc.h>
  53 #include <sys/buf.h>
  54 #include <sys/kmem.h>
  55 #include <sys/mem.h>
  56 #include <sys/kstat.h>
  57 
  58 #include <sys/reboot.h>
  59 
  60 #include <sys/cred.h>
  61 #include <sys/vnode.h>
  62 #include <sys/file.h>
  63 
  64 #include <sys/procfs.h>
  65 
  66 #include <sys/vfs.h>
  67 #include <sys/cmn_err.h>
  68 #include <sys/utsname.h>
  69 #include <sys/debug.h>
  70 #include <sys/kdi.h>
  71 
  72 #include <sys/dumphdr.h>
  73 #include <sys/bootconf.h>
  74 #include <sys/memlist_plat.h>
  75 #include <sys/varargs.h>
  76 #include <sys/promif.h>
  77 #include <sys/modctl.h>
  78 
  79 #include <sys/sunddi.h>
  80 #include <sys/sunndi.h>
  81 #include <sys/ndi_impldefs.h>
  82 #include <sys/ddidmareq.h>
  83 #include <sys/psw.h>
  84 #include <sys/regset.h>
  85 #include <sys/clock.h>
  86 #include <sys/pte.h>
  87 #include <sys/tss.h>
  88 #include <sys/stack.h>
  89 #include <sys/trap.h>
  90 #include <sys/fp.h>
  91 #include <vm/kboot_mmu.h>
  92 #include <vm/anon.h>
  93 #include <vm/as.h>
  94 #include <vm/page.h>
  95 #include <vm/seg.h>
  96 #include <vm/seg_dev.h>
  97 #include <vm/seg_kmem.h>
  98 #include <vm/seg_kpm.h>
  99 #include <vm/seg_map.h>
 100 #include <vm/seg_vn.h>
 101 #include <vm/seg_kp.h>
 102 #include <sys/memnode.h>
 103 #include <vm/vm_dep.h>
 104 #include <sys/thread.h>
 105 #include <sys/sysconf.h>
 106 #include <sys/vm_machparam.h>
 107 #include <sys/archsystm.h>
 108 #include <sys/machsystm.h>
 109 #include <vm/hat.h>
 110 #include <vm/hat_i86.h>
 111 #include <sys/pmem.h>
 112 #include <sys/smp_impldefs.h>
 113 #include <sys/x86_archext.h>
 114 #include <sys/cpuvar.h>
 115 #include <sys/segments.h>
 116 #include <sys/clconf.h>
 117 #include <sys/kobj.h>
 118 #include <sys/kobj_lex.h>
 119 #include <sys/cpc_impl.h>
 120 #include <sys/cpu_module.h>
 121 #include <sys/smbios.h>
 122 #include <sys/debug_info.h>
 123 #include <sys/bootinfo.h>
 124 #include <sys/ddi_periodic.h>
 125 #include <sys/systeminfo.h>
 126 #include <sys/multiboot.h>
 127 #include <sys/ramdisk.h>
 128 
 129 #ifdef  __xpv
 130 
 131 #include <sys/hypervisor.h>
 132 #include <sys/xen_mmu.h>
 133 #include <sys/evtchn_impl.h>
 134 #include <sys/gnttab.h>
 135 #include <sys/xpv_panic.h>
 136 #include <xen/sys/xenbus_comms.h>
 137 #include <xen/public/physdev.h>
 138 
 139 extern void xen_late_startup(void);
 140 
 141 struct xen_evt_data cpu0_evt_data;
 142 
 143 #else   /* __xpv */
 144 #include <sys/memlist_impl.h>
 145 
 146 extern void mem_config_init(void);
 147 #endif /* __xpv */
 148 
 149 extern void progressbar_init(void);
 150 extern void brand_init(void);
 151 extern void pcf_init(void);
 152 extern void pg_init(void);
 153 extern void ssp_init(void);
 154 
 155 extern int size_pse_array(pgcnt_t, int);
 156 
 157 #if defined(_SOFT_HOSTID)
 158 
 159 #include <sys/rtc.h>
 160 
 161 static int32_t set_soft_hostid(void);
 162 static char hostid_file[] = "/etc/hostid";
 163 
 164 #endif
 165 
 166 void *gfx_devinfo_list;
 167 
 168 #if defined(__amd64) && !defined(__xpv)
 169 extern void immu_startup(void);
 170 #endif
 171 
 172 /*
 173  * XXX make declaration below "static" when drivers no longer use this
 174  * interface.
 175  */
 176 extern caddr_t p0_va;   /* Virtual address for accessing physical page 0 */
 177 
 178 /*
 179  * segkp
 180  */
 181 extern int segkp_fromheap;
 182 
 183 static void kvm_init(void);
 184 static void startup_init(void);
 185 static void startup_memlist(void);
 186 static void startup_kmem(void);
 187 static void startup_modules(void);
 188 static void startup_vm(void);
 189 static void startup_end(void);
 190 static void layout_kernel_va(void);
 191 
 192 /*
 193  * Declare these as initialized data so we can patch them.
 194  */
 195 #ifdef __i386
 196 
 197 /*
 198  * Due to virtual address space limitations running in 32 bit mode, restrict
 199  * the amount of physical memory configured to a max of PHYSMEM pages (16g).
 200  *
 201  * If the physical max memory size of 64g were allowed to be configured, the
 202  * size of user virtual address space will be less than 1g. A limited user
 203  * address space greatly reduces the range of applications that can run.
 204  *
 205  * If more physical memory than PHYSMEM is required, users should preferably
 206  * run in 64 bit mode which has far looser virtual address space limitations.
 207  *
 208  * If 64 bit mode is not available (as in IA32) and/or more physical memory
 209  * than PHYSMEM is required in 32 bit mode, physmem can be set to the desired
 210  * value or to 0 (to configure all available memory) via eeprom(1M). kernelbase
 211  * should also be carefully tuned to balance out the need of the user
 212  * application while minimizing the risk of kernel heap exhaustion due to
 213  * kernelbase being set too high.
 214  */
 215 #define PHYSMEM 0x400000
 216 
 217 #else /* __amd64 */
 218 
 219 /*
 220  * For now we can handle memory with physical addresses up to about
 221  * 64 Terabytes. This keeps the kernel above the VA hole, leaving roughly
 222  * half the VA space for seg_kpm. When systems get bigger than 64TB this
 223  * code will need revisiting. There is an implicit assumption that there
 224  * are no *huge* holes in the physical address space too.
 225  */
 226 #define TERABYTE                (1ul << 40)
 227 #define PHYSMEM_MAX64           mmu_btop(64 * TERABYTE)
 228 #define PHYSMEM                 PHYSMEM_MAX64
 229 #define AMD64_VA_HOLE_END       0xFFFF800000000000ul
 230 
 231 #endif /* __amd64 */
 232 
 233 pgcnt_t physmem = PHYSMEM;
 234 pgcnt_t obp_pages;      /* Memory used by PROM for its text and data */
 235 
 236 char *kobj_file_buf;
 237 int kobj_file_bufsize;  /* set in /etc/system */
 238 
 239 /* Global variables for MP support. Used in mp_startup */
 240 caddr_t rm_platter_va = 0;
 241 uint32_t rm_platter_pa;
 242 
 243 int     auto_lpg_disable = 1;
 244 
 245 /*
 246  * Some CPUs have holes in the middle of the 64-bit virtual address range.
 247  */
 248 uintptr_t hole_start, hole_end;
 249 
 250 /*
 251  * kpm mapping window
 252  */
 253 caddr_t kpm_vbase;
 254 size_t  kpm_size;
 255 static int kpm_desired;
 256 #ifdef __amd64
 257 static uintptr_t segkpm_base = (uintptr_t)SEGKPM_BASE;
 258 #endif
 259 
 260 /*
 261  * Configuration parameters set at boot time.
 262  */
 263 
 264 caddr_t econtig;                /* end of first block of contiguous kernel */
 265 
 266 struct bootops          *bootops = 0;   /* passed in from boot */
 267 struct bootops          **bootopsp;
 268 struct boot_syscalls    *sysp;          /* passed in from boot */
 269 
 270 char bootblock_fstype[16];
 271 
 272 char kern_bootargs[OBP_MAXPATHLEN];
 273 char kern_bootfile[OBP_MAXPATHLEN];
 274 
 275 /*
 276  * ZFS zio segment.  This allows us to exclude large portions of ZFS data that
 277  * gets cached in kmem caches on the heap.  If this is set to zero, we allocate
 278  * zio buffers from their own segment, otherwise they are allocated from the
 279  * heap.  The optimization of allocating zio buffers from their own segment is
 280  * only valid on 64-bit kernels.
 281  */
 282 #if defined(__amd64)
 283 int segzio_fromheap = 0;
 284 #else
 285 int segzio_fromheap = 1;
 286 #endif
 287 
 288 /*
 289  * Give folks an escape hatch for disabling SMAP via kmdb. Doesn't work
 290  * post-boot.
 291  */
 292 int disable_smap = 0;
 293 
 294 /*
 295  * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this
 296  * depends on number of BOP_ALLOC calls made and requested size, memory size
 297  * combination and whether boot.bin memory needs to be freed.
 298  */
 299 #define POSS_NEW_FRAGMENTS      12
 300 
 301 /*
 302  * VM data structures
 303  */
 304 long page_hashsz;               /* Size of page hash table (power of two) */
 305 unsigned int page_hashsz_shift; /* log2(page_hashsz) */
 306 struct page *pp_base;           /* Base of initial system page struct array */
 307 struct page **page_hash;        /* Page hash table */
 308 pad_mutex_t *pse_mutex;         /* Locks protecting pp->p_selock */
 309 size_t pse_table_size;          /* Number of mutexes in pse_mutex[] */
 310 int pse_shift;                  /* log2(pse_table_size) */
 311 struct seg ktextseg;            /* Segment used for kernel executable image */
 312 struct seg kvalloc;             /* Segment used for "valloc" mapping */
 313 struct seg kpseg;               /* Segment used for pageable kernel virt mem */
 314 struct seg kmapseg;             /* Segment used for generic kernel mappings */
 315 struct seg kdebugseg;           /* Segment used for the kernel debugger */
 316 
 317 struct seg *segkmap = &kmapseg;     /* Kernel generic mapping segment */
 318 static struct seg *segmap = &kmapseg;       /* easier to use name for in here */
 319 
 320 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */
 321 
 322 #if defined(__amd64)
 323 struct seg kvseg_core;          /* Segment used for the core heap */
 324 struct seg kpmseg;              /* Segment used for physical mapping */
 325 struct seg *segkpm = &kpmseg;       /* 64bit kernel physical mapping segment */
 326 #else
 327 struct seg *segkpm = NULL;      /* Unused on IA32 */
 328 #endif
 329 
 330 caddr_t segkp_base;             /* Base address of segkp */
 331 caddr_t segzio_base;            /* Base address of segzio */
 332 #if defined(__amd64)
 333 pgcnt_t segkpsize = btop(SEGKPDEFSIZE); /* size of segkp segment in pages */
 334 #else
 335 pgcnt_t segkpsize = 0;
 336 #endif
 337 pgcnt_t segziosize = 0;         /* size of zio segment in pages */
 338 
 339 /*
 340  * A static DR page_t VA map is reserved that can map the page structures
 341  * for a domain's entire RA space. The pages that back this space are
 342  * dynamically allocated and need not be physically contiguous.  The DR
 343  * map size is derived from KPM size.
 344  * This mechanism isn't used by x86 yet, so just stubs here.
 345  */
 346 int ppvm_enable = 0;            /* Static virtual map for page structs */
 347 page_t *ppvm_base = NULL;       /* Base of page struct map */
 348 pgcnt_t ppvm_size = 0;          /* Size of page struct map */
 349 
 350 /*
 351  * VA range available to the debugger
 352  */
 353 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
 354 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
 355 
 356 struct memseg *memseg_base;
 357 struct vnode unused_pages_vp;
 358 
 359 #define FOURGB  0x100000000LL
 360 
 361 struct memlist *memlist;
 362 
 363 caddr_t s_text;         /* start of kernel text segment */
 364 caddr_t e_text;         /* end of kernel text segment */
 365 caddr_t s_data;         /* start of kernel data segment */
 366 caddr_t e_data;         /* end of kernel data segment */
 367 caddr_t modtext;        /* start of loadable module text reserved */
 368 caddr_t e_modtext;      /* end of loadable module text reserved */
 369 caddr_t moddata;        /* start of loadable module data reserved */
 370 caddr_t e_moddata;      /* end of loadable module data reserved */
 371 
 372 struct memlist *phys_install;   /* Total installed physical memory */
 373 struct memlist *phys_avail;     /* Total available physical memory */
 374 struct memlist *bios_rsvd;      /* Bios reserved memory */
 375 
 376 /*
 377  * kphysm_init returns the number of pages that were processed
 378  */
 379 static pgcnt_t kphysm_init(page_t *, pgcnt_t);
 380 
 381 #define IO_PROP_SIZE    64      /* device property size */
 382 
 383 /*
 384  * a couple useful roundup macros
 385  */
 386 #define ROUND_UP_PAGE(x)        \
 387         ((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)MMU_PAGESIZE))
 388 #define ROUND_UP_LPAGE(x)       \
 389         ((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[1]))
 390 #define ROUND_UP_4MEG(x)        \
 391         ((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)FOUR_MEG))
 392 #define ROUND_UP_TOPLEVEL(x)    \
 393         ((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[mmu.max_level]))
 394 
 395 /*
 396  *      32-bit Kernel's Virtual memory layout.
 397  *              +-----------------------+
 398  *              |                       |
 399  * 0xFFC00000  -|-----------------------|- ARGSBASE
 400  *              |       debugger        |
 401  * 0xFF800000  -|-----------------------|- SEGDEBUGBASE
 402  *              |      Kernel Data      |
 403  * 0xFEC00000  -|-----------------------|
 404  *              |      Kernel Text      |
 405  * 0xFE800000  -|-----------------------|- KERNEL_TEXT (0xFB400000 on Xen)
 406  *              |---       GDT       ---|- GDT page (GDT_VA)
 407  *              |---    debug info   ---|- debug info (DEBUG_INFO_VA)
 408  *              |                       |
 409  *              |   page_t structures   |
 410  *              |   memsegs, memlists,  |
 411  *              |   page hash, etc.     |
 412  * ---         -|-----------------------|- ekernelheap, valloc_base (floating)
 413  *              |                       |  (segkp is just an arena in the heap)
 414  *              |                       |
 415  *              |       kvseg           |
 416  *              |                       |
 417  *              |                       |
 418  * ---         -|-----------------------|- kernelheap (floating)
 419  *              |        Segkmap        |
 420  * 0xC3002000  -|-----------------------|- segmap_start (floating)
 421  *              |       Red Zone        |
 422  * 0xC3000000  -|-----------------------|- kernelbase / userlimit (floating)
 423  *              |                       |                       ||
 424  *              |     Shared objects    |                       \/
 425  *              |                       |
 426  *              :                       :
 427  *              |       user data       |
 428  *              |-----------------------|
 429  *              |       user text       |
 430  * 0x08048000  -|-----------------------|
 431  *              |       user stack      |
 432  *              :                       :
 433  *              |       invalid         |
 434  * 0x00000000   +-----------------------+
 435  *
 436  *
 437  *              64-bit Kernel's Virtual memory layout. (assuming 64 bit app)
 438  *                      +-----------------------+
 439  *                      |                       |
 440  * 0xFFFFFFFF.FFC00000  |-----------------------|- ARGSBASE
 441  *                      |       debugger (?)    |
 442  * 0xFFFFFFFF.FF800000  |-----------------------|- SEGDEBUGBASE
 443  *                      |      unused           |
 444  *                      +-----------------------+
 445  *                      |      Kernel Data      |
 446  * 0xFFFFFFFF.FBC00000  |-----------------------|
 447  *                      |      Kernel Text      |
 448  * 0xFFFFFFFF.FB800000  |-----------------------|- KERNEL_TEXT
 449  *                      |---    debug info   ---|- debug info (DEBUG_INFO_VA)
 450  *                      |---       GDT       ---|- GDT page (GDT_VA)
 451  *                      |---       IDT       ---|- IDT page (IDT_VA)
 452  *                      |---       LDT       ---|- LDT pages (LDT_VA)
 453  *                      |                       |
 454  *                      |      Core heap        | (used for loadable modules)
 455  * 0xFFFFFFFF.C0000000  |-----------------------|- core_base / ekernelheap
 456  *                      |        Kernel         |
 457  *                      |         heap          |
 458  * 0xFFFFFXXX.XXX00000  |-----------------------|- kernelheap (floating)
 459  *                      |        segmap         |
 460  * 0xFFFFFXXX.XXX00000  |-----------------------|- segmap_start (floating)
 461  *                      |    device mappings    |
 462  * 0xFFFFFXXX.XXX00000  |-----------------------|- toxic_addr (floating)
 463  *                      |         segzio        |
 464  * 0xFFFFFXXX.XXX00000  |-----------------------|- segzio_base (floating)
 465  *                      |         segkp         |
 466  * ---                  |-----------------------|- segkp_base (floating)
 467  *                      |   page_t structures   |  valloc_base + valloc_sz
 468  *                      |   memsegs, memlists,  |
 469  *                      |   page hash, etc.     |
 470  * 0xFFFFFF00.00000000  |-----------------------|- valloc_base (lower if >256GB)
 471  *                      |        segkpm         |
 472  * 0xFFFFFE00.00000000  |-----------------------|
 473  *                      |       Red Zone        |
 474  * 0xFFFFFD80.00000000  |-----------------------|- KERNELBASE (lower if >256GB)
 475  *                      |     User stack        |- User space memory
 476  *                      |                       |
 477  *                      | shared objects, etc   |       (grows downwards)
 478  *                      :                       :
 479  *                      |                       |
 480  * 0xFFFF8000.00000000  |-----------------------|
 481  *                      |                       |
 482  *                      | VA Hole / unused      |
 483  *                      |                       |
 484  * 0x00008000.00000000  |-----------------------|
 485  *                      |                       |
 486  *                      |                       |
 487  *                      :                       :
 488  *                      |       user heap       |       (grows upwards)
 489  *                      |                       |
 490  *                      |       user data       |
 491  *                      |-----------------------|
 492  *                      |       user text       |
 493  * 0x00000000.04000000  |-----------------------|
 494  *                      |       invalid         |
 495  * 0x00000000.00000000  +-----------------------+
 496  *
 497  * A 32 bit app on the 64 bit kernel sees the same layout as on the 32 bit
 498  * kernel, except that userlimit is raised to 0xfe000000
 499  *
 500  * Floating values:
 501  *
 502  * valloc_base: start of the kernel's memory management/tracking data
 503  * structures.  This region contains page_t structures for
 504  * physical memory, memsegs, memlists, and the page hash.
 505  *
 506  * core_base: start of the kernel's "core" heap area on 64-bit systems.
 507  * This area is intended to be used for global data as well as for module
 508  * text/data that does not fit into the nucleus pages.  The core heap is
 509  * restricted to a 2GB range, allowing every address within it to be
 510  * accessed using rip-relative addressing
 511  *
 512  * ekernelheap: end of kernelheap and start of segmap.
 513  *
 514  * kernelheap: start of kernel heap.  On 32-bit systems, this starts right
 515  * above a red zone that separates the user's address space from the
 516  * kernel's.  On 64-bit systems, it sits above segkp and segkpm.
 517  *
 518  * segmap_start: start of segmap. The length of segmap can be modified
 519  * through eeprom. The default length is 16MB on 32-bit systems and 64MB
 520  * on 64-bit systems.
 521  *
 522  * kernelbase: On a 32-bit kernel the default value of 0xd4000000 will be
 523  * decreased by 2X the size required for page_t.  This allows the kernel
 524  * heap to grow in size with physical memory.  With sizeof(page_t) == 80
 525  * bytes, the following shows the values of kernelbase and kernel heap
 526  * sizes for different memory configurations (assuming default segmap and
 527  * segkp sizes).
 528  *
 529  *      mem     size for        kernelbase      kernel heap
 530  *      size    page_t's                        size
 531  *      ----    ---------       ----------      -----------
 532  *      1gb     0x01400000      0xd1800000      684MB
 533  *      2gb     0x02800000      0xcf000000      704MB
 534  *      4gb     0x05000000      0xca000000      744MB
 535  *      6gb     0x07800000      0xc5000000      784MB
 536  *      8gb     0x0a000000      0xc0000000      824MB
 537  *      16gb    0x14000000      0xac000000      984MB
 538  *      32gb    0x28000000      0x84000000      1304MB
 539  *      64gb    0x50000000      0x34000000      1944MB (*)
 540  *
 541  * kernelbase is less than the abi minimum of 0xc0000000 for memory
 542  * configurations above 8gb.
 543  *
 544  * (*) support for memory configurations above 32gb will require manual tuning
 545  * of kernelbase to balance out the need of user applications.
 546  */
 547 
 548 /* real-time-clock initialization parameters */
 549 extern time_t process_rtc_config_file(void);
 550 
 551 uintptr_t       kernelbase;
 552 uintptr_t       postbootkernelbase;     /* not set till boot loader is gone */
 553 uintptr_t       eprom_kernelbase;
 554 size_t          segmapsize;
 555 uintptr_t       segmap_start;
 556 int             segmapfreelists;
 557 pgcnt_t         npages;
 558 pgcnt_t         orig_npages;
 559 size_t          core_size;              /* size of "core" heap */
 560 uintptr_t       core_base;              /* base address of "core" heap */
 561 
 562 /*
 563  * List of bootstrap pages. We mark these as allocated in startup.
 564  * release_bootstrap() will free them when we're completely done with
 565  * the bootstrap.
 566  */
 567 static page_t *bootpages;
 568 
 569 /*
 570  * boot time pages that have a vnode from the ramdisk will keep that forever.
 571  */
 572 static page_t *rd_pages;
 573 
 574 /*
 575  * Lower 64K
 576  */
 577 static page_t *lower_pages = NULL;
 578 static int lower_pages_count = 0;
 579 
 580 struct system_hardware system_hardware;
 581 
 582 /*
 583  * Enable some debugging messages concerning memory usage...
 584  */
 585 static void
 586 print_memlist(char *title, struct memlist *mp)
 587 {
 588         prom_printf("MEMLIST: %s:\n", title);
 589         while (mp != NULL)  {
 590                 prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n",
 591                     mp->ml_address, mp->ml_size);
 592                 mp = mp->ml_next;
 593         }
 594 }
 595 
 596 /*
 597  * XX64 need a comment here.. are these just default values, surely
 598  * we read the "cpuid" type information to figure this out.
 599  */
 600 int     l2cache_sz = 0x80000;
 601 int     l2cache_linesz = 0x40;
 602 int     l2cache_assoc = 1;
 603 
 604 static size_t   textrepl_min_gb = 10;
 605 
 606 /*
 607  * on 64 bit we use a predifined VA range for mapping devices in the kernel
 608  * on 32 bit the mappings are intermixed in the heap, so we use a bit map
 609  */
 610 #ifdef __amd64
 611 
 612 vmem_t          *device_arena;
 613 uintptr_t       toxic_addr = (uintptr_t)NULL;
 614 size_t          toxic_size = 1024 * 1024 * 1024; /* Sparc uses 1 gig too */
 615 
 616 #else   /* __i386 */
 617 
 618 ulong_t         *toxic_bit_map; /* one bit for each 4k of VA in heap_arena */
 619 size_t          toxic_bit_map_len = 0;  /* in bits */
 620 
 621 #endif  /* __i386 */
 622 
 623 /*
 624  * Simple boot time debug facilities
 625  */
 626 static char *prm_dbg_str[] = {
 627         "%s:%d: '%s' is 0x%x\n",
 628         "%s:%d: '%s' is 0x%llx\n"
 629 };
 630 
 631 int prom_debug;
 632 
 633 #define PRM_DEBUG(q)    if (prom_debug)         \
 634         prom_printf(prm_dbg_str[sizeof (q) >> 3], "startup.c", __LINE__, #q, q);
 635 #define PRM_POINT(q)    if (prom_debug)         \
 636         prom_printf("%s:%d: %s\n", "startup.c", __LINE__, q);
 637 
 638 /*
 639  * This structure is used to keep track of the intial allocations
 640  * done in startup_memlist(). The value of NUM_ALLOCATIONS needs to
 641  * be >= the number of ADD_TO_ALLOCATIONS() executed in the code.
 642  */
 643 #define NUM_ALLOCATIONS 8
 644 int num_allocations = 0;
 645 struct {
 646         void **al_ptr;
 647         size_t al_size;
 648 } allocations[NUM_ALLOCATIONS];
 649 size_t valloc_sz = 0;
 650 uintptr_t valloc_base;
 651 
 652 #define ADD_TO_ALLOCATIONS(ptr, size) {                                 \
 653                 size = ROUND_UP_PAGE(size);                             \
 654                 if (num_allocations == NUM_ALLOCATIONS)                 \
 655                         panic("too many ADD_TO_ALLOCATIONS()");         \
 656                 allocations[num_allocations].al_ptr = (void**)&ptr; \
 657                 allocations[num_allocations].al_size = size;            \
 658                 valloc_sz += size;                                      \
 659                 ++num_allocations;                                      \
 660         }
 661 
 662 /*
 663  * Allocate all the initial memory needed by the page allocator.
 664  */
 665 static void
 666 perform_allocations(void)
 667 {
 668         caddr_t mem;
 669         int i;
 670         int valloc_align;
 671 
 672         PRM_DEBUG(valloc_base);
 673         PRM_DEBUG(valloc_sz);
 674         valloc_align = mmu.level_size[mmu.max_page_level > 0];
 675         mem = BOP_ALLOC(bootops, (caddr_t)valloc_base, valloc_sz, valloc_align);
 676         if (mem != (caddr_t)valloc_base)
 677                 panic("BOP_ALLOC() failed");
 678         bzero(mem, valloc_sz);
 679         for (i = 0; i < num_allocations; ++i) {
 680                 *allocations[i].al_ptr = (void *)mem;
 681                 mem += allocations[i].al_size;
 682         }
 683 }
 684 
 685 /*
 686  * Set up and enable SMAP now before we start other CPUs, but after the kernel's
 687  * VM has been set up so we can use hot_patch_kernel_text().
 688  *
 689  * We can only patch 1, 2, or 4 bytes, but not three bytes. So instead, we
 690  * replace the four byte word at the patch point. See uts/intel/ia32/ml/copy.s
 691  * for more information on what's going on here.
 692  */
 693 static void
 694 startup_smap(void)
 695 {
 696         int i;
 697         uint32_t inst;
 698         uint8_t *instp;
 699         char sym[128];
 700 
 701         extern int _smap_enable_patch_count;
 702         extern int _smap_disable_patch_count;
 703 
 704         if (disable_smap != 0)
 705                 remove_x86_feature(x86_featureset, X86FSET_SMAP);
 706 
 707         if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE)
 708                 return;
 709 
 710         for (i = 0; i < _smap_enable_patch_count; i++) {
 711                 int sizep;
 712 
 713                 VERIFY3U(i, <, _smap_enable_patch_count);
 714                 VERIFY(snprintf(sym, sizeof (sym), "_smap_enable_patch_%d", i) <
 715                     sizeof (sym));
 716                 instp = (uint8_t *)(void *)kobj_getelfsym(sym, NULL, &sizep);
 717                 VERIFY(instp != 0);
 718                 inst = (instp[3] << 24) | (SMAP_CLAC_INSTR & 0x00ffffff);
 719                 hot_patch_kernel_text((caddr_t)instp, inst, 4);
 720         }
 721 
 722         for (i = 0; i < _smap_disable_patch_count; i++) {
 723                 int sizep;
 724 
 725                 VERIFY(snprintf(sym, sizeof (sym), "_smap_disable_patch_%d",
 726                     i) < sizeof (sym));
 727                 instp = (uint8_t *)(void *)kobj_getelfsym(sym, NULL, &sizep);
 728                 VERIFY(instp != 0);
 729                 inst = (instp[3] << 24) | (SMAP_STAC_INSTR & 0x00ffffff);
 730                 hot_patch_kernel_text((caddr_t)instp, inst, 4);
 731         }
 732 
 733         hot_patch_kernel_text((caddr_t)smap_enable, SMAP_CLAC_INSTR, 4);
 734         hot_patch_kernel_text((caddr_t)smap_disable, SMAP_STAC_INSTR, 4);
 735         setcr4(getcr4() | CR4_SMAP);
 736         smap_enable();
 737 }
 738 
 739 /*
 740  * Our world looks like this at startup time.
 741  *
 742  * In a 32-bit OS, boot loads the kernel text at 0xfe800000 and kernel data
 743  * at 0xfec00000.  On a 64-bit OS, kernel text and data are loaded at
 744  * 0xffffffff.fe800000 and 0xffffffff.fec00000 respectively.  Those
 745  * addresses are fixed in the binary at link time.
 746  *
 747  * On the text page:
 748  * unix/genunix/krtld/module text loads.
 749  *
 750  * On the data page:
 751  * unix/genunix/krtld/module data loads.
 752  *
 753  * Machine-dependent startup code
 754  */
 755 void
 756 startup(void)
 757 {
 758 #if !defined(__xpv)
 759         extern void startup_pci_bios(void);
 760 #endif
 761         extern cpuset_t cpu_ready_set;
 762 
 763         /*
 764          * Make sure that nobody tries to use sekpm until we have
 765          * initialized it properly.
 766          */
 767 #if defined(__amd64)
 768         kpm_desired = 1;
 769 #endif
 770         kpm_enable = 0;
 771         CPUSET_ONLY(cpu_ready_set, 0);  /* cpu 0 is boot cpu */
 772 
 773 #if defined(__xpv)      /* XXPV fix me! */
 774         {
 775                 extern int segvn_use_regions;
 776                 segvn_use_regions = 0;
 777         }
 778 #endif
 779         ssp_init();
 780         progressbar_init();
 781         startup_init();
 782 #if defined(__xpv)
 783         startup_xen_version();
 784 #endif
 785         startup_memlist();
 786         startup_kmem();
 787         startup_vm();
 788 #if !defined(__xpv)
 789         /*
 790          * Note we need to do this even on fast reboot in order to access
 791          * the irq routing table (used for pci labels).
 792          */
 793         startup_pci_bios();
 794         startup_smap();
 795 #endif
 796 #if defined(__xpv)
 797         startup_xen_mca();
 798 #endif
 799         startup_modules();
 800 
 801         startup_end();
 802 }
 803 
 804 static void
 805 startup_init()
 806 {
 807         PRM_POINT("startup_init() starting...");
 808 
 809         /*
 810          * Complete the extraction of cpuid data
 811          */
 812         cpuid_pass2(CPU);
 813 
 814         (void) check_boot_version(BOP_GETVERSION(bootops));
 815 
 816         /*
 817          * Check for prom_debug in boot environment
 818          */
 819         if (BOP_GETPROPLEN(bootops, "prom_debug") >= 0) {
 820                 ++prom_debug;
 821                 PRM_POINT("prom_debug found in boot enviroment");
 822         }
 823 
 824         /*
 825          * Collect node, cpu and memory configuration information.
 826          */
 827         get_system_configuration();
 828 
 829         /*
 830          * Halt if this is an unsupported processor.
 831          */
 832         if (x86_type == X86_TYPE_486 || x86_type == X86_TYPE_CYRIX_486) {
 833                 printf("\n486 processor (\"%s\") detected.\n",
 834                     CPU->cpu_brandstr);
 835                 halt("This processor is not supported by this release "
 836                     "of Solaris.");
 837         }
 838 
 839         PRM_POINT("startup_init() done");
 840 }
 841 
 842 /*
 843  * Callback for copy_memlist_filter() to filter nucleus, kadb/kmdb, (ie.
 844  * everything mapped above KERNEL_TEXT) pages from phys_avail. Note it
 845  * also filters out physical page zero.  There is some reliance on the
 846  * boot loader allocating only a few contiguous physical memory chunks.
 847  */
 848 static void
 849 avail_filter(uint64_t *addr, uint64_t *size)
 850 {
 851         uintptr_t va;
 852         uintptr_t next_va;
 853         pfn_t pfn;
 854         uint64_t pfn_addr;
 855         uint64_t pfn_eaddr;
 856         uint_t prot;
 857         size_t len;
 858         uint_t change;
 859 
 860         if (prom_debug)
 861                 prom_printf("\tFilter: in: a=%" PRIx64 ", s=%" PRIx64 "\n",
 862                     *addr, *size);
 863 
 864         /*
 865          * page zero is required for BIOS.. never make it available
 866          */
 867         if (*addr == 0) {
 868                 *addr += MMU_PAGESIZE;
 869                 *size -= MMU_PAGESIZE;
 870         }
 871 
 872         /*
 873          * First we trim from the front of the range. Since kbm_probe()
 874          * walks ranges in virtual order, but addr/size are physical, we need
 875          * to the list until no changes are seen.  This deals with the case
 876          * where page "p" is mapped at v, page "p + PAGESIZE" is mapped at w
 877          * but w < v.
 878          */
 879         do {
 880                 change = 0;
 881                 for (va = KERNEL_TEXT;
 882                     *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0;
 883                     va = next_va) {
 884 
 885                         next_va = va + len;
 886                         pfn_addr = pfn_to_pa(pfn);
 887                         pfn_eaddr = pfn_addr + len;
 888 
 889                         if (pfn_addr <= *addr && pfn_eaddr > *addr) {
 890                                 change = 1;
 891                                 while (*size > 0 && len > 0) {
 892                                         *addr += MMU_PAGESIZE;
 893                                         *size -= MMU_PAGESIZE;
 894                                         len -= MMU_PAGESIZE;
 895                                 }
 896                         }
 897                 }
 898                 if (change && prom_debug)
 899                         prom_printf("\t\ttrim: a=%" PRIx64 ", s=%" PRIx64 "\n",
 900                             *addr, *size);
 901         } while (change);
 902 
 903         /*
 904          * Trim pages from the end of the range.
 905          */
 906         for (va = KERNEL_TEXT;
 907             *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0;
 908             va = next_va) {
 909 
 910                 next_va = va + len;
 911                 pfn_addr = pfn_to_pa(pfn);
 912 
 913                 if (pfn_addr >= *addr && pfn_addr < *addr + *size)
 914                         *size = pfn_addr - *addr;
 915         }
 916 
 917         if (prom_debug)
 918                 prom_printf("\tFilter out: a=%" PRIx64 ", s=%" PRIx64 "\n",
 919                     *addr, *size);
 920 }
 921 
 922 static void
 923 kpm_init()
 924 {
 925         struct segkpm_crargs b;
 926 
 927         /*
 928          * These variables were all designed for sfmmu in which segkpm is
 929          * mapped using a single pagesize - either 8KB or 4MB.  On x86, we
 930          * might use 2+ page sizes on a single machine, so none of these
 931          * variables have a single correct value.  They are set up as if we
 932          * always use a 4KB pagesize, which should do no harm.  In the long
 933          * run, we should get rid of KPM's assumption that only a single
 934          * pagesize is used.
 935          */
 936         kpm_pgshft = MMU_PAGESHIFT;
 937         kpm_pgsz =  MMU_PAGESIZE;
 938         kpm_pgoff = MMU_PAGEOFFSET;
 939         kpmp2pshft = 0;
 940         kpmpnpgs = 1;
 941         ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0);
 942 
 943         PRM_POINT("about to create segkpm");
 944         rw_enter(&kas.a_lock, RW_WRITER);
 945 
 946         if (seg_attach(&kas, kpm_vbase, kpm_size, segkpm) < 0)
 947                 panic("cannot attach segkpm");
 948 
 949         b.prot = PROT_READ | PROT_WRITE;
 950         b.nvcolors = 1;
 951 
 952         if (segkpm_create(segkpm, (caddr_t)&b) != 0)
 953                 panic("segkpm_create segkpm");
 954 
 955         rw_exit(&kas.a_lock);
 956 
 957         kpm_enable = 1;
 958 
 959         /*
 960          * As the KPM was disabled while setting up the system, go back and fix
 961          * CPU zero's access to its user page table. This is a bit gross, but
 962          * we have a chicken and egg problem otherwise.
 963          */
 964         ASSERT(CPU->cpu_hat_info->hci_user_l3ptes == NULL);
 965         CPU->cpu_hat_info->hci_user_l3ptes =
 966             (x86pte_t *)hat_kpm_mapin_pfn(CPU->cpu_hat_info->hci_user_l3pfn);
 967 }
 968 
 969 /*
 970  * The debug info page provides enough information to allow external
 971  * inspectors (e.g. when running under a hypervisor) to bootstrap
 972  * themselves into allowing full-blown kernel debugging.
 973  */
 974 static void
 975 init_debug_info(void)
 976 {
 977         caddr_t mem;
 978         debug_info_t *di;
 979 
 980 #ifndef __lint
 981         ASSERT(sizeof (debug_info_t) < MMU_PAGESIZE);
 982 #endif
 983 
 984         mem = BOP_ALLOC(bootops, (caddr_t)DEBUG_INFO_VA, MMU_PAGESIZE,
 985             MMU_PAGESIZE);
 986 
 987         if (mem != (caddr_t)DEBUG_INFO_VA)
 988                 panic("BOP_ALLOC() failed");
 989         bzero(mem, MMU_PAGESIZE);
 990 
 991         di = (debug_info_t *)mem;
 992 
 993         di->di_magic = DEBUG_INFO_MAGIC;
 994         di->di_version = DEBUG_INFO_VERSION;
 995         di->di_modules = (uintptr_t)&modules;
 996         di->di_s_text = (uintptr_t)s_text;
 997         di->di_e_text = (uintptr_t)e_text;
 998         di->di_s_data = (uintptr_t)s_data;
 999         di->di_e_data = (uintptr_t)e_data;
1000         di->di_hat_htable_off = offsetof(hat_t, hat_htable);
1001         di->di_ht_pfn_off = offsetof(htable_t, ht_pfn);
1002 }
1003 
1004 /*
1005  * Build the memlists and other kernel essential memory system data structures.
1006  * This is everything at valloc_base.
1007  */
1008 static void
1009 startup_memlist(void)
1010 {
1011         size_t memlist_sz;
1012         size_t memseg_sz;
1013         size_t pagehash_sz;
1014         size_t pp_sz;
1015         uintptr_t va;
1016         size_t len;
1017         uint_t prot;
1018         pfn_t pfn;
1019         int memblocks;
1020         pfn_t rsvd_high_pfn;
1021         pgcnt_t rsvd_pgcnt;
1022         size_t rsvdmemlist_sz;
1023         int rsvdmemblocks;
1024         caddr_t pagecolor_mem;
1025         size_t pagecolor_memsz;
1026         caddr_t page_ctrs_mem;
1027         size_t page_ctrs_size;
1028         size_t pse_table_alloc_size;
1029         struct memlist *current;
1030         extern void startup_build_mem_nodes(struct memlist *);
1031 
1032         /* XX64 fix these - they should be in include files */
1033         extern size_t page_coloring_init(uint_t, int, int);
1034         extern void page_coloring_setup(caddr_t);
1035 
1036         PRM_POINT("startup_memlist() starting...");
1037 
1038         /*
1039          * Use leftover large page nucleus text/data space for loadable modules.
1040          * Use at most MODTEXT/MODDATA.
1041          */
1042         len = kbm_nucleus_size;
1043         ASSERT(len > MMU_PAGESIZE);
1044 
1045         moddata = (caddr_t)ROUND_UP_PAGE(e_data);
1046         e_moddata = (caddr_t)P2ROUNDUP((uintptr_t)e_data, (uintptr_t)len);
1047         if (e_moddata - moddata > MODDATA)
1048                 e_moddata = moddata + MODDATA;
1049 
1050         modtext = (caddr_t)ROUND_UP_PAGE(e_text);
1051         e_modtext = (caddr_t)P2ROUNDUP((uintptr_t)e_text, (uintptr_t)len);
1052         if (e_modtext - modtext > MODTEXT)
1053                 e_modtext = modtext + MODTEXT;
1054 
1055         econtig = e_moddata;
1056 
1057         PRM_DEBUG(modtext);
1058         PRM_DEBUG(e_modtext);
1059         PRM_DEBUG(moddata);
1060         PRM_DEBUG(e_moddata);
1061         PRM_DEBUG(econtig);
1062 
1063         /*
1064          * Examine the boot loader physical memory map to find out:
1065          * - total memory in system - physinstalled
1066          * - the max physical address - physmax
1067          * - the number of discontiguous segments of memory.
1068          */
1069         if (prom_debug)
1070                 print_memlist("boot physinstalled",
1071                     bootops->boot_mem->physinstalled);
1072         installed_top_size_ex(bootops->boot_mem->physinstalled, &physmax,
1073             &physinstalled, &memblocks);
1074         PRM_DEBUG(physmax);
1075         PRM_DEBUG(physinstalled);
1076         PRM_DEBUG(memblocks);
1077 
1078         /*
1079          * Compute maximum physical address for memory DR operations.
1080          * Memory DR operations are unsupported on xpv or 32bit OSes.
1081          */
1082 #ifdef  __amd64
1083         if (plat_dr_support_memory()) {
1084                 if (plat_dr_physmax == 0) {
1085                         uint_t pabits = UINT_MAX;
1086 
1087                         cpuid_get_addrsize(CPU, &pabits, NULL);
1088                         plat_dr_physmax = btop(1ULL << pabits);
1089                 }
1090                 if (plat_dr_physmax > PHYSMEM_MAX64)
1091                         plat_dr_physmax = PHYSMEM_MAX64;
1092         } else
1093 #endif
1094                 plat_dr_physmax = 0;
1095 
1096         /*
1097          * Examine the bios reserved memory to find out:
1098          * - the number of discontiguous segments of memory.
1099          */
1100         if (prom_debug)
1101                 print_memlist("boot reserved mem",
1102                     bootops->boot_mem->rsvdmem);
1103         installed_top_size_ex(bootops->boot_mem->rsvdmem, &rsvd_high_pfn,
1104             &rsvd_pgcnt, &rsvdmemblocks);
1105         PRM_DEBUG(rsvd_high_pfn);
1106         PRM_DEBUG(rsvd_pgcnt);
1107         PRM_DEBUG(rsvdmemblocks);
1108 
1109         /*
1110          * Initialize hat's mmu parameters.
1111          * Check for enforce-prot-exec in boot environment. It's used to
1112          * enable/disable support for the page table entry NX bit.
1113          * The default is to enforce PROT_EXEC on processors that support NX.
1114          * Boot seems to round up the "len", but 8 seems to be big enough.
1115          */
1116         mmu_init();
1117 
1118 #ifdef  __i386
1119         /*
1120          * physmax is lowered if there is more memory than can be
1121          * physically addressed in 32 bit (PAE/non-PAE) modes.
1122          */
1123         if (mmu.pae_hat) {
1124                 if (PFN_ABOVE64G(physmax)) {
1125                         physinstalled -= (physmax - (PFN_64G - 1));
1126                         physmax = PFN_64G - 1;
1127                 }
1128         } else {
1129                 if (PFN_ABOVE4G(physmax)) {
1130                         physinstalled -= (physmax - (PFN_4G - 1));
1131                         physmax = PFN_4G - 1;
1132                 }
1133         }
1134 #endif
1135 
1136         startup_build_mem_nodes(bootops->boot_mem->physinstalled);
1137 
1138         if (BOP_GETPROPLEN(bootops, "enforce-prot-exec") >= 0) {
1139                 int len = BOP_GETPROPLEN(bootops, "enforce-prot-exec");
1140                 char value[8];
1141 
1142                 if (len < 8)
1143                         (void) BOP_GETPROP(bootops, "enforce-prot-exec", value);
1144                 else
1145                         (void) strcpy(value, "");
1146                 if (strcmp(value, "off") == 0)
1147                         mmu.pt_nx = 0;
1148         }
1149         PRM_DEBUG(mmu.pt_nx);
1150 
1151         /*
1152          * We will need page_t's for every page in the system, except for
1153          * memory mapped at or above above the start of the kernel text segment.
1154          *
1155          * pages above e_modtext are attributed to kernel debugger (obp_pages)
1156          */
1157         npages = physinstalled - 1; /* avail_filter() skips page 0, so "- 1" */
1158         obp_pages = 0;
1159         va = KERNEL_TEXT;
1160         while (kbm_probe(&va, &len, &pfn, &prot) != 0) {
1161                 npages -= len >> MMU_PAGESHIFT;
1162                 if (va >= (uintptr_t)e_moddata)
1163                         obp_pages += len >> MMU_PAGESHIFT;
1164                 va += len;
1165         }
1166         PRM_DEBUG(npages);
1167         PRM_DEBUG(obp_pages);
1168 
1169         /*
1170          * If physmem is patched to be non-zero, use it instead of the computed
1171          * value unless it is larger than the actual amount of memory on hand.
1172          */
1173         if (physmem == 0 || physmem > npages) {
1174                 physmem = npages;
1175         } else if (physmem < npages) {
1176                 orig_npages = npages;
1177                 npages = physmem;
1178         }
1179         PRM_DEBUG(physmem);
1180 
1181         /*
1182          * We now compute the sizes of all the  initial allocations for
1183          * structures the kernel needs in order do kmem_alloc(). These
1184          * include:
1185          *      memsegs
1186          *      memlists
1187          *      page hash table
1188          *      page_t's
1189          *      page coloring data structs
1190          */
1191         memseg_sz = sizeof (struct memseg) * (memblocks + POSS_NEW_FRAGMENTS);
1192         ADD_TO_ALLOCATIONS(memseg_base, memseg_sz);
1193         PRM_DEBUG(memseg_sz);
1194 
1195         /*
1196          * Reserve space for memlists. There's no real good way to know exactly
1197          * how much room we'll need, but this should be a good upper bound.
1198          */
1199         memlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) *
1200             (memblocks + POSS_NEW_FRAGMENTS));
1201         ADD_TO_ALLOCATIONS(memlist, memlist_sz);
1202         PRM_DEBUG(memlist_sz);
1203 
1204         /*
1205          * Reserve space for bios reserved memlists.
1206          */
1207         rsvdmemlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) *
1208             (rsvdmemblocks + POSS_NEW_FRAGMENTS));
1209         ADD_TO_ALLOCATIONS(bios_rsvd, rsvdmemlist_sz);
1210         PRM_DEBUG(rsvdmemlist_sz);
1211 
1212         /* LINTED */
1213         ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), sizeof (struct page)));
1214         /*
1215          * The page structure hash table size is a power of 2
1216          * such that the average hash chain length is PAGE_HASHAVELEN.
1217          */
1218         page_hashsz = npages / PAGE_HASHAVELEN;
1219         page_hashsz_shift = highbit(page_hashsz);
1220         page_hashsz = 1 << page_hashsz_shift;
1221         pagehash_sz = sizeof (struct page *) * page_hashsz;
1222         ADD_TO_ALLOCATIONS(page_hash, pagehash_sz);
1223         PRM_DEBUG(pagehash_sz);
1224 
1225         /*
1226          * Set aside room for the page structures themselves.
1227          */
1228         PRM_DEBUG(npages);
1229         pp_sz = sizeof (struct page) * npages;
1230         ADD_TO_ALLOCATIONS(pp_base, pp_sz);
1231         PRM_DEBUG(pp_sz);
1232 
1233         /*
1234          * determine l2 cache info and memory size for page coloring
1235          */
1236         (void) getl2cacheinfo(CPU,
1237             &l2cache_sz, &l2cache_linesz, &l2cache_assoc);
1238         pagecolor_memsz =
1239             page_coloring_init(l2cache_sz, l2cache_linesz, l2cache_assoc);
1240         ADD_TO_ALLOCATIONS(pagecolor_mem, pagecolor_memsz);
1241         PRM_DEBUG(pagecolor_memsz);
1242 
1243         page_ctrs_size = page_ctrs_sz();
1244         ADD_TO_ALLOCATIONS(page_ctrs_mem, page_ctrs_size);
1245         PRM_DEBUG(page_ctrs_size);
1246 
1247         /*
1248          * Allocate the array that protects pp->p_selock.
1249          */
1250         pse_shift = size_pse_array(physmem, max_ncpus);
1251         pse_table_size = 1 << pse_shift;
1252         pse_table_alloc_size = pse_table_size * sizeof (pad_mutex_t);
1253         ADD_TO_ALLOCATIONS(pse_mutex, pse_table_alloc_size);
1254 
1255 #if defined(__amd64)
1256         valloc_sz = ROUND_UP_LPAGE(valloc_sz);
1257         valloc_base = VALLOC_BASE;
1258 
1259         /*
1260          * The default values of VALLOC_BASE and SEGKPM_BASE should work
1261          * for values of physmax up to 256GB (1/4 TB). They need adjusting when
1262          * memory is at addresses above 256GB. When adjusted, segkpm_base must
1263          * be aligned on KERNEL_REDZONE_SIZE boundary (span of top level pte).
1264          *
1265          * In the general case (>256GB), we use (4 * physmem) for the
1266          * kernel's virtual addresses, which is divided approximately
1267          * as follows:
1268          *  - 1 * physmem for segkpm
1269          *  - 1.5 * physmem for segzio
1270          *  - 1.5 * physmem for heap
1271          * Total: 4.0 * physmem
1272          *
1273          * Note that the segzio and heap sizes are more than physmem so that
1274          * VA fragmentation does not prevent either of them from being
1275          * able to use nearly all of physmem.  The value of 1.5x is determined
1276          * experimentally and may need to change if the workload changes.
1277          */
1278         if (physmax + 1 > mmu_btop(TERABYTE / 4) ||
1279             plat_dr_physmax > mmu_btop(TERABYTE / 4)) {
1280                 uint64_t kpm_resv_amount = mmu_ptob(physmax + 1);
1281 
1282                 if (kpm_resv_amount < mmu_ptob(plat_dr_physmax)) {
1283                         kpm_resv_amount = mmu_ptob(plat_dr_physmax);
1284                 }
1285 
1286                 /*
1287                  * This is what actually controls the KVA : UVA split.
1288                  * The kernel uses high VA, and this is lowering the
1289                  * boundary, thus increasing the amount of VA for the kernel.
1290                  * This gives the kernel 4 * (amount of physical memory) VA.
1291                  *
1292                  * The maximum VA is UINT64_MAX and we are using
1293                  * 64-bit 2's complement math, so e.g. if you have 512GB
1294                  * of memory, segkpm_base = -(4 * 512GB) == -2TB ==
1295                  * UINT64_MAX - 2TB (approximately).  So the kernel's
1296                  * VA is [UINT64_MAX-2TB to UINT64_MAX].
1297                  */
1298                 segkpm_base = -(P2ROUNDUP((4 * kpm_resv_amount),
1299                     KERNEL_REDZONE_SIZE));
1300 
1301                 /* make sure we leave some space for user apps above hole */
1302                 segkpm_base = MAX(segkpm_base, AMD64_VA_HOLE_END + TERABYTE);
1303                 if (segkpm_base > SEGKPM_BASE)
1304                         segkpm_base = SEGKPM_BASE;
1305                 PRM_DEBUG(segkpm_base);
1306 
1307                 valloc_base = segkpm_base + P2ROUNDUP(kpm_resv_amount, ONE_GIG);
1308                 if (valloc_base < segkpm_base)
1309                         panic("not enough kernel VA to support memory size");
1310                 PRM_DEBUG(valloc_base);
1311         }
1312 #else   /* __i386 */
1313         valloc_base = (uintptr_t)(MISC_VA_BASE - valloc_sz);
1314         valloc_base = P2ALIGN(valloc_base, mmu.level_size[1]);
1315         PRM_DEBUG(valloc_base);
1316 #endif  /* __i386 */
1317 
1318         /*
1319          * do all the initial allocations
1320          */
1321         perform_allocations();
1322 
1323         /*
1324          * Build phys_install and phys_avail in kernel memspace.
1325          * - phys_install should be all memory in the system.
1326          * - phys_avail is phys_install minus any memory mapped before this
1327          *    point above KERNEL_TEXT.
1328          */
1329         current = phys_install = memlist;
1330         copy_memlist_filter(bootops->boot_mem->physinstalled, &current, NULL);
1331         if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1332                 panic("physinstalled was too big!");
1333         if (prom_debug)
1334                 print_memlist("phys_install", phys_install);
1335 
1336         phys_avail = current;
1337         PRM_POINT("Building phys_avail:\n");
1338         copy_memlist_filter(bootops->boot_mem->physinstalled, &current,
1339             avail_filter);
1340         if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1341                 panic("physavail was too big!");
1342         if (prom_debug)
1343                 print_memlist("phys_avail", phys_avail);
1344 #ifndef __xpv
1345         /*
1346          * Free unused memlist items, which may be used by memory DR driver
1347          * at runtime.
1348          */
1349         if ((caddr_t)current < (caddr_t)memlist + memlist_sz) {
1350                 memlist_free_block((caddr_t)current,
1351                     (caddr_t)memlist + memlist_sz - (caddr_t)current);
1352         }
1353 #endif
1354 
1355         /*
1356          * Build bios reserved memspace
1357          */
1358         current = bios_rsvd;
1359         copy_memlist_filter(bootops->boot_mem->rsvdmem, &current, NULL);
1360         if ((caddr_t)current > (caddr_t)bios_rsvd + rsvdmemlist_sz)
1361                 panic("bios_rsvd was too big!");
1362         if (prom_debug)
1363                 print_memlist("bios_rsvd", bios_rsvd);
1364 #ifndef __xpv
1365         /*
1366          * Free unused memlist items, which may be used by memory DR driver
1367          * at runtime.
1368          */
1369         if ((caddr_t)current < (caddr_t)bios_rsvd + rsvdmemlist_sz) {
1370                 memlist_free_block((caddr_t)current,
1371                     (caddr_t)bios_rsvd + rsvdmemlist_sz - (caddr_t)current);
1372         }
1373 #endif
1374 
1375         /*
1376          * setup page coloring
1377          */
1378         page_coloring_setup(pagecolor_mem);
1379         page_lock_init();       /* currently a no-op */
1380 
1381         /*
1382          * free page list counters
1383          */
1384         (void) page_ctrs_alloc(page_ctrs_mem);
1385 
1386         /*
1387          * Size the pcf array based on the number of cpus in the box at
1388          * boot time.
1389          */
1390 
1391         pcf_init();
1392 
1393         /*
1394          * Initialize the page structures from the memory lists.
1395          */
1396         availrmem_initial = availrmem = freemem = 0;
1397         PRM_POINT("Calling kphysm_init()...");
1398         npages = kphysm_init(pp_base, npages);
1399         PRM_POINT("kphysm_init() done");
1400         PRM_DEBUG(npages);
1401 
1402         init_debug_info();
1403 
1404         /*
1405          * Now that page_t's have been initialized, remove all the
1406          * initial allocation pages from the kernel free page lists.
1407          */
1408         boot_mapin((caddr_t)valloc_base, valloc_sz);
1409         boot_mapin((caddr_t)MISC_VA_BASE, MISC_VA_SIZE);
1410         PRM_POINT("startup_memlist() done");
1411 
1412         PRM_DEBUG(valloc_sz);
1413 
1414 #if defined(__amd64)
1415         if ((availrmem >> (30 - MMU_PAGESHIFT)) >=
1416             textrepl_min_gb && l2cache_sz <= 2 << 20) {
1417                 extern size_t textrepl_size_thresh;
1418                 textrepl_size_thresh = (16 << 20) - 1;
1419         }
1420 #endif
1421 }
1422 
1423 /*
1424  * Layout the kernel's part of address space and initialize kmem allocator.
1425  */
1426 static void
1427 startup_kmem(void)
1428 {
1429         extern void page_set_colorequiv_arr(void);
1430 #if !defined(__xpv)
1431         extern uint64_t kpti_kbase;
1432 #endif
1433 
1434         PRM_POINT("startup_kmem() starting...");
1435 
1436 #if defined(__amd64)
1437         if (eprom_kernelbase && eprom_kernelbase != KERNELBASE)
1438                 cmn_err(CE_NOTE, "!kernelbase cannot be changed on 64-bit "
1439                     "systems.");
1440         kernelbase = segkpm_base - KERNEL_REDZONE_SIZE;
1441         core_base = (uintptr_t)COREHEAP_BASE;
1442         core_size = (size_t)MISC_VA_BASE - COREHEAP_BASE;
1443 #else   /* __i386 */
1444         /*
1445          * We configure kernelbase based on:
1446          *
1447          * 1. user specified kernelbase via eeprom command. Value cannot exceed
1448          *    KERNELBASE_MAX. we large page align eprom_kernelbase
1449          *
1450          * 2. Default to KERNELBASE and adjust to 2X less the size for page_t.
1451          *    On large memory systems we must lower kernelbase to allow
1452          *    enough room for page_t's for all of memory.
1453          *
1454          * The value set here, might be changed a little later.
1455          */
1456         if (eprom_kernelbase) {
1457                 kernelbase = eprom_kernelbase & mmu.level_mask[1];
1458                 if (kernelbase > KERNELBASE_MAX)
1459                         kernelbase = KERNELBASE_MAX;
1460         } else {
1461                 kernelbase = (uintptr_t)KERNELBASE;
1462                 kernelbase -= ROUND_UP_4MEG(2 * valloc_sz);
1463         }
1464         ASSERT((kernelbase & mmu.level_offset[1]) == 0);
1465         core_base = valloc_base;
1466         core_size = 0;
1467 #endif  /* __i386 */
1468 
1469         PRM_DEBUG(core_base);
1470         PRM_DEBUG(core_size);
1471         PRM_DEBUG(kernelbase);
1472 
1473 #if defined(__i386)
1474         segkp_fromheap = 1;
1475 #endif  /* __i386 */
1476 
1477         ekernelheap = (char *)core_base;
1478         PRM_DEBUG(ekernelheap);
1479 
1480         /*
1481          * Now that we know the real value of kernelbase,
1482          * update variables that were initialized with a value of
1483          * KERNELBASE (in common/conf/param.c).
1484          *
1485          * XXX  The problem with this sort of hackery is that the
1486          *      compiler just may feel like putting the const declarations
1487          *      (in param.c) into the .text section.  Perhaps they should
1488          *      just be declared as variables there?
1489          */
1490 
1491         *(uintptr_t *)&_kernelbase = kernelbase;
1492         *(uintptr_t *)&_userlimit = kernelbase;
1493 #if defined(__amd64)
1494         *(uintptr_t *)&_userlimit -= KERNELBASE - USERLIMIT;
1495 #if !defined(__xpv)
1496         kpti_kbase = kernelbase;
1497 #endif
1498 #else
1499         *(uintptr_t *)&_userlimit32 = _userlimit;
1500 #endif
1501         PRM_DEBUG(_kernelbase);
1502         PRM_DEBUG(_userlimit);
1503         PRM_DEBUG(_userlimit32);
1504 
1505         /* We have to re-do this now that we've modified _userlimit. */
1506         mmu_calc_user_slots();
1507 
1508         layout_kernel_va();
1509 
1510 #if defined(__i386)
1511         /*
1512          * If segmap is too large we can push the bottom of the kernel heap
1513          * higher than the base.  Or worse, it could exceed the top of the
1514          * VA space entirely, causing it to wrap around.
1515          */
1516         if (kernelheap >= ekernelheap || (uintptr_t)kernelheap < kernelbase)
1517                 panic("too little address space available for kernelheap,"
1518                     " use eeprom for lower kernelbase or smaller segmapsize");
1519 #endif  /* __i386 */
1520 
1521         /*
1522          * Initialize the kernel heap. Note 3rd argument must be > 1st.
1523          */
1524         kernelheap_init(kernelheap, ekernelheap,
1525             kernelheap + MMU_PAGESIZE,
1526             (void *)core_base, (void *)(core_base + core_size));
1527 
1528 #if defined(__xpv)
1529         /*
1530          * Link pending events struct into cpu struct
1531          */
1532         CPU->cpu_m.mcpu_evt_pend = &cpu0_evt_data;
1533 #endif
1534         /*
1535          * Initialize kernel memory allocator.
1536          */
1537         kmem_init();
1538 
1539         /*
1540          * Factor in colorequiv to check additional 'equivalent' bins
1541          */
1542         page_set_colorequiv_arr();
1543 
1544         /*
1545          * print this out early so that we know what's going on
1546          */
1547         print_x86_featureset(x86_featureset);
1548 
1549         /*
1550          * Initialize bp_mapin().
1551          */
1552         bp_init(MMU_PAGESIZE, HAT_STORECACHING_OK);
1553 
1554         /*
1555          * orig_npages is non-zero if physmem has been configured for less
1556          * than the available memory.
1557          */
1558         if (orig_npages) {
1559                 cmn_err(CE_WARN, "!%slimiting physmem to 0x%lx of 0x%lx pages",
1560                     (npages == PHYSMEM ? "Due to virtual address space " : ""),
1561                     npages, orig_npages);
1562         }
1563 #if defined(__i386)
1564         if (eprom_kernelbase && (eprom_kernelbase != kernelbase))
1565                 cmn_err(CE_WARN, "kernelbase value, User specified 0x%lx, "
1566                     "System using 0x%lx",
1567                     (uintptr_t)eprom_kernelbase, (uintptr_t)kernelbase);
1568 #endif
1569 
1570 #ifdef  KERNELBASE_ABI_MIN
1571         if (kernelbase < (uintptr_t)KERNELBASE_ABI_MIN) {
1572                 cmn_err(CE_NOTE, "!kernelbase set to 0x%lx, system is not "
1573                     "i386 ABI compliant.", (uintptr_t)kernelbase);
1574         }
1575 #endif
1576 
1577 #ifndef __xpv
1578         if (plat_dr_support_memory()) {
1579                 mem_config_init();
1580         }
1581 #else   /* __xpv */
1582         /*
1583          * Some of the xen start information has to be relocated up
1584          * into the kernel's permanent address space.
1585          */
1586         PRM_POINT("calling xen_relocate_start_info()");
1587         xen_relocate_start_info();
1588         PRM_POINT("xen_relocate_start_info() done");
1589 
1590         /*
1591          * (Update the vcpu pointer in our cpu structure to point into
1592          * the relocated shared info.)
1593          */
1594         CPU->cpu_m.mcpu_vcpu_info =
1595             &HYPERVISOR_shared_info->vcpu_info[CPU->cpu_id];
1596 #endif  /* __xpv */
1597 
1598         PRM_POINT("startup_kmem() done");
1599 }
1600 
1601 #ifndef __xpv
1602 /*
1603  * If we have detected that we are running in an HVM environment, we need
1604  * to prepend the PV driver directory to the module search path.
1605  */
1606 #define HVM_MOD_DIR "/platform/i86hvm/kernel"
1607 static void
1608 update_default_path()
1609 {
1610         char *current, *newpath;
1611         int newlen;
1612 
1613         /*
1614          * We are about to resync with krtld.  krtld will reset its
1615          * internal module search path iff Solaris has set default_path.
1616          * We want to be sure we're prepending this new directory to the
1617          * right search path.
1618          */
1619         current = (default_path == NULL) ? kobj_module_path : default_path;
1620 
1621         newlen = strlen(HVM_MOD_DIR) + strlen(current) + 2;
1622         newpath = kmem_alloc(newlen, KM_SLEEP);
1623         (void) strcpy(newpath, HVM_MOD_DIR);
1624         (void) strcat(newpath, " ");
1625         (void) strcat(newpath, current);
1626 
1627         default_path = newpath;
1628 }
1629 #endif
1630 
1631 static void
1632 startup_modules(void)
1633 {
1634         int cnt;
1635         extern void prom_setup(void);
1636         int32_t v, h;
1637         char d[11];
1638         char *cp;
1639         cmi_hdl_t hdl;
1640 
1641         PRM_POINT("startup_modules() starting...");
1642 
1643 #ifndef __xpv
1644         /*
1645          * Initialize ten-micro second timer so that drivers will
1646          * not get short changed in their init phase. This was
1647          * not getting called until clkinit which, on fast cpu's
1648          * caused the drv_usecwait to be way too short.
1649          */
1650         microfind();
1651 
1652         if ((get_hwenv() & HW_XEN_HVM) != 0)
1653                 update_default_path();
1654 #endif
1655 
1656         /*
1657          * Read the GMT lag from /etc/rtc_config.
1658          */
1659         sgmtl(process_rtc_config_file());
1660 
1661         /*
1662          * Calculate default settings of system parameters based upon
1663          * maxusers, yet allow to be overridden via the /etc/system file.
1664          */
1665         param_calc(0);
1666 
1667         mod_setup();
1668 
1669         /*
1670          * Initialize system parameters.
1671          */
1672         param_init();
1673 
1674         /*
1675          * Initialize the default brands
1676          */
1677         brand_init();
1678 
1679         /*
1680          * maxmem is the amount of physical memory we're playing with.
1681          */
1682         maxmem = physmem;
1683 
1684         /*
1685          * Initialize segment management stuff.
1686          */
1687         seg_init();
1688 
1689         if (modload("fs", "specfs") == -1)
1690                 halt("Can't load specfs");
1691 
1692         if (modload("fs", "devfs") == -1)
1693                 halt("Can't load devfs");
1694 
1695         if (modload("fs", "dev") == -1)
1696                 halt("Can't load dev");
1697 
1698         if (modload("fs", "procfs") == -1)
1699                 halt("Can't load procfs");
1700 
1701         (void) modloadonly("sys", "lbl_edition");
1702 
1703         dispinit();
1704 
1705         /* Read cluster configuration data. */
1706         clconf_init();
1707 
1708 #if defined(__xpv)
1709         (void) ec_init();
1710         gnttab_init();
1711         (void) xs_early_init();
1712 #endif /* __xpv */
1713 
1714         /*
1715          * Create a kernel device tree. First, create rootnex and
1716          * then invoke bus specific code to probe devices.
1717          */
1718         setup_ddi();
1719 
1720 #ifdef __xpv
1721         if (DOMAIN_IS_INITDOMAIN(xen_info))
1722 #endif
1723         {
1724                 id_t smid;
1725                 smbios_system_t smsys;
1726                 smbios_info_t sminfo;
1727                 char *mfg;
1728                 /*
1729                  * Load the System Management BIOS into the global ksmbios
1730                  * handle, if an SMBIOS is present on this system.
1731                  * Also set "si-hw-provider" property, if not already set.
1732                  */
1733                 ksmbios = smbios_open(NULL, SMB_VERSION, ksmbios_flags, NULL);
1734                 if (ksmbios != NULL &&
1735                     ((smid = smbios_info_system(ksmbios, &smsys)) != SMB_ERR) &&
1736                     (smbios_info_common(ksmbios, smid, &sminfo)) != SMB_ERR) {
1737                         mfg = (char *)sminfo.smbi_manufacturer;
1738                         if (BOP_GETPROPLEN(bootops, "si-hw-provider") < 0) {
1739                                 extern char hw_provider[];
1740                                 int i;
1741                                 for (i = 0; i < SYS_NMLN; i++) {
1742                                         if (isprint(mfg[i]))
1743                                                 hw_provider[i] = mfg[i];
1744                                         else {
1745                                                 hw_provider[i] = '\0';
1746                                                 break;
1747                                         }
1748                                 }
1749                                 hw_provider[SYS_NMLN - 1] = '\0';
1750                         }
1751                 }
1752         }
1753 
1754 
1755         /*
1756          * Originally clconf_init() apparently needed the hostid.  But
1757          * this no longer appears to be true - it uses its own nodeid.
1758          * By placing the hostid logic here, we are able to make use of
1759          * the SMBIOS UUID.
1760          */
1761         if ((h = set_soft_hostid()) == HW_INVALID_HOSTID) {
1762                 cmn_err(CE_WARN, "Unable to set hostid");
1763         } else {
1764                 for (v = h, cnt = 0; cnt < 10; cnt++) {
1765                         d[cnt] = (char)(v % 10);
1766                         v /= 10;
1767                         if (v == 0)
1768                                 break;
1769                 }
1770                 for (cp = hw_serial; cnt >= 0; cnt--)
1771                         *cp++ = d[cnt] + '0';
1772                 *cp = 0;
1773         }
1774 
1775         /*
1776          * Set up the CPU module subsystem for the boot cpu in the native
1777          * case, and all physical cpu resource in the xpv dom0 case.
1778          * Modifies the device tree, so this must be done after
1779          * setup_ddi().
1780          */
1781 #ifdef __xpv
1782         /*
1783          * If paravirtualized and on dom0 then we initialize all physical
1784          * cpu handles now;  if paravirtualized on a domU then do not
1785          * initialize.
1786          */
1787         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1788                 xen_mc_lcpu_cookie_t cpi;
1789 
1790                 for (cpi = xen_physcpu_next(NULL); cpi != NULL;
1791                     cpi = xen_physcpu_next(cpi)) {
1792                         if ((hdl = cmi_init(CMI_HDL_SOLARIS_xVM_MCA,
1793                             xen_physcpu_chipid(cpi), xen_physcpu_coreid(cpi),
1794                             xen_physcpu_strandid(cpi))) != NULL &&
1795                             is_x86_feature(x86_featureset, X86FSET_MCA))
1796                                 cmi_mca_init(hdl);
1797                 }
1798         }
1799 #else
1800         /*
1801          * Initialize a handle for the boot cpu - others will initialize
1802          * as they startup.
1803          */
1804         if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
1805             cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) {
1806                 if (is_x86_feature(x86_featureset, X86FSET_MCA))
1807                         cmi_mca_init(hdl);
1808                 CPU->cpu_m.mcpu_cmi_hdl = hdl;
1809         }
1810 #endif  /* __xpv */
1811 
1812         /*
1813          * Fake a prom tree such that /dev/openprom continues to work
1814          */
1815         PRM_POINT("startup_modules: calling prom_setup...");
1816         prom_setup();
1817         PRM_POINT("startup_modules: done");
1818 
1819         /*
1820          * Load all platform specific modules
1821          */
1822         PRM_POINT("startup_modules: calling psm_modload...");
1823         psm_modload();
1824 
1825         PRM_POINT("startup_modules() done");
1826 }
1827 
1828 /*
1829  * claim a "setaside" boot page for use in the kernel
1830  */
1831 page_t *
1832 boot_claim_page(pfn_t pfn)
1833 {
1834         page_t *pp;
1835 
1836         pp = page_numtopp_nolock(pfn);
1837         ASSERT(pp != NULL);
1838 
1839         if (PP_ISBOOTPAGES(pp)) {
1840                 if (pp->p_next != NULL)
1841                         pp->p_next->p_prev = pp->p_prev;
1842                 if (pp->p_prev == NULL)
1843                         bootpages = pp->p_next;
1844                 else
1845                         pp->p_prev->p_next = pp->p_next;
1846         } else {
1847                 /*
1848                  * htable_attach() expects a base pagesize page
1849                  */
1850                 if (pp->p_szc != 0)
1851                         page_boot_demote(pp);
1852                 pp = page_numtopp(pfn, SE_EXCL);
1853         }
1854         return (pp);
1855 }
1856 
1857 /*
1858  * Walk through the pagetables looking for pages mapped in by boot.  If the
1859  * setaside flag is set the pages are expected to be returned to the
1860  * kernel later in boot, so we add them to the bootpages list.
1861  */
1862 static void
1863 protect_boot_range(uintptr_t low, uintptr_t high, int setaside)
1864 {
1865         uintptr_t va = low;
1866         size_t len;
1867         uint_t prot;
1868         pfn_t pfn;
1869         page_t *pp;
1870         pgcnt_t boot_protect_cnt = 0;
1871 
1872         while (kbm_probe(&va, &len, &pfn, &prot) != 0 && va < high) {
1873                 if (va + len >= high)
1874                         panic("0x%lx byte mapping at 0x%p exceeds boot's "
1875                             "legal range.", len, (void *)va);
1876 
1877                 while (len > 0) {
1878                         pp = page_numtopp_alloc(pfn);
1879                         if (pp != NULL) {
1880                                 if (setaside == 0)
1881                                         panic("Unexpected mapping by boot.  "
1882                                             "addr=%p pfn=%lx\n",
1883                                             (void *)va, pfn);
1884 
1885                                 pp->p_next = bootpages;
1886                                 pp->p_prev = NULL;
1887                                 PP_SETBOOTPAGES(pp);
1888                                 if (bootpages != NULL) {
1889                                         bootpages->p_prev = pp;
1890                                 }
1891                                 bootpages = pp;
1892                                 ++boot_protect_cnt;
1893                         }
1894 
1895                         ++pfn;
1896                         len -= MMU_PAGESIZE;
1897                         va += MMU_PAGESIZE;
1898                 }
1899         }
1900         PRM_DEBUG(boot_protect_cnt);
1901 }
1902 
1903 /*
1904  *
1905  */
1906 static void
1907 layout_kernel_va(void)
1908 {
1909         PRM_POINT("layout_kernel_va() starting...");
1910         /*
1911          * Establish the final size of the kernel's heap, size of segmap,
1912          * segkp, etc.
1913          */
1914 
1915 #if defined(__amd64)
1916 
1917         kpm_vbase = (caddr_t)segkpm_base;
1918         if (physmax + 1 < plat_dr_physmax) {
1919                 kpm_size = ROUND_UP_LPAGE(mmu_ptob(plat_dr_physmax));
1920         } else {
1921                 kpm_size = ROUND_UP_LPAGE(mmu_ptob(physmax + 1));
1922         }
1923         if ((uintptr_t)kpm_vbase + kpm_size > (uintptr_t)valloc_base)
1924                 panic("not enough room for kpm!");
1925         PRM_DEBUG(kpm_size);
1926         PRM_DEBUG(kpm_vbase);
1927 
1928         /*
1929          * By default we create a seg_kp in 64 bit kernels, it's a little
1930          * faster to access than embedding it in the heap.
1931          */
1932         segkp_base = (caddr_t)valloc_base + valloc_sz;
1933         if (!segkp_fromheap) {
1934                 size_t sz = mmu_ptob(segkpsize);
1935 
1936                 /*
1937                  * determine size of segkp
1938                  */
1939                 if (sz < SEGKPMINSIZE || sz > SEGKPMAXSIZE) {
1940                         sz = SEGKPDEFSIZE;
1941                         cmn_err(CE_WARN, "!Illegal value for segkpsize. "
1942                             "segkpsize has been reset to %ld pages",
1943                             mmu_btop(sz));
1944                 }
1945                 sz = MIN(sz, MAX(SEGKPMINSIZE, mmu_ptob(physmem)));
1946 
1947                 segkpsize = mmu_btop(ROUND_UP_LPAGE(sz));
1948         }
1949         PRM_DEBUG(segkp_base);
1950         PRM_DEBUG(segkpsize);
1951 
1952         /*
1953          * segzio is used for ZFS cached data. It uses a distinct VA
1954          * segment (from kernel heap) so that we can easily tell not to
1955          * include it in kernel crash dumps on 64 bit kernels. The trick is
1956          * to give it lots of VA, but not constrain the kernel heap.
1957          * We can use 1.5x physmem for segzio, leaving approximately
1958          * another 1.5x physmem for heap.  See also the comment in
1959          * startup_memlist().
1960          */
1961         segzio_base = segkp_base + mmu_ptob(segkpsize);
1962         if (segzio_fromheap) {
1963                 segziosize = 0;
1964         } else {
1965                 size_t physmem_size = mmu_ptob(physmem);
1966                 size_t size = (segziosize == 0) ?
1967                     physmem_size * 3 / 2 : mmu_ptob(segziosize);
1968 
1969                 if (size < SEGZIOMINSIZE)
1970                         size = SEGZIOMINSIZE;
1971                 segziosize = mmu_btop(ROUND_UP_LPAGE(size));
1972         }
1973         PRM_DEBUG(segziosize);
1974         PRM_DEBUG(segzio_base);
1975 
1976         /*
1977          * Put the range of VA for device mappings next, kmdb knows to not
1978          * grep in this range of addresses.
1979          */
1980         toxic_addr =
1981             ROUND_UP_LPAGE((uintptr_t)segzio_base + mmu_ptob(segziosize));
1982         PRM_DEBUG(toxic_addr);
1983         segmap_start = ROUND_UP_LPAGE(toxic_addr + toxic_size);
1984 #else /* __i386 */
1985         segmap_start = ROUND_UP_LPAGE(kernelbase);
1986 #endif /* __i386 */
1987         PRM_DEBUG(segmap_start);
1988 
1989         /*
1990          * Users can change segmapsize through eeprom. If the variable
1991          * is tuned through eeprom, there is no upper bound on the
1992          * size of segmap.
1993          */
1994         segmapsize = MAX(ROUND_UP_LPAGE(segmapsize), SEGMAPDEFAULT);
1995 
1996 #if defined(__i386)
1997         /*
1998          * 32-bit systems don't have segkpm or segkp, so segmap appears at
1999          * the bottom of the kernel's address range.  Set aside space for a
2000          * small red zone just below the start of segmap.
2001          */
2002         segmap_start += KERNEL_REDZONE_SIZE;
2003         segmapsize -= KERNEL_REDZONE_SIZE;
2004 #endif
2005 
2006         PRM_DEBUG(segmap_start);
2007         PRM_DEBUG(segmapsize);
2008         kernelheap = (caddr_t)ROUND_UP_LPAGE(segmap_start + segmapsize);
2009         PRM_DEBUG(kernelheap);
2010         PRM_POINT("layout_kernel_va() done...");
2011 }
2012 
2013 /*
2014  * Finish initializing the VM system, now that we are no longer
2015  * relying on the boot time memory allocators.
2016  */
2017 static void
2018 startup_vm(void)
2019 {
2020         struct segmap_crargs a;
2021 
2022         extern int use_brk_lpg, use_stk_lpg;
2023 
2024         PRM_POINT("startup_vm() starting...");
2025 
2026         /*
2027          * Initialize the hat layer.
2028          */
2029         hat_init();
2030 
2031         /*
2032          * Do final allocations of HAT data structures that need to
2033          * be allocated before quiescing the boot loader.
2034          */
2035         PRM_POINT("Calling hat_kern_alloc()...");
2036         hat_kern_alloc((caddr_t)segmap_start, segmapsize, ekernelheap);
2037         PRM_POINT("hat_kern_alloc() done");
2038 
2039 #ifndef __xpv
2040         /*
2041          * Setup Page Attribute Table
2042          */
2043         pat_sync();
2044 #endif
2045 
2046         /*
2047          * The next two loops are done in distinct steps in order
2048          * to be sure that any page that is doubly mapped (both above
2049          * KERNEL_TEXT and below kernelbase) is dealt with correctly.
2050          * Note this may never happen, but it might someday.
2051          */
2052         bootpages = NULL;
2053         PRM_POINT("Protecting boot pages");
2054 
2055         /*
2056          * Protect any pages mapped above KERNEL_TEXT that somehow have
2057          * page_t's. This can only happen if something weird allocated
2058          * in this range (like kadb/kmdb).
2059          */
2060         protect_boot_range(KERNEL_TEXT, (uintptr_t)-1, 0);
2061 
2062         /*
2063          * Before we can take over memory allocation/mapping from the boot
2064          * loader we must remove from our free page lists any boot allocated
2065          * pages that stay mapped until release_bootstrap().
2066          */
2067         protect_boot_range(0, kernelbase, 1);
2068 
2069 
2070         /*
2071          * Switch to running on regular HAT (not boot_mmu)
2072          */
2073         PRM_POINT("Calling hat_kern_setup()...");
2074         hat_kern_setup();
2075 
2076         /*
2077          * It is no longer safe to call BOP_ALLOC(), so make sure we don't.
2078          */
2079         bop_no_more_mem();
2080 
2081         PRM_POINT("hat_kern_setup() done");
2082 
2083         hat_cpu_online(CPU);
2084 
2085         /*
2086          * Initialize VM system
2087          */
2088         PRM_POINT("Calling kvm_init()...");
2089         kvm_init();
2090         PRM_POINT("kvm_init() done");
2091 
2092         /*
2093          * Tell kmdb that the VM system is now working
2094          */
2095         if (boothowto & RB_DEBUG)
2096                 kdi_dvec_vmready();
2097 
2098 #if defined(__xpv)
2099         /*
2100          * Populate the I/O pool on domain 0
2101          */
2102         if (DOMAIN_IS_INITDOMAIN(xen_info)) {
2103                 extern long populate_io_pool(void);
2104                 long init_io_pool_cnt;
2105 
2106                 PRM_POINT("Populating reserve I/O page pool");
2107                 init_io_pool_cnt = populate_io_pool();
2108                 PRM_DEBUG(init_io_pool_cnt);
2109         }
2110 #endif
2111         /*
2112          * Mangle the brand string etc.
2113          */
2114         cpuid_pass3(CPU);
2115 
2116 #if defined(__amd64)
2117 
2118         /*
2119          * Create the device arena for toxic (to dtrace/kmdb) mappings.
2120          */
2121         device_arena = vmem_create("device", (void *)toxic_addr,
2122             toxic_size, MMU_PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
2123 
2124 #else   /* __i386 */
2125 
2126         /*
2127          * allocate the bit map that tracks toxic pages
2128          */
2129         toxic_bit_map_len = btop((ulong_t)(valloc_base - kernelbase));
2130         PRM_DEBUG(toxic_bit_map_len);
2131         toxic_bit_map =
2132             kmem_zalloc(BT_SIZEOFMAP(toxic_bit_map_len), KM_NOSLEEP);
2133         ASSERT(toxic_bit_map != NULL);
2134         PRM_DEBUG(toxic_bit_map);
2135 
2136 #endif  /* __i386 */
2137 
2138 
2139         /*
2140          * Now that we've got more VA, as well as the ability to allocate from
2141          * it, tell the debugger.
2142          */
2143         if (boothowto & RB_DEBUG)
2144                 kdi_dvec_memavail();
2145 
2146 #if !defined(__xpv)
2147         /*
2148          * Map page pfn=0 for drivers, such as kd, that need to pick up
2149          * parameters left there by controllers/BIOS.
2150          */
2151         PRM_POINT("setup up p0_va");
2152         p0_va = i86devmap(0, 1, PROT_READ);
2153         PRM_DEBUG(p0_va);
2154 #endif
2155 
2156         cmn_err(CE_CONT, "?mem = %luK (0x%lx)\n",
2157             physinstalled << (MMU_PAGESHIFT - 10), ptob(physinstalled));
2158 
2159         /*
2160          * disable automatic large pages for small memory systems or
2161          * when the disable flag is set.
2162          *
2163          * Do not yet consider page sizes larger than 2m/4m.
2164          */
2165         if (!auto_lpg_disable && mmu.max_page_level > 0) {
2166                 max_uheap_lpsize = LEVEL_SIZE(1);
2167                 max_ustack_lpsize = LEVEL_SIZE(1);
2168                 max_privmap_lpsize = LEVEL_SIZE(1);
2169                 max_uidata_lpsize = LEVEL_SIZE(1);
2170                 max_utext_lpsize = LEVEL_SIZE(1);
2171                 max_shm_lpsize = LEVEL_SIZE(1);
2172         }
2173         if (physmem < privm_lpg_min_physmem || mmu.max_page_level == 0 ||
2174             auto_lpg_disable) {
2175                 use_brk_lpg = 0;
2176                 use_stk_lpg = 0;
2177         }
2178         mcntl0_lpsize = LEVEL_SIZE(mmu.umax_page_level);
2179 
2180         PRM_POINT("Calling hat_init_finish()...");
2181         hat_init_finish();
2182         PRM_POINT("hat_init_finish() done");
2183 
2184         /*
2185          * Initialize the segkp segment type.
2186          */
2187         rw_enter(&kas.a_lock, RW_WRITER);
2188         PRM_POINT("Attaching segkp");
2189         if (segkp_fromheap) {
2190                 segkp->s_as = &kas;
2191         } else if (seg_attach(&kas, (caddr_t)segkp_base, mmu_ptob(segkpsize),
2192             segkp) < 0) {
2193                 panic("startup: cannot attach segkp");
2194                 /*NOTREACHED*/
2195         }
2196         PRM_POINT("Doing segkp_create()");
2197         if (segkp_create(segkp) != 0) {
2198                 panic("startup: segkp_create failed");
2199                 /*NOTREACHED*/
2200         }
2201         PRM_DEBUG(segkp);
2202         rw_exit(&kas.a_lock);
2203 
2204         /*
2205          * kpm segment
2206          */
2207         segmap_kpm = 0;
2208         if (kpm_desired)
2209                 kpm_init();
2210 
2211         /*
2212          * Now create segmap segment.
2213          */
2214         rw_enter(&kas.a_lock, RW_WRITER);
2215         if (seg_attach(&kas, (caddr_t)segmap_start, segmapsize, segmap) < 0) {
2216                 panic("cannot attach segmap");
2217                 /*NOTREACHED*/
2218         }
2219         PRM_DEBUG(segmap);
2220 
2221         a.prot = PROT_READ | PROT_WRITE;
2222         a.shmsize = 0;
2223         a.nfreelist = segmapfreelists;
2224 
2225         if (segmap_create(segmap, (caddr_t)&a) != 0)
2226                 panic("segmap_create segmap");
2227         rw_exit(&kas.a_lock);
2228 
2229         setup_vaddr_for_ppcopy(CPU);
2230 
2231         segdev_init();
2232 #if defined(__xpv)
2233         if (DOMAIN_IS_INITDOMAIN(xen_info))
2234 #endif
2235                 pmem_init();
2236 
2237         PRM_POINT("startup_vm() done");
2238 }
2239 
2240 /*
2241  * Load a tod module for the non-standard tod part found on this system.
2242  */
2243 static void
2244 load_tod_module(char *todmod)
2245 {
2246         if (modload("tod", todmod) == -1)
2247                 halt("Can't load TOD module");
2248 }
2249 
2250 static void
2251 startup_end(void)
2252 {
2253         int i;
2254         extern void setx86isalist(void);
2255         extern void cpu_event_init(void);
2256 
2257         PRM_POINT("startup_end() starting...");
2258 
2259         /*
2260          * Perform tasks that get done after most of the VM
2261          * initialization has been done but before the clock
2262          * and other devices get started.
2263          */
2264         kern_setup1();
2265 
2266         /*
2267          * Perform CPC initialization for this CPU.
2268          */
2269         kcpc_hw_init(CPU);
2270 
2271         /*
2272          * Initialize cpu event framework.
2273          */
2274         cpu_event_init();
2275 
2276 #if defined(OPTERON_WORKAROUND_6323525)
2277         if (opteron_workaround_6323525)
2278                 patch_workaround_6323525();
2279 #endif
2280         /*
2281          * If needed, load TOD module now so that ddi_get_time(9F) etc. work
2282          * (For now, "needed" is defined as set tod_module_name in /etc/system)
2283          */
2284         if (tod_module_name != NULL) {
2285                 PRM_POINT("load_tod_module()");
2286                 load_tod_module(tod_module_name);
2287         }
2288 
2289 #if defined(__xpv)
2290         /*
2291          * Forceload interposing TOD module for the hypervisor.
2292          */
2293         PRM_POINT("load_tod_module()");
2294         load_tod_module("xpvtod");
2295 #endif
2296 
2297         /*
2298          * Configure the system.
2299          */
2300         PRM_POINT("Calling configure()...");
2301         configure();            /* set up devices */
2302         PRM_POINT("configure() done");
2303 
2304         /*
2305          * We can now setup for XSAVE because fpu_probe is done in configure().
2306          */
2307         if (fp_save_mech == FP_XSAVE) {
2308                 xsave_setup_msr(CPU);
2309         }
2310 
2311         /*
2312          * Set the isa_list string to the defined instruction sets we
2313          * support.
2314          */
2315         setx86isalist();
2316         cpu_intr_alloc(CPU, NINTR_THREADS);
2317         psm_install();
2318 
2319         /*
2320          * We're done with bootops.  We don't unmap the bootstrap yet because
2321          * we're still using bootsvcs.
2322          */
2323         PRM_POINT("NULLing out bootops");
2324         *bootopsp = (struct bootops *)NULL;
2325         bootops = (struct bootops *)NULL;
2326 
2327 #if defined(__xpv)
2328         ec_init_debug_irq();
2329         xs_domu_init();
2330 #endif
2331 
2332 #if !defined(__xpv)
2333         /*
2334          * Intel IOMMU has been setup/initialized in ddi_impl.c
2335          * Start it up now.
2336          */
2337         immu_startup();
2338 
2339         /*
2340          * Now that we're no longer going to drop into real mode for a BIOS call
2341          * via bootops, we can enable PCID (which requires CR0.PG).
2342          */
2343         enable_pcid();
2344 #endif
2345 
2346         PRM_POINT("Enabling interrupts");
2347         (*picinitf)();
2348         sti();
2349 #if defined(__xpv)
2350         ASSERT(CPU->cpu_m.mcpu_vcpu_info->evtchn_upcall_mask == 0);
2351         xen_late_startup();
2352 #endif
2353 
2354         (void) add_avsoftintr((void *)&softlevel1_hdl, 1, softlevel1,
2355             "softlevel1", NULL, NULL); /* XXX to be moved later */
2356 
2357         /*
2358          * Register software interrupt handlers for ddi_periodic_add(9F).
2359          * Software interrupts up to the level 10 are supported.
2360          */
2361         for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) {
2362                 (void) add_avsoftintr((void *)&softlevel_hdl[i-1], i,
2363                     (avfunc)ddi_periodic_softintr, "ddi_periodic",
2364                     (caddr_t)(uintptr_t)i, NULL);
2365         }
2366 
2367 #if !defined(__xpv)
2368         if (modload("drv", "amd_iommu") < 0) {
2369                 PRM_POINT("No AMD IOMMU present\n");
2370         } else if (ddi_hold_installed_driver(ddi_name_to_major(
2371             "amd_iommu")) == NULL) {
2372                 prom_printf("ERROR: failed to attach AMD IOMMU\n");
2373         }
2374 #endif
2375         post_startup_cpu_fixups();
2376 
2377         PRM_POINT("startup_end() done");
2378 }
2379 
2380 /*
2381  * Don't remove the following 2 variables.  They are necessary
2382  * for reading the hostid from the legacy file (/kernel/misc/sysinit).
2383  */
2384 char *_hs1107 = hw_serial;
2385 ulong_t  _bdhs34;
2386 
2387 void
2388 post_startup(void)
2389 {
2390         extern void cpupm_init(cpu_t *);
2391         extern void cpu_event_init_cpu(cpu_t *);
2392 
2393         /*
2394          * Set the system wide, processor-specific flags to be passed
2395          * to userland via the aux vector for performance hints and
2396          * instruction set extensions.
2397          */
2398         bind_hwcap();
2399 
2400 #ifdef __xpv
2401         if (DOMAIN_IS_INITDOMAIN(xen_info))
2402 #endif
2403         {
2404 #if defined(__xpv)
2405                 xpv_panic_init();
2406 #else
2407                 /*
2408                  * Startup the memory scrubber.
2409                  * XXPV This should be running somewhere ..
2410                  */
2411                 if ((get_hwenv() & HW_VIRTUAL) == 0)
2412                         memscrub_init();
2413 #endif
2414         }
2415 
2416         /*
2417          * Complete CPU module initialization
2418          */
2419         cmi_post_startup();
2420 
2421         /*
2422          * Perform forceloading tasks for /etc/system.
2423          */
2424         (void) mod_sysctl(SYS_FORCELOAD, NULL);
2425 
2426         /*
2427          * ON4.0: Force /proc module in until clock interrupt handle fixed
2428          * ON4.0: This must be fixed or restated in /etc/systems.
2429          */
2430         (void) modload("fs", "procfs");
2431 
2432         (void) i_ddi_attach_hw_nodes("pit_beep");
2433 
2434 #if defined(__i386)
2435         /*
2436          * Check for required functional Floating Point hardware,
2437          * unless FP hardware explicitly disabled.
2438          */
2439         if (fpu_exists && (fpu_pentium_fdivbug || fp_kind == FP_NO))
2440                 halt("No working FP hardware found");
2441 #endif
2442 
2443         maxmem = freemem;
2444 
2445         cpu_event_init_cpu(CPU);
2446         cpupm_init(CPU);
2447         (void) mach_cpu_create_device_node(CPU, NULL);
2448 
2449         pg_init();
2450 }
2451 
2452 static int
2453 pp_in_range(page_t *pp, uint64_t low_addr, uint64_t high_addr)
2454 {
2455         return ((pp->p_pagenum >= btop(low_addr)) &&
2456             (pp->p_pagenum < btopr(high_addr)));
2457 }
2458 
2459 static int
2460 pp_in_module(page_t *pp, const rd_existing_t *modranges)
2461 {
2462         uint_t i;
2463 
2464         for (i = 0; modranges[i].phys != 0; i++) {
2465                 if (pp_in_range(pp, modranges[i].phys,
2466                     modranges[i].phys + modranges[i].size))
2467                         return (1);
2468         }
2469 
2470         return (0);
2471 }
2472 
2473 void
2474 release_bootstrap(void)
2475 {
2476         int root_is_ramdisk;
2477         page_t *pp;
2478         extern void kobj_boot_unmountroot(void);
2479         extern dev_t rootdev;
2480         uint_t i;
2481         char propname[32];
2482         rd_existing_t *modranges;
2483 #if !defined(__xpv)
2484         pfn_t   pfn;
2485 #endif
2486 
2487         /*
2488          * Save the bootfs module ranges so that we can reserve them below
2489          * for the real bootfs.
2490          */
2491         modranges = kmem_alloc(sizeof (rd_existing_t) * MAX_BOOT_MODULES,
2492             KM_SLEEP);
2493         for (i = 0; ; i++) {
2494                 uint64_t start, size;
2495 
2496                 modranges[i].phys = 0;
2497 
2498                 (void) snprintf(propname, sizeof (propname),
2499                     "module-addr-%u", i);
2500                 if (do_bsys_getproplen(NULL, propname) <= 0)
2501                         break;
2502                 (void) do_bsys_getprop(NULL, propname, &start);
2503 
2504                 (void) snprintf(propname, sizeof (propname),
2505                     "module-size-%u", i);
2506                 if (do_bsys_getproplen(NULL, propname) <= 0)
2507                         break;
2508                 (void) do_bsys_getprop(NULL, propname, &size);
2509 
2510                 modranges[i].phys = start;
2511                 modranges[i].size = size;
2512         }
2513 
2514         /* unmount boot ramdisk and release kmem usage */
2515         kobj_boot_unmountroot();
2516 
2517         /*
2518          * We're finished using the boot loader so free its pages.
2519          */
2520         PRM_POINT("Unmapping lower boot pages");
2521 
2522         clear_boot_mappings(0, _userlimit);
2523 
2524         postbootkernelbase = kernelbase;
2525 
2526         /*
2527          * If root isn't on ramdisk, destroy the hardcoded
2528          * ramdisk node now and release the memory. Else,
2529          * ramdisk memory is kept in rd_pages.
2530          */
2531         root_is_ramdisk = (getmajor(rootdev) == ddi_name_to_major("ramdisk"));
2532         if (!root_is_ramdisk) {
2533                 dev_info_t *dip = ddi_find_devinfo("ramdisk", -1, 0);
2534                 ASSERT(dip && ddi_get_parent(dip) == ddi_root_node());
2535                 ndi_rele_devi(dip);     /* held from ddi_find_devinfo */
2536                 (void) ddi_remove_child(dip, 0);
2537         }
2538 
2539         PRM_POINT("Releasing boot pages");
2540         while (bootpages) {
2541                 extern uint64_t ramdisk_start, ramdisk_end;
2542                 pp = bootpages;
2543                 bootpages = pp->p_next;
2544 
2545 
2546                 /* Keep pages for the lower 64K */
2547                 if (pp_in_range(pp, 0, 0x40000)) {
2548                         pp->p_next = lower_pages;
2549                         lower_pages = pp;
2550                         lower_pages_count++;
2551                         continue;
2552                 }
2553 
2554                 if (root_is_ramdisk && pp_in_range(pp, ramdisk_start,
2555                     ramdisk_end) || pp_in_module(pp, modranges)) {
2556                         pp->p_next = rd_pages;
2557                         rd_pages = pp;
2558                         continue;
2559                 }
2560                 pp->p_next = (struct page *)0;
2561                 pp->p_prev = (struct page *)0;
2562                 PP_CLRBOOTPAGES(pp);
2563                 page_free(pp, 1);
2564         }
2565         PRM_POINT("Boot pages released");
2566 
2567         kmem_free(modranges, sizeof (rd_existing_t) * 99);
2568 
2569 #if !defined(__xpv)
2570 /* XXPV -- note this following bunch of code needs to be revisited in Xen 3.0 */
2571         /*
2572          * Find 1 page below 1 MB so that other processors can boot up or
2573          * so that any processor can resume.
2574          * Make sure it has a kernel VA as well as a 1:1 mapping.
2575          * We should have just free'd one up.
2576          */
2577 
2578         /*
2579          * 0x10 pages is 64K.  Leave the bottom 64K alone
2580          * for BIOS.
2581          */
2582         for (pfn = 0x10; pfn < btop(1*1024*1024); pfn++) {
2583                 if (page_numtopp_alloc(pfn) == NULL)
2584                         continue;
2585                 rm_platter_va = i86devmap(pfn, 1,
2586                     PROT_READ | PROT_WRITE | PROT_EXEC);
2587                 rm_platter_pa = ptob(pfn);
2588                 break;
2589         }
2590         if (pfn == btop(1*1024*1024) && use_mp)
2591                 panic("No page below 1M available for starting "
2592                     "other processors or for resuming from system-suspend");
2593 #endif  /* !__xpv */
2594 }
2595 
2596 /*
2597  * Initialize the platform-specific parts of a page_t.
2598  */
2599 void
2600 add_physmem_cb(page_t *pp, pfn_t pnum)
2601 {
2602         pp->p_pagenum = pnum;
2603         pp->p_mapping = NULL;
2604         pp->p_embed = 0;
2605         pp->p_share = 0;
2606         pp->p_mlentry = 0;
2607 }
2608 
2609 /*
2610  * kphysm_init() initializes physical memory.
2611  */
2612 static pgcnt_t
2613 kphysm_init(
2614         page_t *pp,
2615         pgcnt_t npages)
2616 {
2617         struct memlist  *pmem;
2618         struct memseg   *cur_memseg;
2619         pfn_t           base_pfn;
2620         pfn_t           end_pfn;
2621         pgcnt_t         num;
2622         pgcnt_t         pages_done = 0;
2623         uint64_t        addr;
2624         uint64_t        size;
2625         extern pfn_t    ddiphysmin;
2626         extern int      mnode_xwa;
2627         int             ms = 0, me = 0;
2628 
2629         ASSERT(page_hash != NULL && page_hashsz != 0);
2630 
2631         cur_memseg = memseg_base;
2632         for (pmem = phys_avail; pmem && npages; pmem = pmem->ml_next) {
2633                 /*
2634                  * In a 32 bit kernel can't use higher memory if we're
2635                  * not booting in PAE mode. This check takes care of that.
2636                  */
2637                 addr = pmem->ml_address;
2638                 size = pmem->ml_size;
2639                 if (btop(addr) > physmax)
2640                         continue;
2641 
2642                 /*
2643                  * align addr and size - they may not be at page boundaries
2644                  */
2645                 if ((addr & MMU_PAGEOFFSET) != 0) {
2646                         addr += MMU_PAGEOFFSET;
2647                         addr &= ~(uint64_t)MMU_PAGEOFFSET;
2648                         size -= addr - pmem->ml_address;
2649                 }
2650 
2651                 /* only process pages below or equal to physmax */
2652                 if ((btop(addr + size) - 1) > physmax)
2653                         size = ptob(physmax - btop(addr) + 1);
2654 
2655                 num = btop(size);
2656                 if (num == 0)
2657                         continue;
2658 
2659                 if (num > npages)
2660                         num = npages;
2661 
2662                 npages -= num;
2663                 pages_done += num;
2664                 base_pfn = btop(addr);
2665 
2666                 if (prom_debug)
2667                         prom_printf("MEMSEG addr=0x%" PRIx64
2668                             " pgs=0x%lx pfn 0x%lx-0x%lx\n",
2669                             addr, num, base_pfn, base_pfn + num);
2670 
2671                 /*
2672                  * Ignore pages below ddiphysmin to simplify ddi memory
2673                  * allocation with non-zero addr_lo requests.
2674                  */
2675                 if (base_pfn < ddiphysmin) {
2676                         if (base_pfn + num <= ddiphysmin)
2677                                 continue;
2678                         pp += (ddiphysmin - base_pfn);
2679                         num -= (ddiphysmin - base_pfn);
2680                         base_pfn = ddiphysmin;
2681                 }
2682 
2683                 /*
2684                  * mnode_xwa is greater than 1 when large pages regions can
2685                  * cross memory node boundaries. To prevent the formation
2686                  * of these large pages, configure the memsegs based on the
2687                  * memory node ranges which had been made non-contiguous.
2688                  */
2689                 if (mnode_xwa > 1) {
2690 
2691                         end_pfn = base_pfn + num - 1;
2692                         ms = PFN_2_MEM_NODE(base_pfn);
2693                         me = PFN_2_MEM_NODE(end_pfn);
2694 
2695                         if (ms != me) {
2696                                 /*
2697                                  * current range spans more than 1 memory node.
2698                                  * Set num to only the pfn range in the start
2699                                  * memory node.
2700                                  */
2701                                 num = mem_node_config[ms].physmax - base_pfn
2702                                     + 1;
2703                                 ASSERT(end_pfn > mem_node_config[ms].physmax);
2704                         }
2705                 }
2706 
2707                 for (;;) {
2708                         /*
2709                          * Build the memsegs entry
2710                          */
2711                         cur_memseg->pages = pp;
2712                         cur_memseg->epages = pp + num;
2713                         cur_memseg->pages_base = base_pfn;
2714                         cur_memseg->pages_end = base_pfn + num;
2715 
2716                         /*
2717                          * Insert into memseg list in decreasing pfn range
2718                          * order. Low memory is typically more fragmented such
2719                          * that this ordering keeps the larger ranges at the
2720                          * front of the list for code that searches memseg.
2721                          * This ASSERTS that the memsegs coming in from boot
2722                          * are in increasing physical address order and not
2723                          * contiguous.
2724                          */
2725                         if (memsegs != NULL) {
2726                                 ASSERT(cur_memseg->pages_base >=
2727                                     memsegs->pages_end);
2728                                 cur_memseg->next = memsegs;
2729                         }
2730                         memsegs = cur_memseg;
2731 
2732                         /*
2733                          * add_physmem() initializes the PSM part of the page
2734                          * struct by calling the PSM back with add_physmem_cb().
2735                          * In addition it coalesces pages into larger pages as
2736                          * it initializes them.
2737                          */
2738                         add_physmem(pp, num, base_pfn);
2739                         cur_memseg++;
2740                         availrmem_initial += num;
2741                         availrmem += num;
2742 
2743                         pp += num;
2744                         if (ms >= me)
2745                                 break;
2746 
2747                         /* process next memory node range */
2748                         ms++;
2749                         base_pfn = mem_node_config[ms].physbase;
2750                         num = MIN(mem_node_config[ms].physmax,
2751                             end_pfn) - base_pfn + 1;
2752                 }
2753         }
2754 
2755         PRM_DEBUG(availrmem_initial);
2756         PRM_DEBUG(availrmem);
2757         PRM_DEBUG(freemem);
2758         build_pfn_hash();
2759         return (pages_done);
2760 }
2761 
2762 /*
2763  * Kernel VM initialization.
2764  */
2765 static void
2766 kvm_init(void)
2767 {
2768         ASSERT((((uintptr_t)s_text) & MMU_PAGEOFFSET) == 0);
2769 
2770         /*
2771          * Put the kernel segments in kernel address space.
2772          */
2773         rw_enter(&kas.a_lock, RW_WRITER);
2774         as_avlinit(&kas);
2775 
2776         (void) seg_attach(&kas, s_text, e_moddata - s_text, &ktextseg);
2777         (void) segkmem_create(&ktextseg);
2778 
2779         (void) seg_attach(&kas, (caddr_t)valloc_base, valloc_sz, &kvalloc);
2780         (void) segkmem_create(&kvalloc);
2781 
2782         (void) seg_attach(&kas, kernelheap,
2783             ekernelheap - kernelheap, &kvseg);
2784         (void) segkmem_create(&kvseg);
2785 
2786         if (core_size > 0) {
2787                 PRM_POINT("attaching kvseg_core");
2788                 (void) seg_attach(&kas, (caddr_t)core_base, core_size,
2789                     &kvseg_core);
2790                 (void) segkmem_create(&kvseg_core);
2791         }
2792 
2793         if (segziosize > 0) {
2794                 PRM_POINT("attaching segzio");
2795                 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2796                     &kzioseg);
2797                 (void) segkmem_zio_create(&kzioseg);
2798 
2799                 /* create zio area covering new segment */
2800                 segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2801         }
2802 
2803         (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2804         (void) segkmem_create(&kdebugseg);
2805 
2806         rw_exit(&kas.a_lock);
2807 
2808         /*
2809          * Ensure that the red zone at kernelbase is never accessible.
2810          */
2811         PRM_POINT("protecting redzone");
2812         (void) as_setprot(&kas, (caddr_t)kernelbase, KERNEL_REDZONE_SIZE, 0);
2813 
2814         /*
2815          * Make the text writable so that it can be hot patched by DTrace.
2816          */
2817         (void) as_setprot(&kas, s_text, e_modtext - s_text,
2818             PROT_READ | PROT_WRITE | PROT_EXEC);
2819 
2820         /*
2821          * Make data writable until end.
2822          */
2823         (void) as_setprot(&kas, s_data, e_moddata - s_data,
2824             PROT_READ | PROT_WRITE | PROT_EXEC);
2825 }
2826 
2827 #ifndef __xpv
2828 /*
2829  * Solaris adds an entry for Write Combining caching to the PAT
2830  */
2831 static uint64_t pat_attr_reg = PAT_DEFAULT_ATTRIBUTE;
2832 
2833 void
2834 pat_sync(void)
2835 {
2836         ulong_t cr0, cr0_orig, cr4;
2837 
2838         if (!is_x86_feature(x86_featureset, X86FSET_PAT))
2839                 return;
2840         cr0_orig = cr0 = getcr0();
2841         cr4 = getcr4();
2842 
2843         /* disable caching and flush all caches and TLBs */
2844         cr0 |= CR0_CD;
2845         cr0 &= ~CR0_NW;
2846         setcr0(cr0);
2847         invalidate_cache();
2848         if (cr4 & CR4_PGE) {
2849                 setcr4(cr4 & ~(ulong_t)CR4_PGE);
2850                 setcr4(cr4);
2851         } else {
2852                 reload_cr3();
2853         }
2854 
2855         /* add our entry to the PAT */
2856         wrmsr(REG_PAT, pat_attr_reg);
2857 
2858         /* flush TLBs and cache again, then reenable cr0 caching */
2859         if (cr4 & CR4_PGE) {
2860                 setcr4(cr4 & ~(ulong_t)CR4_PGE);
2861                 setcr4(cr4);
2862         } else {
2863                 reload_cr3();
2864         }
2865         invalidate_cache();
2866         setcr0(cr0_orig);
2867 }
2868 
2869 #endif /* !__xpv */
2870 
2871 #if defined(_SOFT_HOSTID)
2872 /*
2873  * On platforms that do not have a hardware serial number, attempt
2874  * to set one based on the contents of /etc/hostid.  If this file does
2875  * not exist, assume that we are to generate a new hostid and set
2876  * it in the kernel, for subsequent saving by a userland process
2877  * once the system is up and the root filesystem is mounted r/w.
2878  *
2879  * In order to gracefully support upgrade on OpenSolaris, if
2880  * /etc/hostid does not exist, we will attempt to get a serial number
2881  * using the legacy method (/kernel/misc/sysinit).
2882  *
2883  * If that isn't present, we attempt to use an SMBIOS UUID, which is
2884  * a hardware serial number.  Note that we don't automatically trust
2885  * all SMBIOS UUIDs (some older platforms are defective and ship duplicate
2886  * UUIDs in violation of the standard), we check against a blacklist.
2887  *
2888  * In an attempt to make the hostid less prone to abuse
2889  * (for license circumvention, etc), we store it in /etc/hostid
2890  * in rot47 format.
2891  */
2892 extern volatile unsigned long tenmicrodata;
2893 static int atoi(char *);
2894 
2895 /*
2896  * Set this to non-zero in /etc/system if you think your SMBIOS returns a
2897  * UUID that is not unique. (Also report it so that the smbios_uuid_blacklist
2898  * array can be updated.)
2899  */
2900 int smbios_broken_uuid = 0;
2901 
2902 /*
2903  * List of known bad UUIDs.  This is just the lower 32-bit values, since
2904  * that's what we use for the host id.  If your hostid falls here, you need
2905  * to contact your hardware OEM for a fix for your BIOS.
2906  */
2907 static unsigned char
2908 smbios_uuid_blacklist[][16] = {
2909 
2910         {       /* Reported bad UUID (Google search) */
2911                 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05,
2912                 0x00, 0x06, 0x00, 0x07, 0x00, 0x08, 0x00, 0x09,
2913         },
2914         {       /* Known bad DELL UUID */
2915                 0x4C, 0x4C, 0x45, 0x44, 0x00, 0x00, 0x20, 0x10,
2916                 0x80, 0x20, 0x80, 0xC0, 0x4F, 0x20, 0x20, 0x20,
2917         },
2918         {       /* Uninitialized flash */
2919                 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
2920                 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
2921         },
2922         {       /* All zeros */
2923                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2924                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
2925         },
2926 };
2927 
2928 static int32_t
2929 uuid_to_hostid(const uint8_t *uuid)
2930 {
2931         /*
2932          * Although the UUIDs are 128-bits, they may not distribute entropy
2933          * evenly.  We would like to use SHA or MD5, but those are located
2934          * in loadable modules and not available this early in boot.  As we
2935          * don't need the values to be cryptographically strong, we just
2936          * generate 32-bit vaue by xor'ing the various sequences together,
2937          * which ensures that the entire UUID contributes to the hostid.
2938          */
2939         uint32_t        id = 0;
2940 
2941         /* first check against the blacklist */
2942         for (int i = 0; i < (sizeof (smbios_uuid_blacklist) / 16); i++) {
2943                 if (bcmp(smbios_uuid_blacklist[0], uuid, 16) == 0) {
2944                         cmn_err(CE_CONT, "?Broken SMBIOS UUID. "
2945                             "Contact BIOS manufacturer for repair.\n");
2946                         return ((int32_t)HW_INVALID_HOSTID);
2947                 }
2948         }
2949 
2950         for (int i = 0; i < 16; i++)
2951                 id ^= ((uuid[i]) << (8 * (i % sizeof (id))));
2952 
2953         /* Make sure return value is positive */
2954         return (id & 0x7fffffff);
2955 }
2956 
2957 static int32_t
2958 set_soft_hostid(void)
2959 {
2960         struct _buf *file;
2961         char tokbuf[MAXNAMELEN];
2962         token_t token;
2963         int done = 0;
2964         u_longlong_t tmp;
2965         int i;
2966         int32_t hostid = (int32_t)HW_INVALID_HOSTID;
2967         unsigned char *c;
2968         hrtime_t tsc;
2969         smbios_system_t smsys;
2970 
2971         /*
2972          * If /etc/hostid file not found, we'd like to get a pseudo
2973          * random number to use at the hostid.  A nice way to do this
2974          * is to read the real time clock.  To remain xen-compatible,
2975          * we can't poke the real hardware, so we use tsc_read() to
2976          * read the real time clock.  However, there is an ominous
2977          * warning in tsc_read that says it can return zero, so we
2978          * deal with that possibility by falling back to using the
2979          * (hopefully random enough) value in tenmicrodata.
2980          */
2981 
2982         if ((file = kobj_open_file(hostid_file)) == (struct _buf *)-1) {
2983                 /*
2984                  * hostid file not found - try to load sysinit module
2985                  * and see if it has a nonzero hostid value...use that
2986                  * instead of generating a new hostid here if so.
2987                  */
2988                 if ((i = modload("misc", "sysinit")) != -1) {
2989                         if (strlen(hw_serial) > 0)
2990                                 hostid = (int32_t)atoi(hw_serial);
2991                         (void) modunload(i);
2992                 }
2993 
2994                 /*
2995                  * We try to use the SMBIOS UUID. But not if it is blacklisted
2996                  * in /etc/system.
2997                  */
2998                 if ((hostid == HW_INVALID_HOSTID) &&
2999                     (smbios_broken_uuid == 0) &&
3000                     (ksmbios != NULL) &&
3001                     (smbios_info_system(ksmbios, &smsys) != SMB_ERR) &&
3002                     (smsys.smbs_uuidlen >= 16)) {
3003                         hostid = uuid_to_hostid(smsys.smbs_uuid);
3004                 }
3005 
3006                 /*
3007                  * Generate a "random" hostid using the clock.  These
3008                  * hostids will change on each boot if the value is not
3009                  * saved to a persistent /etc/hostid file.
3010                  */
3011                 if (hostid == HW_INVALID_HOSTID) {
3012                         tsc = tsc_read();
3013                         if (tsc == 0)   /* tsc_read can return zero sometimes */
3014                                 hostid = (int32_t)tenmicrodata & 0x0CFFFFF;
3015                         else
3016                                 hostid = (int32_t)tsc & 0x0CFFFFF;
3017                 }
3018         } else {
3019                 /* hostid file found */
3020                 while (!done) {
3021                         token = kobj_lex(file, tokbuf, sizeof (tokbuf));
3022 
3023                         switch (token) {
3024                         case POUND:
3025                                 /*
3026                                  * skip comments
3027                                  */
3028                                 kobj_find_eol(file);
3029                                 break;
3030                         case STRING:
3031                                 /*
3032                                  * un-rot47 - obviously this
3033                                  * nonsense is ascii-specific
3034                                  */
3035                                 for (c = (unsigned char *)tokbuf;
3036                                     *c != '\0'; c++) {
3037                                         *c += 47;
3038                                         if (*c > '~')
3039                                                 *c -= 94;
3040                                         else if (*c < '!')
3041                                                 *c += 94;
3042                                 }
3043                                 /*
3044                                  * now we should have a real number
3045                                  */
3046 
3047                                 if (kobj_getvalue(tokbuf, &tmp) != 0)
3048                                         kobj_file_err(CE_WARN, file,
3049                                             "Bad value %s for hostid",
3050                                             tokbuf);
3051                                 else
3052                                         hostid = (int32_t)tmp;
3053 
3054                                 break;
3055                         case EOF:
3056                                 done = 1;
3057                                 /* FALLTHROUGH */
3058                         case NEWLINE:
3059                                 kobj_newline(file);
3060                                 break;
3061                         default:
3062                                 break;
3063 
3064                         }
3065                 }
3066                 if (hostid == HW_INVALID_HOSTID) /* didn't find a hostid */
3067                         kobj_file_err(CE_WARN, file,
3068                             "hostid missing or corrupt");
3069 
3070                 kobj_close_file(file);
3071         }
3072         /*
3073          * hostid is now the value read from /etc/hostid, or the
3074          * new hostid we generated in this routine or HW_INVALID_HOSTID if not
3075          * set.
3076          */
3077         return (hostid);
3078 }
3079 
3080 static int
3081 atoi(char *p)
3082 {
3083         int i = 0;
3084 
3085         while (*p != '\0')
3086                 i = 10 * i + (*p++ - '0');
3087 
3088         return (i);
3089 }
3090 
3091 #endif /* _SOFT_HOSTID */
3092 
3093 void
3094 get_system_configuration(void)
3095 {
3096         char    prop[32];
3097         u_longlong_t nodes_ll, cpus_pernode_ll, lvalue;
3098 
3099         if (BOP_GETPROPLEN(bootops, "nodes") > sizeof (prop) ||
3100             BOP_GETPROP(bootops, "nodes", prop) < 0 ||
3101             kobj_getvalue(prop, &nodes_ll) == -1 ||
3102             nodes_ll > MAXNODES ||
3103             BOP_GETPROPLEN(bootops, "cpus_pernode") > sizeof (prop) ||
3104             BOP_GETPROP(bootops, "cpus_pernode", prop) < 0 ||
3105             kobj_getvalue(prop, &cpus_pernode_ll) == -1) {
3106                 system_hardware.hd_nodes = 1;
3107                 system_hardware.hd_cpus_per_node = 0;
3108         } else {
3109                 system_hardware.hd_nodes = (int)nodes_ll;
3110                 system_hardware.hd_cpus_per_node = (int)cpus_pernode_ll;
3111         }
3112 
3113         if (BOP_GETPROPLEN(bootops, "kernelbase") > sizeof (prop) ||
3114             BOP_GETPROP(bootops, "kernelbase", prop) < 0 ||
3115             kobj_getvalue(prop, &lvalue) == -1)
3116                 eprom_kernelbase = 0;
3117         else
3118                 eprom_kernelbase = (uintptr_t)lvalue;
3119 
3120         if (BOP_GETPROPLEN(bootops, "segmapsize") > sizeof (prop) ||
3121             BOP_GETPROP(bootops, "segmapsize", prop) < 0 ||
3122             kobj_getvalue(prop, &lvalue) == -1)
3123                 segmapsize = SEGMAPDEFAULT;
3124         else
3125                 segmapsize = (uintptr_t)lvalue;
3126 
3127         if (BOP_GETPROPLEN(bootops, "segmapfreelists") > sizeof (prop) ||
3128             BOP_GETPROP(bootops, "segmapfreelists", prop) < 0 ||
3129             kobj_getvalue(prop, &lvalue) == -1)
3130                 segmapfreelists = 0;    /* use segmap driver default */
3131         else
3132                 segmapfreelists = (int)lvalue;
3133 
3134         /* physmem used to be here, but moved much earlier to fakebop.c */
3135 }
3136 
3137 /*
3138  * Add to a memory list.
3139  * start = start of new memory segment
3140  * len = length of new memory segment in bytes
3141  * new = pointer to a new struct memlist
3142  * memlistp = memory list to which to add segment.
3143  */
3144 void
3145 memlist_add(
3146         uint64_t start,
3147         uint64_t len,
3148         struct memlist *new,
3149         struct memlist **memlistp)
3150 {
3151         struct memlist *cur;
3152         uint64_t end = start + len;
3153 
3154         new->ml_address = start;
3155         new->ml_size = len;
3156 
3157         cur = *memlistp;
3158 
3159         while (cur) {
3160                 if (cur->ml_address >= end) {
3161                         new->ml_next = cur;
3162                         *memlistp = new;
3163                         new->ml_prev = cur->ml_prev;
3164                         cur->ml_prev = new;
3165                         return;
3166                 }
3167                 ASSERT(cur->ml_address + cur->ml_size <= start);
3168                 if (cur->ml_next == NULL) {
3169                         cur->ml_next = new;
3170                         new->ml_prev = cur;
3171                         new->ml_next = NULL;
3172                         return;
3173                 }
3174                 memlistp = &cur->ml_next;
3175                 cur = cur->ml_next;
3176         }
3177 }
3178 
3179 void
3180 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3181 {
3182         size_t tsize = e_modtext - modtext;
3183         size_t dsize = e_moddata - moddata;
3184 
3185         *text_arena = vmem_create("module_text", tsize ? modtext : NULL, tsize,
3186             1, segkmem_alloc, segkmem_free, heaptext_arena, 0, VM_SLEEP);
3187         *data_arena = vmem_create("module_data", dsize ? moddata : NULL, dsize,
3188             1, segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3189 }
3190 
3191 caddr_t
3192 kobj_text_alloc(vmem_t *arena, size_t size)
3193 {
3194         return (vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT));
3195 }
3196 
3197 /*ARGSUSED*/
3198 caddr_t
3199 kobj_texthole_alloc(caddr_t addr, size_t size)
3200 {
3201         panic("unexpected call to kobj_texthole_alloc()");
3202         /*NOTREACHED*/
3203         return (0);
3204 }
3205 
3206 /*ARGSUSED*/
3207 void
3208 kobj_texthole_free(caddr_t addr, size_t size)
3209 {
3210         panic("unexpected call to kobj_texthole_free()");
3211 }
3212 
3213 /*
3214  * This is called just after configure() in startup().
3215  *
3216  * The ISALIST concept is a bit hopeless on Intel, because
3217  * there's no guarantee of an ever-more-capable processor
3218  * given that various parts of the instruction set may appear
3219  * and disappear between different implementations.
3220  *
3221  * While it would be possible to correct it and even enhance
3222  * it somewhat, the explicit hardware capability bitmask allows
3223  * more flexibility.
3224  *
3225  * So, we just leave this alone.
3226  */
3227 void
3228 setx86isalist(void)
3229 {
3230         char *tp;
3231         size_t len;
3232         extern char *isa_list;
3233 
3234 #define TBUFSIZE        1024
3235 
3236         tp = kmem_alloc(TBUFSIZE, KM_SLEEP);
3237         *tp = '\0';
3238 
3239 #if defined(__amd64)
3240         (void) strcpy(tp, "amd64 ");
3241 #endif
3242 
3243         switch (x86_vendor) {
3244         case X86_VENDOR_Intel:
3245         case X86_VENDOR_AMD:
3246         case X86_VENDOR_TM:
3247                 if (is_x86_feature(x86_featureset, X86FSET_CMOV)) {
3248                         /*
3249                          * Pentium Pro or later
3250                          */
3251                         (void) strcat(tp, "pentium_pro");
3252                         (void) strcat(tp,
3253                             is_x86_feature(x86_featureset, X86FSET_MMX) ?
3254                             "+mmx pentium_pro " : " ");
3255                 }
3256                 /*FALLTHROUGH*/
3257         case X86_VENDOR_Cyrix:
3258                 /*
3259                  * The Cyrix 6x86 does not have any Pentium features
3260                  * accessible while not at privilege level 0.
3261                  */
3262                 if (is_x86_feature(x86_featureset, X86FSET_CPUID)) {
3263                         (void) strcat(tp, "pentium");
3264                         (void) strcat(tp,
3265                             is_x86_feature(x86_featureset, X86FSET_MMX) ?
3266                             "+mmx pentium " : " ");
3267                 }
3268                 break;
3269         default:
3270                 break;
3271         }
3272         (void) strcat(tp, "i486 i386 i86");
3273         len = strlen(tp) + 1;   /* account for NULL at end of string */
3274         isa_list = strcpy(kmem_alloc(len, KM_SLEEP), tp);
3275         kmem_free(tp, TBUFSIZE);
3276 
3277 #undef TBUFSIZE
3278 }
3279 
3280 
3281 #ifdef __amd64
3282 
3283 void *
3284 device_arena_alloc(size_t size, int vm_flag)
3285 {
3286         return (vmem_alloc(device_arena, size, vm_flag));
3287 }
3288 
3289 void
3290 device_arena_free(void *vaddr, size_t size)
3291 {
3292         vmem_free(device_arena, vaddr, size);
3293 }
3294 
3295 #else /* __i386 */
3296 
3297 void *
3298 device_arena_alloc(size_t size, int vm_flag)
3299 {
3300         caddr_t vaddr;
3301         uintptr_t v;
3302         size_t  start;
3303         size_t  end;
3304 
3305         vaddr = vmem_alloc(heap_arena, size, vm_flag);
3306         if (vaddr == NULL)
3307                 return (NULL);
3308 
3309         v = (uintptr_t)vaddr;
3310         ASSERT(v >= kernelbase);
3311         ASSERT(v + size <= valloc_base);
3312 
3313         start = btop(v - kernelbase);
3314         end = btop(v + size - 1 - kernelbase);
3315         ASSERT(start < toxic_bit_map_len);
3316         ASSERT(end < toxic_bit_map_len);
3317 
3318         while (start <= end) {
3319                 BT_ATOMIC_SET(toxic_bit_map, start);
3320                 ++start;
3321         }
3322         return (vaddr);
3323 }
3324 
3325 void
3326 device_arena_free(void *vaddr, size_t size)
3327 {
3328         uintptr_t v = (uintptr_t)vaddr;
3329         size_t  start;
3330         size_t  end;
3331 
3332         ASSERT(v >= kernelbase);
3333         ASSERT(v + size <= valloc_base);
3334 
3335         start = btop(v - kernelbase);
3336         end = btop(v + size - 1 - kernelbase);
3337         ASSERT(start < toxic_bit_map_len);
3338         ASSERT(end < toxic_bit_map_len);
3339 
3340         while (start <= end) {
3341                 ASSERT(BT_TEST(toxic_bit_map, start) != 0);
3342                 BT_ATOMIC_CLEAR(toxic_bit_map, start);
3343                 ++start;
3344         }
3345         vmem_free(heap_arena, vaddr, size);
3346 }
3347 
3348 /*
3349  * returns 1st address in range that is in device arena, or NULL
3350  * if len is not NULL it returns the length of the toxic range
3351  */
3352 void *
3353 device_arena_contains(void *vaddr, size_t size, size_t *len)
3354 {
3355         uintptr_t v = (uintptr_t)vaddr;
3356         uintptr_t eaddr = v + size;
3357         size_t start;
3358         size_t end;
3359 
3360         /*
3361          * if called very early by kmdb, just return NULL
3362          */
3363         if (toxic_bit_map == NULL)
3364                 return (NULL);
3365 
3366         /*
3367          * First check if we're completely outside the bitmap range.
3368          */
3369         if (v >= valloc_base || eaddr < kernelbase)
3370                 return (NULL);
3371 
3372         /*
3373          * Trim ends of search to look at only what the bitmap covers.
3374          */
3375         if (v < kernelbase)
3376                 v = kernelbase;
3377         start = btop(v - kernelbase);
3378         end = btop(eaddr - kernelbase);
3379         if (end >= toxic_bit_map_len)
3380                 end = toxic_bit_map_len;
3381 
3382         if (bt_range(toxic_bit_map, &start, &end, end) == 0)
3383                 return (NULL);
3384 
3385         v = kernelbase + ptob(start);
3386         if (len != NULL)
3387                 *len = ptob(end - start);
3388         return ((void *)v);
3389 }
3390 
3391 #endif  /* __i386 */