1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2016 by Delphix. All rights reserved.
  25  */
  26 
  27 #include <sys/machsystm.h>
  28 #include <sys/archsystm.h>
  29 #include <sys/vm.h>
  30 #include <sys/cpu.h>
  31 #include <sys/atomic.h>
  32 #include <sys/reboot.h>
  33 #include <sys/kdi.h>
  34 #include <sys/bootconf.h>
  35 #include <sys/memlist_plat.h>
  36 #include <sys/memlist_impl.h>
  37 #include <sys/prom_plat.h>
  38 #include <sys/prom_isa.h>
  39 #include <sys/autoconf.h>
  40 #include <sys/ivintr.h>
  41 #include <sys/fpu/fpusystm.h>
  42 #include <sys/iommutsb.h>
  43 #include <vm/vm_dep.h>
  44 #include <vm/seg_dev.h>
  45 #include <vm/seg_kmem.h>
  46 #include <vm/seg_kpm.h>
  47 #include <vm/seg_map.h>
  48 #include <vm/seg_kp.h>
  49 #include <sys/sysconf.h>
  50 #include <vm/hat_sfmmu.h>
  51 #include <sys/kobj.h>
  52 #include <sys/sun4asi.h>
  53 #include <sys/clconf.h>
  54 #include <sys/platform_module.h>
  55 #include <sys/panic.h>
  56 #include <sys/cpu_sgnblk_defs.h>
  57 #include <sys/clock.h>
  58 #include <sys/cmn_err.h>
  59 #include <sys/dumphdr.h>
  60 #include <sys/promif.h>
  61 #include <sys/prom_debug.h>
  62 #include <sys/traptrace.h>
  63 #include <sys/memnode.h>
  64 #include <sys/mem_cage.h>
  65 #include <sys/mmu.h>
  66 #include <sys/swap.h>
  67 
  68 extern void setup_trap_table(void);
  69 extern int cpu_intrq_setup(struct cpu *);
  70 extern void cpu_intrq_register(struct cpu *);
  71 extern void contig_mem_init(void);
  72 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
  73 extern void mach_dump_buffer_init(void);
  74 extern void mach_descrip_init(void);
  75 extern void mach_descrip_startup_fini(void);
  76 extern void mach_memscrub(void);
  77 extern void mach_fpras(void);
  78 extern void mach_cpu_halt_idle(void);
  79 extern void mach_hw_copy_limit(void);
  80 extern void load_mach_drivers(void);
  81 extern void load_tod_module(void);
  82 #pragma weak load_tod_module
  83 
  84 extern int ndata_alloc_mmfsa(struct memlist *ndata);
  85 #pragma weak ndata_alloc_mmfsa
  86 
  87 extern void cif_init(void);
  88 #pragma weak cif_init
  89 
  90 extern void parse_idprom(void);
  91 extern void add_vx_handler(char *, int, void (*)(cell_t *));
  92 extern void mem_config_init(void);
  93 extern void memseg_remap_init(void);
  94 
  95 extern void mach_kpm_init(void);
  96 extern void pcf_init();
  97 extern int size_pse_array(pgcnt_t, int);
  98 extern void pg_init();
  99 
 100 /*
 101  * External Data:
 102  */
 103 extern int vac_size;    /* cache size in bytes */
 104 extern uint_t vac_mask; /* VAC alignment consistency mask */
 105 extern uint_t vac_colors;
 106 
 107 /*
 108  * Global Data Definitions:
 109  */
 110 
 111 /*
 112  * XXX - Don't port this to new architectures
 113  * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
 114  * 'romp' has no use with a prom with an IEEE 1275 client interface.
 115  * The driver doesn't use the value, but it depends on the symbol.
 116  */
 117 void *romp;             /* veritas driver won't load without romp 4154976 */
 118 /*
 119  * Declare these as initialized data so we can patch them.
 120  */
 121 pgcnt_t physmem = 0;    /* memory size in pages, patch if you want less */
 122 pgcnt_t segkpsize =
 123     btop(SEGKPDEFSIZE); /* size of segkp segment in pages */
 124 uint_t segmap_percent = 6; /* Size of segmap segment */
 125 
 126 int use_cache = 1;              /* cache not reliable (605 bugs) with MP */
 127 int vac_copyback = 1;
 128 char *cache_mode = NULL;
 129 int use_mix = 1;
 130 int prom_debug = 0;
 131 
 132 caddr_t boot_tba;               /* %tba at boot - used by kmdb */
 133 uint_t  tba_taken_over = 0;
 134 
 135 caddr_t s_text;                 /* start of kernel text segment */
 136 caddr_t e_text;                 /* end of kernel text segment */
 137 caddr_t s_data;                 /* start of kernel data segment */
 138 caddr_t e_data;                 /* end of kernel data segment */
 139 
 140 caddr_t modtext;                /* beginning of module text */
 141 size_t  modtext_sz;             /* size of module text */
 142 caddr_t moddata;                /* beginning of module data reserve */
 143 caddr_t e_moddata;              /* end of module data reserve */
 144 
 145 /*
 146  * End of first block of contiguous kernel in 32-bit virtual address space
 147  */
 148 caddr_t         econtig32;      /* end of first blk of contiguous kernel */
 149 
 150 caddr_t         ncbase;         /* beginning of non-cached segment */
 151 caddr_t         ncend;          /* end of non-cached segment */
 152 
 153 size_t  ndata_remain_sz;        /* bytes from end of data to 4MB boundary */
 154 caddr_t nalloc_base;            /* beginning of nucleus allocation */
 155 caddr_t nalloc_end;             /* end of nucleus allocatable memory */
 156 caddr_t valloc_base;            /* beginning of kvalloc segment */
 157 
 158 caddr_t kmem64_base;            /* base of kernel mem segment in 64-bit space */
 159 caddr_t kmem64_end;             /* end of kernel mem segment in 64-bit space */
 160 size_t  kmem64_sz;              /* bytes in kernel mem segment, 64-bit space */
 161 caddr_t kmem64_aligned_end;     /* end of large page, overmaps 64-bit space */
 162 int     kmem64_szc;             /* page size code */
 163 uint64_t kmem64_pabase = (uint64_t)-1;  /* physical address of kmem64_base */
 164 
 165 uintptr_t shm_alignment;        /* VAC address consistency modulus */
 166 struct memlist *phys_install;   /* Total installed physical memory */
 167 struct memlist *phys_avail;     /* Available (unreserved) physical memory */
 168 struct memlist *virt_avail;     /* Available (unmapped?) virtual memory */
 169 struct memlist *nopp_list;      /* pages with no backing page structs */
 170 struct memlist ndata;           /* memlist of nucleus allocatable memory */
 171 int memexp_flag;                /* memory expansion card flag */
 172 uint64_t ecache_flushaddr;      /* physical address used for flushing E$ */
 173 pgcnt_t obp_pages;              /* Physical pages used by OBP */
 174 
 175 /*
 176  * VM data structures
 177  */
 178 long page_hashsz;               /* Size of page hash table (power of two) */
 179 unsigned int page_hashsz_shift; /* log2(page_hashsz) */
 180 struct page *pp_base;           /* Base of system page struct array */
 181 size_t pp_sz;                   /* Size in bytes of page struct array */
 182 struct page **page_hash;        /* Page hash table */
 183 pad_mutex_t *pse_mutex;         /* Locks protecting pp->p_selock */
 184 size_t pse_table_size;          /* Number of mutexes in pse_mutex[] */
 185 int pse_shift;                  /* log2(pse_table_size) */
 186 struct seg ktextseg;            /* Segment used for kernel executable image */
 187 struct seg kvalloc;             /* Segment used for "valloc" mapping */
 188 struct seg kpseg;               /* Segment used for pageable kernel virt mem */
 189 struct seg ktexthole;           /* Segment used for nucleus text hole */
 190 struct seg kmapseg;             /* Segment used for generic kernel mappings */
 191 struct seg kpmseg;              /* Segment used for physical mapping */
 192 struct seg kdebugseg;           /* Segment used for the kernel debugger */
 193 
 194 void *kpm_pp_base;              /* Base of system kpm_page array */
 195 size_t  kpm_pp_sz;              /* Size of system kpm_page array */
 196 pgcnt_t kpm_npages;             /* How many kpm pages are managed */
 197 
 198 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */
 199 struct seg *segkmap = &kmapseg;     /* Kernel generic mapping segment */
 200 struct seg *segkpm = &kpmseg;       /* 64bit kernel physical mapping segment */
 201 
 202 int segzio_fromheap = 0;        /* zio allocations occur from heap */
 203 caddr_t segzio_base;            /* Base address of segzio */
 204 pgcnt_t segziosize = 0;         /* size of zio segment in pages */
 205 
 206 /*
 207  * A static DR page_t VA map is reserved that can map the page structures
 208  * for a domain's entire RA space. The pages that backs this space are
 209  * dynamically allocated and need not be physically contiguous.  The DR
 210  * map size is derived from KPM size.
 211  */
 212 int ppvm_enable = 0;            /* Static virtual map for page structs */
 213 page_t *ppvm_base;              /* Base of page struct map */
 214 pgcnt_t ppvm_size = 0;          /* Size of page struct map */
 215 
 216 /*
 217  * debugger pages (if allocated)
 218  */
 219 struct vnode kdebugvp;
 220 
 221 /*
 222  * VA range available to the debugger
 223  */
 224 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
 225 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
 226 
 227 /*
 228  * Segment for relocated kernel structures in 64-bit large RAM kernels
 229  */
 230 struct seg kmem64;
 231 
 232 struct memseg *memseg_free;
 233 
 234 struct vnode unused_pages_vp;
 235 
 236 /*
 237  * VM data structures allocated early during boot.
 238  */
 239 size_t pagehash_sz;
 240 uint64_t memlist_sz;
 241 
 242 char tbr_wr_addr_inited = 0;
 243 
 244 caddr_t mpo_heap32_buf = NULL;
 245 size_t  mpo_heap32_bufsz = 0;
 246 
 247 /*
 248  * Static Routines:
 249  */
 250 static int ndata_alloc_memseg(struct memlist *, size_t);
 251 static void memlist_new(uint64_t, uint64_t, struct memlist **);
 252 static void memlist_add(uint64_t, uint64_t,
 253         struct memlist **, struct memlist **);
 254 static void kphysm_init(void);
 255 static void kvm_init(void);
 256 static void install_kmem64_tte(void);
 257 
 258 static void startup_init(void);
 259 static void startup_memlist(void);
 260 static void startup_modules(void);
 261 static void startup_bop_gone(void);
 262 static void startup_vm(void);
 263 static void startup_end(void);
 264 static void setup_cage_params(void);
 265 static void startup_create_io_node(void);
 266 
 267 static pgcnt_t npages;
 268 static struct memlist *memlist;
 269 void *memlist_end;
 270 
 271 static pgcnt_t bop_alloc_pages;
 272 static caddr_t hblk_base;
 273 uint_t hblk_alloc_dynamic = 0;
 274 uint_t hblk1_min = H1MIN;
 275 
 276 
 277 /*
 278  * Hooks for unsupported platforms and down-rev firmware
 279  */
 280 int iam_positron(void);
 281 #pragma weak iam_positron
 282 static void do_prom_version_check(void);
 283 
 284 /*
 285  * After receiving a thermal interrupt, this is the number of seconds
 286  * to delay before shutting off the system, assuming
 287  * shutdown fails.  Use /etc/system to change the delay if this isn't
 288  * large enough.
 289  */
 290 int thermal_powerdown_delay = 1200;
 291 
 292 /*
 293  * Used to hold off page relocations into the cage until OBP has completed
 294  * its boot-time handoff of its resources to the kernel.
 295  */
 296 int page_relocate_ready = 0;
 297 
 298 /*
 299  * Indicate if kmem64 allocation was done in small chunks
 300  */
 301 int kmem64_smchunks = 0;
 302 
 303 /*
 304  * Enable some debugging messages concerning memory usage...
 305  */
 306 #ifdef  DEBUGGING_MEM
 307 static int debugging_mem;
 308 static void
 309 printmemlist(char *title, struct memlist *list)
 310 {
 311         if (!debugging_mem)
 312                 return;
 313 
 314         printf("%s\n", title);
 315 
 316         while (list) {
 317                 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
 318                     (uint32_t)(list->ml_address >> 32),
 319                     (uint32_t)list->ml_address,
 320                     (uint32_t)(list->ml_size >> 32),
 321                     (uint32_t)(list->ml_size));
 322                 list = list->ml_next;
 323         }
 324 }
 325 
 326 void
 327 printmemseg(struct memseg *memseg)
 328 {
 329         if (!debugging_mem)
 330                 return;
 331 
 332         printf("memseg\n");
 333 
 334         while (memseg) {
 335                 prom_printf("\tpage = 0x%p, epage = 0x%p, "
 336                     "pfn = 0x%x, epfn = 0x%x\n",
 337                     memseg->pages, memseg->epages,
 338                     memseg->pages_base, memseg->pages_end);
 339                 memseg = memseg->next;
 340         }
 341 }
 342 
 343 #define debug_pause(str)        halt((str))
 344 #define MPRINTF(str)            if (debugging_mem) prom_printf((str))
 345 #define MPRINTF1(str, a)        if (debugging_mem) prom_printf((str), (a))
 346 #define MPRINTF2(str, a, b)     if (debugging_mem) prom_printf((str), (a), (b))
 347 #define MPRINTF3(str, a, b, c) \
 348         if (debugging_mem) prom_printf((str), (a), (b), (c))
 349 #else   /* DEBUGGING_MEM */
 350 #define MPRINTF(str)
 351 #define MPRINTF1(str, a)
 352 #define MPRINTF2(str, a, b)
 353 #define MPRINTF3(str, a, b, c)
 354 #endif  /* DEBUGGING_MEM */
 355 
 356 
 357 /*
 358  *
 359  *                    Kernel's Virtual Memory Layout.
 360  *                       /-----------------------\
 361  * 0xFFFFFFFF.FFFFFFFF  -|                       |-
 362  *                       |   OBP's virtual page  |
 363  *                       |        tables         |
 364  * 0xFFFFFFFC.00000000  -|-----------------------|-
 365  *                       :                       :
 366  *                       :                       :
 367  *                      -|-----------------------|-
 368  *                       |       segzio          | (base and size vary)
 369  * 0xFFFFFE00.00000000  -|-----------------------|-
 370  *                       |                       |  Ultrasparc I/II support
 371  *                       |    segkpm segment     |  up to 2TB of physical
 372  *                       | (64-bit kernel ONLY)  |  memory, VAC has 2 colors
 373  *                       |                       |
 374  * 0xFFFFFA00.00000000  -|-----------------------|- 2TB segkpm alignment
 375  *                       :                       :
 376  *                       :                       :
 377  * 0xFFFFF810.00000000  -|-----------------------|- hole_end
 378  *                       |                       |      ^
 379  *                       |  UltraSPARC I/II call |      |
 380  *                       | bug requires an extra |      |
 381  *                       | 4 GB of space between |      |
 382  *                       |   hole and used RAM   |      |
 383  *                       |                       |      |
 384  * 0xFFFFF800.00000000  -|-----------------------|-     |
 385  *                       |                       |      |
 386  *                       | Virtual Address Hole  |   UltraSPARC
 387  *                       |  on UltraSPARC I/II   |  I/II * ONLY *
 388  *                       |                       |      |
 389  * 0x00000800.00000000  -|-----------------------|-     |
 390  *                       |                       |      |
 391  *                       |  UltraSPARC I/II call |      |
 392  *                       | bug requires an extra |      |
 393  *                       | 4 GB of space between |      |
 394  *                       |   hole and used RAM   |      |
 395  *                       |                       |      v
 396  * 0x000007FF.00000000  -|-----------------------|- hole_start -----
 397  *                       :                       :                 ^
 398  *                       :                       :                 |
 399  *                       |-----------------------|                 |
 400  *                       |                       |                 |
 401  *                       |  ecache flush area    |                 |
 402  *                       |  (twice largest e$)   |                 |
 403  *                       |                       |                 |
 404  * 0x00000XXX.XXX00000  -|-----------------------|- kmem64_        |
 405  *                       | overmapped area       |   alignend_end  |
 406  *                       | (kmem64_alignsize     |                 |
 407  *                       |  boundary)            |                 |
 408  * 0x00000XXX.XXXXXXXX  -|-----------------------|- kmem64_end     |
 409  *                       |                       |                 |
 410  *                       |   64-bit kernel ONLY  |                 |
 411  *                       |                       |                 |
 412  *                       |    kmem64 segment     |                 |
 413  *                       |                       |                 |
 414  *                       | (Relocated extra HME  |           Approximately
 415  *                       |   block allocations,  |          1 TB of virtual
 416  *                       |   memnode freelists,  |           address space
 417  *                       |    HME hash buckets,  |                 |
 418  *                       | mml_table, kpmp_table,|                 |
 419  *                       |  page_t array and     |                 |
 420  *                       |  hashblock pool to    |                 |
 421  *                       |   avoid hard-coded    |                 |
 422  *                       |     32-bit vaddr      |                 |
 423  *                       |     limitations)      |                 |
 424  *                       |                       |                 v
 425  * 0x00000700.00000000  -|-----------------------|- SYSLIMIT (kmem64_base)
 426  *                       |                       |
 427  *                       |  segkmem segment      | (SYSLIMIT - SYSBASE = 4TB)
 428  *                       |                       |
 429  * 0x00000300.00000000  -|-----------------------|- SYSBASE
 430  *                       :                       :
 431  *                       :                       :
 432  *                      -|-----------------------|-
 433  *                       |                       |
 434  *                       |  segmap segment       |   SEGMAPSIZE (1/8th physmem,
 435  *                       |                       |               256G MAX)
 436  * 0x000002a7.50000000  -|-----------------------|- SEGMAPBASE
 437  *                       :                       :
 438  *                       :                       :
 439  *                      -|-----------------------|-
 440  *                       |                       |
 441  *                       |       segkp           |    SEGKPSIZE (2GB)
 442  *                       |                       |
 443  *                       |                       |
 444  * 0x000002a1.00000000  -|-----------------------|- SEGKPBASE
 445  *                       |                       |
 446  * 0x000002a0.00000000  -|-----------------------|- MEMSCRUBBASE
 447  *                       |                       |       (SEGKPBASE - 0x400000)
 448  * 0x0000029F.FFE00000  -|-----------------------|- ARGSBASE
 449  *                       |                       |       (MEMSCRUBBASE - NCARGS)
 450  * 0x0000029F.FFD80000  -|-----------------------|- PPMAPBASE
 451  *                       |                       |       (ARGSBASE - PPMAPSIZE)
 452  * 0x0000029F.FFD00000  -|-----------------------|- PPMAP_FAST_BASE
 453  *                       |                       |
 454  * 0x0000029F.FF980000  -|-----------------------|- PIOMAPBASE
 455  *                       |                       |
 456  * 0x0000029F.FF580000  -|-----------------------|- NARG_BASE
 457  *                       :                       :
 458  *                       :                       :
 459  * 0x00000000.FFFFFFFF  -|-----------------------|- OFW_END_ADDR
 460  *                       |                       |
 461  *                       |         OBP           |
 462  *                       |                       |
 463  * 0x00000000.F0000000  -|-----------------------|- OFW_START_ADDR
 464  *                       |         kmdb          |
 465  * 0x00000000.EDD00000  -|-----------------------|- SEGDEBUGBASE
 466  *                       :                       :
 467  *                       :                       :
 468  * 0x00000000.7c000000  -|-----------------------|- SYSLIMIT32
 469  *                       |                       |
 470  *                       |  segkmem32 segment    | (SYSLIMIT32 - SYSBASE32 =
 471  *                       |                       |    ~64MB)
 472  *                      -|-----------------------|
 473  *                       |      IVSIZE           |
 474  * 0x00000000.70004000  -|-----------------------|
 475  *                       |     panicbuf          |
 476  * 0x00000000.70002000  -|-----------------------|
 477  *                       |      PAGESIZE         |
 478  * 0x00000000.70000000  -|-----------------------|- SYSBASE32
 479  *                       |       boot-time       |
 480  *                       |    temporary space    |
 481  * 0x00000000.4C000000  -|-----------------------|- BOOTTMPBASE
 482  *                       :                       :
 483  *                       :                       :
 484  *                       |                       |
 485  *                       |-----------------------|- econtig32
 486  *                       |    vm structures      |
 487  * 0x00000000.01C00000   |-----------------------|- nalloc_end
 488  *                       |         TSBs          |
 489  *                       |-----------------------|- end/nalloc_base
 490  *                       |   kernel data & bss   |
 491  * 0x00000000.01800000  -|-----------------------|
 492  *                       :   nucleus text hole   :
 493  * 0x00000000.01400000  -|-----------------------|
 494  *                       :                       :
 495  *                       |-----------------------|
 496  *                       |      module text      |
 497  *                       |-----------------------|- e_text/modtext
 498  *                       |      kernel text      |
 499  *                       |-----------------------|
 500  *                       |    trap table (48k)   |
 501  * 0x00000000.01000000  -|-----------------------|- KERNELBASE
 502  *                       | reserved for trapstat |} TSTAT_TOTAL_SIZE
 503  *                       |-----------------------|
 504  *                       |                       |
 505  *                       |        invalid        |
 506  *                       |                       |
 507  * 0x00000000.00000000  _|_______________________|
 508  *
 509  *
 510  *
 511  *                   32-bit User Virtual Memory Layout.
 512  *                       /-----------------------\
 513  *                       |                       |
 514  *                       |        invalid        |
 515  *                       |                       |
 516  *          0xFFC00000  -|-----------------------|- USERLIMIT
 517  *                       |       user stack      |
 518  *                       :                       :
 519  *                       :                       :
 520  *                       :                       :
 521  *                       |       user data       |
 522  *                      -|-----------------------|-
 523  *                       |       user text       |
 524  *          0x00002000  -|-----------------------|-
 525  *                       |       invalid         |
 526  *          0x00000000  _|_______________________|
 527  *
 528  *
 529  *
 530  *                   64-bit User Virtual Memory Layout.
 531  *                       /-----------------------\
 532  *                       |                       |
 533  *                       |        invalid        |
 534  *                       |                       |
 535  *  0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
 536  *                       |       user stack      |
 537  *                       :                       :
 538  *                       :                       :
 539  *                       :                       :
 540  *                       |       user data       |
 541  *                      -|-----------------------|-
 542  *                       |       user text       |
 543  *  0x00000000.01000000 -|-----------------------|-
 544  *                       |       invalid         |
 545  *  0x00000000.00000000 _|_______________________|
 546  */
 547 
 548 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
 549 extern uint64_t ecache_flush_address(void);
 550 
 551 #pragma weak load_platform_modules
 552 #pragma weak plat_startup_memlist
 553 #pragma weak ecache_init_scrub_flush_area
 554 #pragma weak ecache_flush_address
 555 
 556 
 557 /*
 558  * By default the DR Cage is enabled for maximum OS
 559  * MPSS performance.  Users needing to disable the cage mechanism
 560  * can set this variable to zero via /etc/system.
 561  * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
 562  * will result in loss of DR functionality.
 563  * Platforms wishing to disable kernel Cage by default
 564  * should do so in their set_platform_defaults() routine.
 565  */
 566 int     kernel_cage_enable = 1;
 567 
 568 static void
 569 setup_cage_params(void)
 570 {
 571         void (*func)(void);
 572 
 573         func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
 574         if (func != NULL) {
 575                 (*func)();
 576                 return;
 577         }
 578 
 579         if (kernel_cage_enable == 0) {
 580                 return;
 581         }
 582         kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
 583 
 584         if (kcage_on) {
 585                 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
 586         } else {
 587                 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
 588         }
 589 
 590 }
 591 
 592 /*
 593  * Machine-dependent startup code
 594  */
 595 void
 596 startup(void)
 597 {
 598         startup_init();
 599         if (&startup_platform)
 600                 startup_platform();
 601         startup_memlist();
 602         startup_modules();
 603         setup_cage_params();
 604         startup_bop_gone();
 605         startup_vm();
 606         startup_end();
 607 }
 608 
 609 struct regs sync_reg_buf;
 610 uint64_t sync_tt;
 611 
 612 void
 613 sync_handler(void)
 614 {
 615         struct  panic_trap_info         ti;
 616         int i;
 617 
 618         /*
 619          * Prevent trying to talk to the other CPUs since they are
 620          * sitting in the prom and won't reply.
 621          */
 622         for (i = 0; i < NCPU; i++) {
 623                 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
 624                         cpu[i]->cpu_flags &= ~CPU_READY;
 625                         cpu[i]->cpu_flags |= CPU_QUIESCED;
 626                         CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
 627                 }
 628         }
 629 
 630         /*
 631          * Force a serial dump, since there are no CPUs to help.
 632          */
 633         dump_plat_mincpu = 0;
 634 
 635         /*
 636          * We've managed to get here without going through the
 637          * normal panic code path. Try and save some useful
 638          * information.
 639          */
 640         if (!panicstr && (curthread->t_panic_trap == NULL)) {
 641                 ti.trap_type = sync_tt;
 642                 ti.trap_regs = &sync_reg_buf;
 643                 ti.trap_addr = NULL;
 644                 ti.trap_mmu_fsr = 0x0;
 645 
 646                 curthread->t_panic_trap = &ti;
 647         }
 648 
 649         /*
 650          * If we're re-entering the panic path, update the signature
 651          * block so that the SC knows we're in the second part of panic.
 652          */
 653         if (panicstr)
 654                 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
 655 
 656         nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
 657         panic("sync initiated");
 658 }
 659 
 660 
 661 static void
 662 startup_init(void)
 663 {
 664         /*
 665          * We want to save the registers while we're still in OBP
 666          * so that we know they haven't been fiddled with since.
 667          * (In principle, OBP can't change them just because it
 668          * makes a callback, but we'd rather not depend on that
 669          * behavior.)
 670          */
 671         char            sync_str[] =
 672             "warning @ warning off : sync "
 673             "%%tl-c %%tstate h# %p x! "
 674             "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
 675             "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
 676             "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
 677             "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
 678             "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
 679             "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
 680             "%%y h# %p l! %%tl-c %%tt h# %p x! "
 681             "sync ; warning !";
 682 
 683         /*
 684          * 20 == num of %p substrings
 685          * 16 == max num of chars %p will expand to.
 686          */
 687         char            bp[sizeof (sync_str) + 16 * 20];
 688 
 689         /*
 690          * Initialize ptl1 stack for the 1st CPU.
 691          */
 692         ptl1_init_cpu(&cpu0);
 693 
 694         /*
 695          * Initialize the address map for cache consistent mappings
 696          * to random pages; must be done after vac_size is set.
 697          */
 698         ppmapinit();
 699 
 700         /*
 701          * Initialize the PROM callback handler.
 702          */
 703         init_vx_handler();
 704 
 705         /*
 706          * have prom call sync_callback() to handle the sync and
 707          * save some useful information which will be stored in the
 708          * core file later.
 709          */
 710         (void) sprintf((char *)bp, sync_str,
 711             (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
 712             (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
 713             (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
 714             (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
 715             (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
 716             (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
 717             (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
 718             (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
 719             (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
 720             (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
 721         prom_interpret(bp, 0, 0, 0, 0, 0);
 722         add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
 723 }
 724 
 725 
 726 size_t
 727 calc_pp_sz(pgcnt_t npages)
 728 {
 729 
 730         return (npages * sizeof (struct page));
 731 }
 732 
 733 size_t
 734 calc_kpmpp_sz(pgcnt_t npages)
 735 {
 736 
 737         kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
 738         kpm_pgsz = 1ull << kpm_pgshft;
 739         kpm_pgoff = kpm_pgsz - 1;
 740         kpmp2pshft = kpm_pgshft - PAGESHIFT;
 741         kpmpnpgs = 1 << kpmp2pshft;
 742 
 743         if (kpm_smallpages == 0) {
 744                 /*
 745                  * Avoid fragmentation problems in kphysm_init()
 746                  * by allocating for all of physical memory
 747                  */
 748                 kpm_npages = ptokpmpr(physinstalled);
 749                 return (kpm_npages * sizeof (kpm_page_t));
 750         } else {
 751                 kpm_npages = npages;
 752                 return (kpm_npages * sizeof (kpm_spage_t));
 753         }
 754 }
 755 
 756 size_t
 757 calc_pagehash_sz(pgcnt_t npages)
 758 {
 759         /* LINTED */
 760         ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page))));
 761         /*
 762          * The page structure hash table size is a power of 2
 763          * such that the average hash chain length is PAGE_HASHAVELEN.
 764          */
 765         page_hashsz = npages / PAGE_HASHAVELEN;
 766         page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1),
 767             highbit(page_hashsz));
 768         page_hashsz = 1 << page_hashsz_shift;
 769         return (page_hashsz * sizeof (struct page *));
 770 }
 771 
 772 int testkmem64_smchunks = 0;
 773 
 774 int
 775 alloc_kmem64(caddr_t base, caddr_t end)
 776 {
 777         int i;
 778         caddr_t aligned_end = NULL;
 779 
 780         if (testkmem64_smchunks)
 781                 return (1);
 782 
 783         /*
 784          * Make one large memory alloc after figuring out the 64-bit size. This
 785          * will enable use of the largest page size appropriate for the system
 786          * architecture.
 787          */
 788         ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
 789         ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
 790         for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
 791                 size_t alloc_size, alignsize;
 792 #if !defined(C_OBP)
 793                 unsigned long long pa;
 794 #endif  /* !C_OBP */
 795 
 796                 if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
 797                         continue;
 798                 alignsize = TTEBYTES(i);
 799                 kmem64_szc = i;
 800 
 801                 /* limit page size for small memory */
 802                 if (mmu_btop(alignsize) > (npages >> 2))
 803                         continue;
 804 
 805                 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
 806                 alloc_size = aligned_end - base;
 807 #if !defined(C_OBP)
 808                 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
 809                         if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
 810                                 kmem64_pabase = pa;
 811                                 kmem64_aligned_end = aligned_end;
 812                                 install_kmem64_tte();
 813                                 break;
 814                         } else {
 815                                 prom_free_phys(alloc_size, pa);
 816                         }
 817                 }
 818 #else   /* !C_OBP */
 819                 if (prom_alloc(base, alloc_size, alignsize) == base) {
 820                         kmem64_pabase = va_to_pa(kmem64_base);
 821                         kmem64_aligned_end = aligned_end;
 822                         break;
 823                 }
 824 #endif  /* !C_OBP */
 825                 if (i == TTE8K) {
 826 #ifdef sun4v
 827                         /* return failure to try small allocations */
 828                         return (1);
 829 #else
 830                         prom_panic("kmem64 allocation failure");
 831 #endif
 832                 }
 833         }
 834         ASSERT(aligned_end != NULL);
 835         return (0);
 836 }
 837 
 838 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
 839 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
 840 
 841 #if !defined(C_OBP)
 842 /*
 843  * Install a temporary tte handler in OBP for kmem64 area.
 844  *
 845  * We map kmem64 area with large pages before the trap table is taken
 846  * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
 847  * the same area. Duplicate tlb entries with different page sizes
 848  * cause unpredicatble behavior.  To avoid this, we don't create
 849  * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
 850  * OBP).  Instead, we manage translations with a temporary va>tte-data
 851  * handler (kmem64-tte).  This handler is replaced by unix-tte when
 852  * the trap table is taken over.
 853  *
 854  * The temporary handler knows the physical address of the kmem64
 855  * area. It uses the prom's pgmap@ Forth word for other addresses.
 856  *
 857  * We have to use BOP_ALLOC() method for C-OBP platforms because
 858  * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
 859  * sun4u platforms. On sun4u we flush tlb after trap table is taken
 860  * over if we use large pages for kernel heap and kmem64. Since sun4u
 861  * prom (unlike sun4v) calls va>tte-data first for client address
 862  * translation prom's ttes for kmem64 can't get into TLB even if we
 863  * later switch to prom's trap table again. C-OBP uses 4M pages for
 864  * client mappings when possible so on all platforms we get the
 865  * benefit from large mappings for kmem64 area immediately during
 866  * boot.
 867  *
 868  * pseudo code:
 869  * if (context != 0) {
 870  *      return false
 871  * } else if (miss_va in range[kmem64_base, kmem64_end)) {
 872  *      tte = tte_template +
 873  *              (((miss_va & pagemask) - kmem64_base));
 874  *      return tte, true
 875  * } else {
 876  *      return pgmap@ result
 877  * }
 878  */
 879 char kmem64_obp_str[] =
 880         "h# %lx constant kmem64-base "
 881         "h# %lx constant kmem64-end "
 882         "h# %lx constant kmem64-pagemask "
 883         "h# %lx constant kmem64-template "
 884 
 885         ": kmem64-tte ( addr cnum -- false | tte-data true ) "
 886         "    if                                       ( addr ) "
 887         "       drop false exit then                  ( false ) "
 888         "    dup  kmem64-base kmem64-end  within  if  ( addr ) "
 889         "       kmem64-pagemask and                   ( addr' ) "
 890         "       kmem64-base -                         ( addr' ) "
 891         "       kmem64-template +                     ( tte ) "
 892         "       true                                  ( tte true ) "
 893         "    else                                     ( addr ) "
 894         "       pgmap@                                ( tte ) "
 895         "       dup 0< if true else drop false then   ( tte true  |  false ) "
 896         "    then                                     ( tte true  |  false ) "
 897         "; "
 898 
 899         "' kmem64-tte is va>tte-data "
 900 ;
 901 
 902 static void
 903 install_kmem64_tte()
 904 {
 905         char b[sizeof (kmem64_obp_str) + (4 * 16)];
 906         tte_t tte;
 907 
 908         PRM_DEBUG(kmem64_pabase);
 909         PRM_DEBUG(kmem64_szc);
 910         sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
 911             PROC_DATA | HAT_NOSYNC, kmem64_szc);
 912         PRM_DEBUG(tte.ll);
 913         (void) sprintf(b, kmem64_obp_str,
 914             kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
 915         ASSERT(strlen(b) < sizeof (b));
 916         prom_interpret(b, 0, 0, 0, 0, 0);
 917 }
 918 #endif  /* !C_OBP */
 919 
 920 /*
 921  * As OBP takes up some RAM when the system boots, pages will already be "lost"
 922  * to the system and reflected in npages by the time we see it.
 923  *
 924  * We only want to allocate kernel structures in the 64-bit virtual address
 925  * space on systems with enough RAM to make the overhead of keeping track of
 926  * an extra kernel memory segment worthwhile.
 927  *
 928  * Since OBP has already performed its memory allocations by this point, if we
 929  * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
 930  * memory in the 64-bit virtual address space; otherwise keep allocations
 931  * contiguous with we've mapped so far in the 32-bit virtual address space.
 932  */
 933 #define MINMOVE_RAM_MB  ((size_t)1900)
 934 #define MB_TO_BYTES(mb) ((mb) * 1048576ul)
 935 #define BYTES_TO_MB(b) ((b) / 1048576ul)
 936 
 937 pgcnt_t tune_npages = (pgcnt_t)
 938         (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
 939 
 940 #pragma weak page_set_colorequiv_arr_cpu
 941 extern void page_set_colorequiv_arr_cpu(void);
 942 extern void page_set_colorequiv_arr(void);
 943 
 944 static pgcnt_t ramdisk_npages;
 945 static struct memlist *old_phys_avail;
 946 
 947 kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
 948 
 949 static void
 950 startup_memlist(void)
 951 {
 952         size_t hmehash_sz, pagelist_sz, tt_sz;
 953         size_t psetable_sz;
 954         caddr_t alloc_base;
 955         caddr_t memspace;
 956         struct memlist *cur;
 957         size_t syslimit = (size_t)SYSLIMIT;
 958         size_t sysbase = (size_t)SYSBASE;
 959 
 960         /*
 961          * Initialize enough of the system to allow kmem_alloc to work by
 962          * calling boot to allocate its memory until the time that
 963          * kvm_init is completed.  The page structs are allocated after
 964          * rounding up end to the nearest page boundary; the memsegs are
 965          * initialized and the space they use comes from the kernel heap.
 966          * With appropriate initialization, they can be reallocated later
 967          * to a size appropriate for the machine's configuration.
 968          *
 969          * At this point, memory is allocated for things that will never
 970          * need to be freed, this used to be "valloced".  This allows a
 971          * savings as the pages don't need page structures to describe
 972          * them because them will not be managed by the vm system.
 973          */
 974 
 975         /*
 976          * We're loaded by boot with the following configuration (as
 977          * specified in the sun4u/conf/Mapfile):
 978          *
 979          *      text:           4 MB chunk aligned on a 4MB boundary
 980          *      data & bss: 4 MB chunk aligned on a 4MB boundary
 981          *
 982          * These two chunks will eventually be mapped by 2 locked 4MB
 983          * ttes and will represent the nucleus of the kernel.  This gives
 984          * us some free space that is already allocated, some or all of
 985          * which is made available to kernel module text.
 986          *
 987          * The free space in the data-bss chunk is used for nucleus
 988          * allocatable data structures and we reserve it using the
 989          * nalloc_base and nalloc_end variables.  This space is currently
 990          * being used for hat data structures required for tlb miss
 991          * handling operations.  We align nalloc_base to a l2 cache
 992          * linesize because this is the line size the hardware uses to
 993          * maintain cache coherency.
 994          * 512K is carved out for module data.
 995          */
 996 
 997         moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
 998         e_moddata = moddata + MODDATA;
 999         nalloc_base = e_moddata;
1000 
1001         nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
1002         valloc_base = nalloc_base;
1003 
1004         /*
1005          * Calculate the start of the data segment.
1006          */
1007         if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
1008                 prom_panic("nucleus data overflow");
1009 
1010         PRM_DEBUG(moddata);
1011         PRM_DEBUG(nalloc_base);
1012         PRM_DEBUG(nalloc_end);
1013 
1014         /*
1015          * Remember any slop after e_text so we can give it to the modules.
1016          */
1017         PRM_DEBUG(e_text);
1018         modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
1019         if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
1020                 prom_panic("nucleus text overflow");
1021         modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
1022             modtext;
1023         PRM_DEBUG(modtext);
1024         PRM_DEBUG(modtext_sz);
1025 
1026         init_boot_memlists();
1027         copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1028             &boot_physavail, &boot_physavail_len,
1029             &boot_virtavail, &boot_virtavail_len);
1030 
1031         /*
1032          * Remember what the physically available highest page is
1033          * so that dumpsys works properly, and find out how much
1034          * memory is installed.
1035          */
1036         installed_top_size_memlist_array(boot_physinstalled,
1037             boot_physinstalled_len, &physmax, &physinstalled);
1038         PRM_DEBUG(physinstalled);
1039         PRM_DEBUG(physmax);
1040 
1041         /* Fill out memory nodes config structure */
1042         startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
1043 
1044         /*
1045          * npages is the maximum of available physical memory possible.
1046          * (ie. it will never be more than this)
1047          *
1048          * When we boot from a ramdisk, the ramdisk memory isn't free, so
1049          * using phys_avail will underestimate what will end up being freed.
1050          * A better initial guess is just total memory minus the kernel text
1051          */
1052         npages = physinstalled - btop(MMU_PAGESIZE4M);
1053 
1054         /*
1055          * First allocate things that can go in the nucleus data page
1056          * (fault status, TSBs, dmv, CPUs)
1057          */
1058         ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
1059 
1060         if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
1061                 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
1062 
1063         if (ndata_alloc_tsbs(&ndata, npages) != 0)
1064                 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
1065 
1066         if (ndata_alloc_dmv(&ndata) != 0)
1067                 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
1068 
1069         if (ndata_alloc_page_mutexs(&ndata) != 0)
1070                 cmn_err(CE_PANIC,
1071                     "no more nucleus memory after page free lists alloc");
1072 
1073         if (ndata_alloc_hat(&ndata) != 0)
1074                 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
1075 
1076         if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
1077                 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
1078 
1079         /*
1080          * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1081          *
1082          * There are comments all over the SFMMU code warning of dire
1083          * consequences if the TSBs are moved out of 32-bit space.  This
1084          * is largely because the asm code uses "sethi %hi(addr)"-type
1085          * instructions which will not provide the expected result if the
1086          * address is a 64-bit one.
1087          *
1088          * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1089          */
1090         alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
1091         PRM_DEBUG(alloc_base);
1092 
1093         alloc_base = sfmmu_ktsb_alloc(alloc_base);
1094         alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1095         PRM_DEBUG(alloc_base);
1096 
1097         /*
1098          * Allocate IOMMU TSB array.  We do this here so that the physical
1099          * memory gets deducted from the PROM's physical memory list.
1100          */
1101         alloc_base = iommu_tsb_init(alloc_base);
1102         alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1103         PRM_DEBUG(alloc_base);
1104 
1105         /*
1106          * Allow for an early allocation of physically contiguous memory.
1107          */
1108         alloc_base = contig_mem_prealloc(alloc_base, npages);
1109 
1110         /*
1111          * Platforms like Starcat and OPL need special structures assigned in
1112          * 32-bit virtual address space because their probing routines execute
1113          * FCode, and FCode can't handle 64-bit virtual addresses...
1114          */
1115         if (&plat_startup_memlist) {
1116                 alloc_base = plat_startup_memlist(alloc_base);
1117                 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
1118                     ecache_alignsize);
1119                 PRM_DEBUG(alloc_base);
1120         }
1121 
1122         /*
1123          * Save off where the contiguous allocations to date have ended
1124          * in econtig32.
1125          */
1126         econtig32 = alloc_base;
1127         PRM_DEBUG(econtig32);
1128         if (econtig32 > (caddr_t)KERNEL_LIMIT32)
1129                 cmn_err(CE_PANIC, "econtig32 too big");
1130 
1131         pp_sz = calc_pp_sz(npages);
1132         PRM_DEBUG(pp_sz);
1133         if (kpm_enable) {
1134                 kpm_pp_sz = calc_kpmpp_sz(npages);
1135                 PRM_DEBUG(kpm_pp_sz);
1136         }
1137 
1138         hmehash_sz = calc_hmehash_sz(npages);
1139         PRM_DEBUG(hmehash_sz);
1140 
1141         pagehash_sz = calc_pagehash_sz(npages);
1142         PRM_DEBUG(pagehash_sz);
1143 
1144         pagelist_sz = calc_free_pagelist_sz();
1145         PRM_DEBUG(pagelist_sz);
1146 
1147 #ifdef  TRAPTRACE
1148         tt_sz = calc_traptrace_sz();
1149         PRM_DEBUG(tt_sz);
1150 #else
1151         tt_sz = 0;
1152 #endif  /* TRAPTRACE */
1153 
1154         /*
1155          * Place the array that protects pp->p_selock in the kmem64 wad.
1156          */
1157         pse_shift = size_pse_array(npages, max_ncpus);
1158         PRM_DEBUG(pse_shift);
1159         pse_table_size = 1 << pse_shift;
1160         PRM_DEBUG(pse_table_size);
1161         psetable_sz = roundup(
1162             pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
1163         PRM_DEBUG(psetable_sz);
1164 
1165         /*
1166          * Now allocate the whole wad
1167          */
1168         kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
1169             pagelist_sz + tt_sz + psetable_sz;
1170         kmem64_sz = roundup(kmem64_sz, PAGESIZE);
1171         kmem64_base = (caddr_t)syslimit;
1172         kmem64_end = kmem64_base + kmem64_sz;
1173         if (alloc_kmem64(kmem64_base, kmem64_end)) {
1174                 /*
1175                  * Attempt for kmem64 to allocate one big
1176                  * contiguous chunk of memory failed.
1177                  * We get here because we are sun4v.
1178                  * We will proceed by breaking up
1179                  * the allocation into two attempts.
1180                  * First, we allocate kpm_pp_sz, hmehash_sz,
1181                  * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as
1182                  * one contiguous chunk. This is a much smaller
1183                  * chunk and we should get it, if not we panic.
1184                  * Note that hmehash and tt need to be physically
1185                  * (in the real address sense) contiguous.
1186                  * Next, we use bop_alloc_chunk() to
1187                  * to allocate the page_t structures.
1188                  * This will allow the page_t to be allocated
1189                  * in multiple smaller chunks.
1190                  * In doing so, the assumption that page_t is
1191                  * physically contiguous no longer hold, this is ok
1192                  * for sun4v but not for sun4u.
1193                  */
1194                 size_t  tmp_size;
1195                 caddr_t tmp_base;
1196 
1197                 pp_sz  = roundup(pp_sz, PAGESIZE);
1198 
1199                 /*
1200                  * Allocate kpm_pp_sz, hmehash_sz,
1201                  * pagehash_sz, pagelist_sz, tt_sz & psetable_sz
1202                  */
1203                 tmp_base = kmem64_base + pp_sz;
1204                 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
1205                     pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
1206                 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
1207                         prom_panic("kmem64 prom_alloc contig failed");
1208                 PRM_DEBUG(tmp_base);
1209                 PRM_DEBUG(tmp_size);
1210 
1211                 /*
1212                  * Allocate the page_ts
1213                  */
1214                 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
1215                         prom_panic("kmem64 bop_alloc_chunk page_t failed");
1216                 PRM_DEBUG(kmem64_base);
1217                 PRM_DEBUG(pp_sz);
1218 
1219                 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
1220                 ASSERT(kmem64_aligned_end >= kmem64_end);
1221 
1222                 kmem64_smchunks = 1;
1223         } else {
1224 
1225                 /*
1226                  * We need to adjust pp_sz for the normal
1227                  * case where kmem64 can allocate one large chunk
1228                  */
1229                 if (kpm_smallpages == 0) {
1230                         npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
1231                 } else {
1232                         npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
1233                             sizeof (kpm_spage_t));
1234                 }
1235                 pp_sz = npages * sizeof (struct page);
1236         }
1237 
1238         if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
1239                 cmn_err(CE_PANIC, "not enough kmem64 space");
1240         PRM_DEBUG(kmem64_base);
1241         PRM_DEBUG(kmem64_end);
1242         PRM_DEBUG(kmem64_aligned_end);
1243 
1244         /*
1245          * ... and divy it up
1246          */
1247         alloc_base = kmem64_base;
1248 
1249         pp_base = (page_t *)alloc_base;
1250         alloc_base += pp_sz;
1251         alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1252         PRM_DEBUG(pp_base);
1253         PRM_DEBUG(npages);
1254 
1255         if (kpm_enable) {
1256                 kpm_pp_base = alloc_base;
1257                 if (kpm_smallpages == 0) {
1258                         /* kpm_npages based on physinstalled, don't reset */
1259                         kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
1260                 } else {
1261                         kpm_npages = ptokpmpr(npages);
1262                         kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
1263                 }
1264                 alloc_base += kpm_pp_sz;
1265                 alloc_base =
1266                     (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1267                 PRM_DEBUG(kpm_pp_base);
1268         }
1269 
1270         alloc_base = alloc_hmehash(alloc_base);
1271         alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1272         PRM_DEBUG(alloc_base);
1273 
1274         page_hash = (page_t **)alloc_base;
1275         alloc_base += pagehash_sz;
1276         alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1277         PRM_DEBUG(page_hash);
1278 
1279         alloc_base = alloc_page_freelists(alloc_base);
1280         alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1281         PRM_DEBUG(alloc_base);
1282 
1283 #ifdef  TRAPTRACE
1284         ttrace_buf = alloc_base;
1285         alloc_base += tt_sz;
1286         alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1287         PRM_DEBUG(alloc_base);
1288 #endif  /* TRAPTRACE */
1289 
1290         pse_mutex = (pad_mutex_t *)alloc_base;
1291         alloc_base += psetable_sz;
1292         alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1293         PRM_DEBUG(alloc_base);
1294 
1295         /*
1296          * Note that if we use small chunk allocations for
1297          * kmem64, we need to ensure kmem64_end is the same as
1298          * kmem64_aligned_end to prevent subsequent logic from
1299          * trying to reuse the overmapping.
1300          * Otherwise we adjust kmem64_end to what we really allocated.
1301          */
1302         if (kmem64_smchunks) {
1303                 kmem64_end = kmem64_aligned_end;
1304         } else {
1305                 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
1306         }
1307         kmem64_sz = kmem64_end - kmem64_base;
1308 
1309         if (&ecache_init_scrub_flush_area) {
1310                 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
1311                 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
1312         }
1313 
1314         /*
1315          * If physmem is patched to be non-zero, use it instead of
1316          * the monitor value unless physmem is larger than the total
1317          * amount of memory on hand.
1318          */
1319         if (physmem == 0 || physmem > npages)
1320                 physmem = npages;
1321 
1322         /*
1323          * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
1324          * is mounted as root. This memory is held down by OBP and unlike
1325          * the stub boot_archive is never released.
1326          *
1327          * In order to get things sized correctly on lower memory
1328          * machines (where the memory used by the ramdisk represents
1329          * a significant portion of memory), physmem is adjusted.
1330          *
1331          * This is done by subtracting the ramdisk_size which is set
1332          * to the size of the ramdisk (in Kb) in /etc/system at the
1333          * time the miniroot archive is constructed.
1334          */
1335         if (root_is_ramdisk == B_TRUE) {
1336                 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
1337                 physmem -= ramdisk_npages;
1338         }
1339 
1340         if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
1341                 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
1342 
1343         /*
1344          * Allocate space for the interrupt vector table.
1345          */
1346         memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
1347         if (memspace != (caddr_t)intr_vec_table)
1348                 prom_panic("interrupt vector table allocation failure");
1349 
1350         /*
1351          * Between now and when we finish copying in the memory lists,
1352          * allocations happen so the space gets fragmented and the
1353          * lists longer.  Leave enough space for lists twice as
1354          * long as we have now; then roundup to a pagesize.
1355          */
1356         memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
1357             prom_phys_avail_len() + prom_virt_avail_len());
1358         memlist_sz *= 2;
1359         memlist_sz = roundup(memlist_sz, PAGESIZE);
1360         memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
1361         if (memspace == NULL)
1362                 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
1363 
1364         memlist = (struct memlist *)memspace;
1365         memlist_end = (char *)memspace + memlist_sz;
1366         PRM_DEBUG(memlist);
1367         PRM_DEBUG(memlist_end);
1368 
1369         PRM_DEBUG(sysbase);
1370         PRM_DEBUG(syslimit);
1371         kernelheap_init((void *)sysbase, (void *)syslimit,
1372             (caddr_t)sysbase + PAGESIZE, NULL, NULL);
1373 
1374         /*
1375          * Take the most current snapshot we can by calling mem-update.
1376          */
1377         copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1378             &boot_physavail, &boot_physavail_len,
1379             &boot_virtavail, &boot_virtavail_len);
1380 
1381         /*
1382          * Remove the space used by prom_alloc from the kernel heap
1383          * plus the area actually used by the OBP (if any)
1384          * ignoring virtual addresses in virt_avail, above syslimit.
1385          */
1386         virt_avail = memlist;
1387         copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1388 
1389         for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) {
1390                 uint64_t range_base, range_size;
1391 
1392                 if ((range_base = cur->ml_address + cur->ml_size) <
1393                     (uint64_t)sysbase)
1394                         continue;
1395                 if (range_base >= (uint64_t)syslimit)
1396                         break;
1397                 /*
1398                  * Limit the range to end at syslimit.
1399                  */
1400                 range_size = MIN(cur->ml_next->ml_address,
1401                     (uint64_t)syslimit) - range_base;
1402                 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
1403                     0, 0, (void *)range_base, (void *)(range_base + range_size),
1404                     VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1405         }
1406 
1407         phys_avail = memlist;
1408         copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1409 
1410         /*
1411          * Add any extra memory at the end of the ndata region if there's at
1412          * least a page to add.  There might be a few more pages available in
1413          * the middle of the ndata region, but for now they are ignored.
1414          */
1415         nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
1416         if (nalloc_base == NULL)
1417                 nalloc_base = nalloc_end;
1418         ndata_remain_sz = nalloc_end - nalloc_base;
1419 
1420         /*
1421          * Copy physinstalled list into kernel space.
1422          */
1423         phys_install = memlist;
1424         copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
1425 
1426         /*
1427          * Create list of physical addrs we don't need pp's for:
1428          * kernel text 4M page
1429          * kernel data 4M page - ndata_remain_sz
1430          * kmem64 pages
1431          *
1432          * NB if adding any pages here, make sure no kpm page
1433          * overlaps can occur (see ASSERTs in kphysm_memsegs)
1434          */
1435         nopp_list = memlist;
1436         memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
1437         memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
1438             &memlist, &nopp_list);
1439 
1440         /* Don't add to nopp_list if kmem64 was allocated in smchunks */
1441         if (!kmem64_smchunks)
1442                 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
1443 
1444         if ((caddr_t)memlist > (memspace + memlist_sz))
1445                 prom_panic("memlist overflow");
1446 
1447         /*
1448          * Size the pcf array based on the number of cpus in the box at
1449          * boot time.
1450          */
1451         pcf_init();
1452 
1453         /*
1454          * Initialize the page structures from the memory lists.
1455          */
1456         kphysm_init();
1457 
1458         availrmem_initial = availrmem = freemem;
1459         PRM_DEBUG(availrmem);
1460 
1461         /*
1462          * Some of the locks depend on page_hashsz being set!
1463          * kmem_init() depends on this; so, keep it here.
1464          */
1465         page_lock_init();
1466 
1467         /*
1468          * Initialize kernel memory allocator.
1469          */
1470         kmem_init();
1471 
1472         /*
1473          * Factor in colorequiv to check additional 'equivalent' bins
1474          */
1475         if (&page_set_colorequiv_arr_cpu != NULL)
1476                 page_set_colorequiv_arr_cpu();
1477         else
1478                 page_set_colorequiv_arr();
1479 
1480         /*
1481          * Initialize bp_mapin().
1482          */
1483         bp_init(shm_alignment, HAT_STRICTORDER);
1484 
1485         /*
1486          * Reserve space for MPO mblock structs from the 32-bit heap.
1487          */
1488 
1489         if (mpo_heap32_bufsz > (size_t)0) {
1490                 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
1491                     PAGESIZE, 0, 0, mpo_heap32_buf,
1492                     mpo_heap32_buf + mpo_heap32_bufsz,
1493                     VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1494         }
1495         mem_config_init();
1496 }
1497 
1498 static void
1499 startup_modules(void)
1500 {
1501         int nhblk1, nhblk8;
1502         size_t  nhblksz;
1503         pgcnt_t pages_per_hblk;
1504         size_t hme8blk_sz, hme1blk_sz;
1505 
1506         /*
1507          * The system file /etc/system was read already under startup_memlist.
1508          */
1509         if (&set_platform_defaults)
1510                 set_platform_defaults();
1511 
1512         /*
1513          * Calculate default settings of system parameters based upon
1514          * maxusers, yet allow to be overridden via the /etc/system file.
1515          */
1516         param_calc(0);
1517 
1518         mod_setup();
1519 
1520         /*
1521          * If this is a positron, complain and halt.
1522          */
1523         if (&iam_positron && iam_positron()) {
1524                 cmn_err(CE_WARN, "This hardware platform is not supported"
1525                     " by this release of Solaris.\n");
1526 #ifdef DEBUG
1527                 prom_enter_mon();       /* Type 'go' to resume */
1528                 cmn_err(CE_WARN, "Booting an unsupported platform.\n");
1529                 cmn_err(CE_WARN, "Booting with down-rev firmware.\n");
1530 
1531 #else /* DEBUG */
1532                 halt(0);
1533 #endif /* DEBUG */
1534         }
1535 
1536         /*
1537          * If we are running firmware that isn't 64-bit ready
1538          * then complain and halt.
1539          */
1540         do_prom_version_check();
1541 
1542         /*
1543          * Initialize system parameters
1544          */
1545         param_init();
1546 
1547         /*
1548          * maxmem is the amount of physical memory we're playing with.
1549          */
1550         maxmem = physmem;
1551 
1552         /* Set segkp limits. */
1553         ncbase = kdi_segdebugbase;
1554         ncend = kdi_segdebugbase;
1555 
1556         /*
1557          * Initialize the hat layer.
1558          */
1559         hat_init();
1560 
1561         /*
1562          * Initialize segment management stuff.
1563          */
1564         seg_init();
1565 
1566         /*
1567          * Create the va>tte handler, so the prom can understand
1568          * kernel translations.  The handler is installed later, just
1569          * as we are about to take over the trap table from the prom.
1570          */
1571         create_va_to_tte();
1572 
1573         /*
1574          * Load the forthdebugger (optional)
1575          */
1576         forthdebug_init();
1577 
1578         /*
1579          * Create OBP node for console input callbacks
1580          * if it is needed.
1581          */
1582         startup_create_io_node();
1583 
1584         if (modloadonly("fs", "specfs") == -1)
1585                 halt("Can't load specfs");
1586 
1587         if (modloadonly("fs", "devfs") == -1)
1588                 halt("Can't load devfs");
1589 
1590         if (modloadonly("fs", "procfs") == -1)
1591                 halt("Can't load procfs");
1592 
1593         if (modloadonly("misc", "swapgeneric") == -1)
1594                 halt("Can't load swapgeneric");
1595 
1596         (void) modloadonly("sys", "lbl_edition");
1597 
1598         dispinit();
1599 
1600         /*
1601          * Infer meanings to the members of the idprom buffer.
1602          */
1603         parse_idprom();
1604 
1605         /* Read cluster configuration data. */
1606         clconf_init();
1607 
1608         setup_ddi();
1609 
1610         /*
1611          * Lets take this opportunity to load the root device.
1612          */
1613         if (loadrootmodules() != 0)
1614                 debug_enter("Can't load the root filesystem");
1615 
1616         /*
1617          * Load tod driver module for the tod part found on this system.
1618          * Recompute the cpu frequency/delays based on tod as tod part
1619          * tends to keep time more accurately.
1620          */
1621         if (&load_tod_module)
1622                 load_tod_module();
1623 
1624         /*
1625          * Allow platforms to load modules which might
1626          * be needed after bootops are gone.
1627          */
1628         if (&load_platform_modules)
1629                 load_platform_modules();
1630 
1631         setcpudelay();
1632 
1633         copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1634             &boot_physavail, &boot_physavail_len,
1635             &boot_virtavail, &boot_virtavail_len);
1636 
1637         /*
1638          * Calculation and allocation of hmeblks needed to remap
1639          * the memory allocated by PROM till now.
1640          * Overestimate the number of hblk1 elements by assuming
1641          * worst case of TTE64K mappings.
1642          * sfmmu_hblk_alloc will panic if this calculation is wrong.
1643          */
1644         bop_alloc_pages = btopr(kmem64_end - kmem64_base);
1645         pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
1646         bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1647         nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
1648 
1649         bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
1650 
1651         /* sfmmu_init_nucleus_hblks expects properly aligned data structures */
1652         hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
1653         hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
1654 
1655         bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
1656 
1657         pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
1658         nhblk8 = 0;
1659         while (bop_alloc_pages > 1) {
1660                 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1661                 nhblk8 += bop_alloc_pages /= pages_per_hblk;
1662                 bop_alloc_pages *= hme8blk_sz;
1663                 bop_alloc_pages = btopr(bop_alloc_pages);
1664         }
1665         nhblk8 += 2;
1666 
1667         /*
1668          * Since hblk8's can hold up to 64k of mappings aligned on a 64k
1669          * boundary, the number of hblk8's needed to map the entries in the
1670          * boot_virtavail list needs to be adjusted to take this into
1671          * consideration.  Thus, we need to add additional hblk8's since it
1672          * is possible that an hblk8 will not have all 8 slots used due to
1673          * alignment constraints.  Since there were boot_virtavail_len entries
1674          * in that list, we need to add that many hblk8's to the number
1675          * already calculated to make sure we don't underestimate.
1676          */
1677         nhblk8 += boot_virtavail_len;
1678         nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
1679 
1680         /* Allocate in pagesize chunks */
1681         nhblksz = roundup(nhblksz, MMU_PAGESIZE);
1682         hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
1683         sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
1684 }
1685 
1686 static void
1687 startup_bop_gone(void)
1688 {
1689 
1690         /*
1691          * Destroy the MD initialized at startup
1692          * The startup initializes the MD framework
1693          * using prom and BOP alloc free it now.
1694          */
1695         mach_descrip_startup_fini();
1696 
1697         /*
1698          * We're done with prom allocations.
1699          */
1700         bop_fini();
1701 
1702         copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1703             &boot_physavail, &boot_physavail_len,
1704             &boot_virtavail, &boot_virtavail_len);
1705 
1706         /*
1707          * setup physically contiguous area twice as large as the ecache.
1708          * this is used while doing displacement flush of ecaches
1709          */
1710         if (&ecache_flush_address) {
1711                 ecache_flushaddr = ecache_flush_address();
1712                 if (ecache_flushaddr == (uint64_t)-1) {
1713                         cmn_err(CE_PANIC,
1714                             "startup: no memory to set ecache_flushaddr");
1715                 }
1716         }
1717 
1718         /*
1719          * Virtual available next.
1720          */
1721         ASSERT(virt_avail != NULL);
1722         memlist_free_list(virt_avail);
1723         virt_avail = memlist;
1724         copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1725 
1726 }
1727 
1728 
1729 /*
1730  * startup_fixup_physavail - called from mach_sfmmu.c after the final
1731  * allocations have been performed.  We can't call it in startup_bop_gone
1732  * since later operations can cause obp to allocate more memory.
1733  */
1734 void
1735 startup_fixup_physavail(void)
1736 {
1737         struct memlist *cur;
1738         size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
1739 
1740         PRM_DEBUG(kmem64_overmap_size);
1741 
1742         /*
1743          * take the most current snapshot we can by calling mem-update
1744          */
1745         copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1746             &boot_physavail, &boot_physavail_len,
1747             &boot_virtavail, &boot_virtavail_len);
1748 
1749         /*
1750          * Copy phys_avail list, again.
1751          * Both the kernel/boot and the prom have been allocating
1752          * from the original list we copied earlier.
1753          */
1754         cur = memlist;
1755         copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1756 
1757         /*
1758          * Add any unused kmem64 memory from overmapped page
1759          * (Note: va_to_pa does not work for kmem64_end)
1760          */
1761         if (kmem64_overmap_size) {
1762                 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
1763                     kmem64_overmap_size, &memlist, &cur);
1764         }
1765 
1766         /*
1767          * Add any extra memory after e_data we added to the phys_avail list
1768          * back to the old list.
1769          */
1770         if (ndata_remain_sz >= MMU_PAGESIZE)
1771                 memlist_add(va_to_pa(nalloc_base),
1772                     (uint64_t)ndata_remain_sz, &memlist, &cur);
1773 
1774         /*
1775          * There isn't any bounds checking on the memlist area
1776          * so ensure it hasn't overgrown.
1777          */
1778         if ((caddr_t)memlist > (caddr_t)memlist_end)
1779                 cmn_err(CE_PANIC, "startup: memlist size exceeded");
1780 
1781         /*
1782          * The kernel removes the pages that were allocated for it from
1783          * the freelist, but we now have to find any -extra- pages that
1784          * the prom has allocated for it's own book-keeping, and remove
1785          * them from the freelist too. sigh.
1786          */
1787         sync_memlists(phys_avail, cur);
1788 
1789         ASSERT(phys_avail != NULL);
1790 
1791         old_phys_avail = phys_avail;
1792         phys_avail = cur;
1793 }
1794 
1795 void
1796 update_kcage_ranges(uint64_t addr, uint64_t len)
1797 {
1798         pfn_t base = btop(addr);
1799         pgcnt_t num = btop(len);
1800         int rv;
1801 
1802         rv = kcage_range_add(base, num, kcage_startup_dir);
1803 
1804         if (rv == ENOMEM) {
1805                 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
1806                     (len == 0 ? 0 : BYTES_TO_MB(len)));
1807         } else if (rv != 0) {
1808                 /* catch this in debug kernels */
1809                 ASSERT(0);
1810 
1811                 cmn_err(CE_WARN, "unexpected kcage_range_add"
1812                     " return value %d", rv);
1813         }
1814 }
1815 
1816 static void
1817 startup_vm(void)
1818 {
1819         size_t  i;
1820         struct segmap_crargs a;
1821         struct segkpm_crargs b;
1822 
1823         uint64_t avmem;
1824         caddr_t va;
1825         pgcnt_t max_phys_segkp;
1826         int     mnode;
1827 
1828         extern int use_brk_lpg, use_stk_lpg;
1829 
1830         /*
1831          * get prom's mappings, create hments for them and switch
1832          * to the kernel context.
1833          */
1834         hat_kern_setup();
1835 
1836         /*
1837          * Take over trap table
1838          */
1839         setup_trap_table();
1840 
1841         /*
1842          * Install the va>tte handler, so that the prom can handle
1843          * misses and understand the kernel table layout in case
1844          * we need call into the prom.
1845          */
1846         install_va_to_tte();
1847 
1848         /*
1849          * Set a flag to indicate that the tba has been taken over.
1850          */
1851         tba_taken_over = 1;
1852 
1853         /* initialize MMU primary context register */
1854         mmu_init_kcontext();
1855 
1856         /*
1857          * The boot cpu can now take interrupts, x-calls, x-traps
1858          */
1859         CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
1860         CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
1861 
1862         /*
1863          * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
1864          */
1865         tbr_wr_addr_inited = 1;
1866 
1867         /*
1868          * Initialize VM system, and map kernel address space.
1869          */
1870         kvm_init();
1871 
1872         ASSERT(old_phys_avail != NULL && phys_avail != NULL);
1873         if (kernel_cage_enable) {
1874                 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
1875         }
1876         memlist_free_list(old_phys_avail);
1877 
1878         /*
1879          * If the following is true, someone has patched
1880          * phsymem to be less than the number of pages that
1881          * the system actually has.  Remove pages until system
1882          * memory is limited to the requested amount.  Since we
1883          * have allocated page structures for all pages, we
1884          * correct the amount of memory we want to remove
1885          * by the size of the memory used to hold page structures
1886          * for the non-used pages.
1887          */
1888         if (physmem + ramdisk_npages < npages) {
1889                 pgcnt_t diff, off;
1890                 struct page *pp;
1891                 struct seg kseg;
1892 
1893                 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
1894 
1895                 off = 0;
1896                 diff = npages - (physmem + ramdisk_npages);
1897                 diff -= mmu_btopr(diff * sizeof (struct page));
1898                 kseg.s_as = &kas;
1899                 while (diff--) {
1900                         pp = page_create_va(&unused_pages_vp, (offset_t)off,
1901                             MMU_PAGESIZE, PG_WAIT | PG_EXCL,
1902                             &kseg, (caddr_t)off);
1903                         if (pp == NULL)
1904                                 cmn_err(CE_PANIC, "limited physmem too much!");
1905                         page_io_unlock(pp);
1906                         page_downgrade(pp);
1907                         availrmem--;
1908                         off += MMU_PAGESIZE;
1909                 }
1910         }
1911 
1912         /*
1913          * When printing memory, show the total as physmem less
1914          * that stolen by a debugger.
1915          */
1916         cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
1917             (ulong_t)(physinstalled) << (PAGESHIFT - 10),
1918             (ulong_t)(physinstalled) << (PAGESHIFT - 12));
1919 
1920         avmem = (uint64_t)freemem << PAGESHIFT;
1921         cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
1922 
1923         /*
1924          * For small memory systems disable automatic large pages.
1925          */
1926         if (physmem < privm_lpg_min_physmem) {
1927                 use_brk_lpg = 0;
1928                 use_stk_lpg = 0;
1929         }
1930 
1931         /*
1932          * Perform platform specific freelist processing
1933          */
1934         if (&plat_freelist_process) {
1935                 for (mnode = 0; mnode < max_mem_nodes; mnode++)
1936                         if (mem_node_config[mnode].exists)
1937                                 plat_freelist_process(mnode);
1938         }
1939 
1940         /*
1941          * Initialize the segkp segment type.  We position it
1942          * after the configured tables and buffers (whose end
1943          * is given by econtig) and before V_WKBASE_ADDR.
1944          * Also in this area is segkmap (size SEGMAPSIZE).
1945          */
1946 
1947         /* XXX - cache alignment? */
1948         va = (caddr_t)SEGKPBASE;
1949         ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
1950 
1951         max_phys_segkp = (physmem * 2);
1952 
1953         if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
1954                 segkpsize = btop(SEGKPDEFSIZE);
1955                 cmn_err(CE_WARN, "Illegal value for segkpsize. "
1956                     "segkpsize has been reset to %ld pages", segkpsize);
1957         }
1958 
1959         i = ptob(MIN(segkpsize, max_phys_segkp));
1960 
1961         rw_enter(&kas.a_lock, RW_WRITER);
1962         if (seg_attach(&kas, va, i, segkp) < 0)
1963                 cmn_err(CE_PANIC, "startup: cannot attach segkp");
1964         if (segkp_create(segkp) != 0)
1965                 cmn_err(CE_PANIC, "startup: segkp_create failed");
1966         rw_exit(&kas.a_lock);
1967 
1968         /*
1969          * kpm segment
1970          */
1971         segmap_kpm = kpm_enable &&
1972             segmap_kpm && PAGESIZE == MAXBSIZE;
1973 
1974         if (kpm_enable) {
1975                 rw_enter(&kas.a_lock, RW_WRITER);
1976 
1977                 /*
1978                  * The segkpm virtual range range is larger than the
1979                  * actual physical memory size and also covers gaps in
1980                  * the physical address range for the following reasons:
1981                  * . keep conversion between segkpm and physical addresses
1982                  *   simple, cheap and unambiguous.
1983                  * . avoid extension/shrink of the the segkpm in case of DR.
1984                  * . avoid complexity for handling of virtual addressed
1985                  *   caches, segkpm and the regular mapping scheme must be
1986                  *   kept in sync wrt. the virtual color of mapped pages.
1987                  * Any accesses to virtual segkpm ranges not backed by
1988                  * physical memory will fall through the memseg pfn hash
1989                  * and will be handled in segkpm_fault.
1990                  * Additional kpm_size spaces needed for vac alias prevention.
1991                  */
1992                 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
1993                     segkpm) < 0)
1994                         cmn_err(CE_PANIC, "cannot attach segkpm");
1995 
1996                 b.prot = PROT_READ | PROT_WRITE;
1997                 b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
1998 
1999                 if (segkpm_create(segkpm, (caddr_t)&b) != 0)
2000                         panic("segkpm_create segkpm");
2001 
2002                 rw_exit(&kas.a_lock);
2003 
2004                 mach_kpm_init();
2005         }
2006 
2007         va = kpm_vbase + (kpm_size * vac_colors);
2008 
2009         if (!segzio_fromheap) {
2010                 size_t size;
2011                 size_t physmem_b = mmu_ptob(physmem);
2012 
2013                 /* size is in bytes, segziosize is in pages */
2014                 if (segziosize == 0) {
2015                         size = physmem_b;
2016                 } else {
2017                         size = mmu_ptob(segziosize);
2018                 }
2019 
2020                 if (size < SEGZIOMINSIZE) {
2021                         size = SEGZIOMINSIZE;
2022                 } else if (size > SEGZIOMAXSIZE) {
2023                         size = SEGZIOMAXSIZE;
2024                         /*
2025                          * On 64-bit x86, we only have 2TB of KVA.  This exists
2026                          * for parity with x86.
2027                          *
2028                          * SEGZIOMAXSIZE is capped at 512gb so that segzio
2029                          * doesn't consume all of KVA.  However, if we have a
2030                          * system that has more thant 512gb of physical memory,
2031                          * we can actually consume about half of the difference
2032                          * between 512gb and the rest of the available physical
2033                          * memory.
2034                          */
2035                         if (physmem_b > SEGZIOMAXSIZE) {
2036                                 size += (physmem_b - SEGZIOMAXSIZE) / 2;
2037                 }
2038                 }
2039                 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
2040                 /* put the base of the ZIO segment after the kpm segment */
2041                 segzio_base = va;
2042                 va += mmu_ptob(segziosize);
2043                 PRM_DEBUG(segziosize);
2044                 PRM_DEBUG(segzio_base);
2045 
2046                 /*
2047                  * On some platforms, kvm_init is called after the kpm
2048                  * sizes have been determined.  On SPARC, kvm_init is called
2049                  * before, so we have to attach the kzioseg after kvm is
2050                  * initialized, otherwise we'll try to allocate from the boot
2051                  * area since the kernel heap hasn't yet been configured.
2052                  */
2053                 rw_enter(&kas.a_lock, RW_WRITER);
2054 
2055                 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2056                     &kzioseg);
2057                 (void) segkmem_zio_create(&kzioseg);
2058 
2059                 /* create zio area covering new segment */
2060                 segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2061 
2062                 rw_exit(&kas.a_lock);
2063         }
2064 
2065         if (ppvm_enable) {
2066                 caddr_t ppvm_max;
2067 
2068                 /*
2069                  * ppvm refers to the static VA space used to map
2070                  * the page_t's for dynamically added memory.
2071                  *
2072                  * ppvm_base should not cross a potential VA hole.
2073                  *
2074                  * ppvm_size should be large enough to map the
2075                  * page_t's needed to manage all of KPM range.
2076                  */
2077                 ppvm_size =
2078                     roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t),
2079                     MMU_PAGESIZE);
2080                 ppvm_max = (caddr_t)(0ull - ppvm_size);
2081                 ppvm_base = (page_t *)va;
2082 
2083                 if ((caddr_t)ppvm_base <= hole_end) {
2084                         cmn_err(CE_WARN,
2085                             "Memory DR disabled: invalid DR map base: 0x%p\n",
2086                             (void *)ppvm_base);
2087                         ppvm_enable = 0;
2088                 } else if ((caddr_t)ppvm_base > ppvm_max) {
2089                         uint64_t diff = (caddr_t)ppvm_base - ppvm_max;
2090 
2091                         cmn_err(CE_WARN,
2092                             "Memory DR disabled: insufficient DR map size:"
2093                             " 0x%lx (needed 0x%lx)\n",
2094                             ppvm_size - diff, ppvm_size);
2095                         ppvm_enable = 0;
2096                 }
2097                 PRM_DEBUG(ppvm_size);
2098                 PRM_DEBUG(ppvm_base);
2099         }
2100 
2101         /*
2102          * Now create generic mapping segment.  This mapping
2103          * goes SEGMAPSIZE beyond SEGMAPBASE.  But if the total
2104          * virtual address is greater than the amount of free
2105          * memory that is available, then we trim back the
2106          * segment size to that amount
2107          */
2108         va = (caddr_t)SEGMAPBASE;
2109 
2110         /*
2111          * 1201049: segkmap base address must be MAXBSIZE aligned
2112          */
2113         ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
2114 
2115         /*
2116          * Set size of segmap to percentage of freemem at boot,
2117          * but stay within the allowable range
2118          * Note we take percentage  before converting from pages
2119          * to bytes to avoid an overflow on 32-bit kernels.
2120          */
2121         i = mmu_ptob((freemem * segmap_percent) / 100);
2122 
2123         if (i < MINMAPSIZE)
2124                 i = MINMAPSIZE;
2125 
2126         if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
2127                 i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
2128 
2129         i &= MAXBMASK;      /* 1201049: segkmap size must be MAXBSIZE aligned */
2130 
2131         rw_enter(&kas.a_lock, RW_WRITER);
2132         if (seg_attach(&kas, va, i, segkmap) < 0)
2133                 cmn_err(CE_PANIC, "cannot attach segkmap");
2134 
2135         a.prot = PROT_READ | PROT_WRITE;
2136         a.shmsize = shm_alignment;
2137         a.nfreelist = 0;        /* use segmap driver defaults */
2138 
2139         if (segmap_create(segkmap, (caddr_t)&a) != 0)
2140                 panic("segmap_create segkmap");
2141         rw_exit(&kas.a_lock);
2142 
2143         segdev_init();
2144 }
2145 
2146 static void
2147 startup_end(void)
2148 {
2149         if ((caddr_t)memlist > (caddr_t)memlist_end)
2150                 panic("memlist overflow 2");
2151         memlist_free_block((caddr_t)memlist,
2152             ((caddr_t)memlist_end - (caddr_t)memlist));
2153         memlist = NULL;
2154 
2155         /* enable page_relocation since OBP is now done */
2156         page_relocate_ready = 1;
2157 
2158         /*
2159          * Perform tasks that get done after most of the VM
2160          * initialization has been done but before the clock
2161          * and other devices get started.
2162          */
2163         kern_setup1();
2164 
2165         /*
2166          * Perform CPC initialization for this CPU.
2167          */
2168         kcpc_hw_init();
2169 
2170         /*
2171          * Intialize the VM arenas for allocating physically
2172          * contiguus memory chunk for interrupt queues snd
2173          * allocate/register boot cpu's queues, if any and
2174          * allocate dump buffer for sun4v systems to store
2175          * extra crash information during crash dump
2176          */
2177         contig_mem_init();
2178         mach_descrip_init();
2179 
2180         if (cpu_intrq_setup(CPU)) {
2181                 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
2182         }
2183         cpu_intrq_register(CPU);
2184         mach_htraptrace_setup(CPU->cpu_id);
2185         mach_htraptrace_configure(CPU->cpu_id);
2186         mach_dump_buffer_init();
2187 
2188         /*
2189          * Initialize interrupt related stuff
2190          */
2191         cpu_intr_alloc(CPU, NINTR_THREADS);
2192 
2193         (void) splzs();                 /* allow hi clock ints but not zs */
2194 
2195         /*
2196          * Initialize errors.
2197          */
2198         error_init();
2199 
2200         /*
2201          * Note that we may have already used kernel bcopy before this
2202          * point - but if you really care about this, adb the use_hw_*
2203          * variables to 0 before rebooting.
2204          */
2205         mach_hw_copy_limit();
2206 
2207         /*
2208          * Install the "real" preemption guards before DDI services
2209          * are available.
2210          */
2211         (void) prom_set_preprom(kern_preprom);
2212         (void) prom_set_postprom(kern_postprom);
2213         CPU->cpu_m.mutex_ready = 1;
2214 
2215         /*
2216          * Initialize segnf (kernel support for non-faulting loads).
2217          */
2218         segnf_init();
2219 
2220         /*
2221          * Configure the root devinfo node.
2222          */
2223         configure();            /* set up devices */
2224         mach_cpu_halt_idle();
2225 }
2226 
2227 
2228 void
2229 post_startup(void)
2230 {
2231 #ifdef  PTL1_PANIC_DEBUG
2232         extern void init_ptl1_thread(void);
2233 #endif  /* PTL1_PANIC_DEBUG */
2234         extern void abort_sequence_init(void);
2235 
2236         /*
2237          * Set the system wide, processor-specific flags to be passed
2238          * to userland via the aux vector for performance hints and
2239          * instruction set extensions.
2240          */
2241         bind_hwcap();
2242 
2243         /*
2244          * Startup memory scrubber (if any)
2245          */
2246         mach_memscrub();
2247 
2248         /*
2249          * Allocate soft interrupt to handle abort sequence.
2250          */
2251         abort_sequence_init();
2252 
2253         /*
2254          * Configure the rest of the system.
2255          * Perform forceloading tasks for /etc/system.
2256          */
2257         (void) mod_sysctl(SYS_FORCELOAD, NULL);
2258         /*
2259          * ON4.0: Force /proc module in until clock interrupt handle fixed
2260          * ON4.0: This must be fixed or restated in /etc/systems.
2261          */
2262         (void) modload("fs", "procfs");
2263 
2264         /* load machine class specific drivers */
2265         load_mach_drivers();
2266 
2267         /* load platform specific drivers */
2268         if (&load_platform_drivers)
2269                 load_platform_drivers();
2270 
2271         /* load vis simulation module, if we are running w/fpu off */
2272         if (!fpu_exists) {
2273                 if (modload("misc", "vis") == -1)
2274                         halt("Can't load vis");
2275         }
2276 
2277         mach_fpras();
2278 
2279         maxmem = freemem;
2280 
2281         pg_init();
2282 
2283 #ifdef  PTL1_PANIC_DEBUG
2284         init_ptl1_thread();
2285 #endif  /* PTL1_PANIC_DEBUG */
2286 }
2287 
2288 #ifdef  PTL1_PANIC_DEBUG
2289 int             ptl1_panic_test = 0;
2290 int             ptl1_panic_xc_one_test = 0;
2291 int             ptl1_panic_xc_all_test = 0;
2292 int             ptl1_panic_xt_one_test = 0;
2293 int             ptl1_panic_xt_all_test = 0;
2294 kthread_id_t    ptl1_thread_p = NULL;
2295 kcondvar_t      ptl1_cv;
2296 kmutex_t        ptl1_mutex;
2297 int             ptl1_recurse_count_threshold = 0x40;
2298 int             ptl1_recurse_trap_threshold = 0x3d;
2299 extern void     ptl1_recurse(int, int);
2300 extern void     ptl1_panic_xt(int, int);
2301 
2302 /*
2303  * Called once per second by timeout() to wake up
2304  * the ptl1_panic thread to see if it should cause
2305  * a trap to the ptl1_panic() code.
2306  */
2307 /* ARGSUSED */
2308 static void
2309 ptl1_wakeup(void *arg)
2310 {
2311         mutex_enter(&ptl1_mutex);
2312         cv_signal(&ptl1_cv);
2313         mutex_exit(&ptl1_mutex);
2314 }
2315 
2316 /*
2317  * ptl1_panic cross call function:
2318  *     Needed because xc_one() and xc_some() can pass
2319  *      64 bit args but ptl1_recurse() expects ints.
2320  */
2321 static void
2322 ptl1_panic_xc(void)
2323 {
2324         ptl1_recurse(ptl1_recurse_count_threshold,
2325             ptl1_recurse_trap_threshold);
2326 }
2327 
2328 /*
2329  * The ptl1 thread waits for a global flag to be set
2330  * and uses the recurse thresholds to set the stack depth
2331  * to cause a ptl1_panic() directly via a call to ptl1_recurse
2332  * or indirectly via the cross call and cross trap functions.
2333  *
2334  * This is useful testing stack overflows and normal
2335  * ptl1_panic() states with a know stack frame.
2336  *
2337  * ptl1_recurse() is an asm function in ptl1_panic.s that
2338  * sets the {In, Local, Out, and Global} registers to a
2339  * know state on the stack and just prior to causing a
2340  * test ptl1_panic trap.
2341  */
2342 static void
2343 ptl1_thread(void)
2344 {
2345         mutex_enter(&ptl1_mutex);
2346         while (ptl1_thread_p) {
2347                 cpuset_t        other_cpus;
2348                 int             cpu_id;
2349                 int             my_cpu_id;
2350                 int             target_cpu_id;
2351                 int             target_found;
2352 
2353                 if (ptl1_panic_test) {
2354                         ptl1_recurse(ptl1_recurse_count_threshold,
2355                             ptl1_recurse_trap_threshold);
2356                 }
2357 
2358                 /*
2359                  * Find potential targets for x-call and x-trap,
2360                  * if any exist while preempt is disabled we
2361                  * start a ptl1_panic if requested via a
2362                  * globals.
2363                  */
2364                 kpreempt_disable();
2365                 my_cpu_id = CPU->cpu_id;
2366                 other_cpus = cpu_ready_set;
2367                 CPUSET_DEL(other_cpus, CPU->cpu_id);
2368                 target_found = 0;
2369                 if (!CPUSET_ISNULL(other_cpus)) {
2370                         /*
2371                          * Pick the first one
2372                          */
2373                         for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
2374                                 if (cpu_id == my_cpu_id)
2375                                         continue;
2376 
2377                                 if (CPU_XCALL_READY(cpu_id)) {
2378                                         target_cpu_id = cpu_id;
2379                                         target_found = 1;
2380                                         break;
2381                                 }
2382                         }
2383                         ASSERT(target_found);
2384 
2385                         if (ptl1_panic_xc_one_test) {
2386                                 xc_one(target_cpu_id,
2387                                     (xcfunc_t *)ptl1_panic_xc, 0, 0);
2388                         }
2389                         if (ptl1_panic_xc_all_test) {
2390                                 xc_some(other_cpus,
2391                                     (xcfunc_t *)ptl1_panic_xc, 0, 0);
2392                         }
2393                         if (ptl1_panic_xt_one_test) {
2394                                 xt_one(target_cpu_id,
2395                                     (xcfunc_t *)ptl1_panic_xt, 0, 0);
2396                         }
2397                         if (ptl1_panic_xt_all_test) {
2398                                 xt_some(other_cpus,
2399                                     (xcfunc_t *)ptl1_panic_xt, 0, 0);
2400                         }
2401                 }
2402                 kpreempt_enable();
2403                 (void) timeout(ptl1_wakeup, NULL, hz);
2404                 (void) cv_wait(&ptl1_cv, &ptl1_mutex);
2405         }
2406         mutex_exit(&ptl1_mutex);
2407 }
2408 
2409 /*
2410  * Called during early startup to create the ptl1_thread
2411  */
2412 void
2413 init_ptl1_thread(void)
2414 {
2415         ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
2416             &p0, TS_RUN, 0);
2417 }
2418 #endif  /* PTL1_PANIC_DEBUG */
2419 
2420 
2421 static void
2422 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
2423 {
2424         struct memlist *new;
2425 
2426         new = *memlistp;
2427         new->ml_address = start;
2428         new->ml_size = len;
2429         *memlistp = new + 1;
2430 }
2431 
2432 /*
2433  * Add to a memory list.
2434  * start = start of new memory segment
2435  * len = length of new memory segment in bytes
2436  * memlistp = pointer to array of available memory segment structures
2437  * curmemlistp = memory list to which to add segment.
2438  */
2439 static void
2440 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
2441         struct memlist **curmemlistp)
2442 {
2443         struct memlist *new = *memlistp;
2444 
2445         memlist_new(start, len, memlistp);
2446         memlist_insert(new, curmemlistp);
2447 }
2448 
2449 static int
2450 ndata_alloc_memseg(struct memlist *ndata, size_t avail)
2451 {
2452         int nseg;
2453         size_t memseg_sz;
2454         struct memseg *msp;
2455 
2456         /*
2457          * The memseg list is for the chunks of physical memory that
2458          * will be managed by the vm system.  The number calculated is
2459          * a guess as boot may fragment it more when memory allocations
2460          * are made before kphysm_init().
2461          */
2462         memseg_sz = (avail + 10) * sizeof (struct memseg);
2463         memseg_sz = roundup(memseg_sz, PAGESIZE);
2464         nseg = memseg_sz / sizeof (struct memseg);
2465         msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
2466         if (msp == NULL)
2467                 return (1);
2468         PRM_DEBUG(memseg_free);
2469 
2470         while (nseg--) {
2471                 msp->next = memseg_free;
2472                 memseg_free = msp;
2473                 msp++;
2474         }
2475         return (0);
2476 }
2477 
2478 /*
2479  * In the case of architectures that support dynamic addition of
2480  * memory at run-time there are two cases where memsegs need to
2481  * be initialized and added to the memseg list.
2482  * 1) memsegs that are constructed at startup.
2483  * 2) memsegs that are constructed at run-time on
2484  *    hot-plug capable architectures.
2485  * This code was originally part of the function kphysm_init().
2486  */
2487 
2488 static void
2489 memseg_list_add(struct memseg *memsegp)
2490 {
2491         struct memseg **prev_memsegp;
2492         pgcnt_t num;
2493 
2494         /* insert in memseg list, decreasing number of pages order */
2495 
2496         num = MSEG_NPAGES(memsegp);
2497 
2498         for (prev_memsegp = &memsegs; *prev_memsegp;
2499             prev_memsegp = &((*prev_memsegp)->next)) {
2500                 if (num > MSEG_NPAGES(*prev_memsegp))
2501                         break;
2502         }
2503 
2504         memsegp->next = *prev_memsegp;
2505         *prev_memsegp = memsegp;
2506 
2507         if (kpm_enable) {
2508                 memsegp->nextpa = (memsegp->next) ?
2509                     va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
2510 
2511                 if (prev_memsegp != &memsegs) {
2512                         struct memseg *msp;
2513                         msp = (struct memseg *)((caddr_t)prev_memsegp -
2514                             offsetof(struct memseg, next));
2515                         msp->nextpa = va_to_pa(memsegp);
2516                 } else {
2517                         memsegspa = va_to_pa(memsegs);
2518                 }
2519         }
2520 }
2521 
2522 /*
2523  * PSM add_physmem_cb(). US-II and newer processors have some
2524  * flavor of the prefetch capability implemented. We exploit
2525  * this capability for optimum performance.
2526  */
2527 #define PREFETCH_BYTES  64
2528 
2529 void
2530 add_physmem_cb(page_t *pp, pfn_t pnum)
2531 {
2532         extern void      prefetch_page_w(void *);
2533 
2534         pp->p_pagenum = pnum;
2535 
2536         /*
2537          * Prefetch one more page_t into E$. To prevent future
2538          * mishaps with the sizeof(page_t) changing on us, we
2539          * catch this on debug kernels if we can't bring in the
2540          * entire hpage with 2 PREFETCH_BYTES reads. See
2541          * also, sun4u/cpu/cpu_module.c
2542          */
2543         /*LINTED*/
2544         ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
2545         prefetch_page_w((char *)pp);
2546 }
2547 
2548 /*
2549  * Find memseg with given pfn
2550  */
2551 static struct memseg *
2552 memseg_find(pfn_t base, pfn_t *next)
2553 {
2554         struct memseg *seg;
2555 
2556         if (next != NULL)
2557                 *next = LONG_MAX;
2558         for (seg = memsegs; seg != NULL; seg = seg->next) {
2559                 if (base >= seg->pages_base && base < seg->pages_end)
2560                         return (seg);
2561                 if (next != NULL && seg->pages_base > base &&
2562                     seg->pages_base < *next)
2563                         *next = seg->pages_base;
2564         }
2565         return (NULL);
2566 }
2567 
2568 /*
2569  * Put page allocated by OBP on prom_ppages
2570  */
2571 static void
2572 kphysm_erase(uint64_t addr, uint64_t len)
2573 {
2574         struct page *pp;
2575         struct memseg *seg;
2576         pfn_t base = btop(addr), next;
2577         pgcnt_t num = btop(len);
2578 
2579         while (num != 0) {
2580                 pgcnt_t off, left;
2581 
2582                 seg = memseg_find(base, &next);
2583                 if (seg == NULL) {
2584                         if (next == LONG_MAX)
2585                                 break;
2586                         left = MIN(next - base, num);
2587                         base += left, num -= left;
2588                         continue;
2589                 }
2590                 off = base - seg->pages_base;
2591                 pp = seg->pages + off;
2592                 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
2593                 while (num != left) {
2594                         /*
2595                          * init it, lock it, and hashin on prom_pages vp.
2596                          *
2597                          * Mark it as NONRELOC to let DR know the page
2598                          * is locked long term, otherwise DR hangs when
2599                          * trying to remove those pages.
2600                          *
2601                          * XXX  vnode offsets on the prom_ppages vnode
2602                          *      are page numbers (gack) for >32 bit
2603                          *      physical memory machines.
2604                          */
2605                         PP_SETNORELOC(pp);
2606                         add_physmem_cb(pp, base);
2607                         if (page_trylock(pp, SE_EXCL) == 0)
2608                                 cmn_err(CE_PANIC, "prom page locked");
2609                         (void) page_hashin(pp, &promvp,
2610                             (offset_t)base, NULL);
2611                         (void) page_pp_lock(pp, 0, 1);
2612                         pp++, base++, num--;
2613                 }
2614         }
2615 }
2616 
2617 static page_t *ppnext;
2618 static pgcnt_t ppleft;
2619 
2620 static void *kpm_ppnext;
2621 static pgcnt_t kpm_ppleft;
2622 
2623 /*
2624  * Create a memseg
2625  */
2626 static void
2627 kphysm_memseg(uint64_t addr, uint64_t len)
2628 {
2629         pfn_t base = btop(addr);
2630         pgcnt_t num = btop(len);
2631         struct memseg *seg;
2632 
2633         seg = memseg_free;
2634         memseg_free = seg->next;
2635         ASSERT(seg != NULL);
2636 
2637         seg->pages = ppnext;
2638         seg->epages = ppnext + num;
2639         seg->pages_base = base;
2640         seg->pages_end = base + num;
2641         ppnext += num;
2642         ppleft -= num;
2643 
2644         if (kpm_enable) {
2645                 pgcnt_t kpnum = ptokpmpr(num);
2646 
2647                 if (kpnum > kpm_ppleft)
2648                         panic("kphysm_memseg: kpm_pp overflow");
2649                 seg->pagespa = va_to_pa(seg->pages);
2650                 seg->epagespa = va_to_pa(seg->epages);
2651                 seg->kpm_pbase = kpmptop(ptokpmp(base));
2652                 seg->kpm_nkpmpgs = kpnum;
2653                 /*
2654                  * In the kpm_smallpage case, the kpm array
2655                  * is 1-1 wrt the page array
2656                  */
2657                 if (kpm_smallpages) {
2658                         kpm_spage_t *kpm_pp = kpm_ppnext;
2659 
2660                         kpm_ppnext = kpm_pp + kpnum;
2661                         seg->kpm_spages = kpm_pp;
2662                         seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
2663                 } else {
2664                         kpm_page_t *kpm_pp = kpm_ppnext;
2665 
2666                         kpm_ppnext = kpm_pp + kpnum;
2667                         seg->kpm_pages = kpm_pp;
2668                         seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
2669                         /* ASSERT no kpm overlaps */
2670                         ASSERT(
2671                             memseg_find(base - pmodkpmp(base), NULL) == NULL);
2672                         ASSERT(memseg_find(
2673                             roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
2674                 }
2675                 kpm_ppleft -= kpnum;
2676         }
2677 
2678         memseg_list_add(seg);
2679 }
2680 
2681 /*
2682  * Add range to free list
2683  */
2684 void
2685 kphysm_add(uint64_t addr, uint64_t len, int reclaim)
2686 {
2687         struct page *pp;
2688         struct memseg *seg;
2689         pfn_t base = btop(addr);
2690         pgcnt_t num = btop(len);
2691 
2692         seg = memseg_find(base, NULL);
2693         ASSERT(seg != NULL);
2694         pp = seg->pages + (base - seg->pages_base);
2695 
2696         if (reclaim) {
2697                 struct page *rpp = pp;
2698                 struct page *lpp = pp + num;
2699 
2700                 /*
2701                  * page should be locked on prom_ppages
2702                  * unhash and unlock it
2703                  */
2704                 while (rpp < lpp) {
2705                         ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp);
2706                         ASSERT(PP_ISNORELOC(rpp));
2707                         PP_CLRNORELOC(rpp);
2708                         page_pp_unlock(rpp, 0, 1);
2709                         page_hashout(rpp, NULL);
2710                         page_unlock(rpp);
2711                         rpp++;
2712                 }
2713         }
2714 
2715         /*
2716          * add_physmem() initializes the PSM part of the page
2717          * struct by calling the PSM back with add_physmem_cb().
2718          * In addition it coalesces pages into larger pages as
2719          * it initializes them.
2720          */
2721         add_physmem(pp, num, base);
2722 }
2723 
2724 /*
2725  * kphysm_init() tackles the problem of initializing physical memory.
2726  */
2727 static void
2728 kphysm_init(void)
2729 {
2730         struct memlist *pmem;
2731 
2732         ASSERT(page_hash != NULL && page_hashsz != 0);
2733 
2734         ppnext = pp_base;
2735         ppleft = npages;
2736         kpm_ppnext = kpm_pp_base;
2737         kpm_ppleft = kpm_npages;
2738 
2739         /*
2740          * installed pages not on nopp_memlist go in memseg list
2741          */
2742         diff_memlists(phys_install, nopp_list, kphysm_memseg);
2743 
2744         /*
2745          * Free the avail list
2746          */
2747         for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next)
2748                 kphysm_add(pmem->ml_address, pmem->ml_size, 0);
2749 
2750         /*
2751          * Erase pages that aren't available
2752          */
2753         diff_memlists(phys_install, phys_avail, kphysm_erase);
2754 
2755         build_pfn_hash();
2756 }
2757 
2758 /*
2759  * Kernel VM initialization.
2760  * Assumptions about kernel address space ordering:
2761  *      (1) gap (user space)
2762  *      (2) kernel text
2763  *      (3) kernel data/bss
2764  *      (4) gap
2765  *      (5) kernel data structures
2766  *      (6) gap
2767  *      (7) debugger (optional)
2768  *      (8) monitor
2769  *      (9) gap (possibly null)
2770  *      (10) dvma
2771  *      (11) devices
2772  */
2773 static void
2774 kvm_init(void)
2775 {
2776         /*
2777          * Put the kernel segments in kernel address space.
2778          */
2779         rw_enter(&kas.a_lock, RW_WRITER);
2780         as_avlinit(&kas);
2781 
2782         (void) seg_attach(&kas, (caddr_t)KERNELBASE,
2783             (size_t)(e_moddata - KERNELBASE), &ktextseg);
2784         (void) segkmem_create(&ktextseg);
2785 
2786         (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
2787             (size_t)(MMU_PAGESIZE4M), &ktexthole);
2788         (void) segkmem_create(&ktexthole);
2789 
2790         (void) seg_attach(&kas, (caddr_t)valloc_base,
2791             (size_t)(econtig32 - valloc_base), &kvalloc);
2792         (void) segkmem_create(&kvalloc);
2793 
2794         if (kmem64_base) {
2795                 (void) seg_attach(&kas, (caddr_t)kmem64_base,
2796                     (size_t)(kmem64_end - kmem64_base), &kmem64);
2797                 (void) segkmem_create(&kmem64);
2798         }
2799 
2800         /*
2801          * We're about to map out /boot.  This is the beginning of the
2802          * system resource management transition. We can no longer
2803          * call into /boot for I/O or memory allocations.
2804          */
2805         (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
2806         (void) segkmem_create(&kvseg);
2807         hblk_alloc_dynamic = 1;
2808 
2809         /*
2810          * we need to preallocate pages for DR operations before enabling large
2811          * page kernel heap because of memseg_remap_init() hat_unload() hack.
2812          */
2813         memseg_remap_init();
2814 
2815         /* at this point we are ready to use large page heap */
2816         segkmem_heap_lp_init();
2817 
2818         (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
2819             &kvseg32);
2820         (void) segkmem_create(&kvseg32);
2821 
2822         /*
2823          * Create a segment for the debugger.
2824          */
2825         (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2826         (void) segkmem_create(&kdebugseg);
2827 
2828         rw_exit(&kas.a_lock);
2829 }
2830 
2831 char obp_tte_str[] =
2832         "h# %x constant MMU_PAGESHIFT "
2833         "h# %x constant TTE8K "
2834         "h# %x constant SFHME_SIZE "
2835         "h# %x constant SFHME_TTE "
2836         "h# %x constant HMEBLK_TAG "
2837         "h# %x constant HMEBLK_NEXT "
2838         "h# %x constant HMEBLK_MISC "
2839         "h# %x constant HMEBLK_HME1 "
2840         "h# %x constant NHMENTS "
2841         "h# %x constant HBLK_SZMASK "
2842         "h# %x constant HBLK_RANGE_SHIFT "
2843         "h# %x constant HMEBP_HBLK "
2844         "h# %x constant HMEBLK_ENDPA "
2845         "h# %x constant HMEBUCKET_SIZE "
2846         "h# %x constant HTAG_SFMMUPSZ "
2847         "h# %x constant HTAG_BSPAGE_SHIFT "
2848         "h# %x constant HTAG_REHASH_SHIFT "
2849         "h# %x constant SFMMU_INVALID_SHMERID "
2850         "h# %x constant mmu_hashcnt "
2851         "h# %p constant uhme_hash "
2852         "h# %p constant khme_hash "
2853         "h# %x constant UHMEHASH_SZ "
2854         "h# %x constant KHMEHASH_SZ "
2855         "h# %p constant KCONTEXT "
2856         "h# %p constant KHATID "
2857         "h# %x constant ASI_MEM "
2858 
2859         ": PHYS-X@ ( phys -- data ) "
2860         "   ASI_MEM spacex@ "
2861         "; "
2862 
2863         ": PHYS-W@ ( phys -- data ) "
2864         "   ASI_MEM spacew@ "
2865         "; "
2866 
2867         ": PHYS-L@ ( phys -- data ) "
2868         "   ASI_MEM spaceL@ "
2869         "; "
2870 
2871         ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
2872         "   3 * MMU_PAGESHIFT + "
2873         "; "
2874 
2875         ": TTE_IS_VALID ( ttep -- flag ) "
2876         "   PHYS-X@ 0< "
2877         "; "
2878 
2879         ": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
2880         "   dup TTE8K =  if "
2881         "      drop HBLK_RANGE_SHIFT "
2882         "   else "
2883         "      TTE_PAGE_SHIFT "
2884         "   then "
2885         "; "
2886 
2887         ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
2888         "   tuck >> swap MMU_PAGESHIFT - << "
2889         "; "
2890 
2891         ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
2892         "   >> over xor swap                    ( hash sfmmup ) "
2893         "   KHATID <>  if                       ( hash ) "
2894         "      UHMEHASH_SZ and                  ( bucket ) "
2895         "      HMEBUCKET_SIZE * uhme_hash +     ( hmebp ) "
2896         "   else                                ( hash ) "
2897         "      KHMEHASH_SZ and                  ( bucket ) "
2898         "      HMEBUCKET_SIZE * khme_hash +     ( hmebp ) "
2899         "   then                                ( hmebp ) "
2900         "; "
2901 
2902         ": HME_HASH_TABLE_SEARCH "
2903         "       ( sfmmup hmebp hblktag --  sfmmup null | sfmmup hmeblkp ) "
2904         "   >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
2905         "      dup HMEBLK_ENDPA <> if     ( sfmmup hmeblkp ) ( r: hblktag ) "
2906         "         dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp )     "
2907         "            dup hmeblk_tag + 8 + phys-x@ 2 pick = if             "
2908         "                 true  ( sfmmup hmeblkp true ) ( r: hblktag )    "
2909         "            else                                                 "
2910         "                 hmeblk_next + phys-x@ false                     "
2911         "                       ( sfmmup hmeblkp false ) ( r: hblktag )   "
2912         "            then                                                 "
2913         "         else                                                    "
2914         "            hmeblk_next + phys-x@ false                          "
2915         "                       ( sfmmup hmeblkp false ) ( r: hblktag )   "
2916         "         then                                                    "
2917         "      else                                                       "
2918         "         drop 0 true                                             "
2919         "      then                                                       "
2920         "   until r> drop                                              "
2921         "; "
2922 
2923         ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
2924         "   over HME_HASH_SHIFT HME_HASH_BSPAGE  ( sfmmup rehash bspage ) "
2925         "   HTAG_BSPAGE_SHIFT <<           ( sfmmup rehash htag-bspage )"
2926         "   swap HTAG_REHASH_SHIFT << or   ( sfmmup htag-bspage-rehash )"
2927         "   SFMMU_INVALID_SHMERID or nip         ( hblktag ) "
2928         "; "
2929 
2930         ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
2931         "   over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and  ( hmeblkp addr ttesz ) "
2932         "   TTE8K =  if                            ( hmeblkp addr ) "
2933         "      MMU_PAGESHIFT >> NHMENTS 1- and     ( hmeblkp hme-index ) "
2934         "   else                                   ( hmeblkp addr ) "
2935         "      drop 0                              ( hmeblkp 0 ) "
2936         "   then                                   ( hmeblkp hme-index ) "
2937         "   SFHME_SIZE * + HMEBLK_HME1 +           ( hmep ) "
2938         "   SFHME_TTE +                            ( ttep ) "
2939         "; "
2940 
2941         ": unix-tte ( addr cnum -- false | tte-data true ) "
2942         "    KCONTEXT = if                   ( addr ) "
2943         "       KHATID                       ( addr khatid ) "
2944         "    else                            ( addr ) "
2945         "       drop false exit              ( false ) "
2946         "    then "
2947         "      ( addr khatid ) "
2948         "      mmu_hashcnt 1+ 1  do           ( addr sfmmup ) "
2949         "         2dup swap i HME_HASH_SHIFT  "
2950                                         "( addr sfmmup sfmmup addr hmeshift ) "
2951         "         HME_HASH_FUNCTION           ( addr sfmmup hmebp ) "
2952         "         over i 4 pick               "
2953                                 "( addr sfmmup hmebp sfmmup rehash addr ) "
2954         "         HME_HASH_TAG                ( addr sfmmup hmebp hblktag ) "
2955         "         HME_HASH_TABLE_SEARCH       "
2956                                         "( addr sfmmup { null | hmeblkp } ) "
2957         "         ?dup  if                    ( addr sfmmup hmeblkp ) "
2958         "            nip swap HBLK_TO_TTEP    ( ttep ) "
2959         "            dup TTE_IS_VALID  if     ( valid-ttep ) "
2960         "               PHYS-X@ true          ( tte-data true ) "
2961         "            else                     ( invalid-tte ) "
2962         "               drop false            ( false ) "
2963         "            then                     ( false | tte-data true ) "
2964         "            unloop exit              ( false | tte-data true ) "
2965         "         then                        ( addr sfmmup ) "
2966         "      loop                           ( addr sfmmup ) "
2967         "      2drop false                    ( false ) "
2968         "; "
2969 ;
2970 
2971 void
2972 create_va_to_tte(void)
2973 {
2974         char *bp;
2975         extern int khmehash_num, uhmehash_num;
2976         extern struct hmehash_bucket *khme_hash, *uhme_hash;
2977 
2978 #define OFFSET(type, field)     ((uintptr_t)(&((type *)0)->field))
2979 
2980         bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
2981 
2982         /*
2983          * Teach obp how to parse our sw ttes.
2984          */
2985         (void) sprintf(bp, obp_tte_str,
2986             MMU_PAGESHIFT,
2987             TTE8K,
2988             sizeof (struct sf_hment),
2989             OFFSET(struct sf_hment, hme_tte),
2990             OFFSET(struct hme_blk, hblk_tag),
2991             OFFSET(struct hme_blk, hblk_nextpa),
2992             OFFSET(struct hme_blk, hblk_misc),
2993             OFFSET(struct hme_blk, hblk_hme),
2994             NHMENTS,
2995             HBLK_SZMASK,
2996             HBLK_RANGE_SHIFT,
2997             OFFSET(struct hmehash_bucket, hmeh_nextpa),
2998             HMEBLK_ENDPA,
2999             sizeof (struct hmehash_bucket),
3000             HTAG_SFMMUPSZ,
3001             HTAG_BSPAGE_SHIFT,
3002             HTAG_REHASH_SHIFT,
3003             SFMMU_INVALID_SHMERID,
3004             mmu_hashcnt,
3005             (caddr_t)va_to_pa((caddr_t)uhme_hash),
3006             (caddr_t)va_to_pa((caddr_t)khme_hash),
3007             UHMEHASH_SZ,
3008             KHMEHASH_SZ,
3009             KCONTEXT,
3010             KHATID,
3011             ASI_MEM);
3012         prom_interpret(bp, 0, 0, 0, 0, 0);
3013 
3014         kobj_free(bp, MMU_PAGESIZE);
3015 }
3016 
3017 void
3018 install_va_to_tte(void)
3019 {
3020         /*
3021          * advise prom that it can use unix-tte
3022          */
3023         prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
3024 }
3025 
3026 /*
3027  * Here we add "device-type=console" for /os-io node, for currently
3028  * our kernel console output only supports displaying text and
3029  * performing cursor-positioning operations (through kernel framebuffer
3030  * driver) and it doesn't support other functionalities required for a
3031  * standard "display" device as specified in 1275 spec. The main missing
3032  * interface defined by the 1275 spec is "draw-logo".
3033  * also see the comments above prom_stdout_is_framebuffer().
3034  */
3035 static char *create_node =
3036         "\" /\" find-device "
3037         "new-device "
3038         "\" os-io\" device-name "
3039         "\" "OBP_DISPLAY_CONSOLE"\" device-type "
3040         ": cb-r/w  ( adr,len method$ -- #read/#written ) "
3041         "   2>r swap 2 2r> ['] $callback  catch  if "
3042         "      2drop 3drop 0 "
3043         "   then "
3044         "; "
3045         ": read ( adr,len -- #read ) "
3046         "       \" read\" ['] cb-r/w catch  if  2drop 2drop -2 exit then "
3047         "       ( retN ... ret1 N ) "
3048         "       ?dup  if "
3049         "               swap >r 1-  0  ?do  drop  loop  r> "
3050         "       else "
3051         "               -2 "
3052         "       then "
3053         ";    "
3054         ": write ( adr,len -- #written ) "
3055         "       \" write\" ['] cb-r/w catch  if  2drop 2drop 0 exit  then "
3056         "       ( retN ... ret1 N ) "
3057         "       ?dup  if "
3058         "               swap >r 1-  0  ?do  drop  loop  r> "
3059         "        else "
3060         "               0 "
3061         "       then "
3062         "; "
3063         ": poll-tty ( -- ) ; "
3064         ": install-abort  ( -- )  ['] poll-tty d# 10 alarm ; "
3065         ": remove-abort ( -- )  ['] poll-tty 0 alarm ; "
3066         ": cb-give/take ( $method -- ) "
3067         "       0 -rot ['] $callback catch  ?dup  if "
3068         "               >r 2drop 2drop r> throw "
3069         "       else "
3070         "               0  ?do  drop  loop "
3071         "       then "
3072         "; "
3073         ": give ( -- )  \" exit-input\" cb-give/take ; "
3074         ": take ( -- )  \" enter-input\" cb-give/take ; "
3075         ": open ( -- ok? )  true ; "
3076         ": close ( -- ) ; "
3077         "finish-device "
3078         "device-end ";
3079 
3080 /*
3081  * Create the OBP input/output node (FCode serial driver).
3082  * It is needed for both USB console keyboard and for
3083  * the kernel terminal emulator.  It is too early to check for a
3084  * kernel console compatible framebuffer now, so we create this
3085  * so that we're ready if we need to enable kernel terminal emulation.
3086  *
3087  * When the USB software takes over the input device at the time
3088  * consconfig runs, OBP's stdin is redirected to this node.
3089  * Whenever the FORTH user interface is used after this switch,
3090  * the node will call back into the kernel for console input.
3091  * If a serial device such as ttya or a UART with a Type 5 keyboard
3092  * attached is used, OBP takes over the serial device when the system
3093  * goes to the debugger after the system is booted.  This sharing
3094  * of the relatively simple serial device is difficult but possible.
3095  * Sharing the USB host controller is impossible due its complexity.
3096  *
3097  * Similarly to USB keyboard input redirection, after consconfig_dacf
3098  * configures a kernel console framebuffer as the standard output
3099  * device, OBP's stdout is switched to to vector through the
3100  * /os-io node into the kernel terminal emulator.
3101  */
3102 static void
3103 startup_create_io_node(void)
3104 {
3105         prom_interpret(create_node, 0, 0, 0, 0, 0);
3106 }
3107 
3108 
3109 static void
3110 do_prom_version_check(void)
3111 {
3112         int i;
3113         pnode_t node;
3114         char buf[64];
3115         static char drev[] = "Down-rev firmware detected%s\n"
3116             "\tPlease upgrade to the following minimum version:\n"
3117             "\t\t%s\n";
3118 
3119         i = prom_version_check(buf, sizeof (buf), &node);
3120 
3121         if (i == PROM_VER64_OK)
3122                 return;
3123 
3124         if (i == PROM_VER64_UPGRADE) {
3125                 cmn_err(CE_WARN, drev, "", buf);
3126 
3127 #ifdef  DEBUG
3128                 prom_enter_mon();       /* Type 'go' to continue */
3129                 cmn_err(CE_WARN, "Booting with down-rev firmware\n");
3130                 return;
3131 #else
3132                 halt(0);
3133 #endif
3134         }
3135 
3136         /*
3137          * The other possibility is that this is a server running
3138          * good firmware, but down-rev firmware was detected on at
3139          * least one other cpu board. We just complain if we see
3140          * that.
3141          */
3142         cmn_err(CE_WARN, drev, " on one or more CPU boards", buf);
3143 }
3144 
3145 
3146 /*
3147  * Must be defined in platform dependent code.
3148  */
3149 extern caddr_t modtext;
3150 extern size_t modtext_sz;
3151 extern caddr_t moddata;
3152 
3153 #define HEAPTEXT_ARENA(addr)    \
3154         ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
3155         (((uintptr_t)(addr) - HEAPTEXT_BASE) / \
3156         (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
3157 
3158 #define HEAPTEXT_OVERSIZED(addr)        \
3159         ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
3160 
3161 #define HEAPTEXT_IN_NUCLEUSDATA(addr) \
3162         (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
3163         ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
3164 
3165 vmem_t *texthole_source[HEAPTEXT_NARENAS];
3166 vmem_t *texthole_arena[HEAPTEXT_NARENAS];
3167 kmutex_t texthole_lock;
3168 
3169 char kern_bootargs[OBP_MAXPATHLEN];
3170 char kern_bootfile[OBP_MAXPATHLEN];
3171 
3172 void
3173 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3174 {
3175         uintptr_t addr, limit;
3176 
3177         addr = HEAPTEXT_BASE;
3178         limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
3179 
3180         /*
3181          * Before we initialize the text_arena, we want to punch holes in the
3182          * underlying heaptext_arena.  This guarantees that for any text
3183          * address we can find a text hole less than HEAPTEXT_MAPPED away.
3184          */
3185         for (; addr + HEAPTEXT_UNMAPPED <= limit;
3186             addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
3187                 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
3188                     0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
3189                     VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3190         }
3191 
3192         /*
3193          * Allocate one page at the oversize to break up the text region
3194          * from the oversized region.
3195          */
3196         (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
3197             (void *)limit, (void *)(limit + PAGESIZE),
3198             VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3199 
3200         *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
3201             modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
3202             heaptext_arena, 0, VM_SLEEP);
3203         *data_arena = vmem_create("module_data", moddata, MODDATA, 1,
3204             segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3205 }
3206 
3207 caddr_t
3208 kobj_text_alloc(vmem_t *arena, size_t size)
3209 {
3210         caddr_t rval, better;
3211 
3212         /*
3213          * First, try a sleeping allocation.
3214          */
3215         rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
3216 
3217         if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
3218                 return (rval);
3219 
3220         /*
3221          * We didn't get the area that we wanted.  We're going to try to do an
3222          * allocation with explicit constraints.
3223          */
3224         better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
3225             (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
3226             VM_NOSLEEP | VM_BESTFIT);
3227 
3228         if (better != NULL) {
3229                 /*
3230                  * That worked.  Free our first attempt and return.
3231                  */
3232                 vmem_free(arena, rval, size);
3233                 return (better);
3234         }
3235 
3236         /*
3237          * That didn't work; we'll have to return our first attempt.
3238          */
3239         return (rval);
3240 }
3241 
3242 caddr_t
3243 kobj_texthole_alloc(caddr_t addr, size_t size)
3244 {
3245         int arena = HEAPTEXT_ARENA(addr);
3246         char c[30];
3247         uintptr_t base;
3248 
3249         if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
3250                 /*
3251                  * If this is an oversized allocation or it is allocated in
3252                  * the nucleus data page, there is no text hole available for
3253                  * it; return NULL.
3254                  */
3255                 return (NULL);
3256         }
3257 
3258         mutex_enter(&texthole_lock);
3259 
3260         if (texthole_arena[arena] == NULL) {
3261                 ASSERT(texthole_source[arena] == NULL);
3262 
3263                 if (arena == 0) {
3264                         texthole_source[0] = vmem_create("module_text_holesrc",
3265                             (void *)(KERNELBASE + MMU_PAGESIZE4M),
3266                             MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
3267                             0, VM_SLEEP);
3268                 } else {
3269                         base = HEAPTEXT_BASE +
3270                             (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
3271 
3272                         (void) snprintf(c, sizeof (c),
3273                             "heaptext_holesrc_%d", arena);
3274 
3275                         texthole_source[arena] = vmem_create(c, (void *)base,
3276                             HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
3277                             0, VM_SLEEP);
3278                 }
3279 
3280                 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
3281 
3282                 texthole_arena[arena] = vmem_create(c, NULL, 0,
3283                     sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
3284                     texthole_source[arena], 0, VM_SLEEP);
3285         }
3286 
3287         mutex_exit(&texthole_lock);
3288 
3289         ASSERT(texthole_arena[arena] != NULL);
3290         ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3291         return (vmem_alloc(texthole_arena[arena], size,
3292             VM_BESTFIT | VM_NOSLEEP));
3293 }
3294 
3295 void
3296 kobj_texthole_free(caddr_t addr, size_t size)
3297 {
3298         int arena = HEAPTEXT_ARENA(addr);
3299 
3300         ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3301         ASSERT(texthole_arena[arena] != NULL);
3302         vmem_free(texthole_arena[arena], addr, size);
3303 }
3304 
3305 void
3306 release_bootstrap(void)
3307 {
3308         if (&cif_init)
3309                 cif_init();
3310 }