1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright 2018 Joyent, Inc.
  27  */
  28 
  29 #include <sys/types.h>
  30 #include <sys/systm.h>
  31 #include <sys/archsystm.h>
  32 #include <sys/debug.h>
  33 #include <sys/bootconf.h>
  34 #include <sys/bootsvcs.h>
  35 #include <sys/bootinfo.h>
  36 #include <sys/mman.h>
  37 #include <sys/cmn_err.h>
  38 #include <sys/param.h>
  39 #include <sys/machparam.h>
  40 #include <sys/machsystm.h>
  41 #include <sys/promif.h>
  42 #include <sys/kobj.h>
  43 #ifdef __xpv
  44 #include <sys/hypervisor.h>
  45 #endif
  46 #include <vm/kboot_mmu.h>
  47 #include <vm/hat_pte.h>
  48 #include <vm/hat_i86.h>
  49 #include <vm/seg_kmem.h>
  50 
  51 #if 0
  52 /*
  53  * Joe's debug printing
  54  */
  55 #define DBG(x)    \
  56         bop_printf(NULL, "kboot_mmu.c: %s is %" PRIx64 "\n", #x, (uint64_t)(x));
  57 #else
  58 #define DBG(x)  /* naught */
  59 #endif
  60 
  61 /*
  62  * Page table and memory stuff.
  63  */
  64 static caddr_t window;
  65 static caddr_t pte_to_window;
  66 
  67 /*
  68  * this are needed by mmu_init()
  69  */
  70 int kbm_nx_support = 0;         /* NX bit in PTEs is in use */
  71 int kbm_pae_support = 0;        /* PAE is 64 bit Page table entries */
  72 int kbm_pge_support = 0;        /* PGE is Page table global bit enabled */
  73 int kbm_largepage_support = 0;
  74 uint_t kbm_nucleus_size = 0;
  75 
  76 #define BOOT_SHIFT(l)   (shift_amt[l])
  77 #define BOOT_SZ(l)      ((size_t)1 << BOOT_SHIFT(l))
  78 #define BOOT_OFFSET(l)  (BOOT_SZ(l) - 1)
  79 #define BOOT_MASK(l)    (~BOOT_OFFSET(l))
  80 
  81 /*
  82  * Initialize memory management parameters for boot time page table management
  83  */
  84 void
  85 kbm_init(struct xboot_info *bi)
  86 {
  87         /*
  88          * configure mmu information
  89          */
  90         kbm_nucleus_size = (uintptr_t)bi->bi_kseg_size;
  91         kbm_largepage_support = bi->bi_use_largepage;
  92         kbm_nx_support = bi->bi_use_nx;
  93         kbm_pae_support = bi->bi_use_pae;
  94         kbm_pge_support = bi->bi_use_pge;
  95         window = bi->bi_pt_window;
  96         DBG(window);
  97         pte_to_window = bi->bi_pte_to_pt_window;
  98         DBG(pte_to_window);
  99         if (kbm_pae_support) {
 100                 shift_amt = shift_amt_pae;
 101                 ptes_per_table = 512;
 102                 pte_size = 8;
 103                 lpagesize = TWO_MEG;
 104 #ifdef __amd64
 105                 top_level = 3;
 106 #else
 107                 top_level = 2;
 108 #endif
 109         } else {
 110                 shift_amt = shift_amt_nopae;
 111                 ptes_per_table = 1024;
 112                 pte_size = 4;
 113                 lpagesize = FOUR_MEG;
 114                 top_level = 1;
 115         }
 116 
 117 #ifdef __xpv
 118         xen_info = bi->bi_xen_start_info;
 119         mfn_list = (mfn_t *)xen_info->mfn_list;
 120         DBG(mfn_list);
 121         mfn_count = xen_info->nr_pages;
 122         DBG(mfn_count);
 123 #endif
 124         top_page_table = bi->bi_top_page_table;
 125         DBG(top_page_table);
 126 }
 127 
 128 /*
 129  * Change the addressible page table window to point at a given page
 130  */
 131 /*ARGSUSED*/
 132 void *
 133 kbm_remap_window(paddr_t physaddr, int writeable)
 134 {
 135         x86pte_t pt_bits = PT_NOCONSIST | PT_VALID | PT_WRITABLE;
 136 
 137         DBG(physaddr);
 138 
 139 #ifdef __xpv
 140         if (!writeable)
 141                 pt_bits &= ~PT_WRITABLE;
 142         if (HYPERVISOR_update_va_mapping((uintptr_t)window,
 143             pa_to_ma(physaddr) | pt_bits, UVMF_INVLPG | UVMF_LOCAL) < 0)
 144                 bop_panic("HYPERVISOR_update_va_mapping() failed");
 145 #else
 146         if (kbm_pae_support)
 147                 *((x86pte_t *)pte_to_window) = physaddr | pt_bits;
 148         else
 149                 *((x86pte32_t *)pte_to_window) = physaddr | pt_bits;
 150         mmu_invlpg(window);
 151 #endif
 152         DBG(window);
 153         return (window);
 154 }
 155 
 156 /*
 157  * Add a mapping for the physical page at the given virtual address.
 158  */
 159 void
 160 kbm_map(uintptr_t va, paddr_t pa, uint_t level, uint_t is_kernel)
 161 {
 162         x86pte_t *ptep;
 163         paddr_t pte_physaddr;
 164         x86pte_t pteval;
 165 
 166         if (khat_running)
 167                 panic("kbm_map() called too late");
 168 
 169         pteval = pa_to_ma(pa) | PT_NOCONSIST | PT_VALID | PT_WRITABLE;
 170         if (level >= 1)
 171                 pteval |= PT_PAGESIZE;
 172         if (kbm_pge_support && is_kernel)
 173                 pteval |= PT_GLOBAL;
 174 
 175 #ifdef __xpv
 176         /*
 177          * try update_va_mapping first - fails if page table is missing.
 178          */
 179         if (HYPERVISOR_update_va_mapping(va, pteval,
 180             UVMF_INVLPG | UVMF_LOCAL) == 0)
 181                 return;
 182 #endif
 183 
 184         /*
 185          * Find the pte that will map this address. This creates any
 186          * missing intermediate level page tables.
 187          */
 188         ptep = find_pte(va, &pte_physaddr, level, 0);
 189         if (ptep == NULL)
 190                 bop_panic("kbm_map: find_pte returned NULL");
 191 
 192 #ifdef __xpv
 193         if (HYPERVISOR_update_va_mapping(va, pteval, UVMF_INVLPG | UVMF_LOCAL))
 194                 bop_panic("HYPERVISOR_update_va_mapping() failed");
 195 #else
 196         if (kbm_pae_support)
 197                 *ptep = pteval;
 198         else
 199                 *((x86pte32_t *)ptep) = pteval;
 200         mmu_invlpg((caddr_t)va);
 201 #endif
 202 }
 203 
 204 #ifdef __xpv
 205 
 206 /*
 207  * Add a mapping for the machine page at the given virtual address.
 208  */
 209 void
 210 kbm_map_ma(maddr_t ma, uintptr_t va, uint_t level)
 211 {
 212         paddr_t pte_physaddr;
 213         x86pte_t pteval;
 214 
 215         pteval = ma | PT_NOCONSIST | PT_VALID | PT_REF | PT_WRITABLE;
 216         if (level == 1)
 217                 pteval |= PT_PAGESIZE;
 218 
 219         /*
 220          * try update_va_mapping first - fails if page table is missing.
 221          */
 222         if (HYPERVISOR_update_va_mapping(va,
 223             pteval, UVMF_INVLPG | UVMF_LOCAL) == 0)
 224                 return;
 225 
 226         /*
 227          * Find the pte that will map this address. This creates any
 228          * missing intermediate level page tables
 229          */
 230         (void) find_pte(va, &pte_physaddr, level, 0);
 231 
 232         if (HYPERVISOR_update_va_mapping(va,
 233             pteval, UVMF_INVLPG | UVMF_LOCAL) != 0)
 234                 bop_panic("HYPERVISOR_update_va_mapping failed");
 235 }
 236 
 237 #endif /* __xpv */
 238 
 239 
 240 /*
 241  * Probe the boot time page tables to find the first mapping
 242  * including va (or higher) and return non-zero if one is found.
 243  * va is updated to the starting address and len to the pagesize.
 244  * pp will be set to point to the 1st page_t of the mapped page(s).
 245  *
 246  * Note that if va is in the middle of a large page, the returned va
 247  * will be less than what was asked for.
 248  */
 249 int
 250 kbm_probe(uintptr_t *va, size_t *len, pfn_t *pfn, uint_t *prot)
 251 {
 252         uintptr_t       probe_va;
 253         x86pte_t        *ptep;
 254         paddr_t         pte_physaddr;
 255         x86pte_t        pte_val;
 256         level_t         l;
 257 
 258         if (khat_running)
 259                 panic("kbm_probe() called too late");
 260         *len = 0;
 261         *pfn = PFN_INVALID;
 262         *prot = 0;
 263         probe_va = *va;
 264 restart_new_va:
 265         l = top_level;
 266         for (;;) {
 267                 if (IN_VA_HOLE(probe_va))
 268                         probe_va = mmu.hole_end;
 269 
 270                 if (IN_HYPERVISOR_VA(probe_va))
 271 #if defined(__amd64) && defined(__xpv)
 272                         probe_va = HYPERVISOR_VIRT_END;
 273 #else
 274                         return (0);
 275 #endif
 276 
 277                 /*
 278                  * If we don't have a valid PTP/PTE at this level
 279                  * then we can bump VA by this level's pagesize and try again.
 280                  * When the probe_va wraps around, we are done.
 281                  */
 282                 ptep = find_pte(probe_va, &pte_physaddr, l, 1);
 283                 if (ptep == NULL)
 284                         bop_panic("kbm_probe: find_pte returned NULL");
 285                 if (kbm_pae_support)
 286                         pte_val = *ptep;
 287                 else
 288                         pte_val = *((x86pte32_t *)ptep);
 289                 if (!PTE_ISVALID(pte_val)) {
 290                         probe_va = (probe_va & BOOT_MASK(l)) + BOOT_SZ(l);
 291                         if (probe_va <= *va)
 292                                 return (0);
 293                         goto restart_new_va;
 294                 }
 295 
 296                 /*
 297                  * If this entry is a pointer to a lower level page table
 298                  * go down to it.
 299                  */
 300                 if (!PTE_ISPAGE(pte_val, l)) {
 301                         ASSERT(l > 0);
 302                         --l;
 303                         continue;
 304                 }
 305 
 306                 /*
 307                  * We found a boot level page table entry
 308                  */
 309                 *len = BOOT_SZ(l);
 310                 *va = probe_va & ~(*len - 1);
 311                 *pfn = PTE2PFN(pte_val, l);
 312 
 313 
 314                 *prot = PROT_READ | PROT_EXEC;
 315                 if (PTE_GET(pte_val, PT_WRITABLE))
 316                         *prot |= PROT_WRITE;
 317 
 318                 /*
 319                  * pt_nx is cleared if processor doesn't support NX bit
 320                  */
 321                 if (PTE_GET(pte_val, mmu.pt_nx))
 322                         *prot &= ~PROT_EXEC;
 323 
 324                 return (1);
 325         }
 326 }
 327 
 328 
 329 /*
 330  * Destroy a boot loader page table 4K mapping.
 331  */
 332 void
 333 kbm_unmap(uintptr_t va)
 334 {
 335         if (khat_running)
 336                 panic("kbm_unmap() called too late");
 337         else {
 338 #ifdef __xpv
 339                 (void) HYPERVISOR_update_va_mapping(va, 0,
 340                     UVMF_INVLPG | UVMF_LOCAL);
 341 #else
 342                 x86pte_t *ptep;
 343                 level_t level = 0;
 344                 uint_t  probe_only = 1;
 345 
 346                 ptep = find_pte(va, NULL, level, probe_only);
 347                 if (ptep == NULL)
 348                         return;
 349 
 350                 if (kbm_pae_support)
 351                         *ptep = 0;
 352                 else
 353                         *((x86pte32_t *)ptep) = 0;
 354                 mmu_invlpg((caddr_t)va);
 355 #endif
 356         }
 357 }
 358 
 359 
 360 /*
 361  * Change a boot loader page table 4K mapping.
 362  * Returns the pfn of the old mapping.
 363  */
 364 pfn_t
 365 kbm_remap(uintptr_t va, pfn_t pfn)
 366 {
 367         x86pte_t *ptep;
 368         level_t level = 0;
 369         uint_t  probe_only = 1;
 370         x86pte_t pte_val = pa_to_ma(pfn_to_pa(pfn)) | PT_WRITABLE |
 371             PT_NOCONSIST | PT_VALID;
 372         x86pte_t old_pte;
 373 
 374         if (khat_running)
 375                 panic("kbm_remap() called too late");
 376         ptep = find_pte(va, NULL, level, probe_only);
 377         if (ptep == NULL)
 378                 bop_panic("kbm_remap: find_pte returned NULL");
 379 
 380         if (kbm_pae_support)
 381                 old_pte = *ptep;
 382         else
 383                 old_pte = *((x86pte32_t *)ptep);
 384 
 385 #ifdef __xpv
 386         if (HYPERVISOR_update_va_mapping(va, pte_val, UVMF_INVLPG | UVMF_LOCAL))
 387                 bop_panic("HYPERVISOR_update_va_mapping() failed");
 388 #else
 389         if (kbm_pae_support)
 390                 *((x86pte_t *)ptep) = pte_val;
 391         else
 392                 *((x86pte32_t *)ptep) = pte_val;
 393         mmu_invlpg((caddr_t)va);
 394 #endif
 395 
 396         if (!(old_pte & PT_VALID) || ma_to_pa(old_pte) == -1)
 397                 return (PFN_INVALID);
 398         return (mmu_btop(ma_to_pa(old_pte)));
 399 }
 400 
 401 
 402 /*
 403  * Change a boot loader page table 4K mapping to read only.
 404  */
 405 void
 406 kbm_read_only(uintptr_t va, paddr_t pa)
 407 {
 408         x86pte_t pte_val = pa_to_ma(pa) |
 409             PT_NOCONSIST | PT_REF | PT_MOD | PT_VALID;
 410 
 411 #ifdef __xpv
 412         if (HYPERVISOR_update_va_mapping(va, pte_val, UVMF_INVLPG | UVMF_LOCAL))
 413                 bop_panic("HYPERVISOR_update_va_mapping() failed");
 414 #else
 415         x86pte_t *ptep;
 416         level_t level = 0;
 417 
 418         ptep = find_pte(va, NULL, level, 0);
 419         if (ptep == NULL)
 420                 bop_panic("kbm_read_only: find_pte returned NULL");
 421 
 422         if (kbm_pae_support)
 423                 *ptep = pte_val;
 424         else
 425                 *((x86pte32_t *)ptep) = pte_val;
 426         mmu_invlpg((caddr_t)va);
 427 #endif
 428 }
 429 
 430 /*
 431  * interfaces for kernel debugger to access physical memory
 432  */
 433 static x86pte_t save_pte;
 434 
 435 void *
 436 kbm_push(paddr_t pa)
 437 {
 438         static int first_time = 1;
 439 
 440         if (first_time) {
 441                 first_time = 0;
 442                 return (window);
 443         }
 444 
 445         if (kbm_pae_support)
 446                 save_pte = *((x86pte_t *)pte_to_window);
 447         else
 448                 save_pte = *((x86pte32_t *)pte_to_window);
 449         return (kbm_remap_window(pa, 0));
 450 }
 451 
 452 void
 453 kbm_pop(void)
 454 {
 455 #ifdef __xpv
 456         if (HYPERVISOR_update_va_mapping((uintptr_t)window, save_pte,
 457             UVMF_INVLPG | UVMF_LOCAL) < 0)
 458                 bop_panic("HYPERVISOR_update_va_mapping() failed");
 459 #else
 460         if (kbm_pae_support)
 461                 *((x86pte_t *)pte_to_window) = save_pte;
 462         else
 463                 *((x86pte32_t *)pte_to_window) = save_pte;
 464         mmu_invlpg(window);
 465 #endif
 466 }
 467 
 468 x86pte_t
 469 get_pteval(paddr_t table, uint_t index)
 470 {
 471         void *table_ptr = kbm_remap_window(table, 0);
 472 
 473         if (kbm_pae_support)
 474                 return (((x86pte_t *)table_ptr)[index]);
 475         return (((x86pte32_t *)table_ptr)[index]);
 476 }
 477 
 478 #ifndef __xpv
 479 void
 480 set_pteval(paddr_t table, uint_t index, uint_t level, x86pte_t pteval)
 481 {
 482         void *table_ptr = kbm_remap_window(table, 0);
 483         if (kbm_pae_support)
 484                 ((x86pte_t *)table_ptr)[index] = pteval;
 485         else
 486                 ((x86pte32_t *)table_ptr)[index] = pteval;
 487         if (level == top_level && level == 2)
 488                 reload_cr3();
 489 }
 490 #endif
 491 
 492 paddr_t
 493 make_ptable(x86pte_t *pteval, uint_t level)
 494 {
 495         paddr_t new_table;
 496         void *table_ptr;
 497 
 498         new_table = do_bop_phys_alloc(MMU_PAGESIZE, MMU_PAGESIZE);
 499         table_ptr = kbm_remap_window(new_table, 1);
 500         bzero(table_ptr, MMU_PAGESIZE);
 501 #ifdef __xpv
 502         /* Remove write permission to the new page table.  */
 503         (void) kbm_remap_window(new_table, 0);
 504 #endif
 505 
 506         if (level == top_level && level == 2)
 507                 *pteval = pa_to_ma(new_table) | PT_VALID;
 508         else
 509                 *pteval = pa_to_ma(new_table) |
 510                     PT_VALID | PT_REF | PT_USER | PT_WRITABLE;
 511 
 512         return (new_table);
 513 }
 514 
 515 x86pte_t *
 516 map_pte(paddr_t table, uint_t index)
 517 {
 518         void *table_ptr = kbm_remap_window(table, 0);
 519         return ((x86pte_t *)((caddr_t)table_ptr + index * pte_size));
 520 }