8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>

@@ -21,11 +21,11 @@
 
 /*
  * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2012 DEY Storage Systems, Inc.  All rights reserved.
  * Copyright 2017 Nexenta Systems, Inc.
- * Copyright 2015 Joyent, Inc.
+ * Copyright (c) 2018 Joyent, Inc.
  * Copyright (c) 2015 by Delphix. All rights reserved.
  */
 /*
  * Copyright (c) 2010, Intel Corporation.
  * All rights reserved.

@@ -444,12 +444,14 @@
  *                      +-----------------------+
  *                      |      Kernel Data      |
  * 0xFFFFFFFF.FBC00000  |-----------------------|
  *                      |      Kernel Text      |
  * 0xFFFFFFFF.FB800000  |-----------------------|- KERNEL_TEXT
- *                      |---       GDT       ---|- GDT page (GDT_VA)
  *                      |---    debug info   ---|- debug info (DEBUG_INFO_VA)
+ *                      |---       GDT       ---|- GDT page (GDT_VA)
+ *                      |---       IDT       ---|- IDT page (IDT_VA)
+ *                      |---       LDT       ---|- LDT pages (LDT_VA)
  *                      |                       |
  *                      |      Core heap        | (used for loadable modules)
  * 0xFFFFFFFF.C0000000  |-----------------------|- core_base / ekernelheap
  *                      |        Kernel         |
  *                      |         heap          |

@@ -949,10 +951,21 @@
 
         if (segkpm_create(segkpm, (caddr_t)&b) != 0)
                 panic("segkpm_create segkpm");
 
         rw_exit(&kas.a_lock);
+
+        kpm_enable = 1;
+
+        /*
+         * As the KPM was disabled while setting up the system, go back and fix
+         * CPU zero's access to its user page table. This is a bit gross, but
+         * we have a chicken and egg problem otherwise.
+         */
+        ASSERT(CPU->cpu_hat_info->hci_user_l3ptes == NULL);
+        CPU->cpu_hat_info->hci_user_l3ptes =
+            (x86pte_t *)hat_kpm_mapin_pfn(CPU->cpu_hat_info->hci_user_l3pfn);
 }
 
 /*
  * The debug info page provides enough information to allow external
  * inspectors (e.g. when running under a hypervisor) to bootstrap

@@ -1412,10 +1425,13 @@
  */
 static void
 startup_kmem(void)
 {
         extern void page_set_colorequiv_arr(void);
+#if !defined(__xpv)
+        extern uint64_t kpti_kbase;
+#endif
 
         PRM_POINT("startup_kmem() starting...");
 
 #if defined(__amd64)
         if (eprom_kernelbase && eprom_kernelbase != KERNELBASE)

@@ -1474,17 +1490,23 @@
 
         *(uintptr_t *)&_kernelbase = kernelbase;
         *(uintptr_t *)&_userlimit = kernelbase;
 #if defined(__amd64)
         *(uintptr_t *)&_userlimit -= KERNELBASE - USERLIMIT;
+#if !defined(__xpv)
+        kpti_kbase = kernelbase;
+#endif
 #else
         *(uintptr_t *)&_userlimit32 = _userlimit;
 #endif
         PRM_DEBUG(_kernelbase);
         PRM_DEBUG(_userlimit);
         PRM_DEBUG(_userlimit32);
 
+        /* We have to re-do this now that we've modified _userlimit. */
+        mmu_calc_user_slots();
+
         layout_kernel_va();
 
 #if defined(__i386)
         /*
          * If segmap is too large we can push the bottom of the kernel heap

@@ -2119,36 +2141,10 @@
          * it, tell the debugger.
          */
         if (boothowto & RB_DEBUG)
                 kdi_dvec_memavail();
 
-        /*
-         * The following code installs a special page fault handler (#pf)
-         * to work around a pentium bug.
-         */
-#if !defined(__amd64) && !defined(__xpv)
-        if (x86_type == X86_TYPE_P5) {
-                desctbr_t idtr;
-                gate_desc_t *newidt;
-
-                if ((newidt = kmem_zalloc(MMU_PAGESIZE, KM_NOSLEEP)) == NULL)
-                        panic("failed to install pentium_pftrap");
-
-                bcopy(idt0, newidt, NIDT * sizeof (*idt0));
-                set_gatesegd(&newidt[T_PGFLT], &pentium_pftrap,
-                    KCS_SEL, SDT_SYSIGT, TRP_KPL, 0);
-
-                (void) as_setprot(&kas, (caddr_t)newidt, MMU_PAGESIZE,
-                    PROT_READ | PROT_EXEC);
-
-                CPU->cpu_idt = newidt;
-                idtr.dtr_base = (uintptr_t)CPU->cpu_idt;
-                idtr.dtr_limit = (NIDT * sizeof (*idt0)) - 1;
-                wr_idtr(&idtr);
-        }
-#endif  /* !__amd64 */
-
 #if !defined(__xpv)
         /*
          * Map page pfn=0 for drivers, such as kd, that need to pick up
          * parameters left there by controllers/BIOS.
          */

@@ -2207,14 +2203,12 @@
 
         /*
          * kpm segment
          */
         segmap_kpm = 0;
-        if (kpm_desired) {
+        if (kpm_desired)
                 kpm_init();
-                kpm_enable = 1;
-        }
 
         /*
          * Now create segmap segment.
          */
         rw_enter(&kas.a_lock, RW_WRITER);

@@ -2333,16 +2327,22 @@
 #if defined(__xpv)
         ec_init_debug_irq();
         xs_domu_init();
 #endif
 
-#if defined(__amd64) && !defined(__xpv)
+#if !defined(__xpv)
         /*
          * Intel IOMMU has been setup/initialized in ddi_impl.c
          * Start it up now.
          */
         immu_startup();
+
+        /*
+         * Now that we're no longer going to drop into real mode for a BIOS call
+         * via bootops, we can enable PCID (which requires CR0.PG).
+         */
+        enable_pcid();
 #endif
 
         PRM_POINT("Enabling interrupts");
         (*picinitf)();
         sti();