8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>

@@ -20,10 +20,12 @@
  */
 
 /*
  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright 2018 Joyent, Inc.
  */
 
 /*
  * HAT interfaces used by the kernel debugger to interact with the VM system.
  * These interfaces are invoked when the world is stopped.  As such, no blocking

@@ -175,11 +177,11 @@
          * kdi_pread() to walk the page tables
          */
 #if defined(__xpv)
         *pap = pfn_to_pa(CPU->cpu_current_hat->hat_htable->ht_pfn);
 #else
-        *pap = getcr3() & MMU_PAGEMASK;
+        *pap = getcr3_pa();
 #endif
         for (level = mmu.max_level; ; --level) {
                 index = (va >> LEVEL_SHIFT(level)) & (mmu.ptes_per_table - 1);
                 *pap += index << mmu.pte_size_shift;
                 pte = 0;

@@ -247,11 +249,11 @@
 #else
                 else if (hat_kdi_use_pae)
                         *hat_kdi_pte = pte;
                 else
                         *(x86pte32_t *)hat_kdi_pte = pte;
-                mmu_tlbflush_entry((caddr_t)hat_kdi_page);
+                mmu_flush_tlb_kpage(hat_kdi_page);
 #endif
 
                 bcopy(from, to, sz);
 
                 /*

@@ -266,11 +268,11 @@
 #else
                 else if (hat_kdi_use_pae)
                         *hat_kdi_pte = 0;
                 else
                         *(x86pte32_t *)hat_kdi_pte = 0;
-                mmu_tlbflush_entry((caddr_t)hat_kdi_page);
+                mmu_flush_tlb_kpage(hat_kdi_page);
 #endif
 
                 buf += sz;
                 pa += sz;
                 nbytes -= sz;

@@ -294,10 +296,23 @@
 kdi_pwrite(caddr_t buf, size_t nbytes, uint64_t addr, size_t *ncopiedp)
 {
         return (kdi_prw(buf, nbytes, addr, ncopiedp, 0));
 }
 
+#if !defined(__xpv)
+/*
+ * This gets used for flushing the TLB on all the slaves just prior to doing a
+ * kdi_prw().  It's unclear why this was originally done, since kdi_prw() itself
+ * will flush any lingering hat_kdi_page mappings, but let's presume it was a
+ * good idea.
+ */
+void
+kdi_flush_caches(void)
+{
+        mmu_flush_tlb(FLUSH_TLB_ALL, NULL);
+}
+#endif
 
 /*
  * Return the number of bytes, relative to the beginning of a given range, that
  * are non-toxic (can be read from and written to with relative impunity).
  */