1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2014 by Delphix. All rights reserved.
25 * Copyright 2018 Joyent, Inc.
26 */
27
28 #include <sys/types.h>
29 #include <sys/sysmacros.h>
30 #include <sys/kmem.h>
31 #include <sys/atomic.h>
32 #include <sys/bitmap.h>
33 #include <sys/machparam.h>
34 #include <sys/machsystm.h>
35 #include <sys/mman.h>
36 #include <sys/systm.h>
37 #include <sys/cpuvar.h>
38 #include <sys/thread.h>
39 #include <sys/proc.h>
40 #include <sys/cpu.h>
41 #include <sys/kmem.h>
42 #include <sys/disp.h>
43 #include <sys/vmem.h>
44 #include <sys/vmsystm.h>
45 #include <sys/promif.h>
46 #include <sys/var.h>
47 #include <sys/x86_archext.h>
48 #include <sys/archsystm.h>
49 #include <sys/bootconf.h>
50 #include <sys/dumphdr.h>
51 #include <vm/seg_kmem.h>
52 #include <vm/seg_kpm.h>
53 #include <vm/hat.h>
54 #include <vm/hat_i86.h>
55 #include <sys/cmn_err.h>
56 #include <sys/panic.h>
57
58 #ifdef __xpv
59 #include <sys/hypervisor.h>
60 #include <sys/xpv_panic.h>
61 #endif
62
63 #include <sys/bootinfo.h>
64 #include <vm/kboot_mmu.h>
65
66 static void x86pte_zero(htable_t *dest, uint_t entry, uint_t count);
67
68 kmem_cache_t *htable_cache;
69
70 /*
71 * The variable htable_reserve_amount, rather than HTABLE_RESERVE_AMOUNT,
72 * is used in order to facilitate testing of the htable_steal() code.
73 * By resetting htable_reserve_amount to a lower value, we can force
74 * stealing to occur. The reserve amount is a guess to get us through boot.
75 */
76 #define HTABLE_RESERVE_AMOUNT (200)
77 uint_t htable_reserve_amount = HTABLE_RESERVE_AMOUNT;
78 kmutex_t htable_reserve_mutex;
79 uint_t htable_reserve_cnt;
80 htable_t *htable_reserve_pool;
81
82 /*
83 * Used to hand test htable_steal().
84 */
85 #ifdef DEBUG
86 ulong_t force_steal = 0;
87 ulong_t ptable_cnt = 0;
88 #endif
89
90 /*
91 * This variable is so that we can tune this via /etc/system
92 * Any value works, but a power of two <= mmu.ptes_per_table is best.
93 */
94 uint_t htable_steal_passes = 8;
95
96 /*
97 * mutex stuff for access to htable hash
98 */
99 #define NUM_HTABLE_MUTEX 128
100 kmutex_t htable_mutex[NUM_HTABLE_MUTEX];
101 #define HTABLE_MUTEX_HASH(h) ((h) & (NUM_HTABLE_MUTEX - 1))
102
103 #define HTABLE_ENTER(h) mutex_enter(&htable_mutex[HTABLE_MUTEX_HASH(h)]);
104 #define HTABLE_EXIT(h) mutex_exit(&htable_mutex[HTABLE_MUTEX_HASH(h)]);
105
106 /*
107 * forward declarations
108 */
109 static void link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr);
110 static void unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr);
111 static void htable_free(htable_t *ht);
112 static x86pte_t *x86pte_access_pagetable(htable_t *ht, uint_t index);
113 static void x86pte_release_pagetable(htable_t *ht);
114 static x86pte_t x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old,
115 x86pte_t new);
116
117 /*
118 * A counter to track if we are stealing or reaping htables. When non-zero
119 * htable_free() will directly free htables (either to the reserve or kmem)
120 * instead of putting them in a hat's htable cache.
121 */
122 uint32_t htable_dont_cache = 0;
123
124 /*
125 * Track the number of active pagetables, so we can know how many to reap
126 */
127 static uint32_t active_ptables = 0;
128
129 #ifdef __xpv
130 /*
131 * Deal with hypervisor complications.
132 */
133 void
134 xen_flush_va(caddr_t va)
135 {
136 struct mmuext_op t;
137 uint_t count;
138
139 if (IN_XPV_PANIC()) {
140 mmu_flush_tlb_page((uintptr_t)va);
141 } else {
142 t.cmd = MMUEXT_INVLPG_LOCAL;
143 t.arg1.linear_addr = (uintptr_t)va;
144 if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
145 panic("HYPERVISOR_mmuext_op() failed");
146 ASSERT(count == 1);
147 }
148 }
149
150 void
151 xen_gflush_va(caddr_t va, cpuset_t cpus)
152 {
153 struct mmuext_op t;
154 uint_t count;
155
156 if (IN_XPV_PANIC()) {
157 mmu_flush_tlb_page((uintptr_t)va);
158 return;
159 }
160
161 t.cmd = MMUEXT_INVLPG_MULTI;
162 t.arg1.linear_addr = (uintptr_t)va;
163 /*LINTED: constant in conditional context*/
164 set_xen_guest_handle(t.arg2.vcpumask, &cpus);
165 if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
166 panic("HYPERVISOR_mmuext_op() failed");
167 ASSERT(count == 1);
168 }
169
170 void
171 xen_flush_tlb()
172 {
173 struct mmuext_op t;
174 uint_t count;
175
176 if (IN_XPV_PANIC()) {
177 xpv_panic_reload_cr3();
178 } else {
179 t.cmd = MMUEXT_TLB_FLUSH_LOCAL;
180 if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
181 panic("HYPERVISOR_mmuext_op() failed");
182 ASSERT(count == 1);
183 }
184 }
185
186 void
187 xen_gflush_tlb(cpuset_t cpus)
188 {
189 struct mmuext_op t;
190 uint_t count;
191
192 ASSERT(!IN_XPV_PANIC());
193 t.cmd = MMUEXT_TLB_FLUSH_MULTI;
194 /*LINTED: constant in conditional context*/
195 set_xen_guest_handle(t.arg2.vcpumask, &cpus);
196 if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
197 panic("HYPERVISOR_mmuext_op() failed");
198 ASSERT(count == 1);
199 }
200
201 /*
202 * Install/Adjust a kpm mapping under the hypervisor.
203 * Value of "how" should be:
204 * PT_WRITABLE | PT_VALID - regular kpm mapping
205 * PT_VALID - make mapping read-only
206 * 0 - remove mapping
207 *
208 * returns 0 on success. non-zero for failure.
209 */
210 int
211 xen_kpm_page(pfn_t pfn, uint_t how)
212 {
213 paddr_t pa = mmu_ptob((paddr_t)pfn);
214 x86pte_t pte = PT_NOCONSIST | PT_REF | PT_MOD;
215
216 if (kpm_vbase == NULL)
217 return (0);
218
219 if (how)
220 pte |= pa_to_ma(pa) | how;
221 else
222 pte = 0;
223 return (HYPERVISOR_update_va_mapping((uintptr_t)kpm_vbase + pa,
224 pte, UVMF_INVLPG | UVMF_ALL));
225 }
226
227 void
228 xen_pin(pfn_t pfn, level_t lvl)
229 {
230 struct mmuext_op t;
231 uint_t count;
232
233 t.cmd = MMUEXT_PIN_L1_TABLE + lvl;
234 t.arg1.mfn = pfn_to_mfn(pfn);
235 if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
236 panic("HYPERVISOR_mmuext_op() failed");
237 ASSERT(count == 1);
238 }
239
240 void
241 xen_unpin(pfn_t pfn)
242 {
243 struct mmuext_op t;
244 uint_t count;
245
246 t.cmd = MMUEXT_UNPIN_TABLE;
247 t.arg1.mfn = pfn_to_mfn(pfn);
248 if (HYPERVISOR_mmuext_op(&t, 1, &count, DOMID_SELF) < 0)
249 panic("HYPERVISOR_mmuext_op() failed");
250 ASSERT(count == 1);
251 }
252
253 static void
254 xen_map(uint64_t pte, caddr_t va)
255 {
256 if (HYPERVISOR_update_va_mapping((uintptr_t)va, pte,
257 UVMF_INVLPG | UVMF_LOCAL))
258 panic("HYPERVISOR_update_va_mapping() failed");
259 }
260 #endif /* __xpv */
261
262 /*
263 * Allocate a memory page for a hardware page table.
264 *
265 * A wrapper around page_get_physical(), with some extra checks.
266 */
267 static pfn_t
268 ptable_alloc(uintptr_t seed)
269 {
270 pfn_t pfn;
271 page_t *pp;
272
273 pfn = PFN_INVALID;
274
275 /*
276 * The first check is to see if there is memory in the system. If we
277 * drop to throttlefree, then fail the ptable_alloc() and let the
278 * stealing code kick in. Note that we have to do this test here,
279 * since the test in page_create_throttle() would let the NOSLEEP
280 * allocation go through and deplete the page reserves.
281 *
282 * The !NOMEMWAIT() lets pageout, fsflush, etc. skip this check.
283 */
284 if (!NOMEMWAIT() && freemem <= throttlefree + 1)
285 return (PFN_INVALID);
286
287 #ifdef DEBUG
288 /*
289 * This code makes htable_steal() easier to test. By setting
290 * force_steal we force pagetable allocations to fall
291 * into the stealing code. Roughly 1 in ever "force_steal"
292 * page table allocations will fail.
293 */
294 if (proc_pageout != NULL && force_steal > 1 &&
295 ++ptable_cnt > force_steal) {
296 ptable_cnt = 0;
297 return (PFN_INVALID);
298 }
299 #endif /* DEBUG */
300
301 pp = page_get_physical(seed);
302 if (pp == NULL)
303 return (PFN_INVALID);
304 ASSERT(PAGE_SHARED(pp));
305 pfn = pp->p_pagenum;
306 if (pfn == PFN_INVALID)
307 panic("ptable_alloc(): Invalid PFN!!");
308 atomic_inc_32(&active_ptables);
309 HATSTAT_INC(hs_ptable_allocs);
310 return (pfn);
311 }
312
313 /*
314 * Free an htable's associated page table page. See the comments
315 * for ptable_alloc().
316 */
317 static void
318 ptable_free(pfn_t pfn)
319 {
320 page_t *pp = page_numtopp_nolock(pfn);
321
322 /*
323 * need to destroy the page used for the pagetable
324 */
325 ASSERT(pfn != PFN_INVALID);
326 HATSTAT_INC(hs_ptable_frees);
327 atomic_dec_32(&active_ptables);
328 if (pp == NULL)
329 panic("ptable_free(): no page for pfn!");
330 ASSERT(PAGE_SHARED(pp));
331 ASSERT(pfn == pp->p_pagenum);
332 ASSERT(!IN_XPV_PANIC());
333
334 /*
335 * Get an exclusive lock, might have to wait for a kmem reader.
336 */
337 if (!page_tryupgrade(pp)) {
338 u_offset_t off = pp->p_offset;
339 page_unlock(pp);
340 pp = page_lookup(&kvp, off, SE_EXCL);
341 if (pp == NULL)
342 panic("page not found");
343 }
344 #ifdef __xpv
345 if (kpm_vbase && xen_kpm_page(pfn, PT_VALID | PT_WRITABLE) < 0)
346 panic("failure making kpm r/w pfn=0x%lx", pfn);
347 #endif
348 page_hashout(pp, NULL);
349 page_free(pp, 1);
350 page_unresv(1);
351 }
352
353 /*
354 * Put one htable on the reserve list.
355 */
356 static void
357 htable_put_reserve(htable_t *ht)
358 {
359 ht->ht_hat = NULL; /* no longer tied to a hat */
360 ASSERT(ht->ht_pfn == PFN_INVALID);
361 HATSTAT_INC(hs_htable_rputs);
362 mutex_enter(&htable_reserve_mutex);
363 ht->ht_next = htable_reserve_pool;
364 htable_reserve_pool = ht;
365 ++htable_reserve_cnt;
366 mutex_exit(&htable_reserve_mutex);
367 }
368
369 /*
370 * Take one htable from the reserve.
371 */
372 static htable_t *
373 htable_get_reserve(void)
374 {
375 htable_t *ht = NULL;
376
377 mutex_enter(&htable_reserve_mutex);
378 if (htable_reserve_cnt != 0) {
379 ht = htable_reserve_pool;
380 ASSERT(ht != NULL);
381 ASSERT(ht->ht_pfn == PFN_INVALID);
382 htable_reserve_pool = ht->ht_next;
383 --htable_reserve_cnt;
384 HATSTAT_INC(hs_htable_rgets);
385 }
386 mutex_exit(&htable_reserve_mutex);
387 return (ht);
388 }
389
390 /*
391 * Allocate initial htables and put them on the reserve list
392 */
393 void
394 htable_initial_reserve(uint_t count)
395 {
396 htable_t *ht;
397
398 count += HTABLE_RESERVE_AMOUNT;
399 while (count > 0) {
400 ht = kmem_cache_alloc(htable_cache, KM_NOSLEEP);
401 ASSERT(ht != NULL);
402
403 ASSERT(use_boot_reserve);
404 ht->ht_pfn = PFN_INVALID;
405 htable_put_reserve(ht);
406 --count;
407 }
408 }
409
410 /*
411 * Readjust the reserves after a thread finishes using them.
412 */
413 void
414 htable_adjust_reserve()
415 {
416 htable_t *ht;
417
418 /*
419 * Free any excess htables in the reserve list
420 */
421 while (htable_reserve_cnt > htable_reserve_amount &&
422 !USE_HAT_RESERVES()) {
423 ht = htable_get_reserve();
424 if (ht == NULL)
425 return;
426 ASSERT(ht->ht_pfn == PFN_INVALID);
427 kmem_cache_free(htable_cache, ht);
428 }
429 }
430
431 /*
432 * Search the active htables for one to steal. Start at a different hash
433 * bucket every time to help spread the pain of stealing
434 */
435 static void
436 htable_steal_active(hat_t *hat, uint_t cnt, uint_t threshold,
437 uint_t *stolen, htable_t **list)
438 {
439 static uint_t h_seed = 0;
440 htable_t *higher, *ht;
441 uint_t h, e, h_start;
442 uintptr_t va;
443 x86pte_t pte;
444
445 h = h_start = h_seed++ % hat->hat_num_hash;
446 do {
447 higher = NULL;
448 HTABLE_ENTER(h);
449 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
450
451 /*
452 * Can we rule out reaping?
453 */
454 if (ht->ht_busy != 0 ||
455 (ht->ht_flags & HTABLE_SHARED_PFN) ||
456 ht->ht_level > 0 || ht->ht_valid_cnt > threshold ||
457 ht->ht_lock_cnt != 0)
458 continue;
459
460 /*
461 * Increment busy so the htable can't disappear. We
462 * drop the htable mutex to avoid deadlocks with
463 * hat_pageunload() and the hment mutex while we
464 * call hat_pte_unmap()
465 */
466 ++ht->ht_busy;
467 HTABLE_EXIT(h);
468
469 /*
470 * Try stealing.
471 * - unload and invalidate all PTEs
472 */
473 for (e = 0, va = ht->ht_vaddr;
474 e < HTABLE_NUM_PTES(ht) && ht->ht_valid_cnt > 0 &&
475 ht->ht_busy == 1 && ht->ht_lock_cnt == 0;
476 ++e, va += MMU_PAGESIZE) {
477 pte = x86pte_get(ht, e);
478 if (!PTE_ISVALID(pte))
479 continue;
480 hat_pte_unmap(ht, e, HAT_UNLOAD, pte, NULL,
481 B_TRUE);
482 }
483
484 /*
485 * Reacquire htable lock. If we didn't remove all
486 * mappings in the table, or another thread added a new
487 * mapping behind us, give up on this table.
488 */
489 HTABLE_ENTER(h);
490 if (ht->ht_busy != 1 || ht->ht_valid_cnt != 0 ||
491 ht->ht_lock_cnt != 0) {
492 --ht->ht_busy;
493 continue;
494 }
495
496 /*
497 * Steal it and unlink the page table.
498 */
499 higher = ht->ht_parent;
500 unlink_ptp(higher, ht, ht->ht_vaddr);
501
502 /*
503 * remove from the hash list
504 */
505 if (ht->ht_next)
506 ht->ht_next->ht_prev = ht->ht_prev;
507
508 if (ht->ht_prev) {
509 ht->ht_prev->ht_next = ht->ht_next;
510 } else {
511 ASSERT(hat->hat_ht_hash[h] == ht);
512 hat->hat_ht_hash[h] = ht->ht_next;
513 }
514
515 /*
516 * Break to outer loop to release the
517 * higher (ht_parent) pagetable. This
518 * spreads out the pain caused by
519 * pagefaults.
520 */
521 ht->ht_next = *list;
522 *list = ht;
523 ++*stolen;
524 break;
525 }
526 HTABLE_EXIT(h);
527 if (higher != NULL)
528 htable_release(higher);
529 if (++h == hat->hat_num_hash)
530 h = 0;
531 } while (*stolen < cnt && h != h_start);
532 }
533
534 /*
535 * Move hat to the end of the kas list
536 */
537 static void
538 move_victim(hat_t *hat)
539 {
540 ASSERT(MUTEX_HELD(&hat_list_lock));
541
542 /* unlink victim hat */
543 if (hat->hat_prev)
544 hat->hat_prev->hat_next = hat->hat_next;
545 else
546 kas.a_hat->hat_next = hat->hat_next;
547
548 if (hat->hat_next)
549 hat->hat_next->hat_prev = hat->hat_prev;
550 else
551 kas.a_hat->hat_prev = hat->hat_prev;
552 /* relink at end of hat list */
553 hat->hat_next = NULL;
554 hat->hat_prev = kas.a_hat->hat_prev;
555 if (hat->hat_prev)
556 hat->hat_prev->hat_next = hat;
557 else
558 kas.a_hat->hat_next = hat;
559
560 kas.a_hat->hat_prev = hat;
561 }
562
563 /*
564 * This routine steals htables from user processes. Called by htable_reap
565 * (reap=TRUE) or htable_alloc (reap=FALSE).
566 */
567 static htable_t *
568 htable_steal(uint_t cnt, boolean_t reap)
569 {
570 hat_t *hat = kas.a_hat; /* list starts with khat */
571 htable_t *list = NULL;
572 htable_t *ht;
573 uint_t stolen = 0;
574 uint_t pass, passes;
575 uint_t threshold;
576
577 /*
578 * Limit htable_steal_passes to something reasonable
579 */
580 if (htable_steal_passes == 0)
581 htable_steal_passes = 1;
582 if (htable_steal_passes > mmu.ptes_per_table)
583 htable_steal_passes = mmu.ptes_per_table;
584
585 /*
586 * If we're stealing merely as part of kmem reaping (versus stealing
587 * to assure forward progress), we don't want to actually steal any
588 * active htables. (Stealing active htables merely to give memory
589 * back to the system can inadvertently kick off an htable crime wave
590 * as active processes repeatedly steal htables from one another,
591 * plummeting the system into a kind of HAT lawlessness that can
592 * become so violent as to impede the one thing that can end it: the
593 * freeing of memory via ARC reclaim and other means.) So if we're
594 * reaping, we limit ourselves to the first pass that steals cached
595 * htables that aren't in use -- which gives memory back, but averts
596 * the entire breakdown of social order.
597 */
598 passes = reap ? 0 : htable_steal_passes;
599
600 /*
601 * Loop through all user hats. The 1st pass takes cached htables that
602 * aren't in use. The later passes steal by removing mappings, too.
603 */
604 atomic_inc_32(&htable_dont_cache);
605 for (pass = 0; pass <= passes && stolen < cnt; ++pass) {
606 threshold = pass * mmu.ptes_per_table / htable_steal_passes;
607
608 mutex_enter(&hat_list_lock);
609
610 /* skip the first hat (kernel) */
611 hat = kas.a_hat->hat_next;
612 for (;;) {
613 /*
614 * Skip any hat that is already being stolen from.
615 *
616 * We skip SHARED hats, as these are dummy
617 * hats that host ISM shared page tables.
618 *
619 * We also skip if HAT_FREEING because hat_pte_unmap()
620 * won't zero out the PTE's. That would lead to hitting
621 * stale PTEs either here or under hat_unload() when we
622 * steal and unload the same page table in competing
623 * threads.
624 *
625 * We skip HATs that belong to CPUs, to make our lives
626 * simpler.
627 */
628 while (hat != NULL && (hat->hat_flags &
629 (HAT_VICTIM | HAT_SHARED | HAT_FREEING |
630 HAT_PCP)) != 0) {
631 hat = hat->hat_next;
632 }
633
634 if (hat == NULL)
635 break;
636
637 /*
638 * Mark the HAT as a stealing victim so that it is
639 * not freed from under us, e.g. in as_free()
640 */
641 hat->hat_flags |= HAT_VICTIM;
642 mutex_exit(&hat_list_lock);
643
644 /*
645 * Take any htables from the hat's cached "free" list.
646 */
647 hat_enter(hat);
648 while ((ht = hat->hat_ht_cached) != NULL &&
649 stolen < cnt) {
650 hat->hat_ht_cached = ht->ht_next;
651 ht->ht_next = list;
652 list = ht;
653 ++stolen;
654 }
655 hat_exit(hat);
656
657 /*
658 * Don't steal active htables on first pass.
659 */
660 if (pass != 0 && (stolen < cnt))
661 htable_steal_active(hat, cnt, threshold,
662 &stolen, &list);
663
664 /*
665 * do synchronous teardown for the reap case so that
666 * we can forget hat; at this time, hat is
667 * guaranteed to be around because HAT_VICTIM is set
668 * (see htable_free() for similar code)
669 */
670 for (ht = list; (ht) && (reap); ht = ht->ht_next) {
671 if (ht->ht_hat == NULL)
672 continue;
673 ASSERT(ht->ht_hat == hat);
674 #if defined(__xpv) && defined(__amd64)
675 ASSERT(!(ht->ht_flags & HTABLE_COPIED));
676 if (ht->ht_level == mmu.max_level) {
677 ptable_free(hat->hat_user_ptable);
678 hat->hat_user_ptable = PFN_INVALID;
679 }
680 #endif
681 /*
682 * forget the hat
683 */
684 ht->ht_hat = NULL;
685 }
686
687 mutex_enter(&hat_list_lock);
688
689 /*
690 * Are we finished?
691 */
692 if (stolen == cnt) {
693 /*
694 * Try to spread the pain of stealing,
695 * move victim HAT to the end of the HAT list.
696 */
697 if (pass >= 1 && cnt == 1 &&
698 kas.a_hat->hat_prev != hat)
699 move_victim(hat);
700 /*
701 * We are finished
702 */
703 }
704
705 /*
706 * Clear the victim flag, hat can go away now (once
707 * the lock is dropped)
708 */
709 if (hat->hat_flags & HAT_VICTIM) {
710 ASSERT(hat != kas.a_hat);
711 hat->hat_flags &= ~HAT_VICTIM;
712 cv_broadcast(&hat_list_cv);
713 }
714
715 /* move on to the next hat */
716 hat = hat->hat_next;
717 }
718
719 mutex_exit(&hat_list_lock);
720
721 }
722 ASSERT(!MUTEX_HELD(&hat_list_lock));
723
724 atomic_dec_32(&htable_dont_cache);
725 return (list);
726 }
727
728 /*
729 * This is invoked from kmem when the system is low on memory. We try
730 * to free hments, htables, and ptables to improve the memory situation.
731 */
732 /*ARGSUSED*/
733 static void
734 htable_reap(void *handle)
735 {
736 uint_t reap_cnt;
737 htable_t *list;
738 htable_t *ht;
739
740 HATSTAT_INC(hs_reap_attempts);
741 if (!can_steal_post_boot)
742 return;
743
744 /*
745 * Try to reap 5% of the page tables bounded by a maximum of
746 * 5% of physmem and a minimum of 10.
747 */
748 reap_cnt = MAX(MIN(physmem / 20, active_ptables / 20), 10);
749
750 /*
751 * Note: htable_dont_cache should be set at the time of
752 * invoking htable_free()
753 */
754 atomic_inc_32(&htable_dont_cache);
755 /*
756 * Let htable_steal() do the work, we just call htable_free()
757 */
758 XPV_DISALLOW_MIGRATE();
759 list = htable_steal(reap_cnt, B_TRUE);
760 XPV_ALLOW_MIGRATE();
761 while ((ht = list) != NULL) {
762 list = ht->ht_next;
763 HATSTAT_INC(hs_reaped);
764 htable_free(ht);
765 }
766 atomic_dec_32(&htable_dont_cache);
767
768 /*
769 * Free up excess reserves
770 */
771 htable_adjust_reserve();
772 hment_adjust_reserve();
773 }
774
775 /*
776 * Allocate an htable, stealing one or using the reserve if necessary
777 */
778 static htable_t *
779 htable_alloc(
780 hat_t *hat,
781 uintptr_t vaddr,
782 level_t level,
783 htable_t *shared)
784 {
785 htable_t *ht = NULL;
786 uint_t is_copied;
787 uint_t is_bare = 0;
788 uint_t need_to_zero = 1;
789 int kmflags = (can_steal_post_boot ? KM_NOSLEEP : KM_SLEEP);
790
791 if (level < 0 || level > TOP_LEVEL(hat))
792 panic("htable_alloc(): level %d out of range\n", level);
793
794 is_copied = (hat->hat_flags & HAT_COPIED) &&
795 level == hat->hat_max_level;
796 if (is_copied || shared != NULL)
797 is_bare = 1;
798
799 /*
800 * First reuse a cached htable from the hat_ht_cached field, this
801 * avoids unnecessary trips through kmem/page allocators.
802 */
803 if (hat->hat_ht_cached != NULL && !is_bare) {
804 hat_enter(hat);
805 ht = hat->hat_ht_cached;
806 if (ht != NULL) {
807 hat->hat_ht_cached = ht->ht_next;
808 need_to_zero = 0;
809 /* XX64 ASSERT() they're all zero somehow */
810 ASSERT(ht->ht_pfn != PFN_INVALID);
811 }
812 hat_exit(hat);
813 }
814
815 if (ht == NULL) {
816 /*
817 * Allocate an htable, possibly refilling the reserves.
818 */
819 if (USE_HAT_RESERVES()) {
820 ht = htable_get_reserve();
821 } else {
822 /*
823 * Donate successful htable allocations to the reserve.
824 */
825 for (;;) {
826 ht = kmem_cache_alloc(htable_cache, kmflags);
827 if (ht == NULL)
828 break;
829 ht->ht_pfn = PFN_INVALID;
830 if (USE_HAT_RESERVES() ||
831 htable_reserve_cnt >= htable_reserve_amount)
832 break;
833 htable_put_reserve(ht);
834 }
835 }
836
837 /*
838 * allocate a page for the hardware page table if needed
839 */
840 if (ht != NULL && !is_bare) {
841 ht->ht_hat = hat;
842 ht->ht_pfn = ptable_alloc((uintptr_t)ht);
843 if (ht->ht_pfn == PFN_INVALID) {
844 if (USE_HAT_RESERVES())
845 htable_put_reserve(ht);
846 else
847 kmem_cache_free(htable_cache, ht);
848 ht = NULL;
849 }
850 }
851 }
852
853 /*
854 * If allocations failed, kick off a kmem_reap() and resort to
855 * htable steal(). We may spin here if the system is very low on
856 * memory. If the kernel itself has consumed all memory and kmem_reap()
857 * can't free up anything, then we'll really get stuck here.
858 * That should only happen in a system where the administrator has
859 * misconfigured VM parameters via /etc/system.
860 */
861 while (ht == NULL && can_steal_post_boot) {
862 kmem_reap();
863 ht = htable_steal(1, B_FALSE);
864 HATSTAT_INC(hs_steals);
865
866 /*
867 * If we stole for a bare htable, release the pagetable page.
868 */
869 if (ht != NULL) {
870 if (is_bare) {
871 ptable_free(ht->ht_pfn);
872 ht->ht_pfn = PFN_INVALID;
873 #if defined(__xpv) && defined(__amd64)
874 /*
875 * make stolen page table writable again in kpm
876 */
877 } else if (kpm_vbase && xen_kpm_page(ht->ht_pfn,
878 PT_VALID | PT_WRITABLE) < 0) {
879 panic("failure making kpm r/w pfn=0x%lx",
880 ht->ht_pfn);
881 #endif
882 }
883 }
884 }
885
886 /*
887 * All attempts to allocate or steal failed. This should only happen
888 * if we run out of memory during boot, due perhaps to a huge
889 * boot_archive. At this point there's no way to continue.
890 */
891 if (ht == NULL)
892 panic("htable_alloc(): couldn't steal\n");
893
894 #if defined(__amd64) && defined(__xpv)
895 /*
896 * Under the 64-bit hypervisor, we have 2 top level page tables.
897 * If this allocation fails, we'll resort to stealing.
898 * We use the stolen page indirectly, by freeing the
899 * stolen htable first.
900 */
901 if (level == mmu.max_level) {
902 for (;;) {
903 htable_t *stolen;
904
905 hat->hat_user_ptable = ptable_alloc((uintptr_t)ht + 1);
906 if (hat->hat_user_ptable != PFN_INVALID)
907 break;
908 stolen = htable_steal(1, B_FALSE);
909 if (stolen == NULL)
910 panic("2nd steal ptable failed\n");
911 htable_free(stolen);
912 }
913 block_zero_no_xmm(kpm_vbase + pfn_to_pa(hat->hat_user_ptable),
914 MMU_PAGESIZE);
915 }
916 #endif
917
918 /*
919 * Shared page tables have all entries locked and entries may not
920 * be added or deleted.
921 */
922 ht->ht_flags = 0;
923 if (shared != NULL) {
924 ASSERT(shared->ht_valid_cnt > 0);
925 ht->ht_flags |= HTABLE_SHARED_PFN;
926 ht->ht_pfn = shared->ht_pfn;
927 ht->ht_lock_cnt = 0;
928 ht->ht_valid_cnt = 0; /* updated in hat_share() */
929 ht->ht_shares = shared;
930 need_to_zero = 0;
931 } else {
932 ht->ht_shares = NULL;
933 ht->ht_lock_cnt = 0;
934 ht->ht_valid_cnt = 0;
935 }
936
937 /*
938 * setup flags, etc. for copied page tables.
939 */
940 if (is_copied) {
941 ht->ht_flags |= HTABLE_COPIED;
942 ASSERT(ht->ht_pfn == PFN_INVALID);
943 need_to_zero = 0;
944 }
945
946 /*
947 * fill in the htable
948 */
949 ht->ht_hat = hat;
950 ht->ht_parent = NULL;
951 ht->ht_vaddr = vaddr;
952 ht->ht_level = level;
953 ht->ht_busy = 1;
954 ht->ht_next = NULL;
955 ht->ht_prev = NULL;
956
957 /*
958 * Zero out any freshly allocated page table
959 */
960 if (need_to_zero)
961 x86pte_zero(ht, 0, mmu.ptes_per_table);
962
963 #if defined(__amd64) && defined(__xpv)
964 if (!is_bare && kpm_vbase) {
965 (void) xen_kpm_page(ht->ht_pfn, PT_VALID);
966 if (level == mmu.max_level)
967 (void) xen_kpm_page(hat->hat_user_ptable, PT_VALID);
968 }
969 #endif
970
971 return (ht);
972 }
973
974 /*
975 * Free up an htable, either to a hat's cached list, the reserves or
976 * back to kmem.
977 */
978 static void
979 htable_free(htable_t *ht)
980 {
981 hat_t *hat = ht->ht_hat;
982
983 /*
984 * If the process isn't exiting, cache the free htable in the hat
985 * structure. We always do this for the boot time reserve. We don't
986 * do this if the hat is exiting or we are stealing/reaping htables.
987 */
988 if (hat != NULL &&
989 !(ht->ht_flags & HTABLE_SHARED_PFN) &&
990 (use_boot_reserve ||
991 (!(hat->hat_flags & HAT_FREEING) && !htable_dont_cache))) {
992 ASSERT((ht->ht_flags & HTABLE_COPIED) == 0);
993 ASSERT(ht->ht_pfn != PFN_INVALID);
994 hat_enter(hat);
995 ht->ht_next = hat->hat_ht_cached;
996 hat->hat_ht_cached = ht;
997 hat_exit(hat);
998 return;
999 }
1000
1001 /*
1002 * If we have a hardware page table, free it.
1003 * We don't free page tables that are accessed by sharing.
1004 */
1005 if (ht->ht_flags & HTABLE_SHARED_PFN) {
1006 ASSERT(ht->ht_pfn != PFN_INVALID);
1007 } else if (!(ht->ht_flags & HTABLE_COPIED)) {
1008 ptable_free(ht->ht_pfn);
1009 #if defined(__amd64) && defined(__xpv)
1010 if (ht->ht_level == mmu.max_level && hat != NULL) {
1011 ptable_free(hat->hat_user_ptable);
1012 hat->hat_user_ptable = PFN_INVALID;
1013 }
1014 #endif
1015 }
1016 ht->ht_pfn = PFN_INVALID;
1017
1018 /*
1019 * Free it or put into reserves.
1020 */
1021 if (USE_HAT_RESERVES() || htable_reserve_cnt < htable_reserve_amount) {
1022 htable_put_reserve(ht);
1023 } else {
1024 kmem_cache_free(htable_cache, ht);
1025 htable_adjust_reserve();
1026 }
1027 }
1028
1029
1030 /*
1031 * This is called when a hat is being destroyed or swapped out. We reap all
1032 * the remaining htables in the hat cache. If destroying all left over
1033 * htables are also destroyed.
1034 *
1035 * We also don't need to invalidate any of the PTPs nor do any demapping.
1036 */
1037 void
1038 htable_purge_hat(hat_t *hat)
1039 {
1040 htable_t *ht;
1041 int h;
1042
1043 /*
1044 * Purge the htable cache if just reaping.
1045 */
1046 if (!(hat->hat_flags & HAT_FREEING)) {
1047 atomic_inc_32(&htable_dont_cache);
1048 for (;;) {
1049 hat_enter(hat);
1050 ht = hat->hat_ht_cached;
1051 if (ht == NULL) {
1052 hat_exit(hat);
1053 break;
1054 }
1055 hat->hat_ht_cached = ht->ht_next;
1056 hat_exit(hat);
1057 htable_free(ht);
1058 }
1059 atomic_dec_32(&htable_dont_cache);
1060 return;
1061 }
1062
1063 /*
1064 * if freeing, no locking is needed
1065 */
1066 while ((ht = hat->hat_ht_cached) != NULL) {
1067 hat->hat_ht_cached = ht->ht_next;
1068 htable_free(ht);
1069 }
1070
1071 /*
1072 * walk thru the htable hash table and free all the htables in it.
1073 */
1074 for (h = 0; h < hat->hat_num_hash; ++h) {
1075 while ((ht = hat->hat_ht_hash[h]) != NULL) {
1076 if (ht->ht_next)
1077 ht->ht_next->ht_prev = ht->ht_prev;
1078
1079 if (ht->ht_prev) {
1080 ht->ht_prev->ht_next = ht->ht_next;
1081 } else {
1082 ASSERT(hat->hat_ht_hash[h] == ht);
1083 hat->hat_ht_hash[h] = ht->ht_next;
1084 }
1085 htable_free(ht);
1086 }
1087 }
1088 }
1089
1090 /*
1091 * Unlink an entry for a table at vaddr and level out of the existing table
1092 * one level higher. We are always holding the HASH_ENTER() when doing this.
1093 */
1094 static void
1095 unlink_ptp(htable_t *higher, htable_t *old, uintptr_t vaddr)
1096 {
1097 uint_t entry = htable_va2entry(vaddr, higher);
1098 x86pte_t expect = MAKEPTP(old->ht_pfn, old->ht_level);
1099 x86pte_t found;
1100 hat_t *hat = old->ht_hat;
1101
1102 ASSERT(higher->ht_busy > 0);
1103 ASSERT(higher->ht_valid_cnt > 0);
1104 ASSERT(old->ht_valid_cnt == 0);
1105 found = x86pte_cas(higher, entry, expect, 0);
1106 #ifdef __xpv
1107 /*
1108 * This is weird, but Xen apparently automatically unlinks empty
1109 * pagetables from the upper page table. So allow PTP to be 0 already.
1110 */
1111 if (found != expect && found != 0)
1112 #else
1113 if (found != expect)
1114 #endif
1115 panic("Bad PTP found=" FMT_PTE ", expected=" FMT_PTE,
1116 found, expect);
1117
1118 /*
1119 * When a top level PTE changes for a copied htable, we must trigger a
1120 * hat_pcp_update() on all HAT CPUs.
1121 *
1122 * If we don't need do do that, then we still have to INVLPG against an
1123 * address covered by the inner page table, as the latest processors
1124 * have TLB-like caches for non-leaf page table entries.
1125 */
1126 if (!(hat->hat_flags & HAT_FREEING)) {
1127 hat_tlb_inval(hat, (higher->ht_flags & HTABLE_COPIED) ?
1128 DEMAP_ALL_ADDR : old->ht_vaddr);
1129 }
1130
1131 HTABLE_DEC(higher->ht_valid_cnt);
1132 }
1133
1134 /*
1135 * Link an entry for a new table at vaddr and level into the existing table
1136 * one level higher. We are always holding the HASH_ENTER() when doing this.
1137 */
1138 static void
1139 link_ptp(htable_t *higher, htable_t *new, uintptr_t vaddr)
1140 {
1141 uint_t entry = htable_va2entry(vaddr, higher);
1142 x86pte_t newptp = MAKEPTP(new->ht_pfn, new->ht_level);
1143 x86pte_t found;
1144
1145 ASSERT(higher->ht_busy > 0);
1146
1147 ASSERT(new->ht_level != mmu.max_level);
1148
1149 HTABLE_INC(higher->ht_valid_cnt);
1150
1151 found = x86pte_cas(higher, entry, 0, newptp);
1152 if ((found & ~PT_REF) != 0)
1153 panic("HAT: ptp not 0, found=" FMT_PTE, found);
1154
1155 /*
1156 * When a top level PTE changes for a copied htable, we must trigger a
1157 * hat_pcp_update() on all HAT CPUs.
1158 *
1159 * We also need to do this for the kernel hat on PAE 32 bit kernel.
1160 */
1161 if (
1162 #ifdef __i386
1163 (higher->ht_hat == kas.a_hat &&
1164 higher->ht_level == higher->ht_hat->hat_max_level) ||
1165 #endif
1166 (higher->ht_flags & HTABLE_COPIED))
1167 hat_tlb_inval(higher->ht_hat, DEMAP_ALL_ADDR);
1168 }
1169
1170 /*
1171 * Release of hold on an htable. If this is the last use and the pagetable
1172 * is empty we may want to free it, then recursively look at the pagetable
1173 * above it. The recursion is handled by the outer while() loop.
1174 *
1175 * On the metal, during process exit, we don't bother unlinking the tables from
1176 * upper level pagetables. They are instead handled in bulk by hat_free_end().
1177 * We can't do this on the hypervisor as we need the page table to be
1178 * implicitly unpinnned before it goes to the free page lists. This can't
1179 * happen unless we fully unlink it from the page table hierarchy.
1180 */
1181 void
1182 htable_release(htable_t *ht)
1183 {
1184 uint_t hashval;
1185 htable_t *shared;
1186 htable_t *higher;
1187 hat_t *hat;
1188 uintptr_t va;
1189 level_t level;
1190
1191 while (ht != NULL) {
1192 shared = NULL;
1193 for (;;) {
1194 hat = ht->ht_hat;
1195 va = ht->ht_vaddr;
1196 level = ht->ht_level;
1197 hashval = HTABLE_HASH(hat, va, level);
1198
1199 /*
1200 * The common case is that this isn't the last use of
1201 * an htable so we don't want to free the htable.
1202 */
1203 HTABLE_ENTER(hashval);
1204 ASSERT(ht->ht_valid_cnt >= 0);
1205 ASSERT(ht->ht_busy > 0);
1206 if (ht->ht_valid_cnt > 0)
1207 break;
1208 if (ht->ht_busy > 1)
1209 break;
1210 ASSERT(ht->ht_lock_cnt == 0);
1211
1212 #if !defined(__xpv)
1213 /*
1214 * we always release empty shared htables
1215 */
1216 if (!(ht->ht_flags & HTABLE_SHARED_PFN)) {
1217
1218 /*
1219 * don't release if in address space tear down
1220 */
1221 if (hat->hat_flags & HAT_FREEING)
1222 break;
1223
1224 /*
1225 * At and above max_page_level, free if it's for
1226 * a boot-time kernel mapping below kernelbase.
1227 */
1228 if (level >= mmu.max_page_level &&
1229 (hat != kas.a_hat || va >= kernelbase))
1230 break;
1231 }
1232 #endif /* __xpv */
1233
1234 /*
1235 * Remember if we destroy an htable that shares its PFN
1236 * from elsewhere.
1237 */
1238 if (ht->ht_flags & HTABLE_SHARED_PFN) {
1239 ASSERT(shared == NULL);
1240 shared = ht->ht_shares;
1241 HATSTAT_INC(hs_htable_unshared);
1242 }
1243
1244 /*
1245 * Handle release of a table and freeing the htable_t.
1246 * Unlink it from the table higher (ie. ht_parent).
1247 */
1248 higher = ht->ht_parent;
1249 ASSERT(higher != NULL);
1250
1251 /*
1252 * Unlink the pagetable.
1253 */
1254 unlink_ptp(higher, ht, va);
1255
1256 /*
1257 * remove this htable from its hash list
1258 */
1259 if (ht->ht_next)
1260 ht->ht_next->ht_prev = ht->ht_prev;
1261
1262 if (ht->ht_prev) {
1263 ht->ht_prev->ht_next = ht->ht_next;
1264 } else {
1265 ASSERT(hat->hat_ht_hash[hashval] == ht);
1266 hat->hat_ht_hash[hashval] = ht->ht_next;
1267 }
1268 HTABLE_EXIT(hashval);
1269 htable_free(ht);
1270 ht = higher;
1271 }
1272
1273 ASSERT(ht->ht_busy >= 1);
1274 --ht->ht_busy;
1275 HTABLE_EXIT(hashval);
1276
1277 /*
1278 * If we released a shared htable, do a release on the htable
1279 * from which it shared
1280 */
1281 ht = shared;
1282 }
1283 }
1284
1285 /*
1286 * Find the htable for the pagetable at the given level for the given address.
1287 * If found acquires a hold that eventually needs to be htable_release()d
1288 */
1289 htable_t *
1290 htable_lookup(hat_t *hat, uintptr_t vaddr, level_t level)
1291 {
1292 uintptr_t base;
1293 uint_t hashval;
1294 htable_t *ht = NULL;
1295
1296 ASSERT(level >= 0);
1297 ASSERT(level <= TOP_LEVEL(hat));
1298
1299 if (level == TOP_LEVEL(hat)) {
1300 #if defined(__amd64)
1301 /*
1302 * 32 bit address spaces on 64 bit kernels need to check
1303 * for overflow of the 32 bit address space
1304 */
1305 if ((hat->hat_flags & HAT_COPIED_32) &&
1306 vaddr >= ((uint64_t)1 << 32))
1307 return (NULL);
1308 #endif
1309 base = 0;
1310 } else {
1311 base = vaddr & LEVEL_MASK(level + 1);
1312 }
1313
1314 hashval = HTABLE_HASH(hat, base, level);
1315 HTABLE_ENTER(hashval);
1316 for (ht = hat->hat_ht_hash[hashval]; ht; ht = ht->ht_next) {
1317 if (ht->ht_hat == hat &&
1318 ht->ht_vaddr == base &&
1319 ht->ht_level == level)
1320 break;
1321 }
1322 if (ht)
1323 ++ht->ht_busy;
1324
1325 HTABLE_EXIT(hashval);
1326 return (ht);
1327 }
1328
1329 /*
1330 * Acquires a hold on a known htable (from a locked hment entry).
1331 */
1332 void
1333 htable_acquire(htable_t *ht)
1334 {
1335 hat_t *hat = ht->ht_hat;
1336 level_t level = ht->ht_level;
1337 uintptr_t base = ht->ht_vaddr;
1338 uint_t hashval = HTABLE_HASH(hat, base, level);
1339
1340 HTABLE_ENTER(hashval);
1341 #ifdef DEBUG
1342 /*
1343 * make sure the htable is there
1344 */
1345 {
1346 htable_t *h;
1347
1348 for (h = hat->hat_ht_hash[hashval];
1349 h && h != ht;
1350 h = h->ht_next)
1351 ;
1352 ASSERT(h == ht);
1353 }
1354 #endif /* DEBUG */
1355 ++ht->ht_busy;
1356 HTABLE_EXIT(hashval);
1357 }
1358
1359 /*
1360 * Find the htable for the pagetable at the given level for the given address.
1361 * If found acquires a hold that eventually needs to be htable_release()d
1362 * If not found the table is created.
1363 *
1364 * Since we can't hold a hash table mutex during allocation, we have to
1365 * drop it and redo the search on a create. Then we may have to free the newly
1366 * allocated htable if another thread raced in and created it ahead of us.
1367 */
1368 htable_t *
1369 htable_create(
1370 hat_t *hat,
1371 uintptr_t vaddr,
1372 level_t level,
1373 htable_t *shared)
1374 {
1375 uint_t h;
1376 level_t l;
1377 uintptr_t base;
1378 htable_t *ht;
1379 htable_t *higher = NULL;
1380 htable_t *new = NULL;
1381
1382 if (level < 0 || level > TOP_LEVEL(hat))
1383 panic("htable_create(): level %d out of range\n", level);
1384
1385 /*
1386 * Create the page tables in top down order.
1387 */
1388 for (l = TOP_LEVEL(hat); l >= level; --l) {
1389 new = NULL;
1390 if (l == TOP_LEVEL(hat))
1391 base = 0;
1392 else
1393 base = vaddr & LEVEL_MASK(l + 1);
1394
1395 h = HTABLE_HASH(hat, base, l);
1396 try_again:
1397 /*
1398 * look up the htable at this level
1399 */
1400 HTABLE_ENTER(h);
1401 if (l == TOP_LEVEL(hat)) {
1402 ht = hat->hat_htable;
1403 } else {
1404 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
1405 ASSERT(ht->ht_hat == hat);
1406 if (ht->ht_vaddr == base &&
1407 ht->ht_level == l)
1408 break;
1409 }
1410 }
1411
1412 /*
1413 * if we found the htable, increment its busy cnt
1414 * and if we had allocated a new htable, free it.
1415 */
1416 if (ht != NULL) {
1417 /*
1418 * If we find a pre-existing shared table, it must
1419 * share from the same place.
1420 */
1421 if (l == level && shared && ht->ht_shares &&
1422 ht->ht_shares != shared) {
1423 panic("htable shared from wrong place "
1424 "found htable=%p shared=%p",
1425 (void *)ht, (void *)shared);
1426 }
1427 ++ht->ht_busy;
1428 HTABLE_EXIT(h);
1429 if (new)
1430 htable_free(new);
1431 if (higher != NULL)
1432 htable_release(higher);
1433 higher = ht;
1434
1435 /*
1436 * if we didn't find it on the first search
1437 * allocate a new one and search again
1438 */
1439 } else if (new == NULL) {
1440 HTABLE_EXIT(h);
1441 new = htable_alloc(hat, base, l,
1442 l == level ? shared : NULL);
1443 goto try_again;
1444
1445 /*
1446 * 2nd search and still not there, use "new" table
1447 * Link new table into higher, when not at top level.
1448 */
1449 } else {
1450 ht = new;
1451 if (higher != NULL) {
1452 link_ptp(higher, ht, base);
1453 ht->ht_parent = higher;
1454 }
1455 ht->ht_next = hat->hat_ht_hash[h];
1456 ASSERT(ht->ht_prev == NULL);
1457 if (hat->hat_ht_hash[h])
1458 hat->hat_ht_hash[h]->ht_prev = ht;
1459 hat->hat_ht_hash[h] = ht;
1460 HTABLE_EXIT(h);
1461
1462 /*
1463 * Note we don't do htable_release(higher).
1464 * That happens recursively when "new" is removed by
1465 * htable_release() or htable_steal().
1466 */
1467 higher = ht;
1468
1469 /*
1470 * If we just created a new shared page table we
1471 * increment the shared htable's busy count, so that
1472 * it can't be the victim of a steal even if it's empty.
1473 */
1474 if (l == level && shared) {
1475 (void) htable_lookup(shared->ht_hat,
1476 shared->ht_vaddr, shared->ht_level);
1477 HATSTAT_INC(hs_htable_shared);
1478 }
1479 }
1480 }
1481
1482 return (ht);
1483 }
1484
1485 /*
1486 * Inherit initial pagetables from the boot program. On the 64-bit
1487 * hypervisor we also temporarily mark the p_index field of page table
1488 * pages, so we know not to try making them writable in seg_kpm.
1489 */
1490 void
1491 htable_attach(
1492 hat_t *hat,
1493 uintptr_t base,
1494 level_t level,
1495 htable_t *parent,
1496 pfn_t pfn)
1497 {
1498 htable_t *ht;
1499 uint_t h;
1500 uint_t i;
1501 x86pte_t pte;
1502 x86pte_t *ptep;
1503 page_t *pp;
1504 extern page_t *boot_claim_page(pfn_t);
1505
1506 ht = htable_get_reserve();
1507 if (level == mmu.max_level)
1508 kas.a_hat->hat_htable = ht;
1509 ht->ht_hat = hat;
1510 ht->ht_parent = parent;
1511 ht->ht_vaddr = base;
1512 ht->ht_level = level;
1513 ht->ht_busy = 1;
1514 ht->ht_next = NULL;
1515 ht->ht_prev = NULL;
1516 ht->ht_flags = 0;
1517 ht->ht_pfn = pfn;
1518 ht->ht_lock_cnt = 0;
1519 ht->ht_valid_cnt = 0;
1520 if (parent != NULL)
1521 ++parent->ht_busy;
1522
1523 h = HTABLE_HASH(hat, base, level);
1524 HTABLE_ENTER(h);
1525 ht->ht_next = hat->hat_ht_hash[h];
1526 ASSERT(ht->ht_prev == NULL);
1527 if (hat->hat_ht_hash[h])
1528 hat->hat_ht_hash[h]->ht_prev = ht;
1529 hat->hat_ht_hash[h] = ht;
1530 HTABLE_EXIT(h);
1531
1532 /*
1533 * make sure the page table physical page is not FREE
1534 */
1535 if (page_resv(1, KM_NOSLEEP) == 0)
1536 panic("page_resv() failed in ptable alloc");
1537
1538 pp = boot_claim_page(pfn);
1539 ASSERT(pp != NULL);
1540
1541 /*
1542 * Page table pages that were allocated by dboot or
1543 * in very early startup didn't go through boot_mapin()
1544 * and so won't have vnode/offsets. Fix that here.
1545 */
1546 if (pp->p_vnode == NULL) {
1547 /* match offset calculation in page_get_physical() */
1548 u_offset_t offset = (uintptr_t)ht;
1549 if (offset > kernelbase)
1550 offset -= kernelbase;
1551 offset <<= MMU_PAGESHIFT;
1552 #if defined(__amd64)
1553 offset += mmu.hole_start; /* something in VA hole */
1554 #else
1555 offset += 1ULL << 40; /* something > 4 Gig */
1556 #endif
1557 ASSERT(page_exists(&kvp, offset) == NULL);
1558 (void) page_hashin(pp, &kvp, offset, NULL);
1559 }
1560 page_downgrade(pp);
1561 #if defined(__xpv) && defined(__amd64)
1562 /*
1563 * Record in the page_t that is a pagetable for segkpm setup.
1564 */
1565 if (kpm_vbase)
1566 pp->p_index = 1;
1567 #endif
1568
1569 /*
1570 * Count valid mappings and recursively attach lower level pagetables.
1571 */
1572 ptep = kbm_remap_window(pfn_to_pa(pfn), 0);
1573 for (i = 0; i < HTABLE_NUM_PTES(ht); ++i) {
1574 if (mmu.pae_hat)
1575 pte = ptep[i];
1576 else
1577 pte = ((x86pte32_t *)ptep)[i];
1578 if (!IN_HYPERVISOR_VA(base) && PTE_ISVALID(pte)) {
1579 ++ht->ht_valid_cnt;
1580 if (!PTE_ISPAGE(pte, level)) {
1581 htable_attach(hat, base, level - 1,
1582 ht, PTE2PFN(pte, level));
1583 ptep = kbm_remap_window(pfn_to_pa(pfn), 0);
1584 }
1585 }
1586 base += LEVEL_SIZE(level);
1587 if (base == mmu.hole_start)
1588 base = (mmu.hole_end + MMU_PAGEOFFSET) & MMU_PAGEMASK;
1589 }
1590
1591 /*
1592 * As long as all the mappings we had were below kernel base
1593 * we can release the htable.
1594 */
1595 if (base < kernelbase)
1596 htable_release(ht);
1597 }
1598
1599 /*
1600 * Walk through a given htable looking for the first valid entry. This
1601 * routine takes both a starting and ending address. The starting address
1602 * is required to be within the htable provided by the caller, but there is
1603 * no such restriction on the ending address.
1604 *
1605 * If the routine finds a valid entry in the htable (at or beyond the
1606 * starting address), the PTE (and its address) will be returned.
1607 * This PTE may correspond to either a page or a pagetable - it is the
1608 * caller's responsibility to determine which. If no valid entry is
1609 * found, 0 (and invalid PTE) and the next unexamined address will be
1610 * returned.
1611 *
1612 * The loop has been carefully coded for optimization.
1613 */
1614 static x86pte_t
1615 htable_scan(htable_t *ht, uintptr_t *vap, uintptr_t eaddr)
1616 {
1617 uint_t e;
1618 x86pte_t found_pte = (x86pte_t)0;
1619 caddr_t pte_ptr;
1620 caddr_t end_pte_ptr;
1621 int l = ht->ht_level;
1622 uintptr_t va = *vap & LEVEL_MASK(l);
1623 size_t pgsize = LEVEL_SIZE(l);
1624
1625 ASSERT(va >= ht->ht_vaddr);
1626 ASSERT(va <= HTABLE_LAST_PAGE(ht));
1627
1628 /*
1629 * Compute the starting index and ending virtual address
1630 */
1631 e = htable_va2entry(va, ht);
1632
1633 /*
1634 * The following page table scan code knows that the valid
1635 * bit of a PTE is in the lowest byte AND that x86 is little endian!!
1636 */
1637 pte_ptr = (caddr_t)x86pte_access_pagetable(ht, 0);
1638 end_pte_ptr = (caddr_t)PT_INDEX_PTR(pte_ptr, HTABLE_NUM_PTES(ht));
1639 pte_ptr = (caddr_t)PT_INDEX_PTR((x86pte_t *)pte_ptr, e);
1640 while (!PTE_ISVALID(*pte_ptr)) {
1641 va += pgsize;
1642 if (va >= eaddr)
1643 break;
1644 pte_ptr += mmu.pte_size;
1645 ASSERT(pte_ptr <= end_pte_ptr);
1646 if (pte_ptr == end_pte_ptr)
1647 break;
1648 }
1649
1650 /*
1651 * if we found a valid PTE, load the entire PTE
1652 */
1653 if (va < eaddr && pte_ptr != end_pte_ptr)
1654 found_pte = GET_PTE((x86pte_t *)pte_ptr);
1655 x86pte_release_pagetable(ht);
1656
1657 #if defined(__amd64)
1658 /*
1659 * deal with VA hole on amd64
1660 */
1661 if (l == mmu.max_level && va >= mmu.hole_start && va <= mmu.hole_end)
1662 va = mmu.hole_end + va - mmu.hole_start;
1663 #endif /* __amd64 */
1664
1665 *vap = va;
1666 return (found_pte);
1667 }
1668
1669 /*
1670 * Find the address and htable for the first populated translation at or
1671 * above the given virtual address. The caller may also specify an upper
1672 * limit to the address range to search. Uses level information to quickly
1673 * skip unpopulated sections of virtual address spaces.
1674 *
1675 * If not found returns NULL. When found, returns the htable and virt addr
1676 * and has a hold on the htable.
1677 */
1678 x86pte_t
1679 htable_walk(
1680 struct hat *hat,
1681 htable_t **htp,
1682 uintptr_t *vaddr,
1683 uintptr_t eaddr)
1684 {
1685 uintptr_t va = *vaddr;
1686 htable_t *ht;
1687 htable_t *prev = *htp;
1688 level_t l;
1689 level_t max_mapped_level;
1690 x86pte_t pte;
1691
1692 ASSERT(eaddr > va);
1693
1694 /*
1695 * If this is a user address, then we know we need not look beyond
1696 * kernelbase.
1697 */
1698 ASSERT(hat == kas.a_hat || eaddr <= kernelbase ||
1699 eaddr == HTABLE_WALK_TO_END);
1700 if (hat != kas.a_hat && eaddr == HTABLE_WALK_TO_END)
1701 eaddr = kernelbase;
1702
1703 /*
1704 * If we're coming in with a previous page table, search it first
1705 * without doing an htable_lookup(), this should be frequent.
1706 */
1707 if (prev) {
1708 ASSERT(prev->ht_busy > 0);
1709 ASSERT(prev->ht_vaddr <= va);
1710 l = prev->ht_level;
1711 if (va <= HTABLE_LAST_PAGE(prev)) {
1712 pte = htable_scan(prev, &va, eaddr);
1713
1714 if (PTE_ISPAGE(pte, l)) {
1715 *vaddr = va;
1716 *htp = prev;
1717 return (pte);
1718 }
1719 }
1720
1721 /*
1722 * We found nothing in the htable provided by the caller,
1723 * so fall through and do the full search
1724 */
1725 htable_release(prev);
1726 }
1727
1728 /*
1729 * Find the level of the largest pagesize used by this HAT.
1730 */
1731 if (hat->hat_ism_pgcnt > 0) {
1732 max_mapped_level = mmu.umax_page_level;
1733 } else {
1734 max_mapped_level = 0;
1735 for (l = 1; l <= mmu.max_page_level; ++l)
1736 if (hat->hat_pages_mapped[l] != 0)
1737 max_mapped_level = l;
1738 }
1739
1740 while (va < eaddr && va >= *vaddr) {
1741 /*
1742 * Find lowest table with any entry for given address.
1743 */
1744 for (l = 0; l <= TOP_LEVEL(hat); ++l) {
1745 ht = htable_lookup(hat, va, l);
1746 if (ht != NULL) {
1747 pte = htable_scan(ht, &va, eaddr);
1748 if (PTE_ISPAGE(pte, l)) {
1749 VERIFY(!IN_VA_HOLE(va));
1750 *vaddr = va;
1751 *htp = ht;
1752 return (pte);
1753 }
1754 htable_release(ht);
1755 break;
1756 }
1757
1758 /*
1759 * No htable at this level for the address. If there
1760 * is no larger page size that could cover it, we can
1761 * skip right to the start of the next page table.
1762 */
1763 ASSERT(l < TOP_LEVEL(hat));
1764 if (l >= max_mapped_level) {
1765 va = NEXT_ENTRY_VA(va, l + 1);
1766 if (va >= eaddr)
1767 break;
1768 }
1769 }
1770 }
1771
1772 *vaddr = 0;
1773 *htp = NULL;
1774 return (0);
1775 }
1776
1777 /*
1778 * Find the htable and page table entry index of the given virtual address
1779 * with pagesize at or below given level.
1780 * If not found returns NULL. When found, returns the htable, sets
1781 * entry, and has a hold on the htable.
1782 */
1783 htable_t *
1784 htable_getpte(
1785 struct hat *hat,
1786 uintptr_t vaddr,
1787 uint_t *entry,
1788 x86pte_t *pte,
1789 level_t level)
1790 {
1791 htable_t *ht;
1792 level_t l;
1793 uint_t e;
1794
1795 ASSERT(level <= mmu.max_page_level);
1796
1797 for (l = 0; l <= level; ++l) {
1798 ht = htable_lookup(hat, vaddr, l);
1799 if (ht == NULL)
1800 continue;
1801 e = htable_va2entry(vaddr, ht);
1802 if (entry != NULL)
1803 *entry = e;
1804 if (pte != NULL)
1805 *pte = x86pte_get(ht, e);
1806 return (ht);
1807 }
1808 return (NULL);
1809 }
1810
1811 /*
1812 * Find the htable and page table entry index of the given virtual address.
1813 * There must be a valid page mapped at the given address.
1814 * If not found returns NULL. When found, returns the htable, sets
1815 * entry, and has a hold on the htable.
1816 */
1817 htable_t *
1818 htable_getpage(struct hat *hat, uintptr_t vaddr, uint_t *entry)
1819 {
1820 htable_t *ht;
1821 uint_t e;
1822 x86pte_t pte;
1823
1824 ht = htable_getpte(hat, vaddr, &e, &pte, mmu.max_page_level);
1825 if (ht == NULL)
1826 return (NULL);
1827
1828 if (entry)
1829 *entry = e;
1830
1831 if (PTE_ISPAGE(pte, ht->ht_level))
1832 return (ht);
1833 htable_release(ht);
1834 return (NULL);
1835 }
1836
1837
1838 void
1839 htable_init()
1840 {
1841 /*
1842 * To save on kernel VA usage, we avoid debug information in 32 bit
1843 * kernels.
1844 */
1845 #if defined(__amd64)
1846 int kmem_flags = KMC_NOHASH;
1847 #elif defined(__i386)
1848 int kmem_flags = KMC_NOHASH | KMC_NODEBUG;
1849 #endif
1850
1851 /*
1852 * initialize kmem caches
1853 */
1854 htable_cache = kmem_cache_create("htable_t",
1855 sizeof (htable_t), 0, NULL, NULL,
1856 htable_reap, NULL, hat_memload_arena, kmem_flags);
1857 }
1858
1859 /*
1860 * get the pte index for the virtual address in the given htable's pagetable
1861 */
1862 uint_t
1863 htable_va2entry(uintptr_t va, htable_t *ht)
1864 {
1865 level_t l = ht->ht_level;
1866
1867 ASSERT(va >= ht->ht_vaddr);
1868 ASSERT(va <= HTABLE_LAST_PAGE(ht));
1869 return ((va >> LEVEL_SHIFT(l)) & (HTABLE_NUM_PTES(ht) - 1));
1870 }
1871
1872 /*
1873 * Given an htable and the index of a pte in it, return the virtual address
1874 * of the page.
1875 */
1876 uintptr_t
1877 htable_e2va(htable_t *ht, uint_t entry)
1878 {
1879 level_t l = ht->ht_level;
1880 uintptr_t va;
1881
1882 ASSERT(entry < HTABLE_NUM_PTES(ht));
1883 va = ht->ht_vaddr + ((uintptr_t)entry << LEVEL_SHIFT(l));
1884
1885 /*
1886 * Need to skip over any VA hole in top level table
1887 */
1888 #if defined(__amd64)
1889 if (ht->ht_level == mmu.max_level && va >= mmu.hole_start)
1890 va += ((mmu.hole_end - mmu.hole_start) + 1);
1891 #endif
1892
1893 return (va);
1894 }
1895
1896 /*
1897 * The code uses compare and swap instructions to read/write PTE's to
1898 * avoid atomicity problems, since PTEs can be 8 bytes on 32 bit systems.
1899 * will naturally be atomic.
1900 *
1901 * The combination of using kpreempt_disable()/_enable() and the hci_mutex
1902 * are used to ensure that an interrupt won't overwrite a temporary mapping
1903 * while it's in use. If an interrupt thread tries to access a PTE, it will
1904 * yield briefly back to the pinned thread which holds the cpu's hci_mutex.
1905 */
1906 void
1907 x86pte_cpu_init(cpu_t *cpu)
1908 {
1909 struct hat_cpu_info *hci;
1910
1911 hci = kmem_zalloc(sizeof (*hci), KM_SLEEP);
1912 mutex_init(&hci->hci_mutex, NULL, MUTEX_DEFAULT, NULL);
1913 cpu->cpu_hat_info = hci;
1914 }
1915
1916 void
1917 x86pte_cpu_fini(cpu_t *cpu)
1918 {
1919 struct hat_cpu_info *hci = cpu->cpu_hat_info;
1920
1921 kmem_free(hci, sizeof (*hci));
1922 cpu->cpu_hat_info = NULL;
1923 }
1924
1925 #ifdef __i386
1926 /*
1927 * On 32 bit kernels, loading a 64 bit PTE is a little tricky
1928 */
1929 x86pte_t
1930 get_pte64(x86pte_t *ptr)
1931 {
1932 volatile uint32_t *p = (uint32_t *)ptr;
1933 x86pte_t t;
1934
1935 ASSERT(mmu.pae_hat != 0);
1936 for (;;) {
1937 t = p[0];
1938 t |= (uint64_t)p[1] << 32;
1939 if ((t & 0xffffffff) == p[0])
1940 return (t);
1941 }
1942 }
1943 #endif /* __i386 */
1944
1945 /*
1946 * Disable preemption and establish a mapping to the pagetable with the
1947 * given pfn. This is optimized for there case where it's the same
1948 * pfn as we last used referenced from this CPU.
1949 */
1950 static x86pte_t *
1951 x86pte_access_pagetable(htable_t *ht, uint_t index)
1952 {
1953 /*
1954 * HTABLE_COPIED pagetables are contained in the hat_t
1955 */
1956 if (ht->ht_flags & HTABLE_COPIED) {
1957 ASSERT3U(index, <, ht->ht_hat->hat_num_copied);
1958 return (PT_INDEX_PTR(ht->ht_hat->hat_copied_ptes, index));
1959 }
1960 return (x86pte_mapin(ht->ht_pfn, index, ht));
1961 }
1962
1963 /*
1964 * map the given pfn into the page table window.
1965 */
1966 /*ARGSUSED*/
1967 x86pte_t *
1968 x86pte_mapin(pfn_t pfn, uint_t index, htable_t *ht)
1969 {
1970 x86pte_t *pteptr;
1971 x86pte_t pte = 0;
1972 x86pte_t newpte;
1973 int x;
1974
1975 ASSERT(pfn != PFN_INVALID);
1976
1977 if (!khat_running) {
1978 caddr_t va = kbm_remap_window(pfn_to_pa(pfn), 1);
1979 return (PT_INDEX_PTR(va, index));
1980 }
1981
1982 /*
1983 * If kpm is available, use it.
1984 */
1985 if (kpm_vbase)
1986 return (PT_INDEX_PTR(hat_kpm_pfn2va(pfn), index));
1987
1988 /*
1989 * Disable preemption and grab the CPU's hci_mutex
1990 */
1991 kpreempt_disable();
1992
1993 ASSERT(CPU->cpu_hat_info != NULL);
1994 ASSERT(!(getcr4() & CR4_PCIDE));
1995
1996 mutex_enter(&CPU->cpu_hat_info->hci_mutex);
1997 x = PWIN_TABLE(CPU->cpu_id);
1998 pteptr = (x86pte_t *)PWIN_PTE_VA(x);
1999 #ifndef __xpv
2000 if (mmu.pae_hat)
2001 pte = *pteptr;
2002 else
2003 pte = *(x86pte32_t *)pteptr;
2004 #endif
2005
2006 newpte = MAKEPTE(pfn, 0) | mmu.pt_global | mmu.pt_nx;
2007
2008 /*
2009 * For hardware we can use a writable mapping.
2010 */
2011 #ifdef __xpv
2012 if (IN_XPV_PANIC())
2013 #endif
2014 newpte |= PT_WRITABLE;
2015
2016 if (!PTE_EQUIV(newpte, pte)) {
2017
2018 #ifdef __xpv
2019 if (!IN_XPV_PANIC()) {
2020 xen_map(newpte, PWIN_VA(x));
2021 } else
2022 #endif
2023 {
2024 XPV_ALLOW_PAGETABLE_UPDATES();
2025 if (mmu.pae_hat)
2026 *pteptr = newpte;
2027 else
2028 *(x86pte32_t *)pteptr = newpte;
2029 XPV_DISALLOW_PAGETABLE_UPDATES();
2030 mmu_flush_tlb_kpage((uintptr_t)PWIN_VA(x));
2031 }
2032 }
2033 return (PT_INDEX_PTR(PWIN_VA(x), index));
2034 }
2035
2036 /*
2037 * Release access to a page table.
2038 */
2039 static void
2040 x86pte_release_pagetable(htable_t *ht)
2041 {
2042 if (ht->ht_flags & HTABLE_COPIED)
2043 return;
2044
2045 x86pte_mapout();
2046 }
2047
2048 void
2049 x86pte_mapout(void)
2050 {
2051 if (kpm_vbase != NULL || !khat_running)
2052 return;
2053
2054 /*
2055 * Drop the CPU's hci_mutex and restore preemption.
2056 */
2057 #ifdef __xpv
2058 if (!IN_XPV_PANIC()) {
2059 uintptr_t va;
2060
2061 /*
2062 * We need to always clear the mapping in case a page
2063 * that was once a page table page is ballooned out.
2064 */
2065 va = (uintptr_t)PWIN_VA(PWIN_TABLE(CPU->cpu_id));
2066 (void) HYPERVISOR_update_va_mapping(va, 0,
2067 UVMF_INVLPG | UVMF_LOCAL);
2068 }
2069 #endif
2070 mutex_exit(&CPU->cpu_hat_info->hci_mutex);
2071 kpreempt_enable();
2072 }
2073
2074 /*
2075 * Atomic retrieval of a pagetable entry
2076 */
2077 x86pte_t
2078 x86pte_get(htable_t *ht, uint_t entry)
2079 {
2080 x86pte_t pte;
2081 x86pte_t *ptep;
2082
2083 /*
2084 * Be careful that loading PAE entries in 32 bit kernel is atomic.
2085 */
2086 ASSERT(entry < mmu.ptes_per_table);
2087 ptep = x86pte_access_pagetable(ht, entry);
2088 pte = GET_PTE(ptep);
2089 x86pte_release_pagetable(ht);
2090 return (pte);
2091 }
2092
2093 /*
2094 * Atomic unconditional set of a page table entry, it returns the previous
2095 * value. For pre-existing mappings if the PFN changes, then we don't care
2096 * about the old pte's REF / MOD bits. If the PFN remains the same, we leave
2097 * the MOD/REF bits unchanged.
2098 *
2099 * If asked to overwrite a link to a lower page table with a large page
2100 * mapping, this routine returns the special value of LPAGE_ERROR. This
2101 * allows the upper HAT layers to retry with a smaller mapping size.
2102 */
2103 x86pte_t
2104 x86pte_set(htable_t *ht, uint_t entry, x86pte_t new, void *ptr)
2105 {
2106 x86pte_t old;
2107 x86pte_t prev;
2108 x86pte_t *ptep;
2109 level_t l = ht->ht_level;
2110 x86pte_t pfn_mask = (l != 0) ? PT_PADDR_LGPG : PT_PADDR;
2111 x86pte_t n;
2112 uintptr_t addr = htable_e2va(ht, entry);
2113 hat_t *hat = ht->ht_hat;
2114
2115 ASSERT(new != 0); /* don't use to invalidate a PTE, see x86pte_update */
2116 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2117 if (ptr == NULL)
2118 ptep = x86pte_access_pagetable(ht, entry);
2119 else
2120 ptep = ptr;
2121
2122 /*
2123 * Install the new PTE. If remapping the same PFN, then
2124 * copy existing REF/MOD bits to new mapping.
2125 */
2126 do {
2127 prev = GET_PTE(ptep);
2128 n = new;
2129 if (PTE_ISVALID(n) && (prev & pfn_mask) == (new & pfn_mask))
2130 n |= prev & (PT_REF | PT_MOD);
2131
2132 /*
2133 * Another thread may have installed this mapping already,
2134 * flush the local TLB and be done.
2135 */
2136 if (prev == n) {
2137 old = new;
2138 #ifdef __xpv
2139 if (!IN_XPV_PANIC())
2140 xen_flush_va((caddr_t)addr);
2141 else
2142 #endif
2143 mmu_flush_tlb_page(addr);
2144 goto done;
2145 }
2146
2147 /*
2148 * Detect if we have a collision of installing a large
2149 * page mapping where there already is a lower page table.
2150 */
2151 if (l > 0 && (prev & PT_VALID) && !(prev & PT_PAGESIZE)) {
2152 old = LPAGE_ERROR;
2153 goto done;
2154 }
2155
2156 XPV_ALLOW_PAGETABLE_UPDATES();
2157 old = CAS_PTE(ptep, prev, n);
2158 XPV_DISALLOW_PAGETABLE_UPDATES();
2159 } while (old != prev);
2160
2161 /*
2162 * Do a TLB demap if needed, ie. the old pte was valid.
2163 *
2164 * Note that a stale TLB writeback to the PTE here either can't happen
2165 * or doesn't matter. The PFN can only change for NOSYNC|NOCONSIST
2166 * mappings, but they were created with REF and MOD already set, so
2167 * no stale writeback will happen.
2168 *
2169 * Segmap is the only place where remaps happen on the same pfn and for
2170 * that we want to preserve the stale REF/MOD bits.
2171 */
2172 if (old & PT_REF)
2173 hat_tlb_inval(hat, addr);
2174
2175 done:
2176 if (ptr == NULL)
2177 x86pte_release_pagetable(ht);
2178 return (old);
2179 }
2180
2181 /*
2182 * Atomic compare and swap of a page table entry. No TLB invalidates are done.
2183 * This is used for links between pagetables of different levels.
2184 * Note we always create these links with dirty/access set, so they should
2185 * never change.
2186 */
2187 x86pte_t
2188 x86pte_cas(htable_t *ht, uint_t entry, x86pte_t old, x86pte_t new)
2189 {
2190 x86pte_t pte;
2191 x86pte_t *ptep;
2192 #ifdef __xpv
2193 /*
2194 * We can't use writable pagetables for upper level tables, so fake it.
2195 */
2196 mmu_update_t t[2];
2197 int cnt = 1;
2198 int count;
2199 maddr_t ma;
2200
2201 if (!IN_XPV_PANIC()) {
2202 ASSERT(!(ht->ht_flags & HTABLE_COPIED));
2203 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2204 t[0].ptr = ma | MMU_NORMAL_PT_UPDATE;
2205 t[0].val = new;
2206
2207 #if defined(__amd64)
2208 /*
2209 * On the 64-bit hypervisor we need to maintain the user mode
2210 * top page table too.
2211 */
2212 if (ht->ht_level == mmu.max_level && ht->ht_hat != kas.a_hat) {
2213 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(
2214 ht->ht_hat->hat_user_ptable), entry));
2215 t[1].ptr = ma | MMU_NORMAL_PT_UPDATE;
2216 t[1].val = new;
2217 ++cnt;
2218 }
2219 #endif /* __amd64 */
2220
2221 if (HYPERVISOR_mmu_update(t, cnt, &count, DOMID_SELF))
2222 panic("HYPERVISOR_mmu_update() failed");
2223 ASSERT(count == cnt);
2224 return (old);
2225 }
2226 #endif
2227 ptep = x86pte_access_pagetable(ht, entry);
2228 XPV_ALLOW_PAGETABLE_UPDATES();
2229 pte = CAS_PTE(ptep, old, new);
2230 XPV_DISALLOW_PAGETABLE_UPDATES();
2231 x86pte_release_pagetable(ht);
2232 return (pte);
2233 }
2234
2235 /*
2236 * Invalidate a page table entry as long as it currently maps something that
2237 * matches the value determined by expect.
2238 *
2239 * If tlb is set, also invalidates any TLB entries.
2240 *
2241 * Returns the previous value of the PTE.
2242 */
2243 x86pte_t
2244 x86pte_inval(
2245 htable_t *ht,
2246 uint_t entry,
2247 x86pte_t expect,
2248 x86pte_t *pte_ptr,
2249 boolean_t tlb)
2250 {
2251 x86pte_t *ptep;
2252 x86pte_t oldpte;
2253 x86pte_t found;
2254
2255 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2256 ASSERT(ht->ht_level <= mmu.max_page_level);
2257
2258 if (pte_ptr != NULL)
2259 ptep = pte_ptr;
2260 else
2261 ptep = x86pte_access_pagetable(ht, entry);
2262
2263 #if defined(__xpv)
2264 /*
2265 * If exit()ing just use HYPERVISOR_mmu_update(), as we can't be racing
2266 * with anything else.
2267 */
2268 if ((ht->ht_hat->hat_flags & HAT_FREEING) && !IN_XPV_PANIC()) {
2269 int count;
2270 mmu_update_t t[1];
2271 maddr_t ma;
2272
2273 oldpte = GET_PTE(ptep);
2274 if (expect != 0 && (oldpte & PT_PADDR) != (expect & PT_PADDR))
2275 goto done;
2276 ma = pa_to_ma(PT_INDEX_PHYSADDR(pfn_to_pa(ht->ht_pfn), entry));
2277 t[0].ptr = ma | MMU_NORMAL_PT_UPDATE;
2278 t[0].val = 0;
2279 if (HYPERVISOR_mmu_update(t, 1, &count, DOMID_SELF))
2280 panic("HYPERVISOR_mmu_update() failed");
2281 ASSERT(count == 1);
2282 goto done;
2283 }
2284 #endif /* __xpv */
2285
2286 /*
2287 * Note that the loop is needed to handle changes due to h/w updating
2288 * of PT_MOD/PT_REF.
2289 */
2290 do {
2291 oldpte = GET_PTE(ptep);
2292 if (expect != 0 && (oldpte & PT_PADDR) != (expect & PT_PADDR))
2293 goto done;
2294 XPV_ALLOW_PAGETABLE_UPDATES();
2295 found = CAS_PTE(ptep, oldpte, 0);
2296 XPV_DISALLOW_PAGETABLE_UPDATES();
2297 } while (found != oldpte);
2298 if (tlb && (oldpte & (PT_REF | PT_MOD)))
2299 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
2300
2301 done:
2302 if (pte_ptr == NULL)
2303 x86pte_release_pagetable(ht);
2304 return (oldpte);
2305 }
2306
2307 /*
2308 * Change a page table entry af it currently matches the value in expect.
2309 */
2310 x86pte_t
2311 x86pte_update(
2312 htable_t *ht,
2313 uint_t entry,
2314 x86pte_t expect,
2315 x86pte_t new)
2316 {
2317 x86pte_t *ptep;
2318 x86pte_t found;
2319
2320 ASSERT(new != 0);
2321 ASSERT(!(ht->ht_flags & HTABLE_SHARED_PFN));
2322 ASSERT(ht->ht_level <= mmu.max_page_level);
2323
2324 ptep = x86pte_access_pagetable(ht, entry);
2325 XPV_ALLOW_PAGETABLE_UPDATES();
2326 found = CAS_PTE(ptep, expect, new);
2327 XPV_DISALLOW_PAGETABLE_UPDATES();
2328 if (found == expect) {
2329 hat_tlb_inval(ht->ht_hat, htable_e2va(ht, entry));
2330
2331 /*
2332 * When removing write permission *and* clearing the
2333 * MOD bit, check if a write happened via a stale
2334 * TLB entry before the TLB shootdown finished.
2335 *
2336 * If it did happen, simply re-enable write permission and
2337 * act like the original CAS failed.
2338 */
2339 if ((expect & (PT_WRITABLE | PT_MOD)) == PT_WRITABLE &&
2340 (new & (PT_WRITABLE | PT_MOD)) == 0 &&
2341 (GET_PTE(ptep) & PT_MOD) != 0) {
2342 do {
2343 found = GET_PTE(ptep);
2344 XPV_ALLOW_PAGETABLE_UPDATES();
2345 found =
2346 CAS_PTE(ptep, found, found | PT_WRITABLE);
2347 XPV_DISALLOW_PAGETABLE_UPDATES();
2348 } while ((found & PT_WRITABLE) == 0);
2349 }
2350 }
2351 x86pte_release_pagetable(ht);
2352 return (found);
2353 }
2354
2355 #ifndef __xpv
2356 /*
2357 * Copy page tables - this is just a little more complicated than the
2358 * previous routines. Note that it's also not atomic! It also is never
2359 * used for HTABLE_COPIED pagetables.
2360 */
2361 void
2362 x86pte_copy(htable_t *src, htable_t *dest, uint_t entry, uint_t count)
2363 {
2364 caddr_t src_va;
2365 caddr_t dst_va;
2366 size_t size;
2367 x86pte_t *pteptr;
2368 x86pte_t pte;
2369
2370 ASSERT(khat_running);
2371 ASSERT(!(dest->ht_flags & HTABLE_COPIED));
2372 ASSERT(!(src->ht_flags & HTABLE_COPIED));
2373 ASSERT(!(src->ht_flags & HTABLE_SHARED_PFN));
2374 ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN));
2375
2376 /*
2377 * Acquire access to the CPU pagetable windows for the dest and source.
2378 */
2379 dst_va = (caddr_t)x86pte_access_pagetable(dest, entry);
2380 if (kpm_vbase) {
2381 src_va = (caddr_t)
2382 PT_INDEX_PTR(hat_kpm_pfn2va(src->ht_pfn), entry);
2383 } else {
2384 uint_t x = PWIN_SRC(CPU->cpu_id);
2385
2386 ASSERT(!(getcr4() & CR4_PCIDE));
2387
2388 /*
2389 * Finish defining the src pagetable mapping
2390 */
2391 src_va = (caddr_t)PT_INDEX_PTR(PWIN_VA(x), entry);
2392 pte = MAKEPTE(src->ht_pfn, 0) | mmu.pt_global | mmu.pt_nx;
2393 pteptr = (x86pte_t *)PWIN_PTE_VA(x);
2394 if (mmu.pae_hat)
2395 *pteptr = pte;
2396 else
2397 *(x86pte32_t *)pteptr = pte;
2398 mmu_flush_tlb_kpage((uintptr_t)PWIN_VA(x));
2399 }
2400
2401 /*
2402 * now do the copy
2403 */
2404 size = count << mmu.pte_size_shift;
2405 bcopy(src_va, dst_va, size);
2406
2407 x86pte_release_pagetable(dest);
2408 }
2409
2410 #else /* __xpv */
2411
2412 /*
2413 * The hypervisor only supports writable pagetables at level 0, so we have
2414 * to install these 1 by 1 the slow way.
2415 */
2416 void
2417 x86pte_copy(htable_t *src, htable_t *dest, uint_t entry, uint_t count)
2418 {
2419 caddr_t src_va;
2420 x86pte_t pte;
2421
2422 ASSERT(!IN_XPV_PANIC());
2423 src_va = (caddr_t)x86pte_access_pagetable(src, entry);
2424 while (count) {
2425 if (mmu.pae_hat)
2426 pte = *(x86pte_t *)src_va;
2427 else
2428 pte = *(x86pte32_t *)src_va;
2429 if (pte != 0) {
2430 set_pteval(pfn_to_pa(dest->ht_pfn), entry,
2431 dest->ht_level, pte);
2432 #ifdef __amd64
2433 if (dest->ht_level == mmu.max_level &&
2434 htable_e2va(dest, entry) < HYPERVISOR_VIRT_END)
2435 set_pteval(
2436 pfn_to_pa(dest->ht_hat->hat_user_ptable),
2437 entry, dest->ht_level, pte);
2438 #endif
2439 }
2440 --count;
2441 ++entry;
2442 src_va += mmu.pte_size;
2443 }
2444 x86pte_release_pagetable(src);
2445 }
2446 #endif /* __xpv */
2447
2448 /*
2449 * Zero page table entries - Note this doesn't use atomic stores!
2450 */
2451 static void
2452 x86pte_zero(htable_t *dest, uint_t entry, uint_t count)
2453 {
2454 caddr_t dst_va;
2455 size_t size;
2456 #ifdef __xpv
2457 int x;
2458 x86pte_t newpte;
2459 #endif
2460
2461 /*
2462 * Map in the page table to be zeroed.
2463 */
2464 ASSERT(!(dest->ht_flags & HTABLE_SHARED_PFN));
2465 ASSERT(!(dest->ht_flags & HTABLE_COPIED));
2466
2467 /*
2468 * On the hypervisor we don't use x86pte_access_pagetable() since
2469 * in this case the page is not pinned yet.
2470 */
2471 #ifdef __xpv
2472 if (kpm_vbase == NULL) {
2473 kpreempt_disable();
2474 ASSERT(CPU->cpu_hat_info != NULL);
2475 mutex_enter(&CPU->cpu_hat_info->hci_mutex);
2476 x = PWIN_TABLE(CPU->cpu_id);
2477 newpte = MAKEPTE(dest->ht_pfn, 0) | PT_WRITABLE;
2478 xen_map(newpte, PWIN_VA(x));
2479 dst_va = (caddr_t)PT_INDEX_PTR(PWIN_VA(x), entry);
2480 } else
2481 #endif
2482 dst_va = (caddr_t)x86pte_access_pagetable(dest, entry);
2483
2484 size = count << mmu.pte_size_shift;
2485 ASSERT(size > BLOCKZEROALIGN);
2486 #ifdef __i386
2487 if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
2488 bzero(dst_va, size);
2489 else
2490 #endif
2491 block_zero_no_xmm(dst_va, size);
2492
2493 #ifdef __xpv
2494 if (kpm_vbase == NULL) {
2495 xen_map(0, PWIN_VA(x));
2496 mutex_exit(&CPU->cpu_hat_info->hci_mutex);
2497 kpreempt_enable();
2498 } else
2499 #endif
2500 x86pte_release_pagetable(dest);
2501 }
2502
2503 /*
2504 * Called to ensure that all pagetables are in the system dump
2505 */
2506 void
2507 hat_dump(void)
2508 {
2509 hat_t *hat;
2510 uint_t h;
2511 htable_t *ht;
2512
2513 /*
2514 * Dump all page tables
2515 */
2516 for (hat = kas.a_hat; hat != NULL; hat = hat->hat_next) {
2517 for (h = 0; h < hat->hat_num_hash; ++h) {
2518 for (ht = hat->hat_ht_hash[h]; ht; ht = ht->ht_next) {
2519 if ((ht->ht_flags & HTABLE_COPIED) == 0)
2520 dump_page(ht->ht_pfn);
2521 }
2522 }
2523 }
2524 }