Print this page 
9208 hati_demap_func should take pagesize into account
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Tim Kordas <tim.kordas@joyent.com>
Split Close
Expand all
Collapse all
          --- old/usr/src/uts/i86pc/vm/hat_i86.c
          +++ new/usr/src/uts/i86pc/vm/hat_i86.c
↓ open down ↓ 19 lines elided ↑ open up ↑ 
  20   20   */
  21   21  /*
  22   22   * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  23   23   */
  24   24  /*
  25   25   * Copyright (c) 2010, Intel Corporation.
  26   26   * All rights reserved.
  27   27   */
  28   28  /*
  29   29   * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
       30 + * Copyright 2017 Joyent, Inc.  All rights reserved.
  30   31   * Copyright (c) 2014, 2015 by Delphix. All rights reserved.
  31   32   */
  32   33  
  33   34  /*
  34   35   * VM - Hardware Address Translation management for i386 and amd64
  35   36   *
  36   37   * Implementation of the interfaces described in <common/vm/hat.h>
  37   38   *
  38   39   * Nearly all the details of how the hardware is managed should not be
  39   40   * visible outside this layer except for misc. machine specific functions
↓ open down ↓ 1867 lines elided ↑ open up ↑ 
1907 1908  }
1908 1909  
1909 1910  /* ARGSUSED */
1910 1911  void
1911 1912  hat_unlock_region(struct hat *hat, caddr_t addr, size_t len,
1912 1913      hat_region_cookie_t rcookie)
1913 1914  {
1914 1915          panic("No shared region support on x86");
1915 1916  }
1916 1917  
     1918 +/*
     1919 + * A range of virtual pages for purposes of demapping.
     1920 + */
     1921 +typedef struct range_info {
     1922 +        uintptr_t       rng_va;         /* address of page */
     1923 +        ulong_t         rng_cnt;        /* number of pages in range */
     1924 +        level_t         rng_level;      /* page table level */
     1925 +} range_info_t;
     1926 +
1917 1927  #if !defined(__xpv)
1918 1928  /*
1919      - * Cross call service routine to demap a virtual page on
1920      - * the current CPU or flush all mappings in TLB.
     1929 + * Cross call service routine to demap a range of virtual
     1930 + * pages on the current CPU or flush all mappings in TLB.
1921 1931   */
1922 1932  /*ARGSUSED*/
1923 1933  static int
1924 1934  hati_demap_func(xc_arg_t a1, xc_arg_t a2, xc_arg_t a3)
1925 1935  {
1926      -        hat_t   *hat = (hat_t *)a1;
1927      -        caddr_t addr = (caddr_t)a2;
1928      -        size_t len = (size_t)a3;
     1936 +        hat_t           *hat = (hat_t *)a1;
     1937 +        range_info_t    *range = (range_info_t *)a2;
     1938 +        size_t          len = (size_t)a3;
     1939 +        caddr_t         addr = (caddr_t)range->rng_va;
     1940 +        size_t          pgsz = LEVEL_SIZE(range->rng_level);
1929 1941  
1930 1942          /*
1931 1943           * If the target hat isn't the kernel and this CPU isn't operating
1932 1944           * in the target hat, we can ignore the cross call.
1933 1945           */
1934 1946          if (hat != kas.a_hat && hat != CPU->cpu_current_hat)
1935 1947                  return (0);
1936 1948  
1937 1949          /*
1938 1950           * For a normal address, we flush a range of contiguous mappings
1939 1951           */
1940 1952          if ((uintptr_t)addr != DEMAP_ALL_ADDR) {
1941      -                for (size_t i = 0; i < len; i += MMU_PAGESIZE)
     1953 +                for (size_t i = 0; i < len; i += pgsz)
1942 1954                          mmu_tlbflush_entry(addr + i);
1943 1955                  return (0);
1944 1956          }
1945 1957  
1946 1958          /*
1947 1959           * Otherwise we reload cr3 to effect a complete TLB flush.
1948 1960           *
1949 1961           * A reload of cr3 on a VLP process also means we must also recopy in
1950 1962           * the pte values from the struct hat
1951 1963           */
↓ open down ↓ 75 lines elided ↑ open up ↑ 
2027 2039                          flush_all_tlb_entries();
2028 2040          }
2029 2041  }
2030 2042  #endif /* !__xpv */
2031 2043  
2032 2044  /*
2033 2045   * Internal routine to do cross calls to invalidate a range of pages on
2034 2046   * all CPUs using a given hat.
2035 2047   */
2036 2048  void
2037      -hat_tlb_inval_range(hat_t *hat, uintptr_t va, size_t len)
     2049 +hat_tlb_inval_range(hat_t *hat, range_info_t *range)
2038 2050  {
2039 2051          extern int      flushes_require_xcalls; /* from mp_startup.c */
2040 2052          cpuset_t        justme;
2041 2053          cpuset_t        cpus_to_shootdown;
     2054 +        uintptr_t       va = range->rng_va;
     2055 +        size_t          len = range->rng_cnt << LEVEL_SHIFT(range->rng_level);
2042 2056  #ifndef __xpv
2043 2057          cpuset_t        check_cpus;
2044 2058          cpu_t           *cpup;
2045 2059          int             c;
2046 2060  #endif
2047 2061  
2048 2062          /*
2049 2063           * If the hat is being destroyed, there are no more users, so
2050 2064           * demap need not do anything.
2051 2065           */
↓ open down ↓ 16 lines elided ↑ open up ↑ 
2068 2082          if (panicstr || !flushes_require_xcalls) {
2069 2083  #ifdef __xpv
2070 2084                  if (va == DEMAP_ALL_ADDR) {
2071 2085                          xen_flush_tlb();
2072 2086                  } else {
2073 2087                          for (size_t i = 0; i < len; i += MMU_PAGESIZE)
2074 2088                                  xen_flush_va((caddr_t)(va + i));
2075 2089                  }
2076 2090  #else
2077 2091                  (void) hati_demap_func((xc_arg_t)hat,
2078      -                    (xc_arg_t)va, (xc_arg_t)len);
     2092 +                    (xc_arg_t)range, (xc_arg_t)len);
2079 2093  #endif
2080 2094                  return;
2081 2095          }
2082 2096  
2083 2097  
2084 2098          /*
2085 2099           * Determine CPUs to shootdown. Kernel changes always do all CPUs.
2086 2100           * Otherwise it's just CPUs currently executing in this hat.
2087 2101           */
2088 2102          kpreempt_disable();
↓ open down ↓ 38 lines elided ↑ open up ↑ 
2127 2141  
2128 2142  #ifdef __xpv
2129 2143                  if (va == DEMAP_ALL_ADDR) {
2130 2144                          xen_flush_tlb();
2131 2145                  } else {
2132 2146                          for (size_t i = 0; i < len; i += MMU_PAGESIZE)
2133 2147                                  xen_flush_va((caddr_t)(va + i));
2134 2148                  }
2135 2149  #else
2136 2150                  (void) hati_demap_func((xc_arg_t)hat,
2137      -                    (xc_arg_t)va, (xc_arg_t)len);
     2151 +                    (xc_arg_t)range, (xc_arg_t)len);
2138 2152  #endif
2139 2153  
2140 2154          } else {
2141 2155  
2142 2156                  CPUSET_ADD(cpus_to_shootdown, CPU->cpu_id);
2143 2157  #ifdef __xpv
2144 2158                  if (va == DEMAP_ALL_ADDR) {
2145 2159                          xen_gflush_tlb(cpus_to_shootdown);
2146 2160                  } else {
2147 2161                          for (size_t i = 0; i < len; i += MMU_PAGESIZE) {
2148 2162                                  xen_gflush_va((caddr_t)(va + i),
2149 2163                                      cpus_to_shootdown);
2150 2164                          }
2151 2165                  }
2152 2166  #else
2153      -                xc_call((xc_arg_t)hat, (xc_arg_t)va, (xc_arg_t)len,
     2167 +                xc_call((xc_arg_t)hat, (xc_arg_t)range, (xc_arg_t)len,
2154 2168                      CPUSET2BV(cpus_to_shootdown), hati_demap_func);
2155 2169  #endif
2156 2170  
2157 2171          }
2158 2172          kpreempt_enable();
2159 2173  }
2160 2174  
2161 2175  void
2162 2176  hat_tlb_inval(hat_t *hat, uintptr_t va)
2163 2177  {
2164      -        hat_tlb_inval_range(hat, va, MMU_PAGESIZE);
     2178 +        /*
     2179 +         * Create range for a single page.
     2180 +         */
     2181 +        range_info_t range;
     2182 +        range.rng_va = va;
     2183 +        range.rng_cnt = 1; /* one page */
     2184 +        range.rng_level = MIN_PAGE_LEVEL; /* pages are MMU_PAGESIZE */
     2185 +
     2186 +        hat_tlb_inval_range(hat, &range);
2165 2187  }
2166 2188  
2167 2189  /*
2168 2190   * Interior routine for HAT_UNLOADs from hat_unload_callback(),
2169 2191   * hat_kmap_unload() OR from hat_steal() code.  This routine doesn't
2170 2192   * handle releasing of the htables.
2171 2193   */
2172 2194  void
2173 2195  hat_pte_unmap(
2174 2196          htable_t        *ht,
↓ open down ↓ 146 lines elided ↑ open up ↑ 
2321 2343          if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
2322 2344                  ASSERT(hat == kas.a_hat);
2323 2345                  hat_kmap_unload(addr, len, flags);
2324 2346          } else {
2325 2347                  hat_unload_callback(hat, addr, len, flags, NULL);
2326 2348          }
2327 2349          XPV_ALLOW_MIGRATE();
2328 2350  }
2329 2351  
2330 2352  /*
2331      - * Do the callbacks for ranges being unloaded.
2332      - */
2333      -typedef struct range_info {
2334      -        uintptr_t       rng_va;
2335      -        ulong_t         rng_cnt;
2336      -        level_t         rng_level;
2337      -} range_info_t;
2338      -
2339      -/*
2340 2353   * Invalidate the TLB, and perform the callback to the upper level VM system,
2341 2354   * for the specified ranges of contiguous pages.
2342 2355   */
2343 2356  static void
2344 2357  handle_ranges(hat_t *hat, hat_callback_t *cb, uint_t cnt, range_info_t *range)
2345 2358  {
2346 2359          while (cnt > 0) {
2347      -                size_t len;
2348      -
2349 2360                  --cnt;
2350      -                len = range[cnt].rng_cnt << LEVEL_SHIFT(range[cnt].rng_level);
2351      -                hat_tlb_inval_range(hat, (uintptr_t)range[cnt].rng_va, len);
     2361 +                hat_tlb_inval_range(hat, &range[cnt]);
2352 2362  
2353 2363                  if (cb != NULL) {
2354 2364                          cb->hcb_start_addr = (caddr_t)range[cnt].rng_va;
2355 2365                          cb->hcb_end_addr = cb->hcb_start_addr;
2356      -                        cb->hcb_end_addr += len;
     2366 +                        cb->hcb_end_addr += range[cnt].rng_cnt <<
     2367 +                            LEVEL_SHIFT(range[cnt].rng_level);
2357 2368                          cb->hcb_function(cb);
2358 2369                  }
2359 2370          }
2360 2371  }
2361 2372  
2362 2373  /*
2363 2374   * Unload a given range of addresses (has optional callback)
2364 2375   *
2365 2376   * Flags:
2366 2377   * define       HAT_UNLOAD              0x00
↓ open down ↓ 2126 lines elided ↑ open up ↑ 
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX