6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 */
27
28 /*
29 * VM - Hardware Address Translation management for Spitfire MMU.
30 *
31 * This file implements the machine specific hardware translation
32 * needed by the VM system. The machine independent interface is
33 * described in <vm/hat.h> while the machine dependent interface
34 * and data structures are described in <vm/hat_sfmmu.h>.
35 *
36 * The hat layer manages the address translation hardware as a cache
37 * driven by calls from the higher levels in the VM system.
38 */
39
40 #include <sys/types.h>
41 #include <sys/kstat.h>
42 #include <vm/hat.h>
43 #include <vm/hat_sfmmu.h>
44 #include <vm/page.h>
45 #include <sys/pte.h>
8173 * is demoted then other threads will succeed in sfmmu_mlspl_enter(). This
8174 * ensures that p_szc changes of the constituent pages appears atomic for all
8175 * threads that use sfmmu_mlspl_enter() to examine p_szc field.
8176 *
8177 * This mechanism is only used for file system pages where it's not always
8178 * possible to get SE_EXCL locks on all constituent pages to demote the size
8179 * code (as is done for anonymous or kernel large pages).
8180 *
8181 * See more comments in front of sfmmu_mlspl_enter().
8182 */
8183 void
8184 hat_page_demote(page_t *pp)
8185 {
8186 int index;
8187 int sz;
8188 cpuset_t cpuset;
8189 int sync = 0;
8190 page_t *rootpp;
8191 struct sf_hment *sfhme;
8192 struct sf_hment *tmphme = NULL;
8193 struct hme_blk *hmeblkp;
8194 uint_t pszc;
8195 page_t *lastpp;
8196 cpuset_t tset;
8197 pgcnt_t npgs;
8198 kmutex_t *pml;
8199 kmutex_t *pmtx = NULL;
8200
8201 ASSERT(PAGE_EXCL(pp));
8202 ASSERT(!PP_ISFREE(pp));
8203 ASSERT(!PP_ISKAS(pp));
8204 ASSERT(page_szc_lock_assert(pp));
8205 pml = sfmmu_mlist_enter(pp);
8206
8207 pszc = pp->p_szc;
8208 if (pszc == 0) {
8209 goto out;
8210 }
8211
8212 index = PP_MAPINDEX(pp) >> 1;
8213
8214 if (index) {
8215 CPUSET_ZERO(cpuset);
8216 sz = TTE64K;
8217 sync = 1;
8218 }
8219
8220 while (index) {
8221 if (!(index & 0x1)) {
8222 index >>= 1;
8223 sz++;
8224 continue;
8225 }
8226 ASSERT(sz <= pszc);
8227 rootpp = PP_GROUPLEADER(pp, sz);
8228 for (sfhme = rootpp->p_mapping; sfhme; sfhme = tmphme) {
8229 tmphme = sfhme->hme_next;
8230 ASSERT(!IS_PAHME(sfhme));
8231 hmeblkp = sfmmu_hmetohblk(sfhme);
8232 if (hme_size(sfhme) != sz) {
8233 continue;
8234 }
8235 tset = sfmmu_pageunload(rootpp, sfhme, sz);
8236 CPUSET_OR(cpuset, tset);
8237 }
8238 if (index >>= 1) {
8239 sz++;
8240 }
8241 }
8242
8243 ASSERT(!PP_ISMAPPED_LARGE(pp));
8244
8245 if (sync) {
8246 xt_sync(cpuset);
8247 #ifdef VAC
8248 if (PP_ISTNC(pp)) {
8249 conv_tnc(rootpp, sz);
8250 }
8251 #endif /* VAC */
13219 */
13220 /* ARGSUSED */
13221 faultcode_t
13222 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp,
13223 uint_t flags)
13224 {
13225 return (FC_NOSUPPORT);
13226 }
13227
13228 /*
13229 * Searchs the mapping list of the page for a mapping of the same size. If not
13230 * found the corresponding bit is cleared in the p_index field. When large
13231 * pages are more prevalent in the system, we can maintain the mapping list
13232 * in order and we don't have to traverse the list each time. Just check the
13233 * next and prev entries, and if both are of different size, we clear the bit.
13234 */
13235 static void
13236 sfmmu_rm_large_mappings(page_t *pp, int ttesz)
13237 {
13238 struct sf_hment *sfhmep;
13239 struct hme_blk *hmeblkp;
13240 int index;
13241 pgcnt_t npgs;
13242
13243 ASSERT(ttesz > TTE8K);
13244
13245 ASSERT(sfmmu_mlist_held(pp));
13246
13247 ASSERT(PP_ISMAPPED_LARGE(pp));
13248
13249 /*
13250 * Traverse mapping list looking for another mapping of same size.
13251 * since we only want to clear index field if all mappings of
13252 * that size are gone.
13253 */
13254
13255 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) {
13256 if (IS_PAHME(sfhmep))
13257 continue;
13258 hmeblkp = sfmmu_hmetohblk(sfhmep);
13259 if (hme_size(sfhmep) == ttesz) {
13260 /*
13261 * another mapping of the same size. don't clear index.
13262 */
13263 return;
13264 }
13265 }
13266
13267 /*
13268 * Clear the p_index bit for large page.
13269 */
13270 index = PAGESZ_TO_INDEX(ttesz);
13271 npgs = TTEPAGES(ttesz);
13272 while (npgs-- > 0) {
13273 ASSERT(pp->p_index & index);
13274 pp->p_index &= ~index;
13275 pp = PP_PAGENEXT(pp);
13276 }
13277 }
13278
|
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 * Copyright 2016 Gary Mills
27 */
28
29 /*
30 * VM - Hardware Address Translation management for Spitfire MMU.
31 *
32 * This file implements the machine specific hardware translation
33 * needed by the VM system. The machine independent interface is
34 * described in <vm/hat.h> while the machine dependent interface
35 * and data structures are described in <vm/hat_sfmmu.h>.
36 *
37 * The hat layer manages the address translation hardware as a cache
38 * driven by calls from the higher levels in the VM system.
39 */
40
41 #include <sys/types.h>
42 #include <sys/kstat.h>
43 #include <vm/hat.h>
44 #include <vm/hat_sfmmu.h>
45 #include <vm/page.h>
46 #include <sys/pte.h>
8174 * is demoted then other threads will succeed in sfmmu_mlspl_enter(). This
8175 * ensures that p_szc changes of the constituent pages appears atomic for all
8176 * threads that use sfmmu_mlspl_enter() to examine p_szc field.
8177 *
8178 * This mechanism is only used for file system pages where it's not always
8179 * possible to get SE_EXCL locks on all constituent pages to demote the size
8180 * code (as is done for anonymous or kernel large pages).
8181 *
8182 * See more comments in front of sfmmu_mlspl_enter().
8183 */
8184 void
8185 hat_page_demote(page_t *pp)
8186 {
8187 int index;
8188 int sz;
8189 cpuset_t cpuset;
8190 int sync = 0;
8191 page_t *rootpp;
8192 struct sf_hment *sfhme;
8193 struct sf_hment *tmphme = NULL;
8194 uint_t pszc;
8195 page_t *lastpp;
8196 cpuset_t tset;
8197 pgcnt_t npgs;
8198 kmutex_t *pml;
8199 kmutex_t *pmtx = NULL;
8200
8201 ASSERT(PAGE_EXCL(pp));
8202 ASSERT(!PP_ISFREE(pp));
8203 ASSERT(!PP_ISKAS(pp));
8204 ASSERT(page_szc_lock_assert(pp));
8205 pml = sfmmu_mlist_enter(pp);
8206
8207 pszc = pp->p_szc;
8208 if (pszc == 0) {
8209 goto out;
8210 }
8211
8212 index = PP_MAPINDEX(pp) >> 1;
8213
8214 if (index) {
8215 CPUSET_ZERO(cpuset);
8216 sz = TTE64K;
8217 sync = 1;
8218 }
8219
8220 while (index) {
8221 if (!(index & 0x1)) {
8222 index >>= 1;
8223 sz++;
8224 continue;
8225 }
8226 ASSERT(sz <= pszc);
8227 rootpp = PP_GROUPLEADER(pp, sz);
8228 for (sfhme = rootpp->p_mapping; sfhme; sfhme = tmphme) {
8229 tmphme = sfhme->hme_next;
8230 ASSERT(!IS_PAHME(sfhme));
8231 if (hme_size(sfhme) != sz) {
8232 continue;
8233 }
8234 tset = sfmmu_pageunload(rootpp, sfhme, sz);
8235 CPUSET_OR(cpuset, tset);
8236 }
8237 if (index >>= 1) {
8238 sz++;
8239 }
8240 }
8241
8242 ASSERT(!PP_ISMAPPED_LARGE(pp));
8243
8244 if (sync) {
8245 xt_sync(cpuset);
8246 #ifdef VAC
8247 if (PP_ISTNC(pp)) {
8248 conv_tnc(rootpp, sz);
8249 }
8250 #endif /* VAC */
13218 */
13219 /* ARGSUSED */
13220 faultcode_t
13221 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp,
13222 uint_t flags)
13223 {
13224 return (FC_NOSUPPORT);
13225 }
13226
13227 /*
13228 * Searchs the mapping list of the page for a mapping of the same size. If not
13229 * found the corresponding bit is cleared in the p_index field. When large
13230 * pages are more prevalent in the system, we can maintain the mapping list
13231 * in order and we don't have to traverse the list each time. Just check the
13232 * next and prev entries, and if both are of different size, we clear the bit.
13233 */
13234 static void
13235 sfmmu_rm_large_mappings(page_t *pp, int ttesz)
13236 {
13237 struct sf_hment *sfhmep;
13238 int index;
13239 pgcnt_t npgs;
13240
13241 ASSERT(ttesz > TTE8K);
13242
13243 ASSERT(sfmmu_mlist_held(pp));
13244
13245 ASSERT(PP_ISMAPPED_LARGE(pp));
13246
13247 /*
13248 * Traverse mapping list looking for another mapping of same size.
13249 * since we only want to clear index field if all mappings of
13250 * that size are gone.
13251 */
13252
13253 for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) {
13254 if (IS_PAHME(sfhmep))
13255 continue;
13256 if (hme_size(sfhmep) == ttesz) {
13257 /*
13258 * another mapping of the same size. don't clear index.
13259 */
13260 return;
13261 }
13262 }
13263
13264 /*
13265 * Clear the p_index bit for large page.
13266 */
13267 index = PAGESZ_TO_INDEX(ttesz);
13268 npgs = TTEPAGES(ttesz);
13269 while (npgs-- > 0) {
13270 ASSERT(pp->p_index & index);
13271 pp->p_index &= ~index;
13272 pp = PP_PAGENEXT(pp);
13273 }
13274 }
13275
|