1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2019 Joyent, Inc.
26 */
27
28 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
29 /* All Rights Reserved */
30
31 /*
32 * University Copyright- Copyright (c) 1982, 1986, 1988
33 * The Regents of the University of California
34 * All Rights Reserved
35 *
36 * University Acknowledgment- Portions of this document are derived from
37 * software developed by the University of California, Berkeley, and its
38 * contributors.
39 */
40
41 /*
42 * VM - segment of a mapped device.
43 *
44 * This segment driver is used when mapping character special devices.
45 */
46
47 #include <sys/types.h>
48 #include <sys/t_lock.h>
49 #include <sys/sysmacros.h>
50 #include <sys/vtrace.h>
51 #include <sys/systm.h>
52 #include <sys/vmsystm.h>
53 #include <sys/mman.h>
54 #include <sys/errno.h>
55 #include <sys/kmem.h>
56 #include <sys/cmn_err.h>
57 #include <sys/vnode.h>
58 #include <sys/proc.h>
59 #include <sys/conf.h>
60 #include <sys/debug.h>
61 #include <sys/ddidevmap.h>
62 #include <sys/ddi_implfuncs.h>
63 #include <sys/lgrp.h>
64
65 #include <vm/page.h>
66 #include <vm/hat.h>
67 #include <vm/as.h>
68 #include <vm/seg.h>
69 #include <vm/seg_dev.h>
70 #include <vm/seg_kp.h>
71 #include <vm/seg_kmem.h>
72 #include <vm/vpage.h>
73
74 #include <sys/sunddi.h>
75 #include <sys/esunddi.h>
76 #include <sys/fs/snode.h>
77
78
79 #if DEBUG
80 int segdev_debug;
81 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
82 #else
83 #define DEBUGF(level, args)
84 #endif
85
86 /* Default timeout for devmap context management */
87 #define CTX_TIMEOUT_VALUE 0
88
89 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
90 { mutex_enter(&dhp->dh_lock); }
91
92 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
93 { mutex_exit(&dhp->dh_lock); }
94
95 #define round_down_p2(a, s) ((a) & ~((s) - 1))
96 #define round_up_p2(a, s) (((a) + (s) - 1) & ~((s) - 1))
97
98 /*
99 * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
100 * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
101 */
102 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize) \
103 (((uvaddr | paddr) & (pgsize - 1)) == 0)
104 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize) \
105 (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
106
107 #define vpgtob(n) ((n) * sizeof (struct vpage)) /* For brevity */
108
109 #define VTOCVP(vp) (VTOS(vp)->s_commonvp) /* we "know" it's an snode */
110
111 static struct devmap_ctx *devmapctx_list = NULL;
112 static struct devmap_softlock *devmap_slist = NULL;
113
114 /*
115 * mutex, vnode and page for the page of zeros we use for the trash mappings.
116 * One trash page is allocated on the first ddi_umem_setup call that uses it
117 * XXX Eventually, we may want to combine this with what segnf does when all
118 * hat layers implement HAT_NOFAULT.
119 *
120 * The trash page is used when the backing store for a userland mapping is
121 * removed but the application semantics do not take kindly to a SIGBUS.
122 * In that scenario, the applications pages are mapped to some dummy page
123 * which returns garbage on read and writes go into a common place.
124 * (Perfect for NO_FAULT semantics)
125 * The device driver is responsible to communicating to the app with some
126 * other mechanism that such remapping has happened and the app should take
127 * corrective action.
128 * We can also use an anonymous memory page as there is no requirement to
129 * keep the page locked, however this complicates the fault code. RFE.
130 */
131 static struct vnode trashvp;
132 static struct page *trashpp;
133
134 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
135 static vmem_t *umem_np_arena;
136
137 /* Set the cookie to a value we know will never be a valid umem_cookie */
138 #define DEVMAP_DEVMEM_COOKIE ((ddi_umem_cookie_t)0x1)
139
140 /*
141 * Macros to check if type of devmap handle
142 */
143 #define cookie_is_devmem(c) \
144 ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
145
146 #define cookie_is_pmem(c) \
147 ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
148
149 #define cookie_is_kpmem(c) (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
150 ((c)->type == KMEM_PAGEABLE))
151
152 #define dhp_is_devmem(dhp) \
153 (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
154
155 #define dhp_is_pmem(dhp) \
156 (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
157
158 #define dhp_is_kpmem(dhp) \
159 (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
160
161 /*
162 * Private seg op routines.
163 */
164 static int segdev_dup(struct seg *, struct seg *);
165 static int segdev_unmap(struct seg *, caddr_t, size_t);
166 static void segdev_free(struct seg *);
167 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
168 enum fault_type, enum seg_rw);
169 static faultcode_t segdev_faulta(struct seg *, caddr_t);
170 static int segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
171 static int segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
172 static void segdev_badop(void);
173 static int segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
174 static size_t segdev_incore(struct seg *, caddr_t, size_t, char *);
175 static int segdev_lockop(struct seg *, caddr_t, size_t, int, int,
176 ulong_t *, size_t);
177 static int segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
178 static u_offset_t segdev_getoffset(struct seg *, caddr_t);
179 static int segdev_gettype(struct seg *, caddr_t);
180 static int segdev_getvp(struct seg *, caddr_t, struct vnode **);
181 static int segdev_advise(struct seg *, caddr_t, size_t, uint_t);
182 static void segdev_dump(struct seg *);
183 static int segdev_pagelock(struct seg *, caddr_t, size_t,
184 struct page ***, enum lock_type, enum seg_rw);
185 static int segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
186 static int segdev_getmemid(struct seg *, caddr_t, memid_t *);
187 static lgrp_mem_policy_info_t *segdev_getpolicy(struct seg *, caddr_t);
188 static int segdev_capable(struct seg *, segcapability_t);
189
190 /*
191 * XXX this struct is used by rootnex_map_fault to identify
192 * the segment it has been passed. So if you make it
193 * "static" you'll need to fix rootnex_map_fault.
194 */
195 struct seg_ops segdev_ops = {
196 segdev_dup,
197 segdev_unmap,
198 segdev_free,
199 segdev_fault,
200 segdev_faulta,
201 segdev_setprot,
202 segdev_checkprot,
203 (int (*)())segdev_badop, /* kluster */
204 (size_t (*)(struct seg *))NULL, /* swapout */
205 segdev_sync, /* sync */
206 segdev_incore,
207 segdev_lockop, /* lockop */
208 segdev_getprot,
209 segdev_getoffset,
210 segdev_gettype,
211 segdev_getvp,
212 segdev_advise,
213 segdev_dump,
214 segdev_pagelock,
215 segdev_setpagesize,
216 segdev_getmemid,
217 segdev_getpolicy,
218 segdev_capable,
219 seg_inherit_notsup
220 };
221
222 /*
223 * Private segdev support routines
224 */
225 static struct segdev_data *sdp_alloc(void);
226
227 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
228 size_t, enum seg_rw);
229
230 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
231 struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
232
233 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
234 size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
235
236 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
237 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
238 static void devmap_softlock_rele(devmap_handle_t *);
239 static void devmap_ctx_rele(devmap_handle_t *);
240
241 static void devmap_ctxto(void *);
242
243 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
244 caddr_t addr);
245
246 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
247 ulong_t *opfn, ulong_t *pagesize);
248
249 static void free_devmap_handle(devmap_handle_t *dhp);
250
251 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
252 struct seg *newseg);
253
254 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
255
256 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
257
258 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
259
260 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
261 offset_t off, size_t len, uint_t flags);
262
263 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
264 caddr_t addr, size_t *llen, caddr_t *laddr);
265
266 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
267
268 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
269 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
270
271 static void *devmap_umem_alloc_np(size_t size, size_t flags);
272 static void devmap_umem_free_np(void *addr, size_t size);
273
274 /*
275 * routines to lock and unlock underlying segkp segment for
276 * KMEM_PAGEABLE type cookies.
277 */
278 static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
279 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
280
281 /*
282 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
283 * drivers with devmap_access callbacks
284 */
285 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
286 enum fault_type);
287 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
288 enum fault_type);
289
290 static kmutex_t devmapctx_lock;
291
292 static kmutex_t devmap_slock;
293
294 /*
295 * Initialize the thread callbacks and thread private data.
296 */
297 static struct devmap_ctx *
298 devmap_ctxinit(dev_t dev, ulong_t id)
299 {
300 struct devmap_ctx *devctx;
301 struct devmap_ctx *tmp;
302 dev_info_t *dip;
303
304 tmp = kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
305
306 mutex_enter(&devmapctx_lock);
307
308 dip = e_ddi_hold_devi_by_dev(dev, 0);
309 ASSERT(dip != NULL);
310 ddi_release_devi(dip);
311
312 for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
313 if ((devctx->dip == dip) && (devctx->id == id))
314 break;
315
316 if (devctx == NULL) {
317 devctx = tmp;
318 devctx->dip = dip;
319 devctx->id = id;
320 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
321 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
322 devctx->next = devmapctx_list;
323 devmapctx_list = devctx;
324 } else
325 kmem_free(tmp, sizeof (struct devmap_ctx));
326
327 mutex_enter(&devctx->lock);
328 devctx->refcnt++;
329 mutex_exit(&devctx->lock);
330 mutex_exit(&devmapctx_lock);
331
332 return (devctx);
333 }
334
335 /*
336 * Timeout callback called if a CPU has not given up the device context
337 * within dhp->dh_timeout_length ticks
338 */
339 static void
340 devmap_ctxto(void *data)
341 {
342 struct devmap_ctx *devctx = data;
343
344 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
345 "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
346 mutex_enter(&devctx->lock);
347 /*
348 * Set oncpu = 0 so the next mapping trying to get the device context
349 * can.
350 */
351 devctx->oncpu = 0;
352 devctx->timeout = 0;
353 cv_signal(&devctx->cv);
354 mutex_exit(&devctx->lock);
355 }
356
357 /*
358 * Create a device segment.
359 */
360 int
361 segdev_create(struct seg **segpp, void *argsp)
362 {
363 struct seg *seg = *segpp;
364 struct segdev_data *sdp;
365 struct segdev_crargs *a = (struct segdev_crargs *)argsp;
366 devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
367 int error;
368
369 /*
370 * Since the address space is "write" locked, we
371 * don't need the segment lock to protect "segdev" data.
372 */
373 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
374
375 hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
376
377 sdp = sdp_alloc();
378
379 sdp->mapfunc = a->mapfunc;
380 sdp->offset = a->offset;
381 sdp->prot = a->prot;
382 sdp->maxprot = a->maxprot;
383 sdp->type = a->type;
384 sdp->pageprot = 0;
385 sdp->softlockcnt = 0;
386 sdp->vpage = NULL;
387
388 if (sdp->mapfunc == NULL)
389 sdp->devmap_data = dhp;
390 else
391 sdp->devmap_data = dhp = NULL;
392
393 sdp->hat_flags = a->hat_flags;
394 sdp->hat_attr = a->hat_attr;
395
396 /*
397 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
398 */
399 ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
400
401 /*
402 * Hold shadow vnode -- segdev only deals with
403 * character (VCHR) devices. We use the common
404 * vp to hang pages on.
405 */
406 sdp->vp = specfind(a->dev, VCHR);
407 ASSERT(sdp->vp != NULL);
408
409 seg->s_ops = &segdev_ops;
410 seg->s_data = sdp;
411
412 while (dhp != NULL) {
413 dhp->dh_seg = seg;
414 dhp = dhp->dh_next;
415 }
416
417 /*
418 * Inform the vnode of the new mapping.
419 */
420 /*
421 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
422 * dhp specific maxprot because spec_addmap does not use maxprot.
423 */
424 error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
425 seg->s_as, seg->s_base, seg->s_size,
426 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
427
428 if (error != 0) {
429 sdp->devmap_data = NULL;
430 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
431 HAT_UNLOAD_UNMAP);
432 } else {
433 /*
434 * Mappings of /dev/null don't count towards the VSZ of a
435 * process. Mappings of /dev/null have no mapping type.
436 */
437 if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
438 MAP_PRIVATE)) == 0) {
439 seg->s_as->a_resvsize -= seg->s_size;
440 }
441 }
442
443 return (error);
444 }
445
446 static struct segdev_data *
447 sdp_alloc(void)
448 {
449 struct segdev_data *sdp;
450
451 sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
452 rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
453
454 return (sdp);
455 }
456
457 /*
458 * Duplicate seg and return new segment in newseg.
459 */
460 static int
461 segdev_dup(struct seg *seg, struct seg *newseg)
462 {
463 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
464 struct segdev_data *newsdp;
465 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
466 size_t npages;
467 int ret;
468
469 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
470 "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
471
472 DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
473 (void *)dhp, (void *)seg));
474
475 /*
476 * Since the address space is "write" locked, we
477 * don't need the segment lock to protect "segdev" data.
478 */
479 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
480
481 newsdp = sdp_alloc();
482
483 newseg->s_ops = seg->s_ops;
484 newseg->s_data = (void *)newsdp;
485
486 VN_HOLD(sdp->vp);
487 newsdp->vp = sdp->vp;
488 newsdp->mapfunc = sdp->mapfunc;
489 newsdp->offset = sdp->offset;
490 newsdp->pageprot = sdp->pageprot;
491 newsdp->prot = sdp->prot;
492 newsdp->maxprot = sdp->maxprot;
493 newsdp->type = sdp->type;
494 newsdp->hat_attr = sdp->hat_attr;
495 newsdp->hat_flags = sdp->hat_flags;
496 newsdp->softlockcnt = 0;
497
498 /*
499 * Initialize per page data if the segment we are
500 * dup'ing has per page information.
501 */
502 npages = seg_pages(newseg);
503
504 if (sdp->vpage != NULL) {
505 size_t nbytes = vpgtob(npages);
506
507 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
508 bcopy(sdp->vpage, newsdp->vpage, nbytes);
509 } else
510 newsdp->vpage = NULL;
511
512 /*
513 * duplicate devmap handles
514 */
515 if (dhp != NULL) {
516 ret = devmap_handle_dup(dhp,
517 (devmap_handle_t **)&newsdp->devmap_data, newseg);
518 if (ret != 0) {
519 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
520 "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
521 ret, (void *)dhp, (void *)seg);
522 DEBUGF(1, (CE_CONT,
523 "segdev_dup: ret %x dhp %p seg %p\n",
524 ret, (void *)dhp, (void *)seg));
525 return (ret);
526 }
527 }
528
529 /*
530 * Inform the common vnode of the new mapping.
531 */
532 return (VOP_ADDMAP(VTOCVP(newsdp->vp),
533 newsdp->offset, newseg->s_as,
534 newseg->s_base, newseg->s_size, newsdp->prot,
535 newsdp->maxprot, sdp->type, CRED(), NULL));
536 }
537
538 /*
539 * duplicate devmap handles
540 */
541 static int
542 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
543 struct seg *newseg)
544 {
545 devmap_handle_t *newdhp_save = NULL;
546 devmap_handle_t *newdhp = NULL;
547 struct devmap_callback_ctl *callbackops;
548
549 while (dhp != NULL) {
550 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
551
552 /* Need to lock the original dhp while copying if REMAP */
553 HOLD_DHP_LOCK(dhp);
554 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
555 RELE_DHP_LOCK(dhp);
556 newdhp->dh_seg = newseg;
557 newdhp->dh_next = NULL;
558 if (newdhp_save != NULL)
559 newdhp_save->dh_next = newdhp;
560 else
561 *new_dhp = newdhp;
562 newdhp_save = newdhp;
563
564 callbackops = &newdhp->dh_callbackops;
565
566 if (dhp->dh_softlock != NULL)
567 newdhp->dh_softlock = devmap_softlock_init(
568 newdhp->dh_dev,
569 (ulong_t)callbackops->devmap_access);
570 if (dhp->dh_ctx != NULL)
571 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
572 (ulong_t)callbackops->devmap_access);
573
574 /*
575 * Initialize dh_lock if we want to do remap.
576 */
577 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
578 mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
579 newdhp->dh_flags |= DEVMAP_LOCK_INITED;
580 }
581
582 if (callbackops->devmap_dup != NULL) {
583 int ret;
584
585 /*
586 * Call the dup callback so that the driver can
587 * duplicate its private data.
588 */
589 ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
590 (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
591
592 if (ret != 0) {
593 /*
594 * We want to free up this segment as the driver
595 * has indicated that we can't dup it. But we
596 * don't want to call the drivers, devmap_unmap,
597 * callback function as the driver does not
598 * think this segment exists. The caller of
599 * devmap_dup will call seg_free on newseg
600 * as it was the caller that allocated the
601 * segment.
602 */
603 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
604 "newdhp %p dhp %p\n", (void *)newdhp,
605 (void *)dhp));
606 callbackops->devmap_unmap = NULL;
607 return (ret);
608 }
609 }
610
611 dhp = dhp->dh_next;
612 }
613
614 return (0);
615 }
616
617 /*
618 * Split a segment at addr for length len.
619 */
620 /*ARGSUSED*/
621 static int
622 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
623 {
624 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
625 register struct segdev_data *nsdp;
626 register struct seg *nseg;
627 register size_t opages; /* old segment size in pages */
628 register size_t npages; /* new segment size in pages */
629 register size_t dpages; /* pages being deleted (unmapped) */
630 register size_t nbytes;
631 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
632 devmap_handle_t *dhpp;
633 devmap_handle_t *newdhp;
634 struct devmap_callback_ctl *callbackops;
635 caddr_t nbase;
636 offset_t off;
637 ulong_t nsize;
638 size_t mlen, sz;
639
640 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
641 "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
642 (void *)dhp, (void *)seg, (void *)addr, len);
643
644 DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
645 (void *)dhp, (void *)seg, (void *)addr, len));
646
647 /*
648 * Since the address space is "write" locked, we
649 * don't need the segment lock to protect "segdev" data.
650 */
651 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
652
653 if ((sz = sdp->softlockcnt) > 0) {
654 /*
655 * Fail the unmap if pages are SOFTLOCKed through this mapping.
656 * softlockcnt is protected from change by the as write lock.
657 */
658 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
659 "segdev_unmap:error softlockcnt = %ld", sz);
660 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
661 return (EAGAIN);
662 }
663
664 /*
665 * Check for bad sizes
666 */
667 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
668 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
669 panic("segdev_unmap");
670
671 if (dhp != NULL) {
672 devmap_handle_t *tdhp;
673 /*
674 * If large page size was used in hat_devload(),
675 * the same page size must be used in hat_unload().
676 */
677 dhpp = tdhp = devmap_find_handle(dhp, addr);
678 while (tdhp != NULL) {
679 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
680 break;
681 }
682 tdhp = tdhp->dh_next;
683 }
684 if (tdhp != NULL) { /* found a dhp using large pages */
685 size_t slen = len;
686 size_t mlen;
687 size_t soff;
688
689 soff = (ulong_t)(addr - dhpp->dh_uvaddr);
690 while (slen != 0) {
691 mlen = MIN(slen, (dhpp->dh_len - soff));
692 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
693 dhpp->dh_len, HAT_UNLOAD_UNMAP);
694 dhpp = dhpp->dh_next;
695 ASSERT(slen >= mlen);
696 slen -= mlen;
697 soff = 0;
698 }
699 } else
700 hat_unload(seg->s_as->a_hat, addr, len,
701 HAT_UNLOAD_UNMAP);
702 } else {
703 /*
704 * Unload any hardware translations in the range
705 * to be taken out.
706 */
707 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
708 }
709
710 /*
711 * get the user offset which will used in the driver callbacks
712 */
713 off = sdp->offset + (offset_t)(addr - seg->s_base);
714
715 /*
716 * Inform the vnode of the unmapping.
717 */
718 ASSERT(sdp->vp != NULL);
719 (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
720 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
721
722 /*
723 * Check for entire segment
724 */
725 if (addr == seg->s_base && len == seg->s_size) {
726 seg_free(seg);
727 return (0);
728 }
729
730 opages = seg_pages(seg);
731 dpages = btop(len);
732 npages = opages - dpages;
733
734 /*
735 * Check for beginning of segment
736 */
737 if (addr == seg->s_base) {
738 if (sdp->vpage != NULL) {
739 register struct vpage *ovpage;
740
741 ovpage = sdp->vpage; /* keep pointer to vpage */
742
743 nbytes = vpgtob(npages);
744 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
745 bcopy(&ovpage[dpages], sdp->vpage, nbytes);
746
747 /* free up old vpage */
748 kmem_free(ovpage, vpgtob(opages));
749 }
750
751 /*
752 * free devmap handles from the beginning of the mapping.
753 */
754 if (dhp != NULL)
755 devmap_handle_unmap_head(dhp, len);
756
757 sdp->offset += (offset_t)len;
758
759 seg->s_base += len;
760 seg->s_size -= len;
761
762 return (0);
763 }
764
765 /*
766 * Check for end of segment
767 */
768 if (addr + len == seg->s_base + seg->s_size) {
769 if (sdp->vpage != NULL) {
770 register struct vpage *ovpage;
771
772 ovpage = sdp->vpage; /* keep pointer to vpage */
773
774 nbytes = vpgtob(npages);
775 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
776 bcopy(ovpage, sdp->vpage, nbytes);
777
778 /* free up old vpage */
779 kmem_free(ovpage, vpgtob(opages));
780 }
781 seg->s_size -= len;
782
783 /*
784 * free devmap handles from addr to the end of the mapping.
785 */
786 if (dhp != NULL)
787 devmap_handle_unmap_tail(dhp, addr);
788
789 return (0);
790 }
791
792 /*
793 * The section to go is in the middle of the segment,
794 * have to make it into two segments. nseg is made for
795 * the high end while seg is cut down at the low end.
796 */
797 nbase = addr + len; /* new seg base */
798 nsize = (seg->s_base + seg->s_size) - nbase; /* new seg size */
799 seg->s_size = addr - seg->s_base; /* shrink old seg */
800 nseg = seg_alloc(seg->s_as, nbase, nsize);
801 if (nseg == NULL)
802 panic("segdev_unmap seg_alloc");
803
804 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
805 "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
806 DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
807 (void *)seg, (void *)nseg));
808 nsdp = sdp_alloc();
809
810 nseg->s_ops = seg->s_ops;
811 nseg->s_data = (void *)nsdp;
812
813 VN_HOLD(sdp->vp);
814 nsdp->mapfunc = sdp->mapfunc;
815 nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
816 nsdp->vp = sdp->vp;
817 nsdp->pageprot = sdp->pageprot;
818 nsdp->prot = sdp->prot;
819 nsdp->maxprot = sdp->maxprot;
820 nsdp->type = sdp->type;
821 nsdp->hat_attr = sdp->hat_attr;
822 nsdp->hat_flags = sdp->hat_flags;
823 nsdp->softlockcnt = 0;
824
825 /*
826 * Initialize per page data if the segment we are
827 * dup'ing has per page information.
828 */
829 if (sdp->vpage != NULL) {
830 /* need to split vpage into two arrays */
831 register size_t nnbytes;
832 register size_t nnpages;
833 register struct vpage *ovpage;
834
835 ovpage = sdp->vpage; /* keep pointer to vpage */
836
837 npages = seg_pages(seg); /* seg has shrunk */
838 nbytes = vpgtob(npages);
839 nnpages = seg_pages(nseg);
840 nnbytes = vpgtob(nnpages);
841
842 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
843 bcopy(ovpage, sdp->vpage, nbytes);
844
845 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
846 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
847
848 /* free up old vpage */
849 kmem_free(ovpage, vpgtob(opages));
850 } else
851 nsdp->vpage = NULL;
852
853 /*
854 * unmap dhps.
855 */
856 if (dhp == NULL) {
857 nsdp->devmap_data = NULL;
858 return (0);
859 }
860 while (dhp != NULL) {
861 callbackops = &dhp->dh_callbackops;
862 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
863 "segdev_unmap: dhp=%p addr=%p", dhp, addr);
864 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
865 (void *)dhp, (void *)addr,
866 (void *)dhp->dh_uvaddr, dhp->dh_len));
867
868 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
869 dhpp = dhp->dh_next;
870 dhp->dh_next = NULL;
871 dhp = dhpp;
872 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
873 dhp = dhp->dh_next;
874 } else if (addr > dhp->dh_uvaddr &&
875 (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
876 /*
877 * <addr, addr+len> is enclosed by dhp.
878 * create a newdhp that begins at addr+len and
879 * ends at dhp->dh_uvaddr+dhp->dh_len.
880 */
881 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
882 HOLD_DHP_LOCK(dhp);
883 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
884 RELE_DHP_LOCK(dhp);
885 newdhp->dh_seg = nseg;
886 newdhp->dh_next = dhp->dh_next;
887 if (dhp->dh_softlock != NULL)
888 newdhp->dh_softlock = devmap_softlock_init(
889 newdhp->dh_dev,
890 (ulong_t)callbackops->devmap_access);
891 if (dhp->dh_ctx != NULL)
892 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
893 (ulong_t)callbackops->devmap_access);
894 if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
895 mutex_init(&newdhp->dh_lock,
896 NULL, MUTEX_DEFAULT, NULL);
897 }
898 if (callbackops->devmap_unmap != NULL)
899 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
900 off, len, dhp, &dhp->dh_pvtp,
901 newdhp, &newdhp->dh_pvtp);
902 mlen = len + (addr - dhp->dh_uvaddr);
903 devmap_handle_reduce_len(newdhp, mlen);
904 nsdp->devmap_data = newdhp;
905 /* XX Changing len should recalculate LARGE flag */
906 dhp->dh_len = addr - dhp->dh_uvaddr;
907 dhpp = dhp->dh_next;
908 dhp->dh_next = NULL;
909 dhp = dhpp;
910 } else if ((addr > dhp->dh_uvaddr) &&
911 ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
912 mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
913 /*
914 * <addr, addr+len> spans over dhps.
915 */
916 if (callbackops->devmap_unmap != NULL)
917 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
918 off, mlen, (devmap_cookie_t *)dhp,
919 &dhp->dh_pvtp, NULL, NULL);
920 /* XX Changing len should recalculate LARGE flag */
921 dhp->dh_len = addr - dhp->dh_uvaddr;
922 dhpp = dhp->dh_next;
923 dhp->dh_next = NULL;
924 dhp = dhpp;
925 nsdp->devmap_data = dhp;
926 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
927 /*
928 * dhp is enclosed by <addr, addr+len>.
929 */
930 dhp->dh_seg = nseg;
931 nsdp->devmap_data = dhp;
932 dhp = devmap_handle_unmap(dhp);
933 nsdp->devmap_data = dhp; /* XX redundant? */
934 } else if (((addr + len) > dhp->dh_uvaddr) &&
935 ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
936 mlen = addr + len - dhp->dh_uvaddr;
937 if (callbackops->devmap_unmap != NULL)
938 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
939 dhp->dh_uoff, mlen, NULL,
940 NULL, dhp, &dhp->dh_pvtp);
941 devmap_handle_reduce_len(dhp, mlen);
942 nsdp->devmap_data = dhp;
943 dhp->dh_seg = nseg;
944 dhp = dhp->dh_next;
945 } else {
946 dhp->dh_seg = nseg;
947 dhp = dhp->dh_next;
948 }
949 }
950 return (0);
951 }
952
953 /*
954 * Utility function handles reducing the length of a devmap handle during unmap
955 * Note that is only used for unmapping the front portion of the handler,
956 * i.e., we are bumping up the offset/pfn etc up by len
957 * Do not use if reducing length at the tail.
958 */
959 static void
960 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
961 {
962 struct ddi_umem_cookie *cp;
963 struct devmap_pmem_cookie *pcp;
964 /*
965 * adjust devmap handle fields
966 */
967 ASSERT(len < dhp->dh_len);
968
969 /* Make sure only page-aligned changes are done */
970 ASSERT((len & PAGEOFFSET) == 0);
971
972 dhp->dh_len -= len;
973 dhp->dh_uoff += (offset_t)len;
974 dhp->dh_roff += (offset_t)len;
975 dhp->dh_uvaddr += len;
976 /* Need to grab dhp lock if REMAP */
977 HOLD_DHP_LOCK(dhp);
978 cp = dhp->dh_cookie;
979 if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
980 if (cookie_is_devmem(cp)) {
981 dhp->dh_pfn += btop(len);
982 } else if (cookie_is_pmem(cp)) {
983 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
984 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
985 dhp->dh_roff < ptob(pcp->dp_npages));
986 } else {
987 ASSERT(dhp->dh_roff < cp->size);
988 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
989 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
990 ASSERT((dhp->dh_cvaddr + len) <=
991 (cp->cvaddr + cp->size));
992
993 dhp->dh_cvaddr += len;
994 }
995 }
996 /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
997 RELE_DHP_LOCK(dhp);
998 }
999
1000 /*
1001 * Free devmap handle, dhp.
1002 * Return the next devmap handle on the linked list.
1003 */
1004 static devmap_handle_t *
1005 devmap_handle_unmap(devmap_handle_t *dhp)
1006 {
1007 struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1008 struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1009 devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1010
1011 ASSERT(dhp != NULL);
1012
1013 /*
1014 * before we free up dhp, call the driver's devmap_unmap entry point
1015 * to free resources allocated for this dhp.
1016 */
1017 if (callbackops->devmap_unmap != NULL) {
1018 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1019 dhp->dh_len, NULL, NULL, NULL, NULL);
1020 }
1021
1022 if (dhpp == dhp) { /* releasing first dhp, change sdp data */
1023 sdp->devmap_data = dhp->dh_next;
1024 } else {
1025 while (dhpp->dh_next != dhp) {
1026 dhpp = dhpp->dh_next;
1027 }
1028 dhpp->dh_next = dhp->dh_next;
1029 }
1030 dhpp = dhp->dh_next; /* return value is next dhp in chain */
1031
1032 if (dhp->dh_softlock != NULL)
1033 devmap_softlock_rele(dhp);
1034
1035 if (dhp->dh_ctx != NULL)
1036 devmap_ctx_rele(dhp);
1037
1038 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1039 mutex_destroy(&dhp->dh_lock);
1040 }
1041 kmem_free(dhp, sizeof (devmap_handle_t));
1042
1043 return (dhpp);
1044 }
1045
1046 /*
1047 * Free complete devmap handles from dhp for len bytes
1048 * dhp can be either the first handle or a subsequent handle
1049 */
1050 static void
1051 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1052 {
1053 struct devmap_callback_ctl *callbackops;
1054
1055 /*
1056 * free the devmap handles covered by len.
1057 */
1058 while (len >= dhp->dh_len) {
1059 len -= dhp->dh_len;
1060 dhp = devmap_handle_unmap(dhp);
1061 }
1062 if (len != 0) { /* partial unmap at head of first remaining dhp */
1063 callbackops = &dhp->dh_callbackops;
1064
1065 /*
1066 * Call the unmap callback so the drivers can make
1067 * adjustment on its private data.
1068 */
1069 if (callbackops->devmap_unmap != NULL)
1070 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1071 dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1072 devmap_handle_reduce_len(dhp, len);
1073 }
1074 }
1075
1076 /*
1077 * Free devmap handles to truncate the mapping after addr
1078 * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1079 * Also could then use the routine in middle unmap case too
1080 */
1081 static void
1082 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1083 {
1084 register struct seg *seg = dhp->dh_seg;
1085 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1086 register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1087 struct devmap_callback_ctl *callbackops;
1088 register devmap_handle_t *dhpp;
1089 size_t maplen;
1090 ulong_t off;
1091 size_t len;
1092
1093 maplen = (size_t)(addr - dhp->dh_uvaddr);
1094 dhph = devmap_find_handle(dhph, addr);
1095
1096 while (dhph != NULL) {
1097 if (maplen == 0) {
1098 dhph = devmap_handle_unmap(dhph);
1099 } else {
1100 callbackops = &dhph->dh_callbackops;
1101 len = dhph->dh_len - maplen;
1102 off = (ulong_t)sdp->offset + (addr - seg->s_base);
1103 /*
1104 * Call the unmap callback so the driver
1105 * can make adjustments on its private data.
1106 */
1107 if (callbackops->devmap_unmap != NULL)
1108 (*callbackops->devmap_unmap)(dhph,
1109 dhph->dh_pvtp, off, len,
1110 (devmap_cookie_t *)dhph,
1111 &dhph->dh_pvtp, NULL, NULL);
1112 /* XXX Reducing len needs to recalculate LARGE flag */
1113 dhph->dh_len = maplen;
1114 maplen = 0;
1115 dhpp = dhph->dh_next;
1116 dhph->dh_next = NULL;
1117 dhph = dhpp;
1118 }
1119 } /* end while */
1120 }
1121
1122 /*
1123 * Free a segment.
1124 */
1125 static void
1126 segdev_free(struct seg *seg)
1127 {
1128 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1129 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1130
1131 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1132 "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1133 DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1134 (void *)dhp, (void *)seg));
1135
1136 /*
1137 * Since the address space is "write" locked, we
1138 * don't need the segment lock to protect "segdev" data.
1139 */
1140 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1141
1142 while (dhp != NULL)
1143 dhp = devmap_handle_unmap(dhp);
1144
1145 VN_RELE(sdp->vp);
1146 if (sdp->vpage != NULL)
1147 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1148
1149 rw_destroy(&sdp->lock);
1150 kmem_free(sdp, sizeof (*sdp));
1151 }
1152
1153 static void
1154 free_devmap_handle(devmap_handle_t *dhp)
1155 {
1156 register devmap_handle_t *dhpp;
1157
1158 /*
1159 * free up devmap handle
1160 */
1161 while (dhp != NULL) {
1162 dhpp = dhp->dh_next;
1163 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1164 mutex_destroy(&dhp->dh_lock);
1165 }
1166
1167 if (dhp->dh_softlock != NULL)
1168 devmap_softlock_rele(dhp);
1169
1170 if (dhp->dh_ctx != NULL)
1171 devmap_ctx_rele(dhp);
1172
1173 kmem_free(dhp, sizeof (devmap_handle_t));
1174 dhp = dhpp;
1175 }
1176 }
1177
1178 /*
1179 * routines to lock and unlock underlying segkp segment for
1180 * KMEM_PAGEABLE type cookies.
1181 * segkp only allows a single pending F_SOFTLOCK
1182 * we keep track of number of locks in the cookie so we can
1183 * have multiple pending faults and manage the calls to segkp.
1184 * RFE: if segkp supports either pagelock or can support multiple
1185 * calls to F_SOFTLOCK, then these routines can go away.
1186 * If pagelock, segdev_faultpage can fault on a page by page basis
1187 * and simplifies the code quite a bit.
1188 * if multiple calls allowed but not partial ranges, then need for
1189 * cookie->lock and locked count goes away, code can call as_fault directly
1190 */
1191 static faultcode_t
1192 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1193 {
1194 int err = 0;
1195 ASSERT(cookie_is_kpmem(cookie));
1196 /*
1197 * Fault in pages in segkp with F_SOFTLOCK.
1198 * We want to hold the lock until all pages have been loaded.
1199 * segkp only allows single caller to hold SOFTLOCK, so cookie
1200 * holds a count so we dont call into segkp multiple times
1201 */
1202 mutex_enter(&cookie->lock);
1203
1204 /*
1205 * Check for overflow in locked field
1206 */
1207 if ((UINT32_MAX - cookie->locked) < npages) {
1208 err = FC_MAKE_ERR(ENOMEM);
1209 } else if (cookie->locked == 0) {
1210 /* First time locking */
1211 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1212 cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1213 }
1214 if (!err) {
1215 cookie->locked += npages;
1216 }
1217 mutex_exit(&cookie->lock);
1218 return (err);
1219 }
1220
1221 static void
1222 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1223 {
1224 mutex_enter(&cookie->lock);
1225 ASSERT(cookie_is_kpmem(cookie));
1226 ASSERT(cookie->locked >= npages);
1227 cookie->locked -= (uint_t)npages;
1228 if (cookie->locked == 0) {
1229 /* Last unlock */
1230 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1231 cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1232 panic("segdev releasing kpmem lock %p", (void *)cookie);
1233 }
1234 mutex_exit(&cookie->lock);
1235 }
1236
1237 /*
1238 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1239 * drivers with devmap_access callbacks
1240 * slock->softlocked basically works like a rw lock
1241 * -ve counts => F_SOFTLOCK in progress
1242 * +ve counts => F_INVAL/F_PROT in progress
1243 * We allow only one F_SOFTLOCK at a time
1244 * but can have multiple pending F_INVAL/F_PROT calls
1245 *
1246 * This routine waits using cv_wait_sig so killing processes is more graceful
1247 * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1248 */
1249 static int devmap_softlock_enter(
1250 struct devmap_softlock *slock,
1251 size_t npages,
1252 enum fault_type type)
1253 {
1254 if (npages == 0)
1255 return (0);
1256 mutex_enter(&(slock->lock));
1257 switch (type) {
1258 case F_SOFTLOCK :
1259 while (slock->softlocked) {
1260 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1261 /* signalled */
1262 mutex_exit(&(slock->lock));
1263 return (EINTR);
1264 }
1265 }
1266 slock->softlocked -= npages; /* -ve count => locked */
1267 break;
1268 case F_INVAL :
1269 case F_PROT :
1270 while (slock->softlocked < 0)
1271 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1272 /* signalled */
1273 mutex_exit(&(slock->lock));
1274 return (EINTR);
1275 }
1276 slock->softlocked += npages; /* +ve count => f_invals */
1277 break;
1278 default:
1279 ASSERT(0);
1280 }
1281 mutex_exit(&(slock->lock));
1282 return (0);
1283 }
1284
1285 static void devmap_softlock_exit(
1286 struct devmap_softlock *slock,
1287 size_t npages,
1288 enum fault_type type)
1289 {
1290 if (slock == NULL)
1291 return;
1292 mutex_enter(&(slock->lock));
1293 switch (type) {
1294 case F_SOFTLOCK :
1295 ASSERT(-slock->softlocked >= npages);
1296 slock->softlocked += npages; /* -ve count is softlocked */
1297 if (slock->softlocked == 0)
1298 cv_signal(&slock->cv);
1299 break;
1300 case F_INVAL :
1301 case F_PROT:
1302 ASSERT(slock->softlocked >= npages);
1303 slock->softlocked -= npages;
1304 if (slock->softlocked == 0)
1305 cv_signal(&slock->cv);
1306 break;
1307 default:
1308 ASSERT(0);
1309 }
1310 mutex_exit(&(slock->lock));
1311 }
1312
1313 /*
1314 * Do a F_SOFTUNLOCK call over the range requested.
1315 * The range must have already been F_SOFTLOCK'ed.
1316 * The segment lock should be held, (but not the segment private lock?)
1317 * The softunlock code below does not adjust for large page sizes
1318 * assumes the caller already did any addr/len adjustments for
1319 * pagesize mappings before calling.
1320 */
1321 /*ARGSUSED*/
1322 static void
1323 segdev_softunlock(
1324 struct hat *hat, /* the hat */
1325 struct seg *seg, /* seg_dev of interest */
1326 caddr_t addr, /* base address of range */
1327 size_t len, /* number of bytes */
1328 enum seg_rw rw) /* type of access at fault */
1329 {
1330 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1331 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1332
1333 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1334 "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1335 dhp_head, sdp, addr, len);
1336 DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1337 "addr %p len %lx\n",
1338 (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1339
1340 hat_unlock(hat, addr, len);
1341
1342 if (dhp_head != NULL) {
1343 devmap_handle_t *dhp;
1344 size_t mlen;
1345 size_t tlen = len;
1346 ulong_t off;
1347
1348 dhp = devmap_find_handle(dhp_head, addr);
1349 ASSERT(dhp != NULL);
1350
1351 off = (ulong_t)(addr - dhp->dh_uvaddr);
1352 while (tlen != 0) {
1353 mlen = MIN(tlen, (dhp->dh_len - off));
1354
1355 /*
1356 * unlock segkp memory, locked during F_SOFTLOCK
1357 */
1358 if (dhp_is_kpmem(dhp)) {
1359 release_kpmem_lock(
1360 (struct ddi_umem_cookie *)dhp->dh_cookie,
1361 btopr(mlen));
1362 }
1363
1364 /*
1365 * Do the softlock accounting for devmap_access
1366 */
1367 if (dhp->dh_callbackops.devmap_access != NULL) {
1368 devmap_softlock_exit(dhp->dh_softlock,
1369 btopr(mlen), F_SOFTLOCK);
1370 }
1371
1372 tlen -= mlen;
1373 dhp = dhp->dh_next;
1374 off = 0;
1375 }
1376 }
1377
1378 mutex_enter(&freemem_lock);
1379 ASSERT(sdp->softlockcnt >= btopr(len));
1380 sdp->softlockcnt -= btopr(len);
1381 mutex_exit(&freemem_lock);
1382 if (sdp->softlockcnt == 0) {
1383 /*
1384 * All SOFTLOCKS are gone. Wakeup any waiting
1385 * unmappers so they can try again to unmap.
1386 * Check for waiters first without the mutex
1387 * held so we don't always grab the mutex on
1388 * softunlocks.
1389 */
1390 if (AS_ISUNMAPWAIT(seg->s_as)) {
1391 mutex_enter(&seg->s_as->a_contents);
1392 if (AS_ISUNMAPWAIT(seg->s_as)) {
1393 AS_CLRUNMAPWAIT(seg->s_as);
1394 cv_broadcast(&seg->s_as->a_cv);
1395 }
1396 mutex_exit(&seg->s_as->a_contents);
1397 }
1398 }
1399
1400 }
1401
1402 /*
1403 * Handle fault for a single page.
1404 * Done in a separate routine so we can handle errors more easily.
1405 * This routine is called only from segdev_faultpages()
1406 * when looping over the range of addresses requested. The segment lock is held.
1407 */
1408 static faultcode_t
1409 segdev_faultpage(
1410 struct hat *hat, /* the hat */
1411 struct seg *seg, /* seg_dev of interest */
1412 caddr_t addr, /* address in as */
1413 struct vpage *vpage, /* pointer to vpage for seg, addr */
1414 enum fault_type type, /* type of fault */
1415 enum seg_rw rw, /* type of access at fault */
1416 devmap_handle_t *dhp) /* devmap handle if any for this page */
1417 {
1418 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1419 uint_t prot;
1420 pfn_t pfnum = PFN_INVALID;
1421 u_offset_t offset;
1422 uint_t hat_flags;
1423 dev_info_t *dip;
1424
1425 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1426 "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1427 DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1428 (void *)dhp, (void *)seg, (void *)addr));
1429
1430 /*
1431 * Initialize protection value for this page.
1432 * If we have per page protection values check it now.
1433 */
1434 if (sdp->pageprot) {
1435 uint_t protchk;
1436
1437 switch (rw) {
1438 case S_READ:
1439 protchk = PROT_READ;
1440 break;
1441 case S_WRITE:
1442 protchk = PROT_WRITE;
1443 break;
1444 case S_EXEC:
1445 protchk = PROT_EXEC;
1446 break;
1447 case S_OTHER:
1448 default:
1449 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1450 break;
1451 }
1452
1453 prot = VPP_PROT(vpage);
1454 if ((prot & protchk) == 0)
1455 return (FC_PROT); /* illegal access type */
1456 } else {
1457 prot = sdp->prot;
1458 /* caller has already done segment level protection check */
1459 }
1460
1461 if (type == F_SOFTLOCK) {
1462 mutex_enter(&freemem_lock);
1463 sdp->softlockcnt++;
1464 mutex_exit(&freemem_lock);
1465 }
1466
1467 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1468 offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1469 /*
1470 * In the devmap framework, sdp->mapfunc is set to NULL. we can get
1471 * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1472 * seg->s_base.
1473 */
1474 if (dhp == NULL) {
1475 /* If segment has devmap_data, then dhp should be non-NULL */
1476 ASSERT(sdp->devmap_data == NULL);
1477 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1478 (off_t)offset, prot);
1479 prot |= sdp->hat_attr;
1480 } else {
1481 ulong_t off;
1482 struct ddi_umem_cookie *cp;
1483 struct devmap_pmem_cookie *pcp;
1484
1485 /* ensure the dhp passed in contains addr. */
1486 ASSERT(dhp == devmap_find_handle(
1487 (devmap_handle_t *)sdp->devmap_data, addr));
1488
1489 off = addr - dhp->dh_uvaddr;
1490
1491 /*
1492 * This routine assumes that the caller makes sure that the
1493 * fields in dhp used below are unchanged due to remap during
1494 * this call. Caller does HOLD_DHP_LOCK if neeed
1495 */
1496 cp = dhp->dh_cookie;
1497 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1498 pfnum = PFN_INVALID;
1499 } else if (cookie_is_devmem(cp)) {
1500 pfnum = dhp->dh_pfn + btop(off);
1501 } else if (cookie_is_pmem(cp)) {
1502 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1503 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1504 dhp->dh_roff < ptob(pcp->dp_npages));
1505 pfnum = page_pptonum(
1506 pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1507 } else {
1508 ASSERT(dhp->dh_roff < cp->size);
1509 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1510 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1511 ASSERT((dhp->dh_cvaddr + off) <=
1512 (cp->cvaddr + cp->size));
1513 ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1514 (cp->cvaddr + cp->size));
1515
1516 switch (cp->type) {
1517 case UMEM_LOCKED :
1518 if (cp->pparray != NULL) {
1519 ASSERT((dhp->dh_roff &
1520 PAGEOFFSET) == 0);
1521 pfnum = page_pptonum(
1522 cp->pparray[btop(off +
1523 dhp->dh_roff)]);
1524 } else {
1525 pfnum = hat_getpfnum(
1526 ((proc_t *)cp->procp)->p_as->a_hat,
1527 cp->cvaddr + off);
1528 }
1529 break;
1530 case UMEM_TRASH :
1531 pfnum = page_pptonum(trashpp);
1532 /*
1533 * We should set hat_flags to HAT_NOFAULT also
1534 * However, not all hat layers implement this
1535 */
1536 break;
1537 case KMEM_PAGEABLE:
1538 case KMEM_NON_PAGEABLE:
1539 pfnum = hat_getpfnum(kas.a_hat,
1540 dhp->dh_cvaddr + off);
1541 break;
1542 default :
1543 pfnum = PFN_INVALID;
1544 break;
1545 }
1546 }
1547 prot |= dhp->dh_hat_attr;
1548 }
1549 if (pfnum == PFN_INVALID) {
1550 return (FC_MAKE_ERR(EFAULT));
1551 }
1552 /* prot should already be OR'ed in with hat_attributes if needed */
1553
1554 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1555 "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1556 pfnum, pf_is_memory(pfnum), prot, hat_flags);
1557 DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1558 "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1559
1560 if (pf_is_memory(pfnum) || (dhp != NULL)) {
1561 /*
1562 * It's not _really_ required here to pass sdp->hat_flags
1563 * to hat_devload even though we do it.
1564 * This is because hat figures it out DEVMEM mappings
1565 * are non-consistent, anyway.
1566 */
1567 hat_devload(hat, addr, PAGESIZE, pfnum,
1568 prot, hat_flags | sdp->hat_flags);
1569 return (0);
1570 }
1571
1572 /*
1573 * Fall through to the case where devmap is not used and need to call
1574 * up the device tree to set up the mapping
1575 */
1576
1577 dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1578 ASSERT(dip);
1579
1580 /*
1581 * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1582 * This is because this calls drivers which may not expect
1583 * prot to have any other values than PROT_ALL
1584 * The root nexus driver has a hack to peek into the segment
1585 * structure and then OR in sdp->hat_attr.
1586 * XX In case the bus_ops interfaces are ever revisited
1587 * we need to fix this. prot should include other hat attributes
1588 */
1589 if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1590 (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1591 return (FC_MAKE_ERR(EFAULT));
1592 }
1593 return (0);
1594 }
1595
1596 static faultcode_t
1597 segdev_fault(
1598 struct hat *hat, /* the hat */
1599 struct seg *seg, /* the seg_dev of interest */
1600 caddr_t addr, /* the address of the fault */
1601 size_t len, /* the length of the range */
1602 enum fault_type type, /* type of fault */
1603 enum seg_rw rw) /* type of access at fault */
1604 {
1605 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1606 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1607 devmap_handle_t *dhp;
1608 struct devmap_softlock *slock = NULL;
1609 ulong_t slpage = 0;
1610 ulong_t off;
1611 caddr_t maddr = addr;
1612 int err;
1613 int err_is_faultcode = 0;
1614
1615 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1616 "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1617 (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1618 DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1619 "addr %p len %lx type %x\n",
1620 (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1621
1622 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1623
1624 /* Handle non-devmap case */
1625 if (dhp_head == NULL)
1626 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1627
1628 /* Find devmap handle */
1629 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1630 return (FC_NOMAP);
1631
1632 /*
1633 * The seg_dev driver does not implement copy-on-write,
1634 * and always loads translations with maximal allowed permissions
1635 * but we got an fault trying to access the device.
1636 * Servicing the fault is not going to result in any better result
1637 * RFE: If we want devmap_access callbacks to be involved in F_PROT
1638 * faults, then the code below is written for that
1639 * Pending resolution of the following:
1640 * - determine if the F_INVAL/F_SOFTLOCK syncing
1641 * is needed for F_PROT also or not. The code below assumes it does
1642 * - If driver sees F_PROT and calls devmap_load with same type,
1643 * then segdev_faultpages will fail with FC_PROT anyway, need to
1644 * change that so calls from devmap_load to segdev_faultpages for
1645 * F_PROT type are retagged to F_INVAL.
1646 * RFE: Today we dont have drivers that use devmap and want to handle
1647 * F_PROT calls. The code in segdev_fault* is written to allow
1648 * this case but is not tested. A driver that needs this capability
1649 * should be able to remove the short-circuit case; resolve the
1650 * above issues and "should" work.
1651 */
1652 if (type == F_PROT) {
1653 return (FC_PROT);
1654 }
1655
1656 /*
1657 * Loop through dhp list calling devmap_access or segdev_faultpages for
1658 * each devmap handle.
1659 * drivers which implement devmap_access can interpose on faults and do
1660 * device-appropriate special actions before calling devmap_load.
1661 */
1662
1663 /*
1664 * Unfortunately, this simple loop has turned out to expose a variety
1665 * of complex problems which results in the following convoluted code.
1666 *
1667 * First, a desire to handle a serialization of F_SOFTLOCK calls
1668 * to the driver within the framework.
1669 * This results in a dh_softlock structure that is on a per device
1670 * (or device instance) basis and serializes devmap_access calls.
1671 * Ideally we would need to do this for underlying
1672 * memory/device regions that are being faulted on
1673 * but that is hard to identify and with REMAP, harder
1674 * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1675 * to F_SOFTLOCK calls to the driver.
1676 * These serializations are to simplify the driver programmer model.
1677 * To support these two features, the code first goes through the
1678 * devmap handles and counts the pages (slpage) that are covered
1679 * by devmap_access callbacks.
1680 * This part ends with a devmap_softlock_enter call
1681 * which allows only one F_SOFTLOCK active on a device instance,
1682 * but multiple F_INVAL/F_PROTs can be active except when a
1683 * F_SOFTLOCK is active
1684 *
1685 * Next, we dont short-circuit the fault code upfront to call
1686 * segdev_softunlock for F_SOFTUNLOCK, because we must use
1687 * the same length when we softlock and softunlock.
1688 *
1689 * -Hat layers may not support softunlocking lengths less than the
1690 * original length when there is large page support.
1691 * -kpmem locking is dependent on keeping the lengths same.
1692 * -if drivers handled F_SOFTLOCK, they probably also expect to
1693 * see an F_SOFTUNLOCK of the same length
1694 * Hence, if extending lengths during softlock,
1695 * softunlock has to make the same adjustments and goes through
1696 * the same loop calling segdev_faultpages/segdev_softunlock
1697 * But some of the synchronization and error handling is different
1698 */
1699
1700 if (type != F_SOFTUNLOCK) {
1701 devmap_handle_t *dhpp = dhp;
1702 size_t slen = len;
1703
1704 /*
1705 * Calculate count of pages that are :
1706 * a) within the (potentially extended) fault region
1707 * b) AND covered by devmap handle with devmap_access
1708 */
1709 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1710 while (slen != 0) {
1711 size_t mlen;
1712
1713 /*
1714 * Softlocking on a region that allows remap is
1715 * unsupported due to unresolved locking issues
1716 * XXX: unclear what these are?
1717 * One potential is that if there is a pending
1718 * softlock, then a remap should not be allowed
1719 * until the unlock is done. This is easily
1720 * fixed by returning error in devmap*remap on
1721 * checking the dh->dh_softlock->softlocked value
1722 */
1723 if ((type == F_SOFTLOCK) &&
1724 (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1725 return (FC_NOSUPPORT);
1726 }
1727
1728 mlen = MIN(slen, (dhpp->dh_len - off));
1729 if (dhpp->dh_callbackops.devmap_access) {
1730 size_t llen;
1731 caddr_t laddr;
1732 /*
1733 * use extended length for large page mappings
1734 */
1735 HOLD_DHP_LOCK(dhpp);
1736 if ((sdp->pageprot == 0) &&
1737 (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1738 devmap_get_large_pgsize(dhpp,
1739 mlen, maddr, &llen, &laddr);
1740 } else {
1741 llen = mlen;
1742 }
1743 RELE_DHP_LOCK(dhpp);
1744
1745 slpage += btopr(llen);
1746 slock = dhpp->dh_softlock;
1747 }
1748 maddr += mlen;
1749 ASSERT(slen >= mlen);
1750 slen -= mlen;
1751 dhpp = dhpp->dh_next;
1752 off = 0;
1753 }
1754 /*
1755 * synchonize with other faulting threads and wait till safe
1756 * devmap_softlock_enter might return due to signal in cv_wait
1757 *
1758 * devmap_softlock_enter has to be called outside of while loop
1759 * to prevent a deadlock if len spans over multiple dhps.
1760 * dh_softlock is based on device instance and if multiple dhps
1761 * use the same device instance, the second dhp's LOCK call
1762 * will hang waiting on the first to complete.
1763 * devmap_setup verifies that slocks in a dhp_chain are same.
1764 * RFE: this deadlock only hold true for F_SOFTLOCK. For
1765 * F_INVAL/F_PROT, since we now allow multiple in parallel,
1766 * we could have done the softlock_enter inside the loop
1767 * and supported multi-dhp mappings with dissimilar devices
1768 */
1769 if (err = devmap_softlock_enter(slock, slpage, type))
1770 return (FC_MAKE_ERR(err));
1771 }
1772
1773 /* reset 'maddr' to the start addr of the range of fault. */
1774 maddr = addr;
1775
1776 /* calculate the offset corresponds to 'addr' in the first dhp. */
1777 off = (ulong_t)(addr - dhp->dh_uvaddr);
1778
1779 /*
1780 * The fault length may span over multiple dhps.
1781 * Loop until the total length is satisfied.
1782 */
1783 while (len != 0) {
1784 size_t llen;
1785 size_t mlen;
1786 caddr_t laddr;
1787
1788 /*
1789 * mlen is the smaller of 'len' and the length
1790 * from addr to the end of mapping defined by dhp.
1791 */
1792 mlen = MIN(len, (dhp->dh_len - off));
1793
1794 HOLD_DHP_LOCK(dhp);
1795 /*
1796 * Pass the extended length and address to devmap_access
1797 * if large pagesize is used for loading address translations.
1798 */
1799 if ((sdp->pageprot == 0) &&
1800 (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1801 devmap_get_large_pgsize(dhp, mlen, maddr,
1802 &llen, &laddr);
1803 ASSERT(maddr == addr || laddr == maddr);
1804 } else {
1805 llen = mlen;
1806 laddr = maddr;
1807 }
1808
1809 if (dhp->dh_callbackops.devmap_access != NULL) {
1810 offset_t aoff;
1811
1812 aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1813
1814 /*
1815 * call driver's devmap_access entry point which will
1816 * call devmap_load/contextmgmt to load the translations
1817 *
1818 * We drop the dhp_lock before calling access so
1819 * drivers can call devmap_*_remap within access
1820 */
1821 RELE_DHP_LOCK(dhp);
1822
1823 err = (*dhp->dh_callbackops.devmap_access)(
1824 dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1825 } else {
1826 /*
1827 * If no devmap_access entry point, then load mappings
1828 * hold dhp_lock across faultpages if REMAP
1829 */
1830 err = segdev_faultpages(hat, seg, laddr, llen,
1831 type, rw, dhp);
1832 err_is_faultcode = 1;
1833 RELE_DHP_LOCK(dhp);
1834 }
1835
1836 if (err) {
1837 if ((type == F_SOFTLOCK) && (maddr > addr)) {
1838 /*
1839 * If not first dhp, use
1840 * segdev_fault(F_SOFTUNLOCK) for prior dhps
1841 * While this is recursion, it is incorrect to
1842 * call just segdev_softunlock
1843 * if we are using either large pages
1844 * or devmap_access. It will be more right
1845 * to go through the same loop as above
1846 * rather than call segdev_softunlock directly
1847 * It will use the right lenghths as well as
1848 * call into the driver devmap_access routines.
1849 */
1850 size_t done = (size_t)(maddr - addr);
1851 (void) segdev_fault(hat, seg, addr, done,
1852 F_SOFTUNLOCK, S_OTHER);
1853 /*
1854 * reduce slpage by number of pages
1855 * released by segdev_softunlock
1856 */
1857 ASSERT(slpage >= btopr(done));
1858 devmap_softlock_exit(slock,
1859 slpage - btopr(done), type);
1860 } else {
1861 devmap_softlock_exit(slock, slpage, type);
1862 }
1863
1864
1865 /*
1866 * Segdev_faultpages() already returns a faultcode,
1867 * hence, result from segdev_faultpages() should be
1868 * returned directly.
1869 */
1870 if (err_is_faultcode)
1871 return (err);
1872 return (FC_MAKE_ERR(err));
1873 }
1874
1875 maddr += mlen;
1876 ASSERT(len >= mlen);
1877 len -= mlen;
1878 dhp = dhp->dh_next;
1879 off = 0;
1880
1881 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1882 }
1883 /*
1884 * release the softlock count at end of fault
1885 * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1886 */
1887 if ((type == F_INVAL) || (type == F_PROT))
1888 devmap_softlock_exit(slock, slpage, type);
1889 return (0);
1890 }
1891
1892 /*
1893 * segdev_faultpages
1894 *
1895 * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1896 * This routine assumes that the callers makes sure that the fields
1897 * in dhp used below are not changed due to remap during this call.
1898 * Caller does HOLD_DHP_LOCK if neeed
1899 * This routine returns a faultcode_t as a return value for segdev_fault.
1900 */
1901 static faultcode_t
1902 segdev_faultpages(
1903 struct hat *hat, /* the hat */
1904 struct seg *seg, /* the seg_dev of interest */
1905 caddr_t addr, /* the address of the fault */
1906 size_t len, /* the length of the range */
1907 enum fault_type type, /* type of fault */
1908 enum seg_rw rw, /* type of access at fault */
1909 devmap_handle_t *dhp) /* devmap handle */
1910 {
1911 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1912 register caddr_t a;
1913 struct vpage *vpage;
1914 struct ddi_umem_cookie *kpmem_cookie = NULL;
1915 int err;
1916
1917 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1918 "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1919 (void *)dhp, (void *)seg, (void *)addr, len);
1920 DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1921 "dhp %p seg %p addr %p len %lx\n",
1922 (void *)dhp, (void *)seg, (void *)addr, len));
1923
1924 /*
1925 * The seg_dev driver does not implement copy-on-write,
1926 * and always loads translations with maximal allowed permissions
1927 * but we got an fault trying to access the device.
1928 * Servicing the fault is not going to result in any better result
1929 * XXX: If we want to allow devmap_access to handle F_PROT calls,
1930 * This code should be removed and let the normal fault handling
1931 * take care of finding the error
1932 */
1933 if (type == F_PROT) {
1934 return (FC_PROT);
1935 }
1936
1937 if (type == F_SOFTUNLOCK) {
1938 segdev_softunlock(hat, seg, addr, len, rw);
1939 return (0);
1940 }
1941
1942 /*
1943 * For kernel pageable memory, fault/lock segkp pages
1944 * We hold this until the completion of this
1945 * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1946 */
1947 if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1948 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1949 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1950 return (err);
1951 }
1952
1953 /*
1954 * If we have the same protections for the entire segment,
1955 * insure that the access being attempted is legitimate.
1956 */
1957 rw_enter(&sdp->lock, RW_READER);
1958 if (sdp->pageprot == 0) {
1959 uint_t protchk;
1960
1961 switch (rw) {
1962 case S_READ:
1963 protchk = PROT_READ;
1964 break;
1965 case S_WRITE:
1966 protchk = PROT_WRITE;
1967 break;
1968 case S_EXEC:
1969 protchk = PROT_EXEC;
1970 break;
1971 case S_OTHER:
1972 default:
1973 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1974 break;
1975 }
1976
1977 if ((sdp->prot & protchk) == 0) {
1978 rw_exit(&sdp->lock);
1979 /* undo kpmem locking */
1980 if (kpmem_cookie != NULL) {
1981 release_kpmem_lock(kpmem_cookie, btopr(len));
1982 }
1983 return (FC_PROT); /* illegal access type */
1984 }
1985 }
1986
1987 /*
1988 * we do a single hat_devload for the range if
1989 * - devmap framework (dhp is not NULL),
1990 * - pageprot == 0, i.e., no per-page protection set and
1991 * - is device pages, irrespective of whether we are using large pages
1992 */
1993 if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1994 pfn_t pfnum;
1995 uint_t hat_flags;
1996
1997 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1998 rw_exit(&sdp->lock);
1999 return (FC_NOMAP);
2000 }
2001
2002 if (type == F_SOFTLOCK) {
2003 mutex_enter(&freemem_lock);
2004 sdp->softlockcnt += btopr(len);
2005 mutex_exit(&freemem_lock);
2006 }
2007
2008 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2009 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2010 ASSERT(!pf_is_memory(pfnum));
2011
2012 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2013 hat_flags | sdp->hat_flags);
2014 rw_exit(&sdp->lock);
2015 return (0);
2016 }
2017
2018 /* Handle cases where we have to loop through fault handling per-page */
2019
2020 if (sdp->vpage == NULL)
2021 vpage = NULL;
2022 else
2023 vpage = &sdp->vpage[seg_page(seg, addr)];
2024
2025 /* loop over the address range handling each fault */
2026 for (a = addr; a < addr + len; a += PAGESIZE) {
2027 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2028 break;
2029 }
2030 if (vpage != NULL)
2031 vpage++;
2032 }
2033 rw_exit(&sdp->lock);
2034 if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2035 size_t done = (size_t)(a - addr); /* pages fault successfully */
2036 if (done > 0) {
2037 /* use softunlock for those pages */
2038 segdev_softunlock(hat, seg, addr, done, S_OTHER);
2039 }
2040 if (kpmem_cookie != NULL) {
2041 /* release kpmem lock for rest of pages */
2042 ASSERT(len >= done);
2043 release_kpmem_lock(kpmem_cookie, btopr(len - done));
2044 }
2045 } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2046 /* for non-SOFTLOCK cases, release kpmem */
2047 release_kpmem_lock(kpmem_cookie, btopr(len));
2048 }
2049 return (err);
2050 }
2051
2052 /*
2053 * Asynchronous page fault. We simply do nothing since this
2054 * entry point is not supposed to load up the translation.
2055 */
2056 /*ARGSUSED*/
2057 static faultcode_t
2058 segdev_faulta(struct seg *seg, caddr_t addr)
2059 {
2060 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2061 "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2062 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2063
2064 return (0);
2065 }
2066
2067 static int
2068 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2069 {
2070 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2071 register devmap_handle_t *dhp;
2072 register struct vpage *vp, *evp;
2073 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2074 ulong_t off;
2075 size_t mlen, sz;
2076
2077 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2078 "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2079 (void *)seg, (void *)addr, len, prot);
2080 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2081
2082 if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2083 /*
2084 * Fail the setprot if pages are SOFTLOCKed through this
2085 * mapping.
2086 * Softlockcnt is protected from change by the as read lock.
2087 */
2088 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2089 "segdev_setprot:error softlockcnt=%lx", sz);
2090 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2091 return (EAGAIN);
2092 }
2093
2094 if (dhp_head != NULL) {
2095 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2096 return (EINVAL);
2097
2098 /*
2099 * check if violate maxprot.
2100 */
2101 off = (ulong_t)(addr - dhp->dh_uvaddr);
2102 mlen = len;
2103 while (dhp) {
2104 if ((dhp->dh_maxprot & prot) != prot)
2105 return (EACCES); /* violated maxprot */
2106
2107 if (mlen > (dhp->dh_len - off)) {
2108 mlen -= dhp->dh_len - off;
2109 dhp = dhp->dh_next;
2110 off = 0;
2111 } else
2112 break;
2113 }
2114 } else {
2115 if ((sdp->maxprot & prot) != prot)
2116 return (EACCES);
2117 }
2118
2119 rw_enter(&sdp->lock, RW_WRITER);
2120 if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2121 if (sdp->prot == prot) {
2122 rw_exit(&sdp->lock);
2123 return (0); /* all done */
2124 }
2125 sdp->prot = (uchar_t)prot;
2126 } else {
2127 sdp->pageprot = 1;
2128 if (sdp->vpage == NULL) {
2129 /*
2130 * First time through setting per page permissions,
2131 * initialize all the vpage structures to prot
2132 */
2133 sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2134 KM_SLEEP);
2135 evp = &sdp->vpage[seg_pages(seg)];
2136 for (vp = sdp->vpage; vp < evp; vp++)
2137 VPP_SETPROT(vp, sdp->prot);
2138 }
2139 /*
2140 * Now go change the needed vpages protections.
2141 */
2142 evp = &sdp->vpage[seg_page(seg, addr + len)];
2143 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2144 VPP_SETPROT(vp, prot);
2145 }
2146 rw_exit(&sdp->lock);
2147
2148 if (dhp_head != NULL) {
2149 devmap_handle_t *tdhp;
2150 /*
2151 * If large page size was used in hat_devload(),
2152 * the same page size must be used in hat_unload().
2153 */
2154 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2155 while (tdhp != NULL) {
2156 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2157 break;
2158 }
2159 tdhp = tdhp->dh_next;
2160 }
2161 if (tdhp) {
2162 size_t slen = len;
2163 size_t mlen;
2164 size_t soff;
2165
2166 soff = (ulong_t)(addr - dhp->dh_uvaddr);
2167 while (slen != 0) {
2168 mlen = MIN(slen, (dhp->dh_len - soff));
2169 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2170 dhp->dh_len, HAT_UNLOAD);
2171 dhp = dhp->dh_next;
2172 ASSERT(slen >= mlen);
2173 slen -= mlen;
2174 soff = 0;
2175 }
2176 return (0);
2177 }
2178 }
2179
2180 if ((prot & ~PROT_USER) == PROT_NONE) {
2181 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2182 } else {
2183 /*
2184 * RFE: the segment should keep track of all attributes
2185 * allowing us to remove the deprecated hat_chgprot
2186 * and use hat_chgattr.
2187 */
2188 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2189 }
2190
2191 return (0);
2192 }
2193
2194 static int
2195 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2196 {
2197 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2198 struct vpage *vp, *evp;
2199
2200 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2201 "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2202 (void *)seg, (void *)addr, len, prot);
2203 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2204
2205 /*
2206 * If segment protection can be used, simply check against them
2207 */
2208 rw_enter(&sdp->lock, RW_READER);
2209 if (sdp->pageprot == 0) {
2210 register int err;
2211
2212 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2213 rw_exit(&sdp->lock);
2214 return (err);
2215 }
2216
2217 /*
2218 * Have to check down to the vpage level
2219 */
2220 evp = &sdp->vpage[seg_page(seg, addr + len)];
2221 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2222 if ((VPP_PROT(vp) & prot) != prot) {
2223 rw_exit(&sdp->lock);
2224 return (EACCES);
2225 }
2226 }
2227 rw_exit(&sdp->lock);
2228 return (0);
2229 }
2230
2231 static int
2232 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2233 {
2234 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2235 size_t pgno;
2236
2237 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2238 "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2239 (void *)seg, (void *)addr, len, (void *)protv);
2240 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2241
2242 pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2243 if (pgno != 0) {
2244 rw_enter(&sdp->lock, RW_READER);
2245 if (sdp->pageprot == 0) {
2246 do {
2247 protv[--pgno] = sdp->prot;
2248 } while (pgno != 0);
2249 } else {
2250 size_t pgoff = seg_page(seg, addr);
2251
2252 do {
2253 pgno--;
2254 protv[pgno] =
2255 VPP_PROT(&sdp->vpage[pgno + pgoff]);
2256 } while (pgno != 0);
2257 }
2258 rw_exit(&sdp->lock);
2259 }
2260 return (0);
2261 }
2262
2263 static u_offset_t
2264 segdev_getoffset(register struct seg *seg, caddr_t addr)
2265 {
2266 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2267
2268 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2269 "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2270
2271 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2272
2273 return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2274 }
2275
2276 /*ARGSUSED*/
2277 static int
2278 segdev_gettype(register struct seg *seg, caddr_t addr)
2279 {
2280 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2281
2282 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2283 "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2284
2285 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2286
2287 return (sdp->type);
2288 }
2289
2290
2291 /*ARGSUSED*/
2292 static int
2293 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2294 {
2295 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2296
2297 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2298 "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2299
2300 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2301
2302 /*
2303 * Note that this vp is the common_vp of the device, where the
2304 * pages are hung ..
2305 */
2306 *vpp = VTOCVP(sdp->vp);
2307
2308 return (0);
2309 }
2310
2311 static void
2312 segdev_badop(void)
2313 {
2314 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2315 "segdev_badop:start");
2316 panic("segdev_badop");
2317 /*NOTREACHED*/
2318 }
2319
2320 /*
2321 * segdev pages are not in the cache, and thus can't really be controlled.
2322 * Hence, syncs are simply always successful.
2323 */
2324 /*ARGSUSED*/
2325 static int
2326 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2327 {
2328 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2329
2330 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2331
2332 return (0);
2333 }
2334
2335 /*
2336 * segdev pages are always "in core".
2337 */
2338 /*ARGSUSED*/
2339 static size_t
2340 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2341 {
2342 size_t v = 0;
2343
2344 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2345
2346 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2347
2348 for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2349 v += PAGESIZE)
2350 *vec++ = 1;
2351 return (v);
2352 }
2353
2354 /*
2355 * segdev pages are not in the cache, and thus can't really be controlled.
2356 * Hence, locks are simply always successful.
2357 */
2358 /*ARGSUSED*/
2359 static int
2360 segdev_lockop(struct seg *seg, caddr_t addr,
2361 size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2362 {
2363 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2364
2365 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2366
2367 return (0);
2368 }
2369
2370 /*
2371 * segdev pages are not in the cache, and thus can't really be controlled.
2372 * Hence, advise is simply always successful.
2373 */
2374 /*ARGSUSED*/
2375 static int
2376 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2377 {
2378 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2379
2380 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2381
2382 return (0);
2383 }
2384
2385 /*
2386 * segdev pages are not dumped, so we just return
2387 */
2388 /*ARGSUSED*/
2389 static void
2390 segdev_dump(struct seg *seg)
2391 {}
2392
2393 /*
2394 * ddi_segmap_setup: Used by drivers who wish specify mapping attributes
2395 * for a segment. Called from a drivers segmap(9E)
2396 * routine.
2397 */
2398 /*ARGSUSED*/
2399 int
2400 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2401 off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2402 ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2403 {
2404 struct segdev_crargs dev_a;
2405 int (*mapfunc)(dev_t dev, off_t off, int prot);
2406 uint_t hat_attr;
2407 pfn_t pfn;
2408 int error, i;
2409
2410 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2411 "ddi_segmap_setup:start");
2412
2413 if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2414 return (ENODEV);
2415
2416 /*
2417 * Character devices that support the d_mmap
2418 * interface can only be mmap'ed shared.
2419 */
2420 if ((flags & MAP_TYPE) != MAP_SHARED)
2421 return (EINVAL);
2422
2423 /*
2424 * Check that this region is indeed mappable on this platform.
2425 * Use the mapping function.
2426 */
2427 if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2428 return (ENXIO);
2429
2430 /*
2431 * Check to ensure that the entire range is
2432 * legal and we are not trying to map in
2433 * more than the device will let us.
2434 */
2435 for (i = 0; i < len; i += PAGESIZE) {
2436 if (i == 0) {
2437 /*
2438 * Save the pfn at offset here. This pfn will be
2439 * used later to get user address.
2440 */
2441 if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2442 maxprot)) == PFN_INVALID)
2443 return (ENXIO);
2444 } else {
2445 if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2446 PFN_INVALID)
2447 return (ENXIO);
2448 }
2449 }
2450
2451 as_rangelock(as);
2452 /* Pick an address w/o worrying about any vac alignment constraints. */
2453 error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2454 if (error != 0) {
2455 as_rangeunlock(as);
2456 return (error);
2457 }
2458
2459 dev_a.mapfunc = mapfunc;
2460 dev_a.dev = dev;
2461 dev_a.offset = (offset_t)offset;
2462 dev_a.type = flags & MAP_TYPE;
2463 dev_a.prot = (uchar_t)prot;
2464 dev_a.maxprot = (uchar_t)maxprot;
2465 dev_a.hat_attr = hat_attr;
2466 dev_a.hat_flags = 0;
2467 dev_a.devmap_data = NULL;
2468
2469 error = as_map(as, *addrp, len, segdev_create, &dev_a);
2470 as_rangeunlock(as);
2471 return (error);
2472
2473 }
2474
2475 /*ARGSUSED*/
2476 static int
2477 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2478 struct page ***ppp, enum lock_type type, enum seg_rw rw)
2479 {
2480 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2481 "segdev_pagelock:start");
2482 return (ENOTSUP);
2483 }
2484
2485 /*ARGSUSED*/
2486 static int
2487 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2488 uint_t szc)
2489 {
2490 return (ENOTSUP);
2491 }
2492
2493 /*
2494 * devmap_device: Used by devmap framework to establish mapping
2495 * called by devmap_seup(9F) during map setup time.
2496 */
2497 /*ARGSUSED*/
2498 static int
2499 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2500 offset_t off, size_t len, uint_t flags)
2501 {
2502 devmap_handle_t *rdhp, *maxdhp;
2503 struct segdev_crargs dev_a;
2504 int err;
2505 uint_t maxprot = PROT_ALL;
2506 offset_t offset = 0;
2507 pfn_t pfn;
2508 struct devmap_pmem_cookie *pcp;
2509
2510 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2511 "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2512 (void *)dhp, (void *)addr, off, len);
2513
2514 DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2515 (void *)dhp, (void *)addr, off, len));
2516
2517 as_rangelock(as);
2518 if ((flags & MAP_FIXED) == 0) {
2519 offset_t aligned_off;
2520
2521 rdhp = maxdhp = dhp;
2522 while (rdhp != NULL) {
2523 maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2524 maxdhp : rdhp;
2525 rdhp = rdhp->dh_next;
2526 maxprot |= dhp->dh_maxprot;
2527 }
2528 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2529
2530 /*
2531 * Use the dhp that has the
2532 * largest len to get user address.
2533 */
2534 /*
2535 * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2536 * use 0 which is as good as any other.
2537 */
2538 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2539 aligned_off = (offset_t)0;
2540 } else if (dhp_is_devmem(maxdhp)) {
2541 aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2542 } else if (dhp_is_pmem(maxdhp)) {
2543 pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2544 pfn = page_pptonum(
2545 pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2546 aligned_off = (offset_t)ptob(pfn) - offset;
2547 } else {
2548 aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2549 offset;
2550 }
2551
2552 /*
2553 * Pick an address aligned to dh_cookie.
2554 * for kernel memory/user memory, cookie is cvaddr.
2555 * for device memory, cookie is physical address.
2556 */
2557 map_addr(addr, len, aligned_off, 1, flags);
2558 if (*addr == NULL) {
2559 as_rangeunlock(as);
2560 return (ENOMEM);
2561 }
2562 } else {
2563 /*
2564 * User-specified address; blow away any previous mappings.
2565 */
2566 (void) as_unmap(as, *addr, len);
2567 }
2568
2569 dev_a.mapfunc = NULL;
2570 dev_a.dev = dhp->dh_dev;
2571 dev_a.type = flags & MAP_TYPE;
2572 dev_a.offset = off;
2573 /*
2574 * sdp->maxprot has the least restrict protection of all dhps.
2575 */
2576 dev_a.maxprot = maxprot;
2577 dev_a.prot = dhp->dh_prot;
2578 /*
2579 * devmap uses dhp->dh_hat_attr for hat.
2580 */
2581 dev_a.hat_flags = 0;
2582 dev_a.hat_attr = 0;
2583 dev_a.devmap_data = (void *)dhp;
2584
2585 err = as_map(as, *addr, len, segdev_create, &dev_a);
2586 as_rangeunlock(as);
2587 return (err);
2588 }
2589
2590 int
2591 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2592 uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2593 size_t, uint_t, uint_t))
2594 {
2595 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2596 struct devmap_ctx *devctx;
2597 int do_timeout = 0;
2598 int ret;
2599
2600 #ifdef lint
2601 pvtp = pvtp;
2602 #endif
2603
2604 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2605 "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2606 (void *)dhp, off, len);
2607 DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2608 (void *)dhp, off, len));
2609
2610 if (ctxmgt == NULL)
2611 return (FC_HWERR);
2612
2613 devctx = dhp->dh_ctx;
2614
2615 /*
2616 * If we are on an MP system with more than one cpu running
2617 * and if a thread on some CPU already has the context, wait
2618 * for it to finish if there is a hysteresis timeout.
2619 *
2620 * We call cv_wait() instead of cv_wait_sig() because
2621 * it does not matter much if it returned due to a signal
2622 * or due to a cv_signal() or cv_broadcast(). In either event
2623 * we need to complete the mapping otherwise the processes
2624 * will die with a SEGV.
2625 */
2626 if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2627 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2628 "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2629 devctx, dhp);
2630 do_timeout = 1;
2631 mutex_enter(&devctx->lock);
2632 while (devctx->oncpu)
2633 cv_wait(&devctx->cv, &devctx->lock);
2634 devctx->oncpu = 1;
2635 mutex_exit(&devctx->lock);
2636 }
2637
2638 /*
2639 * Call the contextmgt callback so that the driver can handle
2640 * the fault.
2641 */
2642 ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2643
2644 /*
2645 * If devmap_access() returned -1, then there was a hardware
2646 * error so we need to convert the return value to something
2647 * that trap() will understand. Otherwise, the return value
2648 * is already a fault code generated by devmap_unload()
2649 * or devmap_load().
2650 */
2651 if (ret) {
2652 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2653 "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2654 ret, dhp, devctx);
2655 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2656 ret, (void *)dhp));
2657 if (devctx->oncpu) {
2658 mutex_enter(&devctx->lock);
2659 devctx->oncpu = 0;
2660 cv_signal(&devctx->cv);
2661 mutex_exit(&devctx->lock);
2662 }
2663 return (FC_HWERR);
2664 }
2665
2666 /*
2667 * Setup the timeout if we need to
2668 */
2669 if (do_timeout) {
2670 mutex_enter(&devctx->lock);
2671 if (dhp->dh_timeout_length > 0) {
2672 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2673 "devmap_do_ctxmgt:timeout set");
2674 devctx->timeout = timeout(devmap_ctxto,
2675 devctx, dhp->dh_timeout_length);
2676 } else {
2677 /*
2678 * We don't want to wait so set oncpu to
2679 * 0 and wake up anyone waiting.
2680 */
2681 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2682 "devmap_do_ctxmgt:timeout not set");
2683 devctx->oncpu = 0;
2684 cv_signal(&devctx->cv);
2685 }
2686 mutex_exit(&devctx->lock);
2687 }
2688
2689 return (DDI_SUCCESS);
2690 }
2691
2692 /*
2693 * end of mapping
2694 * poff fault_offset |
2695 * base | | |
2696 * | | | |
2697 * V V V V
2698 * +-----------+---------------+-------+---------+-------+
2699 * ^ ^ ^ ^
2700 * |<--- offset--->|<-len->| |
2701 * |<--- dh_len(size of mapping) --->|
2702 * |<-- pg -->|
2703 * -->|rlen|<--
2704 */
2705 static ulong_t
2706 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2707 ulong_t *opfn, ulong_t *pagesize)
2708 {
2709 register int level;
2710 ulong_t pg;
2711 ulong_t poff;
2712 ulong_t base;
2713 caddr_t uvaddr;
2714 long rlen;
2715
2716 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2717 "devmap_roundup:start dhp=%p off=%lx len=%lx",
2718 (void *)dhp, offset, len);
2719 DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2720 (void *)dhp, offset, len));
2721
2722 /*
2723 * get the max. pagesize that is aligned within the range
2724 * <dh_pfn, dh_pfn+offset>.
2725 *
2726 * The calculations below use physical address to ddetermine
2727 * the page size to use. The same calculations can use the
2728 * virtual address to determine the page size.
2729 */
2730 base = (ulong_t)ptob(dhp->dh_pfn);
2731 for (level = dhp->dh_mmulevel; level >= 0; level--) {
2732 pg = page_get_pagesize(level);
2733 poff = ((base + offset) & ~(pg - 1));
2734 uvaddr = dhp->dh_uvaddr + (poff - base);
2735 if ((poff >= base) &&
2736 ((poff + pg) <= (base + dhp->dh_len)) &&
2737 VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2738 break;
2739 }
2740
2741 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2742 "devmap_roundup: base=%lx poff=%lx dhp=%p",
2743 base, poff, dhp);
2744 DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2745 base, poff, dhp->dh_pfn));
2746
2747 ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2748 ASSERT(level >= 0);
2749
2750 *pagesize = pg;
2751 *opfn = dhp->dh_pfn + btop(poff - base);
2752
2753 rlen = len + offset - (poff - base + pg);
2754
2755 ASSERT(rlen < (long)len);
2756
2757 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2758 "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2759 (void *)dhp, level, rlen, pagesize, opfn);
2760 DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2761 "level %x rlen %lx psize %lx opfn %lx\n",
2762 (void *)dhp, level, rlen, *pagesize, *opfn));
2763
2764 return ((ulong_t)((rlen > 0) ? rlen : 0));
2765 }
2766
2767 /*
2768 * find the dhp that contains addr.
2769 */
2770 static devmap_handle_t *
2771 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2772 {
2773 devmap_handle_t *dhp;
2774
2775 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2776 "devmap_find_handle:start");
2777
2778 dhp = dhp_head;
2779 while (dhp) {
2780 if (addr >= dhp->dh_uvaddr &&
2781 addr < (dhp->dh_uvaddr + dhp->dh_len))
2782 return (dhp);
2783 dhp = dhp->dh_next;
2784 }
2785
2786 return ((devmap_handle_t *)NULL);
2787 }
2788
2789 /*
2790 * devmap_unload:
2791 * Marks a segdev segment or pages if offset->offset+len
2792 * is not the entire segment as intercept and unloads the
2793 * pages in the range offset -> offset+len.
2794 */
2795 int
2796 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2797 {
2798 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2799 caddr_t addr;
2800 ulong_t size;
2801 ssize_t soff;
2802
2803 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2804 "devmap_unload:start dhp=%p offset=%llx len=%lx",
2805 (void *)dhp, offset, len);
2806 DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2807 (void *)dhp, offset, len));
2808
2809 soff = (ssize_t)(offset - dhp->dh_uoff);
2810 soff = round_down_p2(soff, PAGESIZE);
2811 if (soff < 0 || soff >= dhp->dh_len)
2812 return (FC_MAKE_ERR(EINVAL));
2813
2814 /*
2815 * Address and size must be page aligned. Len is set to the
2816 * number of bytes in the number of pages that are required to
2817 * support len. Offset is set to the byte offset of the first byte
2818 * of the page that contains offset.
2819 */
2820 len = round_up_p2(len, PAGESIZE);
2821
2822 /*
2823 * If len is == 0, then calculate the size by getting
2824 * the number of bytes from offset to the end of the segment.
2825 */
2826 if (len == 0)
2827 size = dhp->dh_len - soff;
2828 else {
2829 size = len;
2830 if ((soff + size) > dhp->dh_len)
2831 return (FC_MAKE_ERR(EINVAL));
2832 }
2833
2834 /*
2835 * The address is offset bytes from the base address of
2836 * the dhp.
2837 */
2838 addr = (caddr_t)(soff + dhp->dh_uvaddr);
2839
2840 /*
2841 * If large page size was used in hat_devload(),
2842 * the same page size must be used in hat_unload().
2843 */
2844 if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2845 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2846 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2847 } else {
2848 hat_unload(dhp->dh_seg->s_as->a_hat, addr, size,
2849 HAT_UNLOAD|HAT_UNLOAD_OTHER);
2850 }
2851
2852 return (0);
2853 }
2854
2855 /*
2856 * calculates the optimal page size that will be used for hat_devload().
2857 */
2858 static void
2859 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2860 size_t *llen, caddr_t *laddr)
2861 {
2862 ulong_t off;
2863 ulong_t pfn;
2864 ulong_t pgsize;
2865 uint_t first = 1;
2866
2867 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2868 "devmap_get_large_pgsize:start");
2869
2870 /*
2871 * RFE - Code only supports large page mappings for devmem
2872 * This code could be changed in future if we want to support
2873 * large page mappings for kernel exported memory.
2874 */
2875 ASSERT(dhp_is_devmem(dhp));
2876 ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2877
2878 *llen = 0;
2879 off = (ulong_t)(addr - dhp->dh_uvaddr);
2880 while ((long)len > 0) {
2881 /*
2882 * get the optimal pfn to minimize address translations.
2883 * devmap_roundup() returns residue bytes for next round
2884 * calculations.
2885 */
2886 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2887
2888 if (first) {
2889 *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2890 first = 0;
2891 }
2892
2893 *llen += pgsize;
2894 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2895 }
2896 /* Large page mapping len/addr cover more range than original fault */
2897 ASSERT(*llen >= len && *laddr <= addr);
2898 ASSERT((*laddr + *llen) >= (addr + len));
2899 }
2900
2901 /*
2902 * Initialize the devmap_softlock structure.
2903 */
2904 static struct devmap_softlock *
2905 devmap_softlock_init(dev_t dev, ulong_t id)
2906 {
2907 struct devmap_softlock *slock;
2908 struct devmap_softlock *tmp;
2909
2910 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2911 "devmap_softlock_init:start");
2912
2913 tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2914 mutex_enter(&devmap_slock);
2915
2916 for (slock = devmap_slist; slock != NULL; slock = slock->next)
2917 if ((slock->dev == dev) && (slock->id == id))
2918 break;
2919
2920 if (slock == NULL) {
2921 slock = tmp;
2922 slock->dev = dev;
2923 slock->id = id;
2924 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2925 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2926 slock->next = devmap_slist;
2927 devmap_slist = slock;
2928 } else
2929 kmem_free(tmp, sizeof (struct devmap_softlock));
2930
2931 mutex_enter(&slock->lock);
2932 slock->refcnt++;
2933 mutex_exit(&slock->lock);
2934 mutex_exit(&devmap_slock);
2935
2936 return (slock);
2937 }
2938
2939 /*
2940 * Wake up processes that sleep on softlocked.
2941 * Free dh_softlock if refcnt is 0.
2942 */
2943 static void
2944 devmap_softlock_rele(devmap_handle_t *dhp)
2945 {
2946 struct devmap_softlock *slock = dhp->dh_softlock;
2947 struct devmap_softlock *tmp;
2948 struct devmap_softlock *parent;
2949
2950 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2951 "devmap_softlock_rele:start");
2952
2953 mutex_enter(&devmap_slock);
2954 mutex_enter(&slock->lock);
2955
2956 ASSERT(slock->refcnt > 0);
2957
2958 slock->refcnt--;
2959
2960 /*
2961 * If no one is using the device, free up the slock data.
2962 */
2963 if (slock->refcnt == 0) {
2964 slock->softlocked = 0;
2965 cv_signal(&slock->cv);
2966
2967 if (devmap_slist == slock)
2968 devmap_slist = slock->next;
2969 else {
2970 parent = devmap_slist;
2971 for (tmp = devmap_slist->next; tmp != NULL;
2972 tmp = tmp->next) {
2973 if (tmp == slock) {
2974 parent->next = tmp->next;
2975 break;
2976 }
2977 parent = tmp;
2978 }
2979 }
2980 mutex_exit(&slock->lock);
2981 mutex_destroy(&slock->lock);
2982 cv_destroy(&slock->cv);
2983 kmem_free(slock, sizeof (struct devmap_softlock));
2984 } else
2985 mutex_exit(&slock->lock);
2986
2987 mutex_exit(&devmap_slock);
2988 }
2989
2990 /*
2991 * Wake up processes that sleep on dh_ctx->locked.
2992 * Free dh_ctx if refcnt is 0.
2993 */
2994 static void
2995 devmap_ctx_rele(devmap_handle_t *dhp)
2996 {
2997 struct devmap_ctx *devctx = dhp->dh_ctx;
2998 struct devmap_ctx *tmp;
2999 struct devmap_ctx *parent;
3000 timeout_id_t tid;
3001
3002 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
3003 "devmap_ctx_rele:start");
3004
3005 mutex_enter(&devmapctx_lock);
3006 mutex_enter(&devctx->lock);
3007
3008 ASSERT(devctx->refcnt > 0);
3009
3010 devctx->refcnt--;
3011
3012 /*
3013 * If no one is using the device, free up the devctx data.
3014 */
3015 if (devctx->refcnt == 0) {
3016 /*
3017 * Untimeout any threads using this mapping as they are about
3018 * to go away.
3019 */
3020 if (devctx->timeout != 0) {
3021 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3022 "devmap_ctx_rele:untimeout ctx->timeout");
3023
3024 tid = devctx->timeout;
3025 mutex_exit(&devctx->lock);
3026 (void) untimeout(tid);
3027 mutex_enter(&devctx->lock);
3028 }
3029
3030 devctx->oncpu = 0;
3031 cv_signal(&devctx->cv);
3032
3033 if (devmapctx_list == devctx)
3034 devmapctx_list = devctx->next;
3035 else {
3036 parent = devmapctx_list;
3037 for (tmp = devmapctx_list->next; tmp != NULL;
3038 tmp = tmp->next) {
3039 if (tmp == devctx) {
3040 parent->next = tmp->next;
3041 break;
3042 }
3043 parent = tmp;
3044 }
3045 }
3046 mutex_exit(&devctx->lock);
3047 mutex_destroy(&devctx->lock);
3048 cv_destroy(&devctx->cv);
3049 kmem_free(devctx, sizeof (struct devmap_ctx));
3050 } else
3051 mutex_exit(&devctx->lock);
3052
3053 mutex_exit(&devmapctx_lock);
3054 }
3055
3056 /*
3057 * devmap_load:
3058 * Marks a segdev segment or pages if offset->offset+len
3059 * is not the entire segment as nointercept and faults in
3060 * the pages in the range offset -> offset+len.
3061 */
3062 int
3063 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3064 uint_t rw)
3065 {
3066 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3067 struct as *asp = dhp->dh_seg->s_as;
3068 caddr_t addr;
3069 ulong_t size;
3070 ssize_t soff; /* offset from the beginning of the segment */
3071 int rc;
3072
3073 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3074 "devmap_load:start dhp=%p offset=%llx len=%lx",
3075 (void *)dhp, offset, len);
3076
3077 DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3078 (void *)dhp, offset, len));
3079
3080 /*
3081 * Hat layer only supports devload to process' context for which
3082 * the as lock is held. Verify here and return error if drivers
3083 * inadvertently call devmap_load on a wrong devmap handle.
3084 */
3085 if ((asp != &kas) && !AS_LOCK_HELD(asp))
3086 return (FC_MAKE_ERR(EINVAL));
3087
3088 soff = (ssize_t)(offset - dhp->dh_uoff);
3089 soff = round_down_p2(soff, PAGESIZE);
3090 if (soff < 0 || soff >= dhp->dh_len)
3091 return (FC_MAKE_ERR(EINVAL));
3092
3093 /*
3094 * Address and size must be page aligned. Len is set to the
3095 * number of bytes in the number of pages that are required to
3096 * support len. Offset is set to the byte offset of the first byte
3097 * of the page that contains offset.
3098 */
3099 len = round_up_p2(len, PAGESIZE);
3100
3101 /*
3102 * If len == 0, then calculate the size by getting
3103 * the number of bytes from offset to the end of the segment.
3104 */
3105 if (len == 0)
3106 size = dhp->dh_len - soff;
3107 else {
3108 size = len;
3109 if ((soff + size) > dhp->dh_len)
3110 return (FC_MAKE_ERR(EINVAL));
3111 }
3112
3113 /*
3114 * The address is offset bytes from the base address of
3115 * the segment.
3116 */
3117 addr = (caddr_t)(soff + dhp->dh_uvaddr);
3118
3119 HOLD_DHP_LOCK(dhp);
3120 rc = segdev_faultpages(asp->a_hat,
3121 dhp->dh_seg, addr, size, type, rw, dhp);
3122 RELE_DHP_LOCK(dhp);
3123 return (rc);
3124 }
3125
3126 int
3127 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3128 size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3129 {
3130 register devmap_handle_t *dhp;
3131 int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3132 size_t *, uint_t);
3133 int (*mmap)(dev_t, off_t, int);
3134 struct devmap_callback_ctl *callbackops;
3135 devmap_handle_t *dhp_head = NULL;
3136 devmap_handle_t *dhp_prev = NULL;
3137 devmap_handle_t *dhp_curr;
3138 caddr_t addr;
3139 int map_flag;
3140 int ret;
3141 ulong_t total_len;
3142 size_t map_len;
3143 size_t resid_len = len;
3144 offset_t map_off = off;
3145 struct devmap_softlock *slock = NULL;
3146
3147 #ifdef lint
3148 cred = cred;
3149 #endif
3150
3151 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3152 "devmap_setup:start off=%llx len=%lx", off, len);
3153 DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3154 off, len));
3155
3156 devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3157 mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3158
3159 /*
3160 * driver must provide devmap(9E) entry point in cb_ops to use the
3161 * devmap framework.
3162 */
3163 if (devmap == NULL || devmap == nulldev || devmap == nodev)
3164 return (EINVAL);
3165
3166 /*
3167 * To protect from an inadvertent entry because the devmap entry point
3168 * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3169 * mmap is NULL.
3170 */
3171 map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3172 if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3173 return (EINVAL);
3174
3175 /*
3176 * devmap allows mmap(2) to map multiple registers.
3177 * one devmap_handle is created for each register mapped.
3178 */
3179 for (total_len = 0; total_len < len; total_len += map_len) {
3180 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3181
3182 if (dhp_prev != NULL)
3183 dhp_prev->dh_next = dhp;
3184 else
3185 dhp_head = dhp;
3186 dhp_prev = dhp;
3187
3188 dhp->dh_prot = prot;
3189 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3190 dhp->dh_dev = dev;
3191 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3192 dhp->dh_uoff = map_off;
3193
3194 /*
3195 * Get mapping specific info from
3196 * the driver, such as rnumber, roff, len, callbackops,
3197 * accattrp and, if the mapping is for kernel memory,
3198 * ddi_umem_cookie.
3199 */
3200 if ((ret = cdev_devmap(dev, dhp, map_off,
3201 resid_len, &map_len, get_udatamodel())) != 0) {
3202 free_devmap_handle(dhp_head);
3203 return (ENXIO);
3204 }
3205
3206 if (map_len & PAGEOFFSET) {
3207 free_devmap_handle(dhp_head);
3208 return (EINVAL);
3209 }
3210
3211 callbackops = &dhp->dh_callbackops;
3212
3213 if ((callbackops->devmap_access == NULL) ||
3214 (callbackops->devmap_access == nulldev) ||
3215 (callbackops->devmap_access == nodev)) {
3216 /*
3217 * Normally devmap does not support MAP_PRIVATE unless
3218 * the drivers provide a valid devmap_access routine.
3219 */
3220 if ((flags & MAP_PRIVATE) != 0) {
3221 free_devmap_handle(dhp_head);
3222 return (EINVAL);
3223 }
3224 } else {
3225 /*
3226 * Initialize dhp_softlock and dh_ctx if the drivers
3227 * provide devmap_access.
3228 */
3229 dhp->dh_softlock = devmap_softlock_init(dev,
3230 (ulong_t)callbackops->devmap_access);
3231 dhp->dh_ctx = devmap_ctxinit(dev,
3232 (ulong_t)callbackops->devmap_access);
3233
3234 /*
3235 * segdev_fault can only work when all
3236 * dh_softlock in a multi-dhp mapping
3237 * are same. see comments in segdev_fault
3238 * This code keeps track of the first
3239 * dh_softlock allocated in slock and
3240 * compares all later allocations and if
3241 * not similar, returns an error.
3242 */
3243 if (slock == NULL)
3244 slock = dhp->dh_softlock;
3245 if (slock != dhp->dh_softlock) {
3246 free_devmap_handle(dhp_head);
3247 return (ENOTSUP);
3248 }
3249 }
3250
3251 map_off += map_len;
3252 resid_len -= map_len;
3253 }
3254
3255 /*
3256 * get the user virtual address and establish the mapping between
3257 * uvaddr and device physical address.
3258 */
3259 if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3260 != 0) {
3261 /*
3262 * free devmap handles if error during the mapping.
3263 */
3264 free_devmap_handle(dhp_head);
3265
3266 return (ret);
3267 }
3268
3269 /*
3270 * call the driver's devmap_map callback to do more after the mapping,
3271 * such as to allocate driver private data for context management.
3272 */
3273 dhp = dhp_head;
3274 map_off = off;
3275 addr = *addrp;
3276 while (dhp != NULL) {
3277 callbackops = &dhp->dh_callbackops;
3278 dhp->dh_uvaddr = addr;
3279 dhp_curr = dhp;
3280 if (callbackops->devmap_map != NULL) {
3281 ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3282 dev, flags, map_off,
3283 dhp->dh_len, &dhp->dh_pvtp);
3284 if (ret != 0) {
3285 struct segdev_data *sdp;
3286
3287 /*
3288 * call driver's devmap_unmap entry point
3289 * to free driver resources.
3290 */
3291 dhp = dhp_head;
3292 map_off = off;
3293 while (dhp != dhp_curr) {
3294 callbackops = &dhp->dh_callbackops;
3295 if (callbackops->devmap_unmap != NULL) {
3296 (*callbackops->devmap_unmap)(
3297 dhp, dhp->dh_pvtp,
3298 map_off, dhp->dh_len,
3299 NULL, NULL, NULL, NULL);
3300 }
3301 map_off += dhp->dh_len;
3302 dhp = dhp->dh_next;
3303 }
3304 sdp = dhp_head->dh_seg->s_data;
3305 sdp->devmap_data = NULL;
3306 free_devmap_handle(dhp_head);
3307 return (ENXIO);
3308 }
3309 }
3310 map_off += dhp->dh_len;
3311 addr += dhp->dh_len;
3312 dhp = dhp->dh_next;
3313 }
3314
3315 return (0);
3316 }
3317
3318 int
3319 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3320 off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3321 {
3322 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3323 "devmap_segmap:start");
3324 return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3325 (size_t)len, prot, maxprot, flags, cred));
3326 }
3327
3328 /*
3329 * Called from devmap_devmem_setup/remap to see if can use large pages for
3330 * this device mapping.
3331 * Also calculate the max. page size for this mapping.
3332 * this page size will be used in fault routine for
3333 * optimal page size calculations.
3334 */
3335 static void
3336 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3337 {
3338 ASSERT(dhp_is_devmem(dhp));
3339 dhp->dh_mmulevel = 0;
3340
3341 /*
3342 * use large page size only if:
3343 * 1. device memory.
3344 * 2. mmu supports multiple page sizes,
3345 * 3. Driver did not disallow it
3346 * 4. dhp length is at least as big as the large pagesize
3347 * 5. the uvaddr and pfn are large pagesize aligned
3348 */
3349 if (page_num_pagesizes() > 1 &&
3350 !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3351 ulong_t base;
3352 int level;
3353
3354 base = (ulong_t)ptob(dhp->dh_pfn);
3355 for (level = 1; level < page_num_pagesizes(); level++) {
3356 size_t pgsize = page_get_pagesize(level);
3357 if ((dhp->dh_len < pgsize) ||
3358 (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3359 base, pgsize))) {
3360 break;
3361 }
3362 }
3363 dhp->dh_mmulevel = level - 1;
3364 }
3365 if (dhp->dh_mmulevel > 0) {
3366 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3367 } else {
3368 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3369 }
3370 }
3371
3372 /*
3373 * Called by driver devmap routine to pass device specific info to
3374 * the framework. used for device memory mapping only.
3375 */
3376 int
3377 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3378 struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3379 size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3380 {
3381 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3382 ddi_acc_handle_t handle;
3383 ddi_map_req_t mr;
3384 ddi_acc_hdl_t *hp;
3385 int err;
3386
3387 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3388 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3389 (void *)dhp, roff, rnumber, (uint_t)len);
3390 DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3391 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3392
3393 /*
3394 * First to check if this function has been called for this dhp.
3395 */
3396 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3397 return (DDI_FAILURE);
3398
3399 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3400 return (DDI_FAILURE);
3401
3402 if (flags & DEVMAP_MAPPING_INVALID) {
3403 /*
3404 * Don't go up the tree to get pfn if the driver specifies
3405 * DEVMAP_MAPPING_INVALID in flags.
3406 *
3407 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3408 * remap permission.
3409 */
3410 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3411 return (DDI_FAILURE);
3412 }
3413 dhp->dh_pfn = PFN_INVALID;
3414 } else {
3415 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3416 if (handle == NULL)
3417 return (DDI_FAILURE);
3418
3419 hp = impl_acc_hdl_get(handle);
3420 hp->ah_vers = VERS_ACCHDL;
3421 hp->ah_dip = dip;
3422 hp->ah_rnumber = rnumber;
3423 hp->ah_offset = roff;
3424 hp->ah_len = len;
3425 if (accattrp != NULL)
3426 hp->ah_acc = *accattrp;
3427
3428 mr.map_op = DDI_MO_MAP_LOCKED;
3429 mr.map_type = DDI_MT_RNUMBER;
3430 mr.map_obj.rnumber = rnumber;
3431 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3432 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3433 mr.map_handlep = hp;
3434 mr.map_vers = DDI_MAP_VERSION;
3435
3436 /*
3437 * up the device tree to get pfn.
3438 * The rootnex_map_regspec() routine in nexus drivers has been
3439 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3440 */
3441 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3442 dhp->dh_hat_attr = hp->ah_hat_flags;
3443 impl_acc_hdl_free(handle);
3444
3445 if (err)
3446 return (DDI_FAILURE);
3447 }
3448 /* Should not be using devmem setup for memory pages */
3449 ASSERT(!pf_is_memory(dhp->dh_pfn));
3450
3451 /* Only some of the flags bits are settable by the driver */
3452 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3453 dhp->dh_len = ptob(btopr(len));
3454
3455 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3456 dhp->dh_roff = ptob(btop(roff));
3457
3458 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3459 devmap_devmem_large_page_setup(dhp);
3460 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3461 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3462
3463
3464 if (callbackops != NULL) {
3465 bcopy(callbackops, &dhp->dh_callbackops,
3466 sizeof (struct devmap_callback_ctl));
3467 }
3468
3469 /*
3470 * Initialize dh_lock if we want to do remap.
3471 */
3472 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3473 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3474 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3475 }
3476
3477 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3478
3479 return (DDI_SUCCESS);
3480 }
3481
3482 int
3483 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3484 uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3485 uint_t flags, ddi_device_acc_attr_t *accattrp)
3486 {
3487 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3488 ddi_acc_handle_t handle;
3489 ddi_map_req_t mr;
3490 ddi_acc_hdl_t *hp;
3491 pfn_t pfn;
3492 uint_t hat_flags;
3493 int err;
3494
3495 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3496 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3497 (void *)dhp, roff, rnumber, (uint_t)len);
3498 DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3499 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3500
3501 /*
3502 * Return failure if setup has not been done or no remap permission
3503 * has been granted during the setup.
3504 */
3505 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3506 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3507 return (DDI_FAILURE);
3508
3509 /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3510 if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3511 return (DDI_FAILURE);
3512
3513 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3514 return (DDI_FAILURE);
3515
3516 if (!(flags & DEVMAP_MAPPING_INVALID)) {
3517 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3518 if (handle == NULL)
3519 return (DDI_FAILURE);
3520 }
3521
3522 HOLD_DHP_LOCK(dhp);
3523
3524 /*
3525 * Unload the old mapping, so next fault will setup the new mappings
3526 * Do this while holding the dhp lock so other faults dont reestablish
3527 * the mappings
3528 */
3529 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3530 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3531
3532 if (flags & DEVMAP_MAPPING_INVALID) {
3533 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3534 dhp->dh_pfn = PFN_INVALID;
3535 } else {
3536 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3537 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3538 hp = impl_acc_hdl_get(handle);
3539 hp->ah_vers = VERS_ACCHDL;
3540 hp->ah_dip = dip;
3541 hp->ah_rnumber = rnumber;
3542 hp->ah_offset = roff;
3543 hp->ah_len = len;
3544 if (accattrp != NULL)
3545 hp->ah_acc = *accattrp;
3546
3547 mr.map_op = DDI_MO_MAP_LOCKED;
3548 mr.map_type = DDI_MT_RNUMBER;
3549 mr.map_obj.rnumber = rnumber;
3550 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3551 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3552 mr.map_handlep = hp;
3553 mr.map_vers = DDI_MAP_VERSION;
3554
3555 /*
3556 * up the device tree to get pfn.
3557 * The rootnex_map_regspec() routine in nexus drivers has been
3558 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3559 */
3560 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3561 hat_flags = hp->ah_hat_flags;
3562 impl_acc_hdl_free(handle);
3563 if (err) {
3564 RELE_DHP_LOCK(dhp);
3565 return (DDI_FAILURE);
3566 }
3567 /*
3568 * Store result of ddi_map first in local variables, as we do
3569 * not want to overwrite the existing dhp with wrong data.
3570 */
3571 dhp->dh_pfn = pfn;
3572 dhp->dh_hat_attr = hat_flags;
3573 }
3574
3575 /* clear the large page size flag */
3576 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3577
3578 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3579 dhp->dh_roff = ptob(btop(roff));
3580
3581 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3582 devmap_devmem_large_page_setup(dhp);
3583 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3584 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3585
3586 RELE_DHP_LOCK(dhp);
3587 return (DDI_SUCCESS);
3588 }
3589
3590 /*
3591 * called by driver devmap routine to pass kernel virtual address mapping
3592 * info to the framework. used only for kernel memory
3593 * allocated from ddi_umem_alloc().
3594 */
3595 int
3596 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3597 struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3598 offset_t off, size_t len, uint_t maxprot, uint_t flags,
3599 ddi_device_acc_attr_t *accattrp)
3600 {
3601 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3602 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3603
3604 #ifdef lint
3605 dip = dip;
3606 #endif
3607
3608 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3609 "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3610 (void *)dhp, off, cookie, len);
3611 DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3612 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3613
3614 if (cookie == NULL)
3615 return (DDI_FAILURE);
3616
3617 /* For UMEM_TRASH, this restriction is not needed */
3618 if ((off + len) > cp->size)
3619 return (DDI_FAILURE);
3620
3621 /* check if the cache attributes are supported */
3622 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3623 return (DDI_FAILURE);
3624
3625 /*
3626 * First to check if this function has been called for this dhp.
3627 */
3628 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3629 return (DDI_FAILURE);
3630
3631 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3632 return (DDI_FAILURE);
3633
3634 if (flags & DEVMAP_MAPPING_INVALID) {
3635 /*
3636 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3637 * remap permission.
3638 */
3639 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3640 return (DDI_FAILURE);
3641 }
3642 } else {
3643 dhp->dh_cookie = cookie;
3644 dhp->dh_roff = ptob(btop(off));
3645 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3646 /* set HAT cache attributes */
3647 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3648 /* set HAT endianess attributes */
3649 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3650 }
3651
3652 /*
3653 * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3654 * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3655 * create consistent mappings but our intention was to create
3656 * non-consistent mappings.
3657 *
3658 * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3659 * mappings.
3660 *
3661 * kernel exported memory: hat figures it out it's memory and always
3662 * creates consistent mappings.
3663 *
3664 * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3665 *
3666 * /dev/kmem: consistent mappings are created unless they are
3667 * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3668 * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3669 * mappings of /dev/kmem. See common/io/mem.c
3670 */
3671
3672 /* Only some of the flags bits are settable by the driver */
3673 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3674
3675 dhp->dh_len = ptob(btopr(len));
3676 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3677 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3678
3679 if (callbackops != NULL) {
3680 bcopy(callbackops, &dhp->dh_callbackops,
3681 sizeof (struct devmap_callback_ctl));
3682 }
3683 /*
3684 * Initialize dh_lock if we want to do remap.
3685 */
3686 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3687 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3688 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3689 }
3690
3691 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3692
3693 return (DDI_SUCCESS);
3694 }
3695
3696 int
3697 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3698 ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3699 uint_t flags, ddi_device_acc_attr_t *accattrp)
3700 {
3701 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3702 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3703
3704 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3705 "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3706 (void *)dhp, off, cookie, len);
3707 DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3708 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3709
3710 #ifdef lint
3711 dip = dip;
3712 accattrp = accattrp;
3713 #endif
3714 /*
3715 * Reture failure if setup has not been done or no remap permission
3716 * has been granted during the setup.
3717 */
3718 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3719 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3720 return (DDI_FAILURE);
3721
3722 /* No flags supported for remap yet */
3723 if (flags != 0)
3724 return (DDI_FAILURE);
3725
3726 /* check if the cache attributes are supported */
3727 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3728 return (DDI_FAILURE);
3729
3730 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3731 return (DDI_FAILURE);
3732
3733 /* For UMEM_TRASH, this restriction is not needed */
3734 if ((off + len) > cp->size)
3735 return (DDI_FAILURE);
3736
3737 HOLD_DHP_LOCK(dhp);
3738 /*
3739 * Unload the old mapping, so next fault will setup the new mappings
3740 * Do this while holding the dhp lock so other faults dont reestablish
3741 * the mappings
3742 */
3743 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3744 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3745
3746 dhp->dh_cookie = cookie;
3747 dhp->dh_roff = ptob(btop(off));
3748 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3749 /* set HAT cache attributes */
3750 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3751 /* set HAT endianess attributes */
3752 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3753
3754 /* clear the large page size flag */
3755 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3756
3757 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3758 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3759 RELE_DHP_LOCK(dhp);
3760 return (DDI_SUCCESS);
3761 }
3762
3763 /*
3764 * to set timeout value for the driver's context management callback, e.g.
3765 * devmap_access().
3766 */
3767 void
3768 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3769 {
3770 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3771
3772 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3773 "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3774 (void *)dhp, ticks);
3775 dhp->dh_timeout_length = ticks;
3776 }
3777
3778 int
3779 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3780 size_t len, uint_t type, uint_t rw)
3781 {
3782 #ifdef lint
3783 pvtp = pvtp;
3784 #endif
3785
3786 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3787 "devmap_default_access:start");
3788 return (devmap_load(dhp, off, len, type, rw));
3789 }
3790
3791 /*
3792 * segkmem_alloc() wrapper to allocate memory which is both
3793 * non-relocatable (for DR) and sharelocked, since the rest
3794 * of this segment driver requires it.
3795 */
3796 static void *
3797 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3798 {
3799 ASSERT(vmp != NULL);
3800 ASSERT(kvseg.s_base != NULL);
3801 vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3802 return (segkmem_alloc(vmp, size, vmflag));
3803 }
3804
3805 /*
3806 * This is where things are a bit incestuous with seg_kmem: unlike
3807 * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3808 * we need to do a bit of a dance around that to prevent duplication of
3809 * code until we decide to bite the bullet and implement a new kernel
3810 * segment for driver-allocated memory that is exported to user space.
3811 */
3812 static void
3813 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3814 {
3815 page_t *pp;
3816 caddr_t addr = inaddr;
3817 caddr_t eaddr;
3818 pgcnt_t npages = btopr(size);
3819
3820 ASSERT(vmp != NULL);
3821 ASSERT(kvseg.s_base != NULL);
3822 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3823
3824 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3825
3826 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3827 /*
3828 * Use page_find() instead of page_lookup() to find the page
3829 * since we know that it is hashed and has a shared lock.
3830 */
3831 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3832
3833 if (pp == NULL)
3834 panic("devmap_free_pages: page not found");
3835 if (!page_tryupgrade(pp)) {
3836 page_unlock(pp);
3837 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3838 SE_EXCL);
3839 if (pp == NULL)
3840 panic("devmap_free_pages: page already freed");
3841 }
3842 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3843 pp->p_lckcnt = 0;
3844 page_destroy(pp, 0);
3845 }
3846 page_unresv(npages);
3847
3848 if (vmp != NULL)
3849 vmem_free(vmp, inaddr, size);
3850 }
3851
3852 /*
3853 * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3854 * allocating non-pageable kmem in response to a ddi_umem_alloc()
3855 * default request. For now we allocate our own pages and we keep
3856 * them long-term sharelocked, since: A) the fault routines expect the
3857 * memory to already be locked; B) pageable umem is already long-term
3858 * locked; C) it's a lot of work to make it otherwise, particularly
3859 * since the nexus layer expects the pages to never fault. An RFE is to
3860 * not keep the pages long-term locked, but instead to be able to
3861 * take faults on them and simply look them up in kvp in case we
3862 * fault on them. Even then, we must take care not to let pageout
3863 * steal them from us since the data must remain resident; if we
3864 * do this we must come up with some way to pin the pages to prevent
3865 * faults while a driver is doing DMA to/from them.
3866 */
3867 static void *
3868 devmap_umem_alloc_np(size_t size, size_t flags)
3869 {
3870 void *buf;
3871 int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3872
3873 buf = vmem_alloc(umem_np_arena, size, vmflags);
3874 if (buf != NULL)
3875 bzero(buf, size);
3876 return (buf);
3877 }
3878
3879 static void
3880 devmap_umem_free_np(void *addr, size_t size)
3881 {
3882 vmem_free(umem_np_arena, addr, size);
3883 }
3884
3885 /*
3886 * allocate page aligned kernel memory for exporting to user land.
3887 * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3888 * to find a user virtual address that is in same color as the address
3889 * allocated here.
3890 */
3891 void *
3892 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3893 {
3894 register size_t len = ptob(btopr(size));
3895 void *buf = NULL;
3896 struct ddi_umem_cookie *cp;
3897 int iflags = 0;
3898
3899 *cookie = NULL;
3900
3901 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3902 "devmap_umem_alloc:start");
3903 if (len == 0)
3904 return ((void *)NULL);
3905
3906 /*
3907 * allocate cookie
3908 */
3909 if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3910 flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3911 ASSERT(flags & DDI_UMEM_NOSLEEP);
3912 return ((void *)NULL);
3913 }
3914
3915 if (flags & DDI_UMEM_PAGEABLE) {
3916 /* Only one of the flags is allowed */
3917 ASSERT(!(flags & DDI_UMEM_TRASH));
3918 /* initialize resource with 0 */
3919 iflags = KPD_ZERO;
3920
3921 /*
3922 * to allocate unlocked pageable memory, use segkp_get() to
3923 * create a segkp segment. Since segkp can only service kas,
3924 * other segment drivers such as segdev have to do
3925 * as_fault(segkp, SOFTLOCK) in its fault routine,
3926 */
3927 if (flags & DDI_UMEM_NOSLEEP)
3928 iflags |= KPD_NOWAIT;
3929
3930 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3931 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3932 return ((void *)NULL);
3933 }
3934 cp->type = KMEM_PAGEABLE;
3935 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3936 cp->locked = 0;
3937 } else if (flags & DDI_UMEM_TRASH) {
3938 /* Only one of the flags is allowed */
3939 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3940 cp->type = UMEM_TRASH;
3941 buf = NULL;
3942 } else {
3943 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3944 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3945 return ((void *)NULL);
3946 }
3947
3948 cp->type = KMEM_NON_PAGEABLE;
3949 }
3950
3951 /*
3952 * need to save size here. size will be used when
3953 * we do kmem_free.
3954 */
3955 cp->size = len;
3956 cp->cvaddr = (caddr_t)buf;
3957
3958 *cookie = (void *)cp;
3959 return (buf);
3960 }
3961
3962 void
3963 ddi_umem_free(ddi_umem_cookie_t cookie)
3964 {
3965 struct ddi_umem_cookie *cp;
3966
3967 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3968 "devmap_umem_free:start");
3969
3970 /*
3971 * if cookie is NULL, no effects on the system
3972 */
3973 if (cookie == NULL)
3974 return;
3975
3976 cp = (struct ddi_umem_cookie *)cookie;
3977
3978 switch (cp->type) {
3979 case KMEM_PAGEABLE :
3980 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3981 /*
3982 * Check if there are still any pending faults on the cookie
3983 * while the driver is deleting it,
3984 * XXX - could change to an ASSERT but wont catch errant drivers
3985 */
3986 mutex_enter(&cp->lock);
3987 if (cp->locked) {
3988 mutex_exit(&cp->lock);
3989 panic("ddi_umem_free for cookie with pending faults %p",
3990 (void *)cp);
3991 }
3992
3993 segkp_release(segkp, cp->cvaddr);
3994
3995 /*
3996 * release mutex associated with this cookie.
3997 */
3998 mutex_destroy(&cp->lock);
3999 break;
4000 case KMEM_NON_PAGEABLE :
4001 ASSERT(cp->cvaddr != NULL && cp->size != 0);
4002 devmap_umem_free_np(cp->cvaddr, cp->size);
4003 break;
4004 case UMEM_TRASH :
4005 break;
4006 case UMEM_LOCKED :
4007 /* Callers should use ddi_umem_unlock for this type */
4008 ddi_umem_unlock(cookie);
4009 /* Frees the cookie too */
4010 return;
4011 default:
4012 /* panic so we can diagnose the underlying cause */
4013 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4014 cp->type);
4015 }
4016
4017 kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4018 }
4019
4020
4021 static int
4022 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4023 {
4024 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4025
4026 /*
4027 * It looks as if it is always mapped shared
4028 */
4029 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4030 "segdev_getmemid:start");
4031 memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4032 memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4033 return (0);
4034 }
4035
4036 /*ARGSUSED*/
4037 static lgrp_mem_policy_info_t *
4038 segdev_getpolicy(struct seg *seg, caddr_t addr)
4039 {
4040 return (NULL);
4041 }
4042
4043 /*ARGSUSED*/
4044 static int
4045 segdev_capable(struct seg *seg, segcapability_t capability)
4046 {
4047 return (0);
4048 }
4049
4050 /*
4051 * ddi_umem_alloc() non-pageable quantum cache max size.
4052 * This is just a SWAG.
4053 */
4054 #define DEVMAP_UMEM_QUANTUM (8*PAGESIZE)
4055
4056 /*
4057 * Initialize seg_dev from boot. This routine sets up the trash page
4058 * and creates the umem_np_arena used to back non-pageable memory
4059 * requests.
4060 */
4061 void
4062 segdev_init(void)
4063 {
4064 struct seg kseg;
4065
4066 umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4067 devmap_alloc_pages, devmap_free_pages, heap_arena,
4068 DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4069
4070 kseg.s_as = &kas;
4071 trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4072 PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4073 if (trashpp == NULL)
4074 panic("segdev_init: failed to create trash page");
4075 pagezero(trashpp, 0, PAGESIZE);
4076 page_downgrade(trashpp);
4077 }
4078
4079 /*
4080 * Invoke platform-dependent support routines so that /proc can have
4081 * the platform code deal with curious hardware.
4082 */
4083 int
4084 segdev_copyfrom(struct seg *seg,
4085 caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4086 {
4087 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4088 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4089
4090 return (e_ddi_copyfromdev(sp->s_dip,
4091 (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4092 }
4093
4094 int
4095 segdev_copyto(struct seg *seg,
4096 caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4097 {
4098 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4099 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4100
4101 return (e_ddi_copytodev(sp->s_dip,
4102 (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4103 }