1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved.
27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
28 */
29
30 /*
31 * x86 root nexus driver
32 */
33
34 #include <sys/sysmacros.h>
35 #include <sys/conf.h>
36 #include <sys/autoconf.h>
37 #include <sys/sysmacros.h>
38 #include <sys/debug.h>
39 #include <sys/psw.h>
40 #include <sys/ddidmareq.h>
41 #include <sys/promif.h>
42 #include <sys/devops.h>
43 #include <sys/kmem.h>
44 #include <sys/cmn_err.h>
45 #include <vm/seg.h>
46 #include <vm/seg_kmem.h>
47 #include <vm/seg_dev.h>
48 #include <sys/vmem.h>
49 #include <sys/mman.h>
50 #include <vm/hat.h>
51 #include <vm/as.h>
52 #include <vm/page.h>
53 #include <sys/avintr.h>
54 #include <sys/errno.h>
55 #include <sys/modctl.h>
56 #include <sys/ddi_impldefs.h>
57 #include <sys/sunddi.h>
58 #include <sys/sunndi.h>
59 #include <sys/mach_intr.h>
60 #include <sys/psm.h>
61 #include <sys/ontrap.h>
62 #include <sys/atomic.h>
63 #include <sys/sdt.h>
64 #include <sys/rootnex.h>
65 #include <vm/hat_i86.h>
66 #include <sys/ddifm.h>
67 #include <sys/ddi_isa.h>
68 #include <sys/apic.h>
69
70 #ifdef __xpv
71 #include <sys/bootinfo.h>
72 #include <sys/hypervisor.h>
73 #include <sys/bootconf.h>
74 #include <vm/kboot_mmu.h>
75 #endif
76
77 #if defined(__amd64) && !defined(__xpv)
78 #include <sys/immu.h>
79 #endif
80
81
82 /*
83 * enable/disable extra checking of function parameters. Useful for debugging
84 * drivers.
85 */
86 #ifdef DEBUG
87 int rootnex_alloc_check_parms = 1;
88 int rootnex_bind_check_parms = 1;
89 int rootnex_bind_check_inuse = 1;
90 int rootnex_unbind_verify_buffer = 0;
91 int rootnex_sync_check_parms = 1;
92 #else
93 int rootnex_alloc_check_parms = 0;
94 int rootnex_bind_check_parms = 0;
95 int rootnex_bind_check_inuse = 0;
96 int rootnex_unbind_verify_buffer = 0;
97 int rootnex_sync_check_parms = 0;
98 #endif
99
100 boolean_t rootnex_dmar_not_setup;
101
102 /* Master Abort and Target Abort panic flag */
103 int rootnex_fm_ma_ta_panic_flag = 0;
104
105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
106 int rootnex_bind_fail = 1;
107 int rootnex_bind_warn = 1;
108 uint8_t *rootnex_warn_list;
109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
110 #define ROOTNEX_BIND_WARNING (0x1 << 0)
111
112 /*
113 * revert back to old broken behavior of always sync'ing entire copy buffer.
114 * This is useful if be have a buggy driver which doesn't correctly pass in
115 * the offset and size into ddi_dma_sync().
116 */
117 int rootnex_sync_ignore_params = 0;
118
119 /*
120 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
121 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
122 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
123 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
124 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
125 * (< 8K). We will still need to allocate the copy buffer during bind though
126 * (if we need one). These can only be modified in /etc/system before rootnex
127 * attach.
128 */
129 #if defined(__amd64)
130 int rootnex_prealloc_cookies = 65;
131 int rootnex_prealloc_windows = 4;
132 int rootnex_prealloc_copybuf = 2;
133 #else
134 int rootnex_prealloc_cookies = 33;
135 int rootnex_prealloc_windows = 4;
136 int rootnex_prealloc_copybuf = 2;
137 #endif
138
139 /* driver global state */
140 static rootnex_state_t *rootnex_state;
141
142 #ifdef DEBUG
143 /* shortcut to rootnex counters */
144 static uint64_t *rootnex_cnt;
145 #endif
146
147 /*
148 * XXX - does x86 even need these or are they left over from the SPARC days?
149 */
150 /* statically defined integer/boolean properties for the root node */
151 static rootnex_intprop_t rootnex_intprp[] = {
152 { "PAGESIZE", PAGESIZE },
153 { "MMU_PAGESIZE", MMU_PAGESIZE },
154 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET },
155 { DDI_RELATIVE_ADDRESSING, 1 },
156 };
157 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
158
159 /*
160 * If we're dom0, we're using a real device so we need to load
161 * the cookies with MFNs instead of PFNs.
162 */
163 #ifdef __xpv
164 typedef maddr_t rootnex_addr_t;
165 #define ROOTNEX_PADDR_TO_RBASE(pa) \
166 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
167 #else
168 typedef paddr_t rootnex_addr_t;
169 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa)
170 #endif
171
172 static struct cb_ops rootnex_cb_ops = {
173 nodev, /* open */
174 nodev, /* close */
175 nodev, /* strategy */
176 nodev, /* print */
177 nodev, /* dump */
178 nodev, /* read */
179 nodev, /* write */
180 nodev, /* ioctl */
181 nodev, /* devmap */
182 nodev, /* mmap */
183 nodev, /* segmap */
184 nochpoll, /* chpoll */
185 ddi_prop_op, /* cb_prop_op */
186 NULL, /* struct streamtab */
187 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
188 CB_REV, /* Rev */
189 nodev, /* cb_aread */
190 nodev /* cb_awrite */
191 };
192
193 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
194 off_t offset, off_t len, caddr_t *vaddrp);
195 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
196 struct hat *hat, struct seg *seg, caddr_t addr,
197 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
198 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
199 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
200 ddi_dma_handle_t *handlep);
201 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
202 ddi_dma_handle_t handle);
203 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
204 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
205 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
206 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
207 ddi_dma_handle_t handle);
208 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
209 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
210 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
211 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
212 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
213 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
214 ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
215 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
216 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
217 ddi_ctl_enum_t ctlop, void *arg, void *result);
218 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
219 ddi_iblock_cookie_t *ibc);
220 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
221 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
222 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
223 void *);
224 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
225
226 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
227 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
228 ddi_dma_handle_t *handlep);
229 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
230 ddi_dma_handle_t handle);
231 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
232 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
233 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
234 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
235 ddi_dma_handle_t handle);
236 #if defined(__amd64) && !defined(__xpv)
237 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
238 ddi_dma_handle_t handle);
239 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
240 ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
241 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
242 ddi_dma_cookie_t *cookiep, uint_t ccount);
243 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
244 ddi_dma_handle_t handle);
245 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
246 #endif
247 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
248 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
249 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
250 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
251 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
252
253 #if defined(__amd64) && !defined(__xpv)
254 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
255 ddi_dma_handle_t handle, void *v);
256 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
257 ddi_dma_handle_t handle);
258 #endif
259
260
261 static struct bus_ops rootnex_bus_ops = {
262 BUSO_REV,
263 rootnex_map,
264 NULL,
265 NULL,
266 NULL,
267 rootnex_map_fault,
268 0,
269 rootnex_dma_allochdl,
270 rootnex_dma_freehdl,
271 rootnex_dma_bindhdl,
272 rootnex_dma_unbindhdl,
273 rootnex_dma_sync,
274 rootnex_dma_win,
275 rootnex_dma_mctl,
276 rootnex_ctlops,
277 ddi_bus_prop_op,
278 i_ddi_rootnex_get_eventcookie,
279 i_ddi_rootnex_add_eventcall,
280 i_ddi_rootnex_remove_eventcall,
281 i_ddi_rootnex_post_event,
282 0, /* bus_intr_ctl */
283 0, /* bus_config */
284 0, /* bus_unconfig */
285 rootnex_fm_init, /* bus_fm_init */
286 NULL, /* bus_fm_fini */
287 NULL, /* bus_fm_access_enter */
288 NULL, /* bus_fm_access_exit */
289 NULL, /* bus_powr */
290 rootnex_intr_ops /* bus_intr_op */
291 };
292
293 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
294 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
295 static int rootnex_quiesce(dev_info_t *dip);
296
297 static struct dev_ops rootnex_ops = {
298 DEVO_REV,
299 0,
300 ddi_no_info,
301 nulldev,
302 nulldev,
303 rootnex_attach,
304 rootnex_detach,
305 nulldev,
306 &rootnex_cb_ops,
307 &rootnex_bus_ops,
308 NULL,
309 rootnex_quiesce, /* quiesce */
310 };
311
312 static struct modldrv rootnex_modldrv = {
313 &mod_driverops,
314 "i86pc root nexus",
315 &rootnex_ops
316 };
317
318 static struct modlinkage rootnex_modlinkage = {
319 MODREV_1,
320 { (void *)&rootnex_modldrv, NULL }
321 };
322
323 #if defined(__amd64) && !defined(__xpv)
324 static iommulib_nexops_t iommulib_nexops = {
325 IOMMU_NEXOPS_VERSION,
326 "Rootnex IOMMU ops Vers 1.1",
327 NULL,
328 rootnex_coredma_allochdl,
329 rootnex_coredma_freehdl,
330 rootnex_coredma_bindhdl,
331 rootnex_coredma_unbindhdl,
332 rootnex_coredma_reset_cookies,
333 rootnex_coredma_get_cookies,
334 rootnex_coredma_set_cookies,
335 rootnex_coredma_clear_cookies,
336 rootnex_coredma_get_sleep_flags,
337 rootnex_coredma_sync,
338 rootnex_coredma_win,
339 rootnex_coredma_hdl_setprivate,
340 rootnex_coredma_hdl_getprivate
341 };
342 #endif
343
344 /*
345 * extern hacks
346 */
347 extern struct seg_ops segdev_ops;
348 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
349 #ifdef DDI_MAP_DEBUG
350 extern int ddi_map_debug_flag;
351 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf
352 #endif
353 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
354 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
355 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
356 psm_intr_op_t, int *);
357 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
358 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
359
360 /*
361 * Use device arena to use for device control register mappings.
362 * Various kernel memory walkers (debugger, dtrace) need to know
363 * to avoid this address range to prevent undesired device activity.
364 */
365 extern void *device_arena_alloc(size_t size, int vm_flag);
366 extern void device_arena_free(void * vaddr, size_t size);
367
368
369 /*
370 * Internal functions
371 */
372 static int rootnex_dma_init();
373 static void rootnex_add_props(dev_info_t *);
374 static int rootnex_ctl_reportdev(dev_info_t *dip);
375 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
376 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
377 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
378 static int rootnex_map_handle(ddi_map_req_t *mp);
379 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
380 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
381 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
382 ddi_dma_attr_t *attr);
383 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
384 rootnex_sglinfo_t *sglinfo);
385 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
386 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
387 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
388 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
389 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
390 rootnex_dma_t *dma, ddi_dma_attr_t *attr);
391 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
392 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
393 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
394 static void rootnex_teardown_windows(rootnex_dma_t *dma);
395 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
396 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
397 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
398 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
399 size_t *copybuf_used, page_t **cur_pp);
400 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
401 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
402 ddi_dma_attr_t *attr, off_t cur_offset);
403 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
404 rootnex_dma_t *dma, rootnex_window_t **windowp,
405 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
406 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
407 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
408 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
409 off_t offset, size_t size, uint_t cache_flags);
410 static int rootnex_verify_buffer(rootnex_dma_t *dma);
411 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
412 const void *comp_addr, const void *not_used);
413 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
414 rootnex_sglinfo_t *sglinfo);
415 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
416
417 /*
418 * _init()
419 *
420 */
421 int
422 _init(void)
423 {
424
425 rootnex_state = NULL;
426 return (mod_install(&rootnex_modlinkage));
427 }
428
429
430 /*
431 * _info()
432 *
433 */
434 int
435 _info(struct modinfo *modinfop)
436 {
437 return (mod_info(&rootnex_modlinkage, modinfop));
438 }
439
440
441 /*
442 * _fini()
443 *
444 */
445 int
446 _fini(void)
447 {
448 return (EBUSY);
449 }
450
451
452 /*
453 * rootnex_attach()
454 *
455 */
456 static int
457 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
458 {
459 int fmcap;
460 int e;
461
462 switch (cmd) {
463 case DDI_ATTACH:
464 break;
465 case DDI_RESUME:
466 #if defined(__amd64) && !defined(__xpv)
467 return (immu_unquiesce());
468 #else
469 return (DDI_SUCCESS);
470 #endif
471 default:
472 return (DDI_FAILURE);
473 }
474
475 /*
476 * We should only have one instance of rootnex. Save it away since we
477 * don't have an easy way to get it back later.
478 */
479 ASSERT(rootnex_state == NULL);
480 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
481
482 rootnex_state->r_dip = dip;
483 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
484 rootnex_state->r_reserved_msg_printed = B_FALSE;
485 #ifdef DEBUG
486 rootnex_cnt = &rootnex_state->r_counters[0];
487 #endif
488
489 /*
490 * Set minimum fm capability level for i86pc platforms and then
491 * initialize error handling. Since we're the rootnex, we don't
492 * care what's returned in the fmcap field.
493 */
494 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
495 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
496 fmcap = ddi_system_fmcap;
497 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
498
499 /* initialize DMA related state */
500 e = rootnex_dma_init();
501 if (e != DDI_SUCCESS) {
502 kmem_free(rootnex_state, sizeof (rootnex_state_t));
503 return (DDI_FAILURE);
504 }
505
506 /* Add static root node properties */
507 rootnex_add_props(dip);
508
509 /* since we can't call ddi_report_dev() */
510 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
511
512 /* Initialize rootnex event handle */
513 i_ddi_rootnex_init_events(dip);
514
515 #if defined(__amd64) && !defined(__xpv)
516 e = iommulib_nexus_register(dip, &iommulib_nexops,
517 &rootnex_state->r_iommulib_handle);
518
519 ASSERT(e == DDI_SUCCESS);
520 #endif
521
522 return (DDI_SUCCESS);
523 }
524
525
526 /*
527 * rootnex_detach()
528 *
529 */
530 /*ARGSUSED*/
531 static int
532 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
533 {
534 switch (cmd) {
535 case DDI_SUSPEND:
536 #if defined(__amd64) && !defined(__xpv)
537 return (immu_quiesce());
538 #else
539 return (DDI_SUCCESS);
540 #endif
541 default:
542 return (DDI_FAILURE);
543 }
544 /*NOTREACHED*/
545
546 }
547
548
549 /*
550 * rootnex_dma_init()
551 *
552 */
553 /*ARGSUSED*/
554 static int
555 rootnex_dma_init()
556 {
557 size_t bufsize;
558
559
560 /*
561 * size of our cookie/window/copybuf state needed in dma bind that we
562 * pre-alloc in dma_alloc_handle
563 */
564 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
565 rootnex_state->r_prealloc_size =
566 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
567 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
568 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
569
570 /*
571 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
572 * allocate 16 extra bytes for struct pointer alignment
573 * (p->dmai_private & dma->dp_prealloc_buffer)
574 */
575 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
576 rootnex_state->r_prealloc_size + 0x10;
577 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
578 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
579 if (rootnex_state->r_dmahdl_cache == NULL) {
580 return (DDI_FAILURE);
581 }
582
583 /*
584 * allocate array to track which major numbers we have printed warnings
585 * for.
586 */
587 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
588 KM_SLEEP);
589
590 return (DDI_SUCCESS);
591 }
592
593
594 /*
595 * rootnex_add_props()
596 *
597 */
598 static void
599 rootnex_add_props(dev_info_t *dip)
600 {
601 rootnex_intprop_t *rpp;
602 int i;
603
604 /* Add static integer/boolean properties to the root node */
605 rpp = rootnex_intprp;
606 for (i = 0; i < NROOT_INTPROPS; i++) {
607 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
608 rpp[i].prop_name, rpp[i].prop_value);
609 }
610 }
611
612
613
614 /*
615 * *************************
616 * ctlops related routines
617 * *************************
618 */
619
620 /*
621 * rootnex_ctlops()
622 *
623 */
624 /*ARGSUSED*/
625 static int
626 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
627 void *arg, void *result)
628 {
629 int n, *ptr;
630 struct ddi_parent_private_data *pdp;
631
632 switch (ctlop) {
633 case DDI_CTLOPS_DMAPMAPC:
634 /*
635 * Return 'partial' to indicate that dma mapping
636 * has to be done in the main MMU.
637 */
638 return (DDI_DMA_PARTIAL);
639
640 case DDI_CTLOPS_BTOP:
641 /*
642 * Convert byte count input to physical page units.
643 * (byte counts that are not a page-size multiple
644 * are rounded down)
645 */
646 *(ulong_t *)result = btop(*(ulong_t *)arg);
647 return (DDI_SUCCESS);
648
649 case DDI_CTLOPS_PTOB:
650 /*
651 * Convert size in physical pages to bytes
652 */
653 *(ulong_t *)result = ptob(*(ulong_t *)arg);
654 return (DDI_SUCCESS);
655
656 case DDI_CTLOPS_BTOPR:
657 /*
658 * Convert byte count input to physical page units
659 * (byte counts that are not a page-size multiple
660 * are rounded up)
661 */
662 *(ulong_t *)result = btopr(*(ulong_t *)arg);
663 return (DDI_SUCCESS);
664
665 case DDI_CTLOPS_INITCHILD:
666 return (impl_ddi_sunbus_initchild(arg));
667
668 case DDI_CTLOPS_UNINITCHILD:
669 impl_ddi_sunbus_removechild(arg);
670 return (DDI_SUCCESS);
671
672 case DDI_CTLOPS_REPORTDEV:
673 return (rootnex_ctl_reportdev(rdip));
674
675 case DDI_CTLOPS_IOMIN:
676 /*
677 * Nothing to do here but reflect back..
678 */
679 return (DDI_SUCCESS);
680
681 case DDI_CTLOPS_REGSIZE:
682 case DDI_CTLOPS_NREGS:
683 break;
684
685 case DDI_CTLOPS_SIDDEV:
686 if (ndi_dev_is_prom_node(rdip))
687 return (DDI_SUCCESS);
688 if (ndi_dev_is_persistent_node(rdip))
689 return (DDI_SUCCESS);
690 return (DDI_FAILURE);
691
692 case DDI_CTLOPS_POWER:
693 return ((*pm_platform_power)((power_req_t *)arg));
694
695 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
696 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
697 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
698 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
699 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
700 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
701 if (!rootnex_state->r_reserved_msg_printed) {
702 rootnex_state->r_reserved_msg_printed = B_TRUE;
703 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
704 "1 or more reserved/obsolete operations.");
705 }
706 return (DDI_FAILURE);
707
708 default:
709 return (DDI_FAILURE);
710 }
711 /*
712 * The rest are for "hardware" properties
713 */
714 if ((pdp = ddi_get_parent_data(rdip)) == NULL)
715 return (DDI_FAILURE);
716
717 if (ctlop == DDI_CTLOPS_NREGS) {
718 ptr = (int *)result;
719 *ptr = pdp->par_nreg;
720 } else {
721 off_t *size = (off_t *)result;
722
723 ptr = (int *)arg;
724 n = *ptr;
725 if (n >= pdp->par_nreg) {
726 return (DDI_FAILURE);
727 }
728 *size = (off_t)pdp->par_reg[n].regspec_size;
729 }
730 return (DDI_SUCCESS);
731 }
732
733
734 /*
735 * rootnex_ctl_reportdev()
736 *
737 */
738 static int
739 rootnex_ctl_reportdev(dev_info_t *dev)
740 {
741 int i, n, len, f_len = 0;
742 char *buf;
743
744 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
745 f_len += snprintf(buf, REPORTDEV_BUFSIZE,
746 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
747 len = strlen(buf);
748
749 for (i = 0; i < sparc_pd_getnreg(dev); i++) {
750
751 struct regspec *rp = sparc_pd_getreg(dev, i);
752
753 if (i == 0)
754 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
755 ": ");
756 else
757 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
758 " and ");
759 len = strlen(buf);
760
761 switch (rp->regspec_bustype) {
762
763 case BTEISA:
764 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
765 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
766 break;
767
768 case BTISA:
769 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
770 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
771 break;
772
773 default:
774 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
775 "space %x offset %x",
776 rp->regspec_bustype, rp->regspec_addr);
777 break;
778 }
779 len = strlen(buf);
780 }
781 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
782 int pri;
783
784 if (i != 0) {
785 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
786 ",");
787 len = strlen(buf);
788 }
789 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
790 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
791 " sparc ipl %d", pri);
792 len = strlen(buf);
793 }
794 #ifdef DEBUG
795 if (f_len + 1 >= REPORTDEV_BUFSIZE) {
796 cmn_err(CE_NOTE, "next message is truncated: "
797 "printed length 1024, real length %d", f_len);
798 }
799 #endif /* DEBUG */
800 cmn_err(CE_CONT, "?%s\n", buf);
801 kmem_free(buf, REPORTDEV_BUFSIZE);
802 return (DDI_SUCCESS);
803 }
804
805
806 /*
807 * ******************
808 * map related code
809 * ******************
810 */
811
812 /*
813 * rootnex_map()
814 *
815 */
816 static int
817 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
818 off_t len, caddr_t *vaddrp)
819 {
820 struct regspec *rp, tmp_reg;
821 ddi_map_req_t mr = *mp; /* Get private copy of request */
822 int error;
823
824 mp = &mr;
825
826 switch (mp->map_op) {
827 case DDI_MO_MAP_LOCKED:
828 case DDI_MO_UNMAP:
829 case DDI_MO_MAP_HANDLE:
830 break;
831 default:
832 #ifdef DDI_MAP_DEBUG
833 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
834 mp->map_op);
835 #endif /* DDI_MAP_DEBUG */
836 return (DDI_ME_UNIMPLEMENTED);
837 }
838
839 if (mp->map_flags & DDI_MF_USER_MAPPING) {
840 #ifdef DDI_MAP_DEBUG
841 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
842 #endif /* DDI_MAP_DEBUG */
843 return (DDI_ME_UNIMPLEMENTED);
844 }
845
846 /*
847 * First, if given an rnumber, convert it to a regspec...
848 * (Presumably, this is on behalf of a child of the root node?)
849 */
850
851 if (mp->map_type == DDI_MT_RNUMBER) {
852
853 int rnumber = mp->map_obj.rnumber;
854 #ifdef DDI_MAP_DEBUG
855 static char *out_of_range =
856 "rootnex_map: Out of range rnumber <%d>, device <%s>";
857 #endif /* DDI_MAP_DEBUG */
858
859 rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
860 if (rp == NULL) {
861 #ifdef DDI_MAP_DEBUG
862 cmn_err(CE_WARN, out_of_range, rnumber,
863 ddi_get_name(rdip));
864 #endif /* DDI_MAP_DEBUG */
865 return (DDI_ME_RNUMBER_RANGE);
866 }
867
868 /*
869 * Convert the given ddi_map_req_t from rnumber to regspec...
870 */
871
872 mp->map_type = DDI_MT_REGSPEC;
873 mp->map_obj.rp = rp;
874 }
875
876 /*
877 * Adjust offset and length correspnding to called values...
878 * XXX: A non-zero length means override the one in the regspec
879 * XXX: (regardless of what's in the parent's range?)
880 */
881
882 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */
883 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */
884
885 #ifdef DDI_MAP_DEBUG
886 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
887 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
888 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
889 len, mp->map_handlep);
890 #endif /* DDI_MAP_DEBUG */
891
892 /*
893 * I/O or memory mapping:
894 *
895 * <bustype=0, addr=x, len=x>: memory
896 * <bustype=1, addr=x, len=x>: i/o
897 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
898 */
899
900 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
901 cmn_err(CE_WARN, "<%s,%s> invalid register spec"
902 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
903 ddi_get_name(rdip), rp->regspec_bustype,
904 rp->regspec_addr, rp->regspec_size);
905 return (DDI_ME_INVAL);
906 }
907
908 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
909 /*
910 * compatibility i/o mapping
911 */
912 rp->regspec_bustype += (uint_t)offset;
913 } else {
914 /*
915 * Normal memory or i/o mapping
916 */
917 rp->regspec_addr += (uint_t)offset;
918 }
919
920 if (len != 0)
921 rp->regspec_size = (uint_t)len;
922
923 #ifdef DDI_MAP_DEBUG
924 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
925 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
926 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
927 offset, len, mp->map_handlep);
928 #endif /* DDI_MAP_DEBUG */
929
930 /*
931 * Apply any parent ranges at this level, if applicable.
932 * (This is where nexus specific regspec translation takes place.
933 * Use of this function is implicit agreement that translation is
934 * provided via ddi_apply_range.)
935 */
936
937 #ifdef DDI_MAP_DEBUG
938 ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
939 ddi_get_name(dip), ddi_get_name(rdip));
940 #endif /* DDI_MAP_DEBUG */
941
942 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
943 return (error);
944
945 switch (mp->map_op) {
946 case DDI_MO_MAP_LOCKED:
947
948 /*
949 * Set up the locked down kernel mapping to the regspec...
950 */
951
952 return (rootnex_map_regspec(mp, vaddrp));
953
954 case DDI_MO_UNMAP:
955
956 /*
957 * Release mapping...
958 */
959
960 return (rootnex_unmap_regspec(mp, vaddrp));
961
962 case DDI_MO_MAP_HANDLE:
963
964 return (rootnex_map_handle(mp));
965
966 default:
967 return (DDI_ME_UNIMPLEMENTED);
968 }
969 }
970
971
972 /*
973 * rootnex_map_fault()
974 *
975 * fault in mappings for requestors
976 */
977 /*ARGSUSED*/
978 static int
979 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
980 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
981 uint_t lock)
982 {
983
984 #ifdef DDI_MAP_DEBUG
985 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
986 ddi_map_debug(" Seg <%s>\n",
987 seg->s_ops == &segdev_ops ? "segdev" :
988 seg == &kvseg ? "segkmem" : "NONE!");
989 #endif /* DDI_MAP_DEBUG */
990
991 /*
992 * This is all terribly broken, but it is a start
993 *
994 * XXX Note that this test means that segdev_ops
995 * must be exported from seg_dev.c.
996 * XXX What about devices with their own segment drivers?
997 */
998 if (seg->s_ops == &segdev_ops) {
999 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1000
1001 if (hat == NULL) {
1002 /*
1003 * This is one plausible interpretation of
1004 * a null hat i.e. use the first hat on the
1005 * address space hat list which by convention is
1006 * the hat of the system MMU. At alternative
1007 * would be to panic .. this might well be better ..
1008 */
1009 ASSERT(AS_READ_HELD(seg->s_as));
1010 hat = seg->s_as->a_hat;
1011 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1012 }
1013 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1014 (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1015 } else if (seg == &kvseg && dp == NULL) {
1016 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1017 HAT_LOAD_LOCK);
1018 } else
1019 return (DDI_FAILURE);
1020 return (DDI_SUCCESS);
1021 }
1022
1023
1024 /*
1025 * rootnex_map_regspec()
1026 * we don't support mapping of I/O cards above 4Gb
1027 */
1028 static int
1029 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1030 {
1031 rootnex_addr_t rbase;
1032 void *cvaddr;
1033 uint_t npages, pgoffset;
1034 struct regspec *rp;
1035 ddi_acc_hdl_t *hp;
1036 ddi_acc_impl_t *ap;
1037 uint_t hat_acc_flags;
1038 paddr_t pbase;
1039
1040 rp = mp->map_obj.rp;
1041 hp = mp->map_handlep;
1042
1043 #ifdef DDI_MAP_DEBUG
1044 ddi_map_debug(
1045 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1046 rp->regspec_bustype, rp->regspec_addr,
1047 rp->regspec_size, mp->map_handlep);
1048 #endif /* DDI_MAP_DEBUG */
1049
1050 /*
1051 * I/O or memory mapping
1052 *
1053 * <bustype=0, addr=x, len=x>: memory
1054 * <bustype=1, addr=x, len=x>: i/o
1055 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1056 */
1057
1058 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1059 cmn_err(CE_WARN, "rootnex: invalid register spec"
1060 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
1061 rp->regspec_addr, rp->regspec_size);
1062 return (DDI_FAILURE);
1063 }
1064
1065 if (rp->regspec_bustype != 0) {
1066 /*
1067 * I/O space - needs a handle.
1068 */
1069 if (hp == NULL) {
1070 return (DDI_FAILURE);
1071 }
1072 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1073 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1074 impl_acc_hdl_init(hp);
1075
1076 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1077 #ifdef DDI_MAP_DEBUG
1078 ddi_map_debug("rootnex_map_regspec: mmap() "
1079 "to I/O space is not supported.\n");
1080 #endif /* DDI_MAP_DEBUG */
1081 return (DDI_ME_INVAL);
1082 } else {
1083 /*
1084 * 1275-compliant vs. compatibility i/o mapping
1085 */
1086 *vaddrp =
1087 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1088 ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1089 ((caddr_t)(uintptr_t)rp->regspec_addr);
1090 #ifdef __xpv
1091 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1092 hp->ah_pfn = xen_assign_pfn(
1093 mmu_btop((ulong_t)rp->regspec_addr &
1094 MMU_PAGEMASK));
1095 } else {
1096 hp->ah_pfn = mmu_btop(
1097 (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1098 }
1099 #else
1100 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1101 MMU_PAGEMASK);
1102 #endif
1103 hp->ah_pnum = mmu_btopr(rp->regspec_size +
1104 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1105 }
1106
1107 #ifdef DDI_MAP_DEBUG
1108 ddi_map_debug(
1109 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1110 rp->regspec_size, *vaddrp);
1111 #endif /* DDI_MAP_DEBUG */
1112 return (DDI_SUCCESS);
1113 }
1114
1115 /*
1116 * Memory space
1117 */
1118
1119 if (hp != NULL) {
1120 /*
1121 * hat layer ignores
1122 * hp->ah_acc.devacc_attr_endian_flags.
1123 */
1124 switch (hp->ah_acc.devacc_attr_dataorder) {
1125 case DDI_STRICTORDER_ACC:
1126 hat_acc_flags = HAT_STRICTORDER;
1127 break;
1128 case DDI_UNORDERED_OK_ACC:
1129 hat_acc_flags = HAT_UNORDERED_OK;
1130 break;
1131 case DDI_MERGING_OK_ACC:
1132 hat_acc_flags = HAT_MERGING_OK;
1133 break;
1134 case DDI_LOADCACHING_OK_ACC:
1135 hat_acc_flags = HAT_LOADCACHING_OK;
1136 break;
1137 case DDI_STORECACHING_OK_ACC:
1138 hat_acc_flags = HAT_STORECACHING_OK;
1139 break;
1140 }
1141 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1142 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1143 impl_acc_hdl_init(hp);
1144 hp->ah_hat_flags = hat_acc_flags;
1145 } else {
1146 hat_acc_flags = HAT_STRICTORDER;
1147 }
1148
1149 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1150 #ifdef __xpv
1151 /*
1152 * If we're dom0, we're using a real device so we need to translate
1153 * the MA to a PA.
1154 */
1155 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1156 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1157 } else {
1158 pbase = rbase;
1159 }
1160 #else
1161 pbase = rbase;
1162 #endif
1163 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1164
1165 if (rp->regspec_size == 0) {
1166 #ifdef DDI_MAP_DEBUG
1167 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1168 #endif /* DDI_MAP_DEBUG */
1169 return (DDI_ME_INVAL);
1170 }
1171
1172 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1173 /* extra cast to make gcc happy */
1174 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1175 } else {
1176 npages = mmu_btopr(rp->regspec_size + pgoffset);
1177
1178 #ifdef DDI_MAP_DEBUG
1179 ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1180 "physical %llx", npages, pbase);
1181 #endif /* DDI_MAP_DEBUG */
1182
1183 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1184 if (cvaddr == NULL)
1185 return (DDI_ME_NORESOURCES);
1186
1187 /*
1188 * Now map in the pages we've allocated...
1189 */
1190 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1191 mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1192 HAT_LOAD_LOCK);
1193 *vaddrp = (caddr_t)cvaddr + pgoffset;
1194
1195 /* save away pfn and npages for FMA */
1196 hp = mp->map_handlep;
1197 if (hp) {
1198 hp->ah_pfn = mmu_btop(pbase);
1199 hp->ah_pnum = npages;
1200 }
1201 }
1202
1203 #ifdef DDI_MAP_DEBUG
1204 ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1205 #endif /* DDI_MAP_DEBUG */
1206 return (DDI_SUCCESS);
1207 }
1208
1209
1210 /*
1211 * rootnex_unmap_regspec()
1212 *
1213 */
1214 static int
1215 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1216 {
1217 caddr_t addr = (caddr_t)*vaddrp;
1218 uint_t npages, pgoffset;
1219 struct regspec *rp;
1220
1221 if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1222 return (0);
1223
1224 rp = mp->map_obj.rp;
1225
1226 if (rp->regspec_size == 0) {
1227 #ifdef DDI_MAP_DEBUG
1228 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1229 #endif /* DDI_MAP_DEBUG */
1230 return (DDI_ME_INVAL);
1231 }
1232
1233 /*
1234 * I/O or memory mapping:
1235 *
1236 * <bustype=0, addr=x, len=x>: memory
1237 * <bustype=1, addr=x, len=x>: i/o
1238 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1239 */
1240 if (rp->regspec_bustype != 0) {
1241 /*
1242 * This is I/O space, which requires no particular
1243 * processing on unmap since it isn't mapped in the
1244 * first place.
1245 */
1246 return (DDI_SUCCESS);
1247 }
1248
1249 /*
1250 * Memory space
1251 */
1252 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1253 npages = mmu_btopr(rp->regspec_size + pgoffset);
1254 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1255 device_arena_free(addr - pgoffset, ptob(npages));
1256
1257 /*
1258 * Destroy the pointer - the mapping has logically gone
1259 */
1260 *vaddrp = NULL;
1261
1262 return (DDI_SUCCESS);
1263 }
1264
1265
1266 /*
1267 * rootnex_map_handle()
1268 *
1269 */
1270 static int
1271 rootnex_map_handle(ddi_map_req_t *mp)
1272 {
1273 rootnex_addr_t rbase;
1274 ddi_acc_hdl_t *hp;
1275 uint_t pgoffset;
1276 struct regspec *rp;
1277 paddr_t pbase;
1278
1279 rp = mp->map_obj.rp;
1280
1281 #ifdef DDI_MAP_DEBUG
1282 ddi_map_debug(
1283 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1284 rp->regspec_bustype, rp->regspec_addr,
1285 rp->regspec_size, mp->map_handlep);
1286 #endif /* DDI_MAP_DEBUG */
1287
1288 /*
1289 * I/O or memory mapping:
1290 *
1291 * <bustype=0, addr=x, len=x>: memory
1292 * <bustype=1, addr=x, len=x>: i/o
1293 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1294 */
1295 if (rp->regspec_bustype != 0) {
1296 /*
1297 * This refers to I/O space, and we don't support "mapping"
1298 * I/O space to a user.
1299 */
1300 return (DDI_FAILURE);
1301 }
1302
1303 /*
1304 * Set up the hat_flags for the mapping.
1305 */
1306 hp = mp->map_handlep;
1307
1308 switch (hp->ah_acc.devacc_attr_endian_flags) {
1309 case DDI_NEVERSWAP_ACC:
1310 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1311 break;
1312 case DDI_STRUCTURE_LE_ACC:
1313 hp->ah_hat_flags = HAT_STRUCTURE_LE;
1314 break;
1315 case DDI_STRUCTURE_BE_ACC:
1316 return (DDI_FAILURE);
1317 default:
1318 return (DDI_REGS_ACC_CONFLICT);
1319 }
1320
1321 switch (hp->ah_acc.devacc_attr_dataorder) {
1322 case DDI_STRICTORDER_ACC:
1323 break;
1324 case DDI_UNORDERED_OK_ACC:
1325 hp->ah_hat_flags |= HAT_UNORDERED_OK;
1326 break;
1327 case DDI_MERGING_OK_ACC:
1328 hp->ah_hat_flags |= HAT_MERGING_OK;
1329 break;
1330 case DDI_LOADCACHING_OK_ACC:
1331 hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1332 break;
1333 case DDI_STORECACHING_OK_ACC:
1334 hp->ah_hat_flags |= HAT_STORECACHING_OK;
1335 break;
1336 default:
1337 return (DDI_FAILURE);
1338 }
1339
1340 rbase = (rootnex_addr_t)rp->regspec_addr &
1341 (~(rootnex_addr_t)MMU_PAGEOFFSET);
1342 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1343
1344 if (rp->regspec_size == 0)
1345 return (DDI_ME_INVAL);
1346
1347 #ifdef __xpv
1348 /*
1349 * If we're dom0, we're using a real device so we need to translate
1350 * the MA to a PA.
1351 */
1352 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1353 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1354 (rbase & MMU_PAGEOFFSET);
1355 } else {
1356 pbase = rbase;
1357 }
1358 #else
1359 pbase = rbase;
1360 #endif
1361
1362 hp->ah_pfn = mmu_btop(pbase);
1363 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1364
1365 return (DDI_SUCCESS);
1366 }
1367
1368
1369
1370 /*
1371 * ************************
1372 * interrupt related code
1373 * ************************
1374 */
1375
1376 /*
1377 * rootnex_intr_ops()
1378 * bus_intr_op() function for interrupt support
1379 */
1380 /* ARGSUSED */
1381 static int
1382 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1383 ddi_intr_handle_impl_t *hdlp, void *result)
1384 {
1385 struct intrspec *ispec;
1386
1387 DDI_INTR_NEXDBG((CE_CONT,
1388 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1389 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1390
1391 /* Process the interrupt operation */
1392 switch (intr_op) {
1393 case DDI_INTROP_GETCAP:
1394 /* First check with pcplusmp */
1395 if (psm_intr_ops == NULL)
1396 return (DDI_FAILURE);
1397
1398 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1399 *(int *)result = 0;
1400 return (DDI_FAILURE);
1401 }
1402 break;
1403 case DDI_INTROP_SETCAP:
1404 if (psm_intr_ops == NULL)
1405 return (DDI_FAILURE);
1406
1407 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1408 return (DDI_FAILURE);
1409 break;
1410 case DDI_INTROP_ALLOC:
1411 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1412 return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1413 case DDI_INTROP_FREE:
1414 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1415 return (rootnex_free_intr_fixed(rdip, hdlp));
1416 case DDI_INTROP_GETPRI:
1417 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1418 return (DDI_FAILURE);
1419 *(int *)result = ispec->intrspec_pri;
1420 break;
1421 case DDI_INTROP_SETPRI:
1422 /* Validate the interrupt priority passed to us */
1423 if (*(int *)result > LOCK_LEVEL)
1424 return (DDI_FAILURE);
1425
1426 /* Ensure that PSM is all initialized and ispec is ok */
1427 if ((psm_intr_ops == NULL) ||
1428 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1429 return (DDI_FAILURE);
1430
1431 /* Change the priority */
1432 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1433 PSM_FAILURE)
1434 return (DDI_FAILURE);
1435
1436 /* update the ispec with the new priority */
1437 ispec->intrspec_pri = *(int *)result;
1438 break;
1439 case DDI_INTROP_ADDISR:
1440 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1441 return (DDI_FAILURE);
1442 ispec->intrspec_func = hdlp->ih_cb_func;
1443 break;
1444 case DDI_INTROP_REMISR:
1445 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1446 return (DDI_FAILURE);
1447 ispec->intrspec_func = (uint_t (*)()) 0;
1448 break;
1449 case DDI_INTROP_ENABLE:
1450 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1451 return (DDI_FAILURE);
1452
1453 /* Call psmi to translate irq with the dip */
1454 if (psm_intr_ops == NULL)
1455 return (DDI_FAILURE);
1456
1457 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1458 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1459 (int *)&hdlp->ih_vector) == PSM_FAILURE)
1460 return (DDI_FAILURE);
1461
1462 /* Add the interrupt handler */
1463 if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1464 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1465 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1466 return (DDI_FAILURE);
1467 break;
1468 case DDI_INTROP_DISABLE:
1469 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1470 return (DDI_FAILURE);
1471
1472 /* Call psm_ops() to translate irq with the dip */
1473 if (psm_intr_ops == NULL)
1474 return (DDI_FAILURE);
1475
1476 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1477 (void) (*psm_intr_ops)(rdip, hdlp,
1478 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1479
1480 /* Remove the interrupt handler */
1481 rem_avintr((void *)hdlp, ispec->intrspec_pri,
1482 hdlp->ih_cb_func, hdlp->ih_vector);
1483 break;
1484 case DDI_INTROP_SETMASK:
1485 if (psm_intr_ops == NULL)
1486 return (DDI_FAILURE);
1487
1488 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1489 return (DDI_FAILURE);
1490 break;
1491 case DDI_INTROP_CLRMASK:
1492 if (psm_intr_ops == NULL)
1493 return (DDI_FAILURE);
1494
1495 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1496 return (DDI_FAILURE);
1497 break;
1498 case DDI_INTROP_GETPENDING:
1499 if (psm_intr_ops == NULL)
1500 return (DDI_FAILURE);
1501
1502 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1503 result)) {
1504 *(int *)result = 0;
1505 return (DDI_FAILURE);
1506 }
1507 break;
1508 case DDI_INTROP_NAVAIL:
1509 case DDI_INTROP_NINTRS:
1510 *(int *)result = i_ddi_get_intx_nintrs(rdip);
1511 if (*(int *)result == 0) {
1512 /*
1513 * Special case for 'pcic' driver' only. This driver
1514 * driver is a child of 'isa' and 'rootnex' drivers.
1515 *
1516 * See detailed comments on this in the function
1517 * rootnex_get_ispec().
1518 *
1519 * Children of 'pcic' send 'NINITR' request all the
1520 * way to rootnex driver. But, the 'pdp->par_nintr'
1521 * field may not initialized. So, we fake it here
1522 * to return 1 (a la what PCMCIA nexus does).
1523 */
1524 if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1525 *(int *)result = 1;
1526 else
1527 return (DDI_FAILURE);
1528 }
1529 break;
1530 case DDI_INTROP_SUPPORTED_TYPES:
1531 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */
1532 break;
1533 default:
1534 return (DDI_FAILURE);
1535 }
1536
1537 return (DDI_SUCCESS);
1538 }
1539
1540
1541 /*
1542 * rootnex_get_ispec()
1543 * convert an interrupt number to an interrupt specification.
1544 * The interrupt number determines which interrupt spec will be
1545 * returned if more than one exists.
1546 *
1547 * Look into the parent private data area of the 'rdip' to find out
1548 * the interrupt specification. First check to make sure there is
1549 * one that matchs "inumber" and then return a pointer to it.
1550 *
1551 * Return NULL if one could not be found.
1552 *
1553 * NOTE: This is needed for rootnex_intr_ops()
1554 */
1555 static struct intrspec *
1556 rootnex_get_ispec(dev_info_t *rdip, int inum)
1557 {
1558 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1559
1560 /*
1561 * Special case handling for drivers that provide their own
1562 * intrspec structures instead of relying on the DDI framework.
1563 *
1564 * A broken hardware driver in ON could potentially provide its
1565 * own intrspec structure, instead of relying on the hardware.
1566 * If these drivers are children of 'rootnex' then we need to
1567 * continue to provide backward compatibility to them here.
1568 *
1569 * Following check is a special case for 'pcic' driver which
1570 * was found to have broken hardwre andby provides its own intrspec.
1571 *
1572 * Verbatim comments from this driver are shown here:
1573 * "Don't use the ddi_add_intr since we don't have a
1574 * default intrspec in all cases."
1575 *
1576 * Since an 'ispec' may not be always created for it,
1577 * check for that and create one if so.
1578 *
1579 * NOTE: Currently 'pcic' is the only driver found to do this.
1580 */
1581 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1582 pdp->par_nintr = 1;
1583 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1584 pdp->par_nintr, KM_SLEEP);
1585 }
1586
1587 /* Validate the interrupt number */
1588 if (inum >= pdp->par_nintr)
1589 return (NULL);
1590
1591 /* Get the interrupt structure pointer and return that */
1592 return ((struct intrspec *)&pdp->par_intr[inum]);
1593 }
1594
1595 /*
1596 * Allocate interrupt vector for FIXED (legacy) type.
1597 */
1598 static int
1599 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1600 void *result)
1601 {
1602 struct intrspec *ispec;
1603 ddi_intr_handle_impl_t info_hdl;
1604 int ret;
1605 int free_phdl = 0;
1606 apic_get_type_t type_info;
1607
1608 if (psm_intr_ops == NULL)
1609 return (DDI_FAILURE);
1610
1611 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1612 return (DDI_FAILURE);
1613
1614 /*
1615 * If the PSM module is "APIX" then pass the request for it
1616 * to allocate the vector now.
1617 */
1618 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1619 info_hdl.ih_private = &type_info;
1620 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1621 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1622 if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1623 free_phdl = 1;
1624 i_ddi_alloc_intr_phdl(hdlp);
1625 }
1626 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1627 ret = (*psm_intr_ops)(rdip, hdlp,
1628 PSM_INTR_OP_ALLOC_VECTORS, result);
1629 if (free_phdl) { /* free up the phdl structure */
1630 free_phdl = 0;
1631 i_ddi_free_intr_phdl(hdlp);
1632 hdlp->ih_private = NULL;
1633 }
1634 } else {
1635 /*
1636 * No APIX module; fall back to the old scheme where the
1637 * interrupt vector is allocated during ddi_enable_intr() call.
1638 */
1639 hdlp->ih_pri = ispec->intrspec_pri;
1640 *(int *)result = hdlp->ih_scratch1;
1641 ret = DDI_SUCCESS;
1642 }
1643
1644 return (ret);
1645 }
1646
1647 /*
1648 * Free up interrupt vector for FIXED (legacy) type.
1649 */
1650 static int
1651 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1652 {
1653 struct intrspec *ispec;
1654 struct ddi_parent_private_data *pdp;
1655 ddi_intr_handle_impl_t info_hdl;
1656 int ret;
1657 apic_get_type_t type_info;
1658
1659 if (psm_intr_ops == NULL)
1660 return (DDI_FAILURE);
1661
1662 /*
1663 * If the PSM module is "APIX" then pass the request for it
1664 * to free up the vector now.
1665 */
1666 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1667 info_hdl.ih_private = &type_info;
1668 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1669 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1670 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1671 return (DDI_FAILURE);
1672 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1673 ret = (*psm_intr_ops)(rdip, hdlp,
1674 PSM_INTR_OP_FREE_VECTORS, NULL);
1675 } else {
1676 /*
1677 * No APIX module; fall back to the old scheme where
1678 * the interrupt vector was already freed during
1679 * ddi_disable_intr() call.
1680 */
1681 ret = DDI_SUCCESS;
1682 }
1683
1684 pdp = ddi_get_parent_data(rdip);
1685
1686 /*
1687 * Special case for 'pcic' driver' only.
1688 * If an intrspec was created for it, clean it up here
1689 * See detailed comments on this in the function
1690 * rootnex_get_ispec().
1691 */
1692 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1693 kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1694 pdp->par_nintr);
1695 /*
1696 * Set it to zero; so that
1697 * DDI framework doesn't free it again
1698 */
1699 pdp->par_intr = NULL;
1700 pdp->par_nintr = 0;
1701 }
1702
1703 return (ret);
1704 }
1705
1706
1707 /*
1708 * ******************
1709 * dma related code
1710 * ******************
1711 */
1712
1713 /*ARGSUSED*/
1714 static int
1715 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1716 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1717 ddi_dma_handle_t *handlep)
1718 {
1719 uint64_t maxsegmentsize_ll;
1720 uint_t maxsegmentsize;
1721 ddi_dma_impl_t *hp;
1722 rootnex_dma_t *dma;
1723 uint64_t count_max;
1724 uint64_t seg;
1725 int kmflag;
1726 int e;
1727
1728
1729 /* convert our sleep flags */
1730 if (waitfp == DDI_DMA_SLEEP) {
1731 kmflag = KM_SLEEP;
1732 } else {
1733 kmflag = KM_NOSLEEP;
1734 }
1735
1736 /*
1737 * We try to do only one memory allocation here. We'll do a little
1738 * pointer manipulation later. If the bind ends up taking more than
1739 * our prealloc's space, we'll have to allocate more memory in the
1740 * bind operation. Not great, but much better than before and the
1741 * best we can do with the current bind interfaces.
1742 */
1743 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1744 if (hp == NULL)
1745 return (DDI_DMA_NORESOURCES);
1746
1747 /* Do our pointer manipulation now, align the structures */
1748 hp->dmai_private = (void *)(((uintptr_t)hp +
1749 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1750 dma = (rootnex_dma_t *)hp->dmai_private;
1751 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1752 sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1753
1754 /* setup the handle */
1755 rootnex_clean_dmahdl(hp);
1756 hp->dmai_error.err_fep = NULL;
1757 hp->dmai_error.err_cf = NULL;
1758 dma->dp_dip = rdip;
1759 dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1760 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1761
1762 /*
1763 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1764 * is being used. Set the upper limit to the seg value.
1765 * There will be enough DVMA space to always get addresses
1766 * that will match the constraints.
1767 */
1768 if (IOMMU_USED(rdip) &&
1769 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1770 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1771 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1772 } else
1773 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1774
1775 hp->dmai_minxfer = attr->dma_attr_minxfer;
1776 hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1777 hp->dmai_rdip = rdip;
1778 hp->dmai_attr = *attr;
1779
1780 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1781 dma->dp_sglinfo.si_cancross = B_FALSE;
1782 else
1783 dma->dp_sglinfo.si_cancross = B_TRUE;
1784
1785 /* we don't need to worry about the SPL since we do a tryenter */
1786 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1787
1788 /*
1789 * Figure out our maximum segment size. If the segment size is greater
1790 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1791 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1792 * dma_attr_count_max are size-1 type values.
1793 *
1794 * Maximum segment size is the largest physically contiguous chunk of
1795 * memory that we can return from a bind (i.e. the maximum size of a
1796 * single cookie).
1797 */
1798
1799 /* handle the rollover cases */
1800 seg = attr->dma_attr_seg + 1;
1801 if (seg < attr->dma_attr_seg) {
1802 seg = attr->dma_attr_seg;
1803 }
1804 count_max = attr->dma_attr_count_max + 1;
1805 if (count_max < attr->dma_attr_count_max) {
1806 count_max = attr->dma_attr_count_max;
1807 }
1808
1809 /*
1810 * granularity may or may not be a power of two. If it isn't, we can't
1811 * use a simple mask.
1812 */
1813 if (!ISP2(attr->dma_attr_granular)) {
1814 dma->dp_granularity_power_2 = B_FALSE;
1815 } else {
1816 dma->dp_granularity_power_2 = B_TRUE;
1817 }
1818
1819 /*
1820 * maxxfer should be a whole multiple of granularity. If we're going to
1821 * break up a window because we're greater than maxxfer, we might as
1822 * well make sure it's maxxfer is a whole multiple so we don't have to
1823 * worry about triming the window later on for this case.
1824 */
1825 if (attr->dma_attr_granular > 1) {
1826 if (dma->dp_granularity_power_2) {
1827 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1828 (attr->dma_attr_maxxfer &
1829 (attr->dma_attr_granular - 1));
1830 } else {
1831 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1832 (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1833 }
1834 } else {
1835 dma->dp_maxxfer = attr->dma_attr_maxxfer;
1836 }
1837
1838 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1839 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1840 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1841 maxsegmentsize = 0xFFFFFFFF;
1842 } else {
1843 maxsegmentsize = maxsegmentsize_ll;
1844 }
1845 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1846 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1847
1848 /* check the ddi_dma_attr arg to make sure it makes a little sense */
1849 if (rootnex_alloc_check_parms) {
1850 e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1851 if (e != DDI_SUCCESS) {
1852 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1853 (void) rootnex_dma_freehdl(dip, rdip,
1854 (ddi_dma_handle_t)hp);
1855 return (e);
1856 }
1857 }
1858
1859 *handlep = (ddi_dma_handle_t)hp;
1860
1861 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1862 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1863 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1864
1865 return (DDI_SUCCESS);
1866 }
1867
1868
1869 /*
1870 * rootnex_dma_allochdl()
1871 * called from ddi_dma_alloc_handle().
1872 */
1873 static int
1874 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1875 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1876 {
1877 int retval = DDI_SUCCESS;
1878 #if defined(__amd64) && !defined(__xpv)
1879
1880 if (IOMMU_UNITIALIZED(rdip)) {
1881 retval = iommulib_nex_open(dip, rdip);
1882
1883 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1884 return (retval);
1885 }
1886
1887 if (IOMMU_UNUSED(rdip)) {
1888 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1889 handlep);
1890 } else {
1891 retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1892 waitfp, arg, handlep);
1893 }
1894 #else
1895 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1896 handlep);
1897 #endif
1898 switch (retval) {
1899 case DDI_DMA_NORESOURCES:
1900 if (waitfp != DDI_DMA_DONTWAIT) {
1901 ddi_set_callback(waitfp, arg,
1902 &rootnex_state->r_dvma_call_list_id);
1903 }
1904 break;
1905 case DDI_SUCCESS:
1906 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1907 break;
1908 default:
1909 break;
1910 }
1911 return (retval);
1912 }
1913
1914 /*ARGSUSED*/
1915 static int
1916 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1917 ddi_dma_handle_t handle)
1918 {
1919 ddi_dma_impl_t *hp;
1920 rootnex_dma_t *dma;
1921
1922
1923 hp = (ddi_dma_impl_t *)handle;
1924 dma = (rootnex_dma_t *)hp->dmai_private;
1925
1926 /* unbind should have been called first */
1927 ASSERT(!dma->dp_inuse);
1928
1929 mutex_destroy(&dma->dp_mutex);
1930 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1931
1932 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1933 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1934 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1935
1936 return (DDI_SUCCESS);
1937 }
1938
1939 /*
1940 * rootnex_dma_freehdl()
1941 * called from ddi_dma_free_handle().
1942 */
1943 static int
1944 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1945 {
1946 int ret;
1947
1948 ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1949 #if defined(__amd64) && !defined(__xpv)
1950 if (IOMMU_USED(rdip))
1951 ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1952 else
1953 #endif
1954 ret = rootnex_coredma_freehdl(dip, rdip, handle);
1955
1956 if (rootnex_state->r_dvma_call_list_id)
1957 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1958
1959 return (ret);
1960 }
1961
1962 /*ARGSUSED*/
1963 static int
1964 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1965 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1966 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1967 {
1968 rootnex_sglinfo_t *sinfo;
1969 ddi_dma_obj_t *dmao;
1970 #if defined(__amd64) && !defined(__xpv)
1971 struct dvmaseg *dvs;
1972 ddi_dma_cookie_t *cookie;
1973 #endif
1974 ddi_dma_attr_t *attr;
1975 ddi_dma_impl_t *hp;
1976 rootnex_dma_t *dma;
1977 int kmflag;
1978 int e;
1979 uint_t ncookies;
1980
1981 hp = (ddi_dma_impl_t *)handle;
1982 dma = (rootnex_dma_t *)hp->dmai_private;
1983 dmao = &dma->dp_dma;
1984 sinfo = &dma->dp_sglinfo;
1985 attr = &hp->dmai_attr;
1986
1987 /* convert the sleep flags */
1988 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1989 dma->dp_sleep_flags = kmflag = KM_SLEEP;
1990 } else {
1991 dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1992 }
1993
1994 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1995
1996 /*
1997 * This is useful for debugging a driver. Not as useful in a production
1998 * system. The only time this will fail is if you have a driver bug.
1999 */
2000 if (rootnex_bind_check_inuse) {
2001 /*
2002 * No one else should ever have this lock unless someone else
2003 * is trying to use this handle. So contention on the lock
2004 * is the same as inuse being set.
2005 */
2006 e = mutex_tryenter(&dma->dp_mutex);
2007 if (e == 0) {
2008 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2009 return (DDI_DMA_INUSE);
2010 }
2011 if (dma->dp_inuse) {
2012 mutex_exit(&dma->dp_mutex);
2013 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2014 return (DDI_DMA_INUSE);
2015 }
2016 dma->dp_inuse = B_TRUE;
2017 mutex_exit(&dma->dp_mutex);
2018 }
2019
2020 /* check the ddi_dma_attr arg to make sure it makes a little sense */
2021 if (rootnex_bind_check_parms) {
2022 e = rootnex_valid_bind_parms(dmareq, attr);
2023 if (e != DDI_SUCCESS) {
2024 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2025 rootnex_clean_dmahdl(hp);
2026 return (e);
2027 }
2028 }
2029
2030 /* save away the original bind info */
2031 dma->dp_dma = dmareq->dmar_object;
2032
2033 #if defined(__amd64) && !defined(__xpv)
2034 if (IOMMU_USED(rdip)) {
2035 dmao = &dma->dp_dvma;
2036 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2037 switch (e) {
2038 case DDI_SUCCESS:
2039 if (sinfo->si_cancross ||
2040 dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2041 dmao->dmao_size > sinfo->si_max_cookie_size) {
2042 dma->dp_dvma_used = B_TRUE;
2043 break;
2044 }
2045 sinfo->si_sgl_size = 1;
2046 hp->dmai_rflags |= DMP_NOSYNC;
2047
2048 dma->dp_dvma_used = B_TRUE;
2049 dma->dp_need_to_free_cookie = B_FALSE;
2050
2051 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2052 cookie = hp->dmai_cookie = dma->dp_cookies =
2053 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2054 cookie->dmac_laddress = dvs->dvs_start +
2055 dmao->dmao_obj.dvma_obj.dv_off;
2056 cookie->dmac_size = dvs->dvs_len;
2057 cookie->dmac_type = 0;
2058
2059 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2060 rdip);
2061 goto fast;
2062 case DDI_ENOTSUP:
2063 break;
2064 default:
2065 rootnex_clean_dmahdl(hp);
2066 return (e);
2067 }
2068 }
2069 #endif
2070
2071 /*
2072 * Figure out a rough estimate of what maximum number of pages
2073 * this buffer could use (a high estimate of course).
2074 */
2075 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2076
2077 if (dma->dp_dvma_used) {
2078 /*
2079 * The number of physical pages is the worst case.
2080 *
2081 * For DVMA, the worst case is the length divided
2082 * by the maximum cookie length, plus 1. Add to that
2083 * the number of segment boundaries potentially crossed, and
2084 * the additional number of DVMA segments that was returned.
2085 *
2086 * In the normal case, for modern devices, si_cancross will
2087 * be false, and dv_nseg will be 1, and the fast path will
2088 * have been taken above.
2089 */
2090 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2091 + 1;
2092 if (sinfo->si_cancross)
2093 ncookies +=
2094 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2095 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2096
2097 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2098 }
2099
2100 /*
2101 * We'll use the pre-allocated cookies for any bind that will *always*
2102 * fit (more important to be consistent, we don't want to create
2103 * additional degenerate cases).
2104 */
2105 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2106 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2107 dma->dp_need_to_free_cookie = B_FALSE;
2108 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2109 uint_t, sinfo->si_max_pages);
2110
2111 /*
2112 * For anything larger than that, we'll go ahead and allocate the
2113 * maximum number of pages we expect to see. Hopefuly, we won't be
2114 * seeing this path in the fast path for high performance devices very
2115 * frequently.
2116 *
2117 * a ddi bind interface that allowed the driver to provide storage to
2118 * the bind interface would speed this case up.
2119 */
2120 } else {
2121 /*
2122 * Save away how much memory we allocated. If we're doing a
2123 * nosleep, the alloc could fail...
2124 */
2125 dma->dp_cookie_size = sinfo->si_max_pages *
2126 sizeof (ddi_dma_cookie_t);
2127 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2128 if (dma->dp_cookies == NULL) {
2129 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2130 rootnex_clean_dmahdl(hp);
2131 return (DDI_DMA_NORESOURCES);
2132 }
2133 dma->dp_need_to_free_cookie = B_TRUE;
2134 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2135 uint_t, sinfo->si_max_pages);
2136 }
2137 hp->dmai_cookie = dma->dp_cookies;
2138
2139 /*
2140 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2141 * looking at the constraints in the dma structure. It will then put
2142 * some additional state about the sgl in the dma struct (i.e. is
2143 * the sgl clean, or do we need to do some munging; how many pages
2144 * need to be copied, etc.)
2145 */
2146 if (dma->dp_dvma_used)
2147 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2148 else
2149 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2150
2151 out:
2152 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2153 /* if we don't need a copy buffer, we don't need to sync */
2154 if (sinfo->si_copybuf_req == 0) {
2155 hp->dmai_rflags |= DMP_NOSYNC;
2156 }
2157
2158 /*
2159 * if we don't need the copybuf and we don't need to do a partial, we
2160 * hit the fast path. All the high performance devices should be trying
2161 * to hit this path. To hit this path, a device should be able to reach
2162 * all of memory, shouldn't try to bind more than it can transfer, and
2163 * the buffer shouldn't require more cookies than the driver/device can
2164 * handle [sgllen]).
2165 *
2166 * Note that negative values of dma_attr_sgllen are supposed
2167 * to mean unlimited, but we just cast them to mean a
2168 * "ridiculous large limit". This saves some extra checks on
2169 * hot paths.
2170 */
2171 if ((sinfo->si_copybuf_req == 0) &&
2172 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2173 (dmao->dmao_size < dma->dp_maxxfer)) {
2174 fast:
2175 /*
2176 * If the driver supports FMA, insert the handle in the FMA DMA
2177 * handle cache.
2178 */
2179 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2180 hp->dmai_error.err_cf = rootnex_dma_check;
2181
2182 /*
2183 * copy out the first cookie and ccountp, set the cookie
2184 * pointer to the second cookie. The first cookie is passed
2185 * back on the stack. Additional cookies are accessed via
2186 * ddi_dma_nextcookie()
2187 */
2188 *cookiep = dma->dp_cookies[0];
2189 *ccountp = sinfo->si_sgl_size;
2190 hp->dmai_cookie++;
2191 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2192 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2193 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2194 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2195 uint_t, dmao->dmao_size, uint_t, *ccountp);
2196
2197
2198 return (DDI_DMA_MAPPED);
2199 }
2200
2201 /*
2202 * go to the slow path, we may need to alloc more memory, create
2203 * multiple windows, and munge up a sgl to make the device happy.
2204 */
2205
2206 /*
2207 * With the IOMMU mapobject method used, we should never hit
2208 * the slow path. If we do, something is seriously wrong.
2209 * Clean up and return an error.
2210 */
2211
2212 #if defined(__amd64) && !defined(__xpv)
2213
2214 if (dma->dp_dvma_used) {
2215 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2216 &dma->dp_dvma);
2217 e = DDI_DMA_NOMAPPING;
2218 } else {
2219 #endif
2220 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2221 kmflag);
2222 #if defined(__amd64) && !defined(__xpv)
2223 }
2224 #endif
2225 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2226 if (dma->dp_need_to_free_cookie) {
2227 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2228 }
2229 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2230 rootnex_clean_dmahdl(hp); /* must be after free cookie */
2231 return (e);
2232 }
2233
2234 /*
2235 * If the driver supports FMA, insert the handle in the FMA DMA handle
2236 * cache.
2237 */
2238 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2239 hp->dmai_error.err_cf = rootnex_dma_check;
2240
2241 /* if the first window uses the copy buffer, sync it for the device */
2242 if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2243 (hp->dmai_rflags & DDI_DMA_WRITE)) {
2244 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2245 DDI_DMA_SYNC_FORDEV);
2246 }
2247
2248 /*
2249 * copy out the first cookie and ccountp, set the cookie pointer to the
2250 * second cookie. Make sure the partial flag is set/cleared correctly.
2251 * If we have a partial map (i.e. multiple windows), the number of
2252 * cookies we return is the number of cookies in the first window.
2253 */
2254 if (e == DDI_DMA_MAPPED) {
2255 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2256 *ccountp = sinfo->si_sgl_size;
2257 hp->dmai_nwin = 1;
2258 } else {
2259 hp->dmai_rflags |= DDI_DMA_PARTIAL;
2260 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2261 ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2262 }
2263 *cookiep = dma->dp_cookies[0];
2264 hp->dmai_cookie++;
2265
2266 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2267 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2268 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2269 dmao->dmao_size, uint_t, *ccountp);
2270 return (e);
2271 }
2272
2273 /*
2274 * rootnex_dma_bindhdl()
2275 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2276 */
2277 static int
2278 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2279 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2280 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2281 {
2282 int ret;
2283 #if defined(__amd64) && !defined(__xpv)
2284 if (IOMMU_USED(rdip))
2285 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2286 cookiep, ccountp);
2287 else
2288 #endif
2289 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2290 cookiep, ccountp);
2291
2292 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2293 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2294 &rootnex_state->r_dvma_call_list_id);
2295 }
2296
2297 return (ret);
2298 }
2299
2300
2301
2302 /*ARGSUSED*/
2303 static int
2304 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2305 ddi_dma_handle_t handle)
2306 {
2307 ddi_dma_impl_t *hp;
2308 rootnex_dma_t *dma;
2309 int e;
2310
2311 hp = (ddi_dma_impl_t *)handle;
2312 dma = (rootnex_dma_t *)hp->dmai_private;
2313
2314 /* make sure the buffer wasn't free'd before calling unbind */
2315 if (rootnex_unbind_verify_buffer) {
2316 e = rootnex_verify_buffer(dma);
2317 if (e != DDI_SUCCESS) {
2318 ASSERT(0);
2319 return (DDI_FAILURE);
2320 }
2321 }
2322
2323 /* sync the current window before unbinding the buffer */
2324 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2325 (hp->dmai_rflags & DDI_DMA_READ)) {
2326 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2327 DDI_DMA_SYNC_FORCPU);
2328 }
2329
2330 /*
2331 * cleanup and copy buffer or window state. if we didn't use the copy
2332 * buffer or windows, there won't be much to do :-)
2333 */
2334 rootnex_teardown_copybuf(dma);
2335 rootnex_teardown_windows(dma);
2336
2337 #if defined(__amd64) && !defined(__xpv)
2338 if (IOMMU_USED(rdip))
2339 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2340 &dma->dp_dvma);
2341 #endif
2342
2343 /*
2344 * If we had to allocate space to for the worse case sgl (it didn't
2345 * fit into our pre-allocate buffer), free that up now
2346 */
2347 if (dma->dp_need_to_free_cookie) {
2348 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2349 }
2350
2351 /*
2352 * clean up the handle so it's ready for the next bind (i.e. if the
2353 * handle is reused).
2354 */
2355 rootnex_clean_dmahdl(hp);
2356 hp->dmai_error.err_cf = NULL;
2357
2358 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2359 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2360 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2361
2362 return (DDI_SUCCESS);
2363 }
2364
2365 /*
2366 * rootnex_dma_unbindhdl()
2367 * called from ddi_dma_unbind_handle()
2368 */
2369 /*ARGSUSED*/
2370 static int
2371 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2372 ddi_dma_handle_t handle)
2373 {
2374 int ret;
2375
2376 #if defined(__amd64) && !defined(__xpv)
2377 if (IOMMU_USED(rdip))
2378 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2379 else
2380 #endif
2381 ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2382
2383 if (rootnex_state->r_dvma_call_list_id)
2384 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2385
2386 return (ret);
2387 }
2388
2389 #if defined(__amd64) && !defined(__xpv)
2390
2391 static int
2392 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2393 {
2394 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2395 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2396
2397 if (dma->dp_sleep_flags != KM_SLEEP &&
2398 dma->dp_sleep_flags != KM_NOSLEEP)
2399 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2400 return (dma->dp_sleep_flags);
2401 }
2402 /*ARGSUSED*/
2403 static void
2404 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2405 {
2406 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2407 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2408 rootnex_window_t *window;
2409
2410 if (dma->dp_window) {
2411 window = &dma->dp_window[dma->dp_current_win];
2412 hp->dmai_cookie = window->wd_first_cookie;
2413 } else {
2414 hp->dmai_cookie = dma->dp_cookies;
2415 }
2416 hp->dmai_cookie++;
2417 }
2418
2419 /*ARGSUSED*/
2420 static int
2421 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2422 ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2423 {
2424 int i;
2425 int km_flags;
2426 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2427 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2428 rootnex_window_t *window;
2429 ddi_dma_cookie_t *cp;
2430 ddi_dma_cookie_t *cookie;
2431
2432 ASSERT(*cookiepp == NULL);
2433 ASSERT(*ccountp == 0);
2434
2435 if (dma->dp_window) {
2436 window = &dma->dp_window[dma->dp_current_win];
2437 cp = window->wd_first_cookie;
2438 *ccountp = window->wd_cookie_cnt;
2439 } else {
2440 cp = dma->dp_cookies;
2441 *ccountp = dma->dp_sglinfo.si_sgl_size;
2442 }
2443
2444 km_flags = rootnex_coredma_get_sleep_flags(handle);
2445 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2446 if (cookie == NULL) {
2447 return (DDI_DMA_NORESOURCES);
2448 }
2449
2450 for (i = 0; i < *ccountp; i++) {
2451 cookie[i].dmac_notused = cp[i].dmac_notused;
2452 cookie[i].dmac_type = cp[i].dmac_type;
2453 cookie[i].dmac_address = cp[i].dmac_address;
2454 cookie[i].dmac_size = cp[i].dmac_size;
2455 }
2456
2457 *cookiepp = cookie;
2458
2459 return (DDI_SUCCESS);
2460 }
2461
2462 /*ARGSUSED*/
2463 static int
2464 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2465 ddi_dma_cookie_t *cookiep, uint_t ccount)
2466 {
2467 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2468 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2469 rootnex_window_t *window;
2470 ddi_dma_cookie_t *cur_cookiep;
2471
2472 ASSERT(cookiep);
2473 ASSERT(ccount != 0);
2474 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2475
2476 if (dma->dp_window) {
2477 window = &dma->dp_window[dma->dp_current_win];
2478 dma->dp_saved_cookies = window->wd_first_cookie;
2479 window->wd_first_cookie = cookiep;
2480 ASSERT(ccount == window->wd_cookie_cnt);
2481 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2482 + window->wd_first_cookie;
2483 } else {
2484 dma->dp_saved_cookies = dma->dp_cookies;
2485 dma->dp_cookies = cookiep;
2486 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2487 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2488 + dma->dp_cookies;
2489 }
2490
2491 dma->dp_need_to_switch_cookies = B_TRUE;
2492 hp->dmai_cookie = cur_cookiep;
2493
2494 return (DDI_SUCCESS);
2495 }
2496
2497 /*ARGSUSED*/
2498 static int
2499 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2500 {
2501 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2502 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2503 rootnex_window_t *window;
2504 ddi_dma_cookie_t *cur_cookiep;
2505 ddi_dma_cookie_t *cookie_array;
2506 uint_t ccount;
2507
2508 /* check if cookies have not been switched */
2509 if (dma->dp_need_to_switch_cookies == B_FALSE)
2510 return (DDI_SUCCESS);
2511
2512 ASSERT(dma->dp_saved_cookies);
2513
2514 if (dma->dp_window) {
2515 window = &dma->dp_window[dma->dp_current_win];
2516 cookie_array = window->wd_first_cookie;
2517 window->wd_first_cookie = dma->dp_saved_cookies;
2518 dma->dp_saved_cookies = NULL;
2519 ccount = window->wd_cookie_cnt;
2520 cur_cookiep = (hp->dmai_cookie - cookie_array)
2521 + window->wd_first_cookie;
2522 } else {
2523 cookie_array = dma->dp_cookies;
2524 dma->dp_cookies = dma->dp_saved_cookies;
2525 dma->dp_saved_cookies = NULL;
2526 ccount = dma->dp_sglinfo.si_sgl_size;
2527 cur_cookiep = (hp->dmai_cookie - cookie_array)
2528 + dma->dp_cookies;
2529 }
2530
2531 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2532
2533 hp->dmai_cookie = cur_cookiep;
2534
2535 dma->dp_need_to_switch_cookies = B_FALSE;
2536
2537 return (DDI_SUCCESS);
2538 }
2539
2540 #endif
2541
2542 static struct as *
2543 rootnex_get_as(ddi_dma_obj_t *dmao)
2544 {
2545 struct as *asp;
2546
2547 switch (dmao->dmao_type) {
2548 case DMA_OTYP_VADDR:
2549 case DMA_OTYP_BUFVADDR:
2550 asp = dmao->dmao_obj.virt_obj.v_as;
2551 if (asp == NULL)
2552 asp = &kas;
2553 break;
2554 default:
2555 asp = NULL;
2556 break;
2557 }
2558 return (asp);
2559 }
2560
2561 /*
2562 * rootnex_verify_buffer()
2563 * verify buffer wasn't free'd
2564 */
2565 static int
2566 rootnex_verify_buffer(rootnex_dma_t *dma)
2567 {
2568 page_t **pplist;
2569 caddr_t vaddr;
2570 uint_t pcnt;
2571 uint_t poff;
2572 page_t *pp;
2573 char b;
2574 int i;
2575
2576 /* Figure out how many pages this buffer occupies */
2577 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2578 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2579 } else {
2580 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2581 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2582 }
2583 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2584
2585 switch (dma->dp_dma.dmao_type) {
2586 case DMA_OTYP_PAGES:
2587 /*
2588 * for a linked list of pp's walk through them to make sure
2589 * they're locked and not free.
2590 */
2591 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2592 for (i = 0; i < pcnt; i++) {
2593 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2594 return (DDI_FAILURE);
2595 }
2596 pp = pp->p_next;
2597 }
2598 break;
2599
2600 case DMA_OTYP_VADDR:
2601 case DMA_OTYP_BUFVADDR:
2602 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2603 /*
2604 * for an array of pp's walk through them to make sure they're
2605 * not free. It's possible that they may not be locked.
2606 */
2607 if (pplist) {
2608 for (i = 0; i < pcnt; i++) {
2609 if (PP_ISFREE(pplist[i])) {
2610 return (DDI_FAILURE);
2611 }
2612 }
2613
2614 /* For a virtual address, try to peek at each page */
2615 } else {
2616 if (rootnex_get_as(&dma->dp_dma) == &kas) {
2617 for (i = 0; i < pcnt; i++) {
2618 if (ddi_peek8(NULL, vaddr, &b) ==
2619 DDI_FAILURE)
2620 return (DDI_FAILURE);
2621 vaddr += MMU_PAGESIZE;
2622 }
2623 }
2624 }
2625 break;
2626
2627 default:
2628 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2629 break;
2630 }
2631
2632 return (DDI_SUCCESS);
2633 }
2634
2635
2636 /*
2637 * rootnex_clean_dmahdl()
2638 * Clean the dma handle. This should be called on a handle alloc and an
2639 * unbind handle. Set the handle state to the default settings.
2640 */
2641 static void
2642 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2643 {
2644 rootnex_dma_t *dma;
2645
2646
2647 dma = (rootnex_dma_t *)hp->dmai_private;
2648
2649 hp->dmai_nwin = 0;
2650 dma->dp_current_cookie = 0;
2651 dma->dp_copybuf_size = 0;
2652 dma->dp_window = NULL;
2653 dma->dp_cbaddr = NULL;
2654 dma->dp_inuse = B_FALSE;
2655 dma->dp_dvma_used = B_FALSE;
2656 dma->dp_need_to_free_cookie = B_FALSE;
2657 dma->dp_need_to_switch_cookies = B_FALSE;
2658 dma->dp_saved_cookies = NULL;
2659 dma->dp_sleep_flags = KM_PANIC;
2660 dma->dp_need_to_free_window = B_FALSE;
2661 dma->dp_partial_required = B_FALSE;
2662 dma->dp_trim_required = B_FALSE;
2663 dma->dp_sglinfo.si_copybuf_req = 0;
2664 #if !defined(__amd64)
2665 dma->dp_cb_remaping = B_FALSE;
2666 dma->dp_kva = NULL;
2667 #endif
2668
2669 /* FMA related initialization */
2670 hp->dmai_fault = 0;
2671 hp->dmai_fault_check = NULL;
2672 hp->dmai_fault_notify = NULL;
2673 hp->dmai_error.err_ena = 0;
2674 hp->dmai_error.err_status = DDI_FM_OK;
2675 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2676 hp->dmai_error.err_ontrap = NULL;
2677 }
2678
2679
2680 /*
2681 * rootnex_valid_alloc_parms()
2682 * Called in ddi_dma_alloc_handle path to validate its parameters.
2683 */
2684 static int
2685 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2686 {
2687 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2688 (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2689 (attr->dma_attr_granular > MMU_PAGESIZE) ||
2690 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2691 return (DDI_DMA_BADATTR);
2692 }
2693
2694 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2695 return (DDI_DMA_BADATTR);
2696 }
2697
2698 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2699 MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2700 attr->dma_attr_sgllen == 0) {
2701 return (DDI_DMA_BADATTR);
2702 }
2703
2704 /* We should be able to DMA into every byte offset in a page */
2705 if (maxsegmentsize < MMU_PAGESIZE) {
2706 return (DDI_DMA_BADATTR);
2707 }
2708
2709 /* if we're bouncing on seg, seg must be <= addr_hi */
2710 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2711 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2712 return (DDI_DMA_BADATTR);
2713 }
2714 return (DDI_SUCCESS);
2715 }
2716
2717 /*
2718 * rootnex_valid_bind_parms()
2719 * Called in ddi_dma_*_bind_handle path to validate its parameters.
2720 */
2721 /* ARGSUSED */
2722 static int
2723 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2724 {
2725 #if !defined(__amd64)
2726 /*
2727 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2728 * we can track the offset for the obsoleted interfaces.
2729 */
2730 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2731 return (DDI_DMA_TOOBIG);
2732 }
2733 #endif
2734
2735 return (DDI_SUCCESS);
2736 }
2737
2738
2739 /*
2740 * rootnex_need_bounce_seg()
2741 * check to see if the buffer lives on both side of the seg.
2742 */
2743 static boolean_t
2744 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2745 {
2746 ddi_dma_atyp_t buftype;
2747 rootnex_addr_t raddr;
2748 boolean_t lower_addr;
2749 boolean_t upper_addr;
2750 uint64_t offset;
2751 page_t **pplist;
2752 uint64_t paddr;
2753 uint32_t psize;
2754 uint32_t size;
2755 caddr_t vaddr;
2756 uint_t pcnt;
2757 page_t *pp;
2758
2759
2760 /* shortcuts */
2761 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2762 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2763 buftype = dmar_object->dmao_type;
2764 size = dmar_object->dmao_size;
2765
2766 lower_addr = B_FALSE;
2767 upper_addr = B_FALSE;
2768 pcnt = 0;
2769
2770 /*
2771 * Process the first page to handle the initial offset of the buffer.
2772 * We'll use the base address we get later when we loop through all
2773 * the pages.
2774 */
2775 if (buftype == DMA_OTYP_PAGES) {
2776 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2777 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2778 MMU_PAGEOFFSET;
2779 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2780 psize = MIN(size, (MMU_PAGESIZE - offset));
2781 pp = pp->p_next;
2782 sglinfo->si_asp = NULL;
2783 } else if (pplist != NULL) {
2784 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2785 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2786 if (sglinfo->si_asp == NULL) {
2787 sglinfo->si_asp = &kas;
2788 }
2789 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2790 paddr += offset;
2791 psize = MIN(size, (MMU_PAGESIZE - offset));
2792 pcnt++;
2793 } else {
2794 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2795 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2796 if (sglinfo->si_asp == NULL) {
2797 sglinfo->si_asp = &kas;
2798 }
2799 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2800 paddr += offset;
2801 psize = MIN(size, (MMU_PAGESIZE - offset));
2802 vaddr += psize;
2803 }
2804
2805 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2806
2807 if ((raddr + psize) > sglinfo->si_segmask) {
2808 upper_addr = B_TRUE;
2809 } else {
2810 lower_addr = B_TRUE;
2811 }
2812 size -= psize;
2813
2814 /*
2815 * Walk through the rest of the pages in the buffer. Track to see
2816 * if we have pages on both sides of the segment boundary.
2817 */
2818 while (size > 0) {
2819 /* partial or full page */
2820 psize = MIN(size, MMU_PAGESIZE);
2821
2822 if (buftype == DMA_OTYP_PAGES) {
2823 /* get the paddr from the page_t */
2824 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2825 paddr = pfn_to_pa(pp->p_pagenum);
2826 pp = pp->p_next;
2827 } else if (pplist != NULL) {
2828 /* index into the array of page_t's to get the paddr */
2829 ASSERT(!PP_ISFREE(pplist[pcnt]));
2830 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2831 pcnt++;
2832 } else {
2833 /* call into the VM to get the paddr */
2834 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2835 vaddr));
2836 vaddr += psize;
2837 }
2838
2839 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2840
2841 if ((raddr + psize) > sglinfo->si_segmask) {
2842 upper_addr = B_TRUE;
2843 } else {
2844 lower_addr = B_TRUE;
2845 }
2846 /*
2847 * if the buffer lives both above and below the segment
2848 * boundary, or the current page is the page immediately
2849 * after the segment, we will use a copy/bounce buffer for
2850 * all pages > seg.
2851 */
2852 if ((lower_addr && upper_addr) ||
2853 (raddr == (sglinfo->si_segmask + 1))) {
2854 return (B_TRUE);
2855 }
2856
2857 size -= psize;
2858 }
2859
2860 return (B_FALSE);
2861 }
2862
2863 /*
2864 * rootnex_get_sgl()
2865 * Called in bind fastpath to get the sgl. Most of this will be replaced
2866 * with a call to the vm layer when vm2.0 comes around...
2867 */
2868 static void
2869 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2870 rootnex_sglinfo_t *sglinfo)
2871 {
2872 ddi_dma_atyp_t buftype;
2873 rootnex_addr_t raddr;
2874 uint64_t last_page;
2875 uint64_t offset;
2876 uint64_t addrhi;
2877 uint64_t addrlo;
2878 uint64_t maxseg;
2879 page_t **pplist;
2880 uint64_t paddr;
2881 uint32_t psize;
2882 uint32_t size;
2883 caddr_t vaddr;
2884 uint_t pcnt;
2885 page_t *pp;
2886 uint_t cnt;
2887
2888
2889 /* shortcuts */
2890 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2891 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2892 maxseg = sglinfo->si_max_cookie_size;
2893 buftype = dmar_object->dmao_type;
2894 addrhi = sglinfo->si_max_addr;
2895 addrlo = sglinfo->si_min_addr;
2896 size = dmar_object->dmao_size;
2897
2898 pcnt = 0;
2899 cnt = 0;
2900
2901
2902 /*
2903 * check to see if we need to use the copy buffer for pages over
2904 * the segment attr.
2905 */
2906 sglinfo->si_bounce_on_seg = B_FALSE;
2907 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2908 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2909 dmar_object, sglinfo);
2910 }
2911
2912 /*
2913 * if we were passed down a linked list of pages, i.e. pointer to
2914 * page_t, use this to get our physical address and buf offset.
2915 */
2916 if (buftype == DMA_OTYP_PAGES) {
2917 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2918 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2919 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2920 MMU_PAGEOFFSET;
2921 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2922 psize = MIN(size, (MMU_PAGESIZE - offset));
2923 pp = pp->p_next;
2924 sglinfo->si_asp = NULL;
2925
2926 /*
2927 * We weren't passed down a linked list of pages, but if we were passed
2928 * down an array of pages, use this to get our physical address and buf
2929 * offset.
2930 */
2931 } else if (pplist != NULL) {
2932 ASSERT((buftype == DMA_OTYP_VADDR) ||
2933 (buftype == DMA_OTYP_BUFVADDR));
2934
2935 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2936 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2937 if (sglinfo->si_asp == NULL) {
2938 sglinfo->si_asp = &kas;
2939 }
2940
2941 ASSERT(!PP_ISFREE(pplist[pcnt]));
2942 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2943 paddr += offset;
2944 psize = MIN(size, (MMU_PAGESIZE - offset));
2945 pcnt++;
2946
2947 /*
2948 * All we have is a virtual address, we'll need to call into the VM
2949 * to get the physical address.
2950 */
2951 } else {
2952 ASSERT((buftype == DMA_OTYP_VADDR) ||
2953 (buftype == DMA_OTYP_BUFVADDR));
2954
2955 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2956 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2957 if (sglinfo->si_asp == NULL) {
2958 sglinfo->si_asp = &kas;
2959 }
2960
2961 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2962 paddr += offset;
2963 psize = MIN(size, (MMU_PAGESIZE - offset));
2964 vaddr += psize;
2965 }
2966
2967 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2968
2969 /*
2970 * Setup the first cookie with the physical address of the page and the
2971 * size of the page (which takes into account the initial offset into
2972 * the page.
2973 */
2974 sgl[cnt].dmac_laddress = raddr;
2975 sgl[cnt].dmac_size = psize;
2976 sgl[cnt].dmac_type = 0;
2977
2978 /*
2979 * Save away the buffer offset into the page. We'll need this later in
2980 * the copy buffer code to help figure out the page index within the
2981 * buffer and the offset into the current page.
2982 */
2983 sglinfo->si_buf_offset = offset;
2984
2985 /*
2986 * If we are using the copy buffer for anything over the segment
2987 * boundary, and this page is over the segment boundary.
2988 * OR
2989 * if the DMA engine can't reach the physical address.
2990 */
2991 if (((sglinfo->si_bounce_on_seg) &&
2992 ((raddr + psize) > sglinfo->si_segmask)) ||
2993 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2994 /*
2995 * Increase how much copy buffer we use. We always increase by
2996 * pagesize so we don't have to worry about converting offsets.
2997 * Set a flag in the cookies dmac_type to indicate that it uses
2998 * the copy buffer. If this isn't the last cookie, go to the
2999 * next cookie (since we separate each page which uses the copy
3000 * buffer in case the copy buffer is not physically contiguous.
3001 */
3002 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3003 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3004 if ((cnt + 1) < sglinfo->si_max_pages) {
3005 cnt++;
3006 sgl[cnt].dmac_laddress = 0;
3007 sgl[cnt].dmac_size = 0;
3008 sgl[cnt].dmac_type = 0;
3009 }
3010 }
3011
3012 /*
3013 * save this page's physical address so we can figure out if the next
3014 * page is physically contiguous. Keep decrementing size until we are
3015 * done with the buffer.
3016 */
3017 last_page = raddr & MMU_PAGEMASK;
3018 size -= psize;
3019
3020 while (size > 0) {
3021 /* Get the size for this page (i.e. partial or full page) */
3022 psize = MIN(size, MMU_PAGESIZE);
3023
3024 if (buftype == DMA_OTYP_PAGES) {
3025 /* get the paddr from the page_t */
3026 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3027 paddr = pfn_to_pa(pp->p_pagenum);
3028 pp = pp->p_next;
3029 } else if (pplist != NULL) {
3030 /* index into the array of page_t's to get the paddr */
3031 ASSERT(!PP_ISFREE(pplist[pcnt]));
3032 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3033 pcnt++;
3034 } else {
3035 /* call into the VM to get the paddr */
3036 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3037 vaddr));
3038 vaddr += psize;
3039 }
3040
3041 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3042
3043 /*
3044 * If we are using the copy buffer for anything over the
3045 * segment boundary, and this page is over the segment
3046 * boundary.
3047 * OR
3048 * if the DMA engine can't reach the physical address.
3049 */
3050 if (((sglinfo->si_bounce_on_seg) &&
3051 ((raddr + psize) > sglinfo->si_segmask)) ||
3052 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3053
3054 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3055
3056 /*
3057 * if there is something in the current cookie, go to
3058 * the next one. We only want one page in a cookie which
3059 * uses the copybuf since the copybuf doesn't have to
3060 * be physically contiguous.
3061 */
3062 if (sgl[cnt].dmac_size != 0) {
3063 cnt++;
3064 }
3065 sgl[cnt].dmac_laddress = raddr;
3066 sgl[cnt].dmac_size = psize;
3067 #if defined(__amd64)
3068 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3069 #else
3070 /*
3071 * save the buf offset for 32-bit kernel. used in the
3072 * obsoleted interfaces.
3073 */
3074 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3075 (dmar_object->dmao_size - size);
3076 #endif
3077 /* if this isn't the last cookie, go to the next one */
3078 if ((cnt + 1) < sglinfo->si_max_pages) {
3079 cnt++;
3080 sgl[cnt].dmac_laddress = 0;
3081 sgl[cnt].dmac_size = 0;
3082 sgl[cnt].dmac_type = 0;
3083 }
3084
3085 /*
3086 * this page didn't need the copy buffer, if it's not physically
3087 * contiguous, or it would put us over a segment boundary, or it
3088 * puts us over the max cookie size, or the current sgl doesn't
3089 * have anything in it.
3090 */
3091 } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3092 !(raddr & sglinfo->si_segmask) ||
3093 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3094 (sgl[cnt].dmac_size == 0)) {
3095 /*
3096 * if we're not already in a new cookie, go to the next
3097 * cookie.
3098 */
3099 if (sgl[cnt].dmac_size != 0) {
3100 cnt++;
3101 }
3102
3103 /* save the cookie information */
3104 sgl[cnt].dmac_laddress = raddr;
3105 sgl[cnt].dmac_size = psize;
3106 #if defined(__amd64)
3107 sgl[cnt].dmac_type = 0;
3108 #else
3109 /*
3110 * save the buf offset for 32-bit kernel. used in the
3111 * obsoleted interfaces.
3112 */
3113 sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3114 #endif
3115
3116 /*
3117 * this page didn't need the copy buffer, it is physically
3118 * contiguous with the last page, and it's <= the max cookie
3119 * size.
3120 */
3121 } else {
3122 sgl[cnt].dmac_size += psize;
3123
3124 /*
3125 * if this exactly == the maximum cookie size, and
3126 * it isn't the last cookie, go to the next cookie.
3127 */
3128 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3129 ((cnt + 1) < sglinfo->si_max_pages)) {
3130 cnt++;
3131 sgl[cnt].dmac_laddress = 0;
3132 sgl[cnt].dmac_size = 0;
3133 sgl[cnt].dmac_type = 0;
3134 }
3135 }
3136
3137 /*
3138 * save this page's physical address so we can figure out if the
3139 * next page is physically contiguous. Keep decrementing size
3140 * until we are done with the buffer.
3141 */
3142 last_page = raddr;
3143 size -= psize;
3144 }
3145
3146 /* we're done, save away how many cookies the sgl has */
3147 if (sgl[cnt].dmac_size == 0) {
3148 ASSERT(cnt < sglinfo->si_max_pages);
3149 sglinfo->si_sgl_size = cnt;
3150 } else {
3151 sglinfo->si_sgl_size = cnt + 1;
3152 }
3153 }
3154
3155 static void
3156 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3157 rootnex_sglinfo_t *sglinfo)
3158 {
3159 uint64_t offset;
3160 uint64_t maxseg;
3161 uint64_t dvaddr;
3162 struct dvmaseg *dvs;
3163 uint64_t paddr;
3164 uint32_t psize, ssize;
3165 uint32_t size;
3166 uint_t cnt;
3167 int physcontig;
3168
3169 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3170
3171 /* shortcuts */
3172 maxseg = sglinfo->si_max_cookie_size;
3173 size = dmar_object->dmao_size;
3174
3175 cnt = 0;
3176 sglinfo->si_bounce_on_seg = B_FALSE;
3177
3178 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3179 offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3180 ssize = dvs->dvs_len;
3181 paddr = dvs->dvs_start;
3182 paddr += offset;
3183 psize = MIN(ssize, (maxseg - offset));
3184 dvaddr = paddr + psize;
3185 ssize -= psize;
3186
3187 sgl[cnt].dmac_laddress = paddr;
3188 sgl[cnt].dmac_size = psize;
3189 sgl[cnt].dmac_type = 0;
3190
3191 size -= psize;
3192 while (size > 0) {
3193 if (ssize == 0) {
3194 dvs++;
3195 ssize = dvs->dvs_len;
3196 dvaddr = dvs->dvs_start;
3197 physcontig = 0;
3198 } else
3199 physcontig = 1;
3200
3201 paddr = dvaddr;
3202 psize = MIN(ssize, maxseg);
3203 dvaddr += psize;
3204 ssize -= psize;
3205
3206 if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3207 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3208 (sgl[cnt].dmac_size == 0)) {
3209 /*
3210 * if we're not already in a new cookie, go to the next
3211 * cookie.
3212 */
3213 if (sgl[cnt].dmac_size != 0) {
3214 cnt++;
3215 }
3216
3217 /* save the cookie information */
3218 sgl[cnt].dmac_laddress = paddr;
3219 sgl[cnt].dmac_size = psize;
3220 sgl[cnt].dmac_type = 0;
3221 } else {
3222 sgl[cnt].dmac_size += psize;
3223
3224 /*
3225 * if this exactly == the maximum cookie size, and
3226 * it isn't the last cookie, go to the next cookie.
3227 */
3228 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3229 ((cnt + 1) < sglinfo->si_max_pages)) {
3230 cnt++;
3231 sgl[cnt].dmac_laddress = 0;
3232 sgl[cnt].dmac_size = 0;
3233 sgl[cnt].dmac_type = 0;
3234 }
3235 }
3236 size -= psize;
3237 }
3238
3239 /* we're done, save away how many cookies the sgl has */
3240 if (sgl[cnt].dmac_size == 0) {
3241 sglinfo->si_sgl_size = cnt;
3242 } else {
3243 sglinfo->si_sgl_size = cnt + 1;
3244 }
3245 }
3246
3247 /*
3248 * rootnex_bind_slowpath()
3249 * Call in the bind path if the calling driver can't use the sgl without
3250 * modifying it. We either need to use the copy buffer and/or we will end up
3251 * with a partial bind.
3252 */
3253 static int
3254 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3255 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3256 {
3257 rootnex_sglinfo_t *sinfo;
3258 rootnex_window_t *window;
3259 ddi_dma_cookie_t *cookie;
3260 size_t copybuf_used;
3261 size_t dmac_size;
3262 boolean_t partial;
3263 off_t cur_offset;
3264 page_t *cur_pp;
3265 major_t mnum;
3266 int e;
3267 int i;
3268
3269
3270 sinfo = &dma->dp_sglinfo;
3271 copybuf_used = 0;
3272 partial = B_FALSE;
3273
3274 /*
3275 * If we're using the copybuf, set the copybuf state in dma struct.
3276 * Needs to be first since it sets the copy buffer size.
3277 */
3278 if (sinfo->si_copybuf_req != 0) {
3279 e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3280 if (e != DDI_SUCCESS) {
3281 return (e);
3282 }
3283 } else {
3284 dma->dp_copybuf_size = 0;
3285 }
3286
3287 /*
3288 * Figure out if we need to do a partial mapping. If so, figure out
3289 * if we need to trim the buffers when we munge the sgl.
3290 */
3291 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3292 (dmao->dmao_size > dma->dp_maxxfer) ||
3293 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3294 dma->dp_partial_required = B_TRUE;
3295 if (attr->dma_attr_granular != 1) {
3296 dma->dp_trim_required = B_TRUE;
3297 }
3298 } else {
3299 dma->dp_partial_required = B_FALSE;
3300 dma->dp_trim_required = B_FALSE;
3301 }
3302
3303 /* If we need to do a partial bind, make sure the driver supports it */
3304 if (dma->dp_partial_required &&
3305 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3306
3307 mnum = ddi_driver_major(dma->dp_dip);
3308 /*
3309 * patchable which allows us to print one warning per major
3310 * number.
3311 */
3312 if ((rootnex_bind_warn) &&
3313 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3314 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3315 cmn_err(CE_WARN, "!%s: coding error detected, the "
3316 "driver is using ddi_dma_attr(9S) incorrectly. "
3317 "There is a small risk of data corruption in "
3318 "particular with large I/Os. The driver should be "
3319 "replaced with a corrected version for proper "
3320 "system operation. To disable this warning, add "
3321 "'set rootnex:rootnex_bind_warn=0' to "
3322 "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3323 }
3324 return (DDI_DMA_TOOBIG);
3325 }
3326
3327 /*
3328 * we might need multiple windows, setup state to handle them. In this
3329 * code path, we will have at least one window.
3330 */
3331 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3332 if (e != DDI_SUCCESS) {
3333 rootnex_teardown_copybuf(dma);
3334 return (e);
3335 }
3336
3337 window = &dma->dp_window[0];
3338 cookie = &dma->dp_cookies[0];
3339 cur_offset = 0;
3340 rootnex_init_win(hp, dma, window, cookie, cur_offset);
3341 if (dmao->dmao_type == DMA_OTYP_PAGES) {
3342 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3343 }
3344
3345 /* loop though all the cookies we got back from get_sgl() */
3346 for (i = 0; i < sinfo->si_sgl_size; i++) {
3347 /*
3348 * If we're using the copy buffer, check this cookie and setup
3349 * its associated copy buffer state. If this cookie uses the
3350 * copy buffer, make sure we sync this window during dma_sync.
3351 */
3352 if (dma->dp_copybuf_size > 0) {
3353 rootnex_setup_cookie(dmao, dma, cookie,
3354 cur_offset, ©buf_used, &cur_pp);
3355 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3356 window->wd_dosync = B_TRUE;
3357 }
3358 }
3359
3360 /*
3361 * save away the cookie size, since it could be modified in
3362 * the windowing code.
3363 */
3364 dmac_size = cookie->dmac_size;
3365
3366 /* if we went over max copybuf size */
3367 if (dma->dp_copybuf_size &&
3368 (copybuf_used > dma->dp_copybuf_size)) {
3369 partial = B_TRUE;
3370 e = rootnex_copybuf_window_boundary(hp, dma, &window,
3371 cookie, cur_offset, ©buf_used);
3372 if (e != DDI_SUCCESS) {
3373 rootnex_teardown_copybuf(dma);
3374 rootnex_teardown_windows(dma);
3375 return (e);
3376 }
3377
3378 /*
3379 * if the coookie uses the copy buffer, make sure the
3380 * new window we just moved to is set to sync.
3381 */
3382 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3383 window->wd_dosync = B_TRUE;
3384 }
3385 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3386 dma->dp_dip);
3387
3388 /* if the cookie cnt == max sgllen, move to the next window */
3389 } else if (window->wd_cookie_cnt >=
3390 (unsigned)attr->dma_attr_sgllen) {
3391 partial = B_TRUE;
3392 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3393 e = rootnex_sgllen_window_boundary(hp, dma, &window,
3394 cookie, attr, cur_offset);
3395 if (e != DDI_SUCCESS) {
3396 rootnex_teardown_copybuf(dma);
3397 rootnex_teardown_windows(dma);
3398 return (e);
3399 }
3400
3401 /*
3402 * if the coookie uses the copy buffer, make sure the
3403 * new window we just moved to is set to sync.
3404 */
3405 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3406 window->wd_dosync = B_TRUE;
3407 }
3408 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3409 dma->dp_dip);
3410
3411 /* else if we will be over maxxfer */
3412 } else if ((window->wd_size + dmac_size) >
3413 dma->dp_maxxfer) {
3414 partial = B_TRUE;
3415 e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3416 cookie);
3417 if (e != DDI_SUCCESS) {
3418 rootnex_teardown_copybuf(dma);
3419 rootnex_teardown_windows(dma);
3420 return (e);
3421 }
3422
3423 /*
3424 * if the coookie uses the copy buffer, make sure the
3425 * new window we just moved to is set to sync.
3426 */
3427 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3428 window->wd_dosync = B_TRUE;
3429 }
3430 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3431 dma->dp_dip);
3432
3433 /* else this cookie fits in the current window */
3434 } else {
3435 window->wd_cookie_cnt++;
3436 window->wd_size += dmac_size;
3437 }
3438
3439 /* track our offset into the buffer, go to the next cookie */
3440 ASSERT(dmac_size <= dmao->dmao_size);
3441 ASSERT(cookie->dmac_size <= dmac_size);
3442 cur_offset += dmac_size;
3443 cookie++;
3444 }
3445
3446 /* if we ended up with a zero sized window in the end, clean it up */
3447 if (window->wd_size == 0) {
3448 hp->dmai_nwin--;
3449 window--;
3450 }
3451
3452 ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3453
3454 if (!partial) {
3455 return (DDI_DMA_MAPPED);
3456 }
3457
3458 ASSERT(dma->dp_partial_required);
3459 return (DDI_DMA_PARTIAL_MAP);
3460 }
3461
3462 /*
3463 * rootnex_setup_copybuf()
3464 * Called in bind slowpath. Figures out if we're going to use the copy
3465 * buffer, and if we do, sets up the basic state to handle it.
3466 */
3467 static int
3468 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3469 rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3470 {
3471 rootnex_sglinfo_t *sinfo;
3472 ddi_dma_attr_t lattr;
3473 size_t max_copybuf;
3474 int cansleep;
3475 int e;
3476 #if !defined(__amd64)
3477 int vmflag;
3478 #endif
3479
3480 ASSERT(!dma->dp_dvma_used);
3481
3482 sinfo = &dma->dp_sglinfo;
3483
3484 /* read this first so it's consistent through the routine */
3485 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3486
3487 /* We need to call into the rootnex on ddi_dma_sync() */
3488 hp->dmai_rflags &= ~DMP_NOSYNC;
3489
3490 /* make sure the copybuf size <= the max size */
3491 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3492 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3493
3494 #if !defined(__amd64)
3495 /*
3496 * if we don't have kva space to copy to/from, allocate the KVA space
3497 * now. We only do this for the 32-bit kernel. We use seg kpm space for
3498 * the 64-bit kernel.
3499 */
3500 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3501 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3502
3503 /* convert the sleep flags */
3504 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3505 vmflag = VM_SLEEP;
3506 } else {
3507 vmflag = VM_NOSLEEP;
3508 }
3509
3510 /* allocate Kernel VA space that we can bcopy to/from */
3511 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3512 vmflag);
3513 if (dma->dp_kva == NULL) {
3514 return (DDI_DMA_NORESOURCES);
3515 }
3516 }
3517 #endif
3518
3519 /* convert the sleep flags */
3520 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3521 cansleep = 1;
3522 } else {
3523 cansleep = 0;
3524 }
3525
3526 /*
3527 * Allocate the actual copy buffer. This needs to fit within the DMA
3528 * engine limits, so we can't use kmem_alloc... We don't need
3529 * contiguous memory (sgllen) since we will be forcing windows on
3530 * sgllen anyway.
3531 */
3532 lattr = *attr;
3533 lattr.dma_attr_align = MMU_PAGESIZE;
3534 lattr.dma_attr_sgllen = -1; /* no limit */
3535 /*
3536 * if we're using the copy buffer because of seg, use that for our
3537 * upper address limit.
3538 */
3539 if (sinfo->si_bounce_on_seg) {
3540 lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3541 }
3542 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3543 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3544 if (e != DDI_SUCCESS) {
3545 #if !defined(__amd64)
3546 if (dma->dp_kva != NULL) {
3547 vmem_free(heap_arena, dma->dp_kva,
3548 dma->dp_copybuf_size);
3549 }
3550 #endif
3551 return (DDI_DMA_NORESOURCES);
3552 }
3553
3554 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3555 size_t, dma->dp_copybuf_size);
3556
3557 return (DDI_SUCCESS);
3558 }
3559
3560
3561 /*
3562 * rootnex_setup_windows()
3563 * Called in bind slowpath to setup the window state. We always have windows
3564 * in the slowpath. Even if the window count = 1.
3565 */
3566 static int
3567 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3568 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3569 {
3570 rootnex_window_t *windowp;
3571 rootnex_sglinfo_t *sinfo;
3572 size_t copy_state_size;
3573 size_t win_state_size;
3574 size_t state_available;
3575 size_t space_needed;
3576 uint_t copybuf_win;
3577 uint_t maxxfer_win;
3578 size_t space_used;
3579 uint_t sglwin;
3580
3581
3582 sinfo = &dma->dp_sglinfo;
3583
3584 dma->dp_current_win = 0;
3585 hp->dmai_nwin = 0;
3586
3587 /* If we don't need to do a partial, we only have one window */
3588 if (!dma->dp_partial_required) {
3589 dma->dp_max_win = 1;
3590
3591 /*
3592 * we need multiple windows, need to figure out the worse case number
3593 * of windows.
3594 */
3595 } else {
3596 /*
3597 * if we need windows because we need more copy buffer that
3598 * we allow, the worse case number of windows we could need
3599 * here would be (copybuf space required / copybuf space that
3600 * we have) plus one for remainder, and plus 2 to handle the
3601 * extra pages on the trim for the first and last pages of the
3602 * buffer (a page is the minimum window size so under the right
3603 * attr settings, you could have a window for each page).
3604 * The last page will only be hit here if the size is not a
3605 * multiple of the granularity (which theoretically shouldn't
3606 * be the case but never has been enforced, so we could have
3607 * broken things without it).
3608 */
3609 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3610 ASSERT(dma->dp_copybuf_size > 0);
3611 copybuf_win = (sinfo->si_copybuf_req /
3612 dma->dp_copybuf_size) + 1 + 2;
3613 } else {
3614 copybuf_win = 0;
3615 }
3616
3617 /*
3618 * if we need windows because we have more cookies than the H/W
3619 * can handle, the number of windows we would need here would
3620 * be (cookie count / cookies count H/W supports minus 1[for
3621 * trim]) plus one for remainder.
3622 */
3623 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3624 sglwin = (sinfo->si_sgl_size /
3625 (attr->dma_attr_sgllen - 1)) + 1;
3626 } else {
3627 sglwin = 0;
3628 }
3629
3630 /*
3631 * if we need windows because we're binding more memory than the
3632 * H/W can transfer at once, the number of windows we would need
3633 * here would be (xfer count / max xfer H/W supports) plus one
3634 * for remainder, and plus 2 to handle the extra pages on the
3635 * trim (see above comment about trim)
3636 */
3637 if (dmao->dmao_size > dma->dp_maxxfer) {
3638 maxxfer_win = (dmao->dmao_size /
3639 dma->dp_maxxfer) + 1 + 2;
3640 } else {
3641 maxxfer_win = 0;
3642 }
3643 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win;
3644 ASSERT(dma->dp_max_win > 0);
3645 }
3646 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3647
3648 /*
3649 * Get space for window and potential copy buffer state. Before we
3650 * go and allocate memory, see if we can get away with using what's
3651 * left in the pre-allocted state or the dynamically allocated sgl.
3652 */
3653 space_used = (uintptr_t)(sinfo->si_sgl_size *
3654 sizeof (ddi_dma_cookie_t));
3655
3656 /* if we dynamically allocated space for the cookies */
3657 if (dma->dp_need_to_free_cookie) {
3658 /* if we have more space in the pre-allocted buffer, use it */
3659 ASSERT(space_used <= dma->dp_cookie_size);
3660 if ((dma->dp_cookie_size - space_used) <=
3661 rootnex_state->r_prealloc_size) {
3662 state_available = rootnex_state->r_prealloc_size;
3663 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3664
3665 /*
3666 * else, we have more free space in the dynamically allocated
3667 * buffer, i.e. the buffer wasn't worse case fragmented so we
3668 * didn't need a lot of cookies.
3669 */
3670 } else {
3671 state_available = dma->dp_cookie_size - space_used;
3672 windowp = (rootnex_window_t *)
3673 &dma->dp_cookies[sinfo->si_sgl_size];
3674 }
3675
3676 /* we used the pre-alloced buffer */
3677 } else {
3678 ASSERT(space_used <= rootnex_state->r_prealloc_size);
3679 state_available = rootnex_state->r_prealloc_size - space_used;
3680 windowp = (rootnex_window_t *)
3681 &dma->dp_cookies[sinfo->si_sgl_size];
3682 }
3683
3684 /*
3685 * figure out how much state we need to track the copy buffer. Add an
3686 * addition 8 bytes for pointer alignemnt later.
3687 */
3688 if (dma->dp_copybuf_size > 0) {
3689 copy_state_size = sinfo->si_max_pages *
3690 sizeof (rootnex_pgmap_t);
3691 } else {
3692 copy_state_size = 0;
3693 }
3694 /* add an additional 8 bytes for pointer alignment */
3695 space_needed = win_state_size + copy_state_size + 0x8;
3696
3697 /* if we have enough space already, use it */
3698 if (state_available >= space_needed) {
3699 dma->dp_window = windowp;
3700 dma->dp_need_to_free_window = B_FALSE;
3701
3702 /* not enough space, need to allocate more. */
3703 } else {
3704 dma->dp_window = kmem_alloc(space_needed, kmflag);
3705 if (dma->dp_window == NULL) {
3706 return (DDI_DMA_NORESOURCES);
3707 }
3708 dma->dp_need_to_free_window = B_TRUE;
3709 dma->dp_window_size = space_needed;
3710 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3711 dma->dp_dip, size_t, space_needed);
3712 }
3713
3714 /*
3715 * we allocate copy buffer state and window state at the same time.
3716 * setup our copy buffer state pointers. Make sure it's aligned.
3717 */
3718 if (dma->dp_copybuf_size > 0) {
3719 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3720 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3721
3722 #if !defined(__amd64)
3723 /*
3724 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3725 * false/NULL. Should be quicker to bzero vs loop and set.
3726 */
3727 bzero(dma->dp_pgmap, copy_state_size);
3728 #endif
3729 } else {
3730 dma->dp_pgmap = NULL;
3731 }
3732
3733 return (DDI_SUCCESS);
3734 }
3735
3736
3737 /*
3738 * rootnex_teardown_copybuf()
3739 * cleans up after rootnex_setup_copybuf()
3740 */
3741 static void
3742 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3743 {
3744 #if !defined(__amd64)
3745 int i;
3746
3747 /*
3748 * if we allocated kernel heap VMEM space, go through all the pages and
3749 * map out any of the ones that we're mapped into the kernel heap VMEM
3750 * arena. Then free the VMEM space.
3751 */
3752 if (dma->dp_kva != NULL) {
3753 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3754 if (dma->dp_pgmap[i].pm_mapped) {
3755 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3756 MMU_PAGESIZE, HAT_UNLOAD);
3757 dma->dp_pgmap[i].pm_mapped = B_FALSE;
3758 }
3759 }
3760
3761 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3762 }
3763
3764 #endif
3765
3766 /* if we allocated a copy buffer, free it */
3767 if (dma->dp_cbaddr != NULL) {
3768 i_ddi_mem_free(dma->dp_cbaddr, NULL);
3769 }
3770 }
3771
3772
3773 /*
3774 * rootnex_teardown_windows()
3775 * cleans up after rootnex_setup_windows()
3776 */
3777 static void
3778 rootnex_teardown_windows(rootnex_dma_t *dma)
3779 {
3780 /*
3781 * if we had to allocate window state on the last bind (because we
3782 * didn't have enough pre-allocated space in the handle), free it.
3783 */
3784 if (dma->dp_need_to_free_window) {
3785 kmem_free(dma->dp_window, dma->dp_window_size);
3786 }
3787 }
3788
3789
3790 /*
3791 * rootnex_init_win()
3792 * Called in bind slow path during creation of a new window. Initializes
3793 * window state to default values.
3794 */
3795 /*ARGSUSED*/
3796 static void
3797 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3798 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3799 {
3800 hp->dmai_nwin++;
3801 window->wd_dosync = B_FALSE;
3802 window->wd_offset = cur_offset;
3803 window->wd_size = 0;
3804 window->wd_first_cookie = cookie;
3805 window->wd_cookie_cnt = 0;
3806 window->wd_trim.tr_trim_first = B_FALSE;
3807 window->wd_trim.tr_trim_last = B_FALSE;
3808 window->wd_trim.tr_first_copybuf_win = B_FALSE;
3809 window->wd_trim.tr_last_copybuf_win = B_FALSE;
3810 #if !defined(__amd64)
3811 window->wd_remap_copybuf = dma->dp_cb_remaping;
3812 #endif
3813 }
3814
3815
3816 /*
3817 * rootnex_setup_cookie()
3818 * Called in the bind slow path when the sgl uses the copy buffer. If any of
3819 * the sgl uses the copy buffer, we need to go through each cookie, figure
3820 * out if it uses the copy buffer, and if it does, save away everything we'll
3821 * need during sync.
3822 */
3823 static void
3824 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3825 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3826 page_t **cur_pp)
3827 {
3828 boolean_t copybuf_sz_power_2;
3829 rootnex_sglinfo_t *sinfo;
3830 paddr_t paddr;
3831 uint_t pidx;
3832 uint_t pcnt;
3833 off_t poff;
3834 #if defined(__amd64)
3835 pfn_t pfn;
3836 #else
3837 page_t **pplist;
3838 #endif
3839
3840 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3841
3842 sinfo = &dma->dp_sglinfo;
3843
3844 /*
3845 * Calculate the page index relative to the start of the buffer. The
3846 * index to the current page for our buffer is the offset into the
3847 * first page of the buffer plus our current offset into the buffer
3848 * itself, shifted of course...
3849 */
3850 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3851 ASSERT(pidx < sinfo->si_max_pages);
3852
3853 /* if this cookie uses the copy buffer */
3854 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3855 /*
3856 * NOTE: we know that since this cookie uses the copy buffer, it
3857 * is <= MMU_PAGESIZE.
3858 */
3859
3860 /*
3861 * get the offset into the page. For the 64-bit kernel, get the
3862 * pfn which we'll use with seg kpm.
3863 */
3864 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3865 #if defined(__amd64)
3866 /* mfn_to_pfn() is a NOP on i86pc */
3867 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3868 #endif /* __amd64 */
3869
3870 /* figure out if the copybuf size is a power of 2 */
3871 if (!ISP2(dma->dp_copybuf_size)) {
3872 copybuf_sz_power_2 = B_FALSE;
3873 } else {
3874 copybuf_sz_power_2 = B_TRUE;
3875 }
3876
3877 /* This page uses the copy buffer */
3878 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3879
3880 /*
3881 * save the copy buffer KVA that we'll use with this page.
3882 * if we still fit within the copybuf, it's a simple add.
3883 * otherwise, we need to wrap over using & or % accordingly.
3884 */
3885 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3886 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3887 *copybuf_used;
3888 } else {
3889 if (copybuf_sz_power_2) {
3890 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3891 (uintptr_t)dma->dp_cbaddr +
3892 (*copybuf_used &
3893 (dma->dp_copybuf_size - 1)));
3894 } else {
3895 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3896 (uintptr_t)dma->dp_cbaddr +
3897 (*copybuf_used % dma->dp_copybuf_size));
3898 }
3899 }
3900
3901 /*
3902 * over write the cookie physical address with the address of
3903 * the physical address of the copy buffer page that we will
3904 * use.
3905 */
3906 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3907 dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3908
3909 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3910
3911 /* if we have a kernel VA, it's easy, just save that address */
3912 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3913 (sinfo->si_asp == &kas)) {
3914 /*
3915 * save away the page aligned virtual address of the
3916 * driver buffer. Offsets are handled in the sync code.
3917 */
3918 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3919 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3920 & MMU_PAGEMASK);
3921 #if !defined(__amd64)
3922 /*
3923 * we didn't need to, and will never need to map this
3924 * page.
3925 */
3926 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3927 #endif
3928
3929 /* we don't have a kernel VA. We need one for the bcopy. */
3930 } else {
3931 #if defined(__amd64)
3932 /*
3933 * for the 64-bit kernel, it's easy. We use seg kpm to
3934 * get a Kernel VA for the corresponding pfn.
3935 */
3936 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3937 #else
3938 /*
3939 * for the 32-bit kernel, this is a pain. First we'll
3940 * save away the page_t or user VA for this page. This
3941 * is needed in rootnex_dma_win() when we switch to a
3942 * new window which requires us to re-map the copy
3943 * buffer.
3944 */
3945 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3946 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3947 dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3948 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3949 } else if (pplist != NULL) {
3950 dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3951 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3952 } else {
3953 dma->dp_pgmap[pidx].pm_pp = NULL;
3954 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3955 (((uintptr_t)
3956 dmar_object->dmao_obj.virt_obj.v_addr +
3957 cur_offset) & MMU_PAGEMASK);
3958 }
3959
3960 /*
3961 * save away the page aligned virtual address which was
3962 * allocated from the kernel heap arena (taking into
3963 * account if we need more copy buffer than we alloced
3964 * and use multiple windows to handle this, i.e. &,%).
3965 * NOTE: there isn't and physical memory backing up this
3966 * virtual address space currently.
3967 */
3968 if ((*copybuf_used + MMU_PAGESIZE) <=
3969 dma->dp_copybuf_size) {
3970 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3971 (((uintptr_t)dma->dp_kva + *copybuf_used) &
3972 MMU_PAGEMASK);
3973 } else {
3974 if (copybuf_sz_power_2) {
3975 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3976 (((uintptr_t)dma->dp_kva +
3977 (*copybuf_used &
3978 (dma->dp_copybuf_size - 1))) &
3979 MMU_PAGEMASK);
3980 } else {
3981 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3982 (((uintptr_t)dma->dp_kva +
3983 (*copybuf_used %
3984 dma->dp_copybuf_size)) &
3985 MMU_PAGEMASK);
3986 }
3987 }
3988
3989 /*
3990 * if we haven't used up the available copy buffer yet,
3991 * map the kva to the physical page.
3992 */
3993 if (!dma->dp_cb_remaping && ((*copybuf_used +
3994 MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3995 dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3996 if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3997 i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3998 dma->dp_pgmap[pidx].pm_kaddr);
3999 } else {
4000 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4001 sinfo->si_asp,
4002 dma->dp_pgmap[pidx].pm_kaddr);
4003 }
4004
4005 /*
4006 * we've used up the available copy buffer, this page
4007 * will have to be mapped during rootnex_dma_win() when
4008 * we switch to a new window which requires a re-map
4009 * the copy buffer. (32-bit kernel only)
4010 */
4011 } else {
4012 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4013 }
4014 #endif
4015 /* go to the next page_t */
4016 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4017 *cur_pp = (*cur_pp)->p_next;
4018 }
4019 }
4020
4021 /* add to the copy buffer count */
4022 *copybuf_used += MMU_PAGESIZE;
4023
4024 /*
4025 * This cookie doesn't use the copy buffer. Walk through the pages this
4026 * cookie occupies to reflect this.
4027 */
4028 } else {
4029 /*
4030 * figure out how many pages the cookie occupies. We need to
4031 * use the original page offset of the buffer and the cookies
4032 * offset in the buffer to do this.
4033 */
4034 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4035 pcnt = mmu_btopr(cookie->dmac_size + poff);
4036
4037 while (pcnt > 0) {
4038 #if !defined(__amd64)
4039 /*
4040 * the 32-bit kernel doesn't have seg kpm, so we need
4041 * to map in the driver buffer (if it didn't come down
4042 * with a kernel VA) on the fly. Since this page doesn't
4043 * use the copy buffer, it's not, or will it ever, have
4044 * to be mapped in.
4045 */
4046 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4047 #endif
4048 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4049
4050 /*
4051 * we need to update pidx and cur_pp or we'll loose
4052 * track of where we are.
4053 */
4054 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4055 *cur_pp = (*cur_pp)->p_next;
4056 }
4057 pidx++;
4058 pcnt--;
4059 }
4060 }
4061 }
4062
4063
4064 /*
4065 * rootnex_sgllen_window_boundary()
4066 * Called in the bind slow path when the next cookie causes us to exceed (in
4067 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4068 * length supported by the DMA H/W.
4069 */
4070 static int
4071 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4072 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4073 off_t cur_offset)
4074 {
4075 off_t new_offset;
4076 size_t trim_sz;
4077 off_t coffset;
4078
4079
4080 /*
4081 * if we know we'll never have to trim, it's pretty easy. Just move to
4082 * the next window and init it. We're done.
4083 */
4084 if (!dma->dp_trim_required) {
4085 (*windowp)++;
4086 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4087 (*windowp)->wd_cookie_cnt++;
4088 (*windowp)->wd_size = cookie->dmac_size;
4089 return (DDI_SUCCESS);
4090 }
4091
4092 /* figure out how much we need to trim from the window */
4093 ASSERT(attr->dma_attr_granular != 0);
4094 if (dma->dp_granularity_power_2) {
4095 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4096 } else {
4097 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4098 }
4099
4100 /* The window's a whole multiple of granularity. We're done */
4101 if (trim_sz == 0) {
4102 (*windowp)++;
4103 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4104 (*windowp)->wd_cookie_cnt++;
4105 (*windowp)->wd_size = cookie->dmac_size;
4106 return (DDI_SUCCESS);
4107 }
4108
4109 /*
4110 * The window's not a whole multiple of granularity, since we know this
4111 * is due to the sgllen, we need to go back to the last cookie and trim
4112 * that one, add the left over part of the old cookie into the new
4113 * window, and then add in the new cookie into the new window.
4114 */
4115
4116 /*
4117 * make sure the driver isn't making us do something bad... Trimming and
4118 * sgllen == 1 don't go together.
4119 */
4120 if (attr->dma_attr_sgllen == 1) {
4121 return (DDI_DMA_NOMAPPING);
4122 }
4123
4124 /*
4125 * first, setup the current window to account for the trim. Need to go
4126 * back to the last cookie for this.
4127 */
4128 cookie--;
4129 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4130 (*windowp)->wd_trim.tr_last_cookie = cookie;
4131 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4132 ASSERT(cookie->dmac_size > trim_sz);
4133 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4134 (*windowp)->wd_size -= trim_sz;
4135
4136 /* save the buffer offsets for the next window */
4137 coffset = cookie->dmac_size - trim_sz;
4138 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4139
4140 /*
4141 * set this now in case this is the first window. all other cases are
4142 * set in dma_win()
4143 */
4144 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4145
4146 /*
4147 * initialize the next window using what's left over in the previous
4148 * cookie.
4149 */
4150 (*windowp)++;
4151 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4152 (*windowp)->wd_cookie_cnt++;
4153 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4154 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4155 (*windowp)->wd_trim.tr_first_size = trim_sz;
4156 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4157 (*windowp)->wd_dosync = B_TRUE;
4158 }
4159
4160 /*
4161 * now go back to the current cookie and add it to the new window. set
4162 * the new window size to the what was left over from the previous
4163 * cookie and what's in the current cookie.
4164 */
4165 cookie++;
4166 (*windowp)->wd_cookie_cnt++;
4167 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4168
4169 /*
4170 * trim plus the next cookie could put us over maxxfer (a cookie can be
4171 * a max size of maxxfer). Handle that case.
4172 */
4173 if ((*windowp)->wd_size > dma->dp_maxxfer) {
4174 /*
4175 * maxxfer is already a whole multiple of granularity, and this
4176 * trim will be <= the previous trim (since a cookie can't be
4177 * larger than maxxfer). Make things simple here.
4178 */
4179 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4180 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4181 (*windowp)->wd_trim.tr_last_cookie = cookie;
4182 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4183 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4184 (*windowp)->wd_size -= trim_sz;
4185 ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4186
4187 /* save the buffer offsets for the next window */
4188 coffset = cookie->dmac_size - trim_sz;
4189 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4190
4191 /* setup the next window */
4192 (*windowp)++;
4193 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4194 (*windowp)->wd_cookie_cnt++;
4195 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4196 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4197 coffset;
4198 (*windowp)->wd_trim.tr_first_size = trim_sz;
4199 }
4200
4201 return (DDI_SUCCESS);
4202 }
4203
4204
4205 /*
4206 * rootnex_copybuf_window_boundary()
4207 * Called in bind slowpath when we get to a window boundary because we used
4208 * up all the copy buffer that we have.
4209 */
4210 static int
4211 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4212 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4213 size_t *copybuf_used)
4214 {
4215 rootnex_sglinfo_t *sinfo;
4216 off_t new_offset;
4217 size_t trim_sz;
4218 paddr_t paddr;
4219 off_t coffset;
4220 uint_t pidx;
4221 off_t poff;
4222
4223
4224 sinfo = &dma->dp_sglinfo;
4225
4226 /*
4227 * the copy buffer should be a whole multiple of page size. We know that
4228 * this cookie is <= MMU_PAGESIZE.
4229 */
4230 ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4231
4232 /*
4233 * from now on, all new windows in this bind need to be re-mapped during
4234 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4235 * space...
4236 */
4237 #if !defined(__amd64)
4238 dma->dp_cb_remaping = B_TRUE;
4239 #endif
4240
4241 /* reset copybuf used */
4242 *copybuf_used = 0;
4243
4244 /*
4245 * if we don't have to trim (since granularity is set to 1), go to the
4246 * next window and add the current cookie to it. We know the current
4247 * cookie uses the copy buffer since we're in this code path.
4248 */
4249 if (!dma->dp_trim_required) {
4250 (*windowp)++;
4251 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4252
4253 /* Add this cookie to the new window */
4254 (*windowp)->wd_cookie_cnt++;
4255 (*windowp)->wd_size += cookie->dmac_size;
4256 *copybuf_used += MMU_PAGESIZE;
4257 return (DDI_SUCCESS);
4258 }
4259
4260 /*
4261 * *** may need to trim, figure it out.
4262 */
4263
4264 /* figure out how much we need to trim from the window */
4265 if (dma->dp_granularity_power_2) {
4266 trim_sz = (*windowp)->wd_size &
4267 (hp->dmai_attr.dma_attr_granular - 1);
4268 } else {
4269 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4270 }
4271
4272 /*
4273 * if the window's a whole multiple of granularity, go to the next
4274 * window, init it, then add in the current cookie. We know the current
4275 * cookie uses the copy buffer since we're in this code path.
4276 */
4277 if (trim_sz == 0) {
4278 (*windowp)++;
4279 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4280
4281 /* Add this cookie to the new window */
4282 (*windowp)->wd_cookie_cnt++;
4283 (*windowp)->wd_size += cookie->dmac_size;
4284 *copybuf_used += MMU_PAGESIZE;
4285 return (DDI_SUCCESS);
4286 }
4287
4288 /*
4289 * *** We figured it out, we definitly need to trim
4290 */
4291
4292 /*
4293 * make sure the driver isn't making us do something bad...
4294 * Trimming and sgllen == 1 don't go together.
4295 */
4296 if (hp->dmai_attr.dma_attr_sgllen == 1) {
4297 return (DDI_DMA_NOMAPPING);
4298 }
4299
4300 /*
4301 * first, setup the current window to account for the trim. Need to go
4302 * back to the last cookie for this. Some of the last cookie will be in
4303 * the current window, and some of the last cookie will be in the new
4304 * window. All of the current cookie will be in the new window.
4305 */
4306 cookie--;
4307 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4308 (*windowp)->wd_trim.tr_last_cookie = cookie;
4309 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4310 ASSERT(cookie->dmac_size > trim_sz);
4311 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4312 (*windowp)->wd_size -= trim_sz;
4313
4314 /*
4315 * we're trimming the last cookie (not the current cookie). So that
4316 * last cookie may have or may not have been using the copy buffer (
4317 * we know the cookie passed in uses the copy buffer since we're in
4318 * this code path).
4319 *
4320 * If the last cookie doesn't use the copy buffer, nothing special to
4321 * do. However, if it does uses the copy buffer, it will be both the
4322 * last page in the current window and the first page in the next
4323 * window. Since we are reusing the copy buffer (and KVA space on the
4324 * 32-bit kernel), this page will use the end of the copy buffer in the
4325 * current window, and the start of the copy buffer in the next window.
4326 * Track that info... The cookie physical address was already set to
4327 * the copy buffer physical address in setup_cookie..
4328 */
4329 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4330 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4331 (*windowp)->wd_size) >> MMU_PAGESHIFT;
4332 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4333 (*windowp)->wd_trim.tr_last_pidx = pidx;
4334 (*windowp)->wd_trim.tr_last_cbaddr =
4335 dma->dp_pgmap[pidx].pm_cbaddr;
4336 #if !defined(__amd64)
4337 (*windowp)->wd_trim.tr_last_kaddr =
4338 dma->dp_pgmap[pidx].pm_kaddr;
4339 #endif
4340 }
4341
4342 /* save the buffer offsets for the next window */
4343 coffset = cookie->dmac_size - trim_sz;
4344 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4345
4346 /*
4347 * set this now in case this is the first window. all other cases are
4348 * set in dma_win()
4349 */
4350 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4351
4352 /*
4353 * initialize the next window using what's left over in the previous
4354 * cookie.
4355 */
4356 (*windowp)++;
4357 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4358 (*windowp)->wd_cookie_cnt++;
4359 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4360 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4361 (*windowp)->wd_trim.tr_first_size = trim_sz;
4362
4363 /*
4364 * again, we're tracking if the last cookie uses the copy buffer.
4365 * read the comment above for more info on why we need to track
4366 * additional state.
4367 *
4368 * For the first cookie in the new window, we need reset the physical
4369 * address to DMA into to the start of the copy buffer plus any
4370 * initial page offset which may be present.
4371 */
4372 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4373 (*windowp)->wd_dosync = B_TRUE;
4374 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4375 (*windowp)->wd_trim.tr_first_pidx = pidx;
4376 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4377 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4378
4379 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4380 poff;
4381 (*windowp)->wd_trim.tr_first_paddr =
4382 ROOTNEX_PADDR_TO_RBASE(paddr);
4383
4384 #if !defined(__amd64)
4385 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4386 #endif
4387 /* account for the cookie copybuf usage in the new window */
4388 *copybuf_used += MMU_PAGESIZE;
4389
4390 /*
4391 * every piece of code has to have a hack, and here is this
4392 * ones :-)
4393 *
4394 * There is a complex interaction between setup_cookie and the
4395 * copybuf window boundary. The complexity had to be in either
4396 * the maxxfer window, or the copybuf window, and I chose the
4397 * copybuf code.
4398 *
4399 * So in this code path, we have taken the last cookie,
4400 * virtually broken it in half due to the trim, and it happens
4401 * to use the copybuf which further complicates life. At the
4402 * same time, we have already setup the current cookie, which
4403 * is now wrong. More background info: the current cookie uses
4404 * the copybuf, so it is only a page long max. So we need to
4405 * fix the current cookies copy buffer address, physical
4406 * address, and kva for the 32-bit kernel. We due this by
4407 * bumping them by page size (of course, we can't due this on
4408 * the physical address since the copy buffer may not be
4409 * physically contiguous).
4410 */
4411 cookie++;
4412 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4413 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4414
4415 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4416 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4417 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4418
4419 #if !defined(__amd64)
4420 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4421 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4422 #endif
4423 } else {
4424 /* go back to the current cookie */
4425 cookie++;
4426 }
4427
4428 /*
4429 * add the current cookie to the new window. set the new window size to
4430 * the what was left over from the previous cookie and what's in the
4431 * current cookie.
4432 */
4433 (*windowp)->wd_cookie_cnt++;
4434 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4435 ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4436
4437 /*
4438 * we know that the cookie passed in always uses the copy buffer. We
4439 * wouldn't be here if it didn't.
4440 */
4441 *copybuf_used += MMU_PAGESIZE;
4442
4443 return (DDI_SUCCESS);
4444 }
4445
4446
4447 /*
4448 * rootnex_maxxfer_window_boundary()
4449 * Called in bind slowpath when we get to a window boundary because we will
4450 * go over maxxfer.
4451 */
4452 static int
4453 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4454 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4455 {
4456 size_t dmac_size;
4457 off_t new_offset;
4458 size_t trim_sz;
4459 off_t coffset;
4460
4461
4462 /*
4463 * calculate how much we have to trim off of the current cookie to equal
4464 * maxxfer. We don't have to account for granularity here since our
4465 * maxxfer already takes that into account.
4466 */
4467 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4468 ASSERT(trim_sz <= cookie->dmac_size);
4469 ASSERT(trim_sz <= dma->dp_maxxfer);
4470
4471 /* save cookie size since we need it later and we might change it */
4472 dmac_size = cookie->dmac_size;
4473
4474 /*
4475 * if we're not trimming the entire cookie, setup the current window to
4476 * account for the trim.
4477 */
4478 if (trim_sz < cookie->dmac_size) {
4479 (*windowp)->wd_cookie_cnt++;
4480 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4481 (*windowp)->wd_trim.tr_last_cookie = cookie;
4482 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4483 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4484 (*windowp)->wd_size = dma->dp_maxxfer;
4485
4486 /*
4487 * set the adjusted cookie size now in case this is the first
4488 * window. All other windows are taken care of in get win
4489 */
4490 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4491 }
4492
4493 /*
4494 * coffset is the current offset within the cookie, new_offset is the
4495 * current offset with the entire buffer.
4496 */
4497 coffset = dmac_size - trim_sz;
4498 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4499
4500 /* initialize the next window */
4501 (*windowp)++;
4502 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4503 (*windowp)->wd_cookie_cnt++;
4504 (*windowp)->wd_size = trim_sz;
4505 if (trim_sz < dmac_size) {
4506 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4507 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4508 coffset;
4509 (*windowp)->wd_trim.tr_first_size = trim_sz;
4510 }
4511
4512 return (DDI_SUCCESS);
4513 }
4514
4515
4516 /*ARGSUSED*/
4517 static int
4518 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4519 off_t off, size_t len, uint_t cache_flags)
4520 {
4521 rootnex_sglinfo_t *sinfo;
4522 rootnex_pgmap_t *cbpage;
4523 rootnex_window_t *win;
4524 ddi_dma_impl_t *hp;
4525 rootnex_dma_t *dma;
4526 caddr_t fromaddr;
4527 caddr_t toaddr;
4528 uint_t psize;
4529 off_t offset;
4530 uint_t pidx;
4531 size_t size;
4532 off_t poff;
4533 int e;
4534
4535
4536 hp = (ddi_dma_impl_t *)handle;
4537 dma = (rootnex_dma_t *)hp->dmai_private;
4538 sinfo = &dma->dp_sglinfo;
4539
4540 /*
4541 * if we don't have any windows, we don't need to sync. A copybuf
4542 * will cause us to have at least one window.
4543 */
4544 if (dma->dp_window == NULL) {
4545 return (DDI_SUCCESS);
4546 }
4547
4548 /* This window may not need to be sync'd */
4549 win = &dma->dp_window[dma->dp_current_win];
4550 if (!win->wd_dosync) {
4551 return (DDI_SUCCESS);
4552 }
4553
4554 /* handle off and len special cases */
4555 if ((off == 0) || (rootnex_sync_ignore_params)) {
4556 offset = win->wd_offset;
4557 } else {
4558 offset = off;
4559 }
4560 if ((len == 0) || (rootnex_sync_ignore_params)) {
4561 size = win->wd_size;
4562 } else {
4563 size = len;
4564 }
4565
4566 /* check the sync args to make sure they make a little sense */
4567 if (rootnex_sync_check_parms) {
4568 e = rootnex_valid_sync_parms(hp, win, offset, size,
4569 cache_flags);
4570 if (e != DDI_SUCCESS) {
4571 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4572 return (DDI_FAILURE);
4573 }
4574 }
4575
4576 /*
4577 * special case the first page to handle the offset into the page. The
4578 * offset to the current page for our buffer is the offset into the
4579 * first page of the buffer plus our current offset into the buffer
4580 * itself, masked of course.
4581 */
4582 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4583 psize = MIN((MMU_PAGESIZE - poff), size);
4584
4585 /* go through all the pages that we want to sync */
4586 while (size > 0) {
4587 /*
4588 * Calculate the page index relative to the start of the buffer.
4589 * The index to the current page for our buffer is the offset
4590 * into the first page of the buffer plus our current offset
4591 * into the buffer itself, shifted of course...
4592 */
4593 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4594 ASSERT(pidx < sinfo->si_max_pages);
4595
4596 /*
4597 * if this page uses the copy buffer, we need to sync it,
4598 * otherwise, go on to the next page.
4599 */
4600 cbpage = &dma->dp_pgmap[pidx];
4601 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4602 (cbpage->pm_uses_copybuf == B_FALSE));
4603 if (cbpage->pm_uses_copybuf) {
4604 /* cbaddr and kaddr should be page aligned */
4605 ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4606 MMU_PAGEOFFSET) == 0);
4607 ASSERT(((uintptr_t)cbpage->pm_kaddr &
4608 MMU_PAGEOFFSET) == 0);
4609
4610 /*
4611 * if we're copying for the device, we are going to
4612 * copy from the drivers buffer and to the rootnex
4613 * allocated copy buffer.
4614 */
4615 if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4616 fromaddr = cbpage->pm_kaddr + poff;
4617 toaddr = cbpage->pm_cbaddr + poff;
4618 ROOTNEX_DPROBE2(rootnex__sync__dev,
4619 dev_info_t *, dma->dp_dip, size_t, psize);
4620
4621 /*
4622 * if we're copying for the cpu/kernel, we are going to
4623 * copy from the rootnex allocated copy buffer to the
4624 * drivers buffer.
4625 */
4626 } else {
4627 fromaddr = cbpage->pm_cbaddr + poff;
4628 toaddr = cbpage->pm_kaddr + poff;
4629 ROOTNEX_DPROBE2(rootnex__sync__cpu,
4630 dev_info_t *, dma->dp_dip, size_t, psize);
4631 }
4632
4633 bcopy(fromaddr, toaddr, psize);
4634 }
4635
4636 /*
4637 * decrement size until we're done, update our offset into the
4638 * buffer, and get the next page size.
4639 */
4640 size -= psize;
4641 offset += psize;
4642 psize = MIN(MMU_PAGESIZE, size);
4643
4644 /* page offset is zero for the rest of this loop */
4645 poff = 0;
4646 }
4647
4648 return (DDI_SUCCESS);
4649 }
4650
4651 /*
4652 * rootnex_dma_sync()
4653 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4654 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4655 * is set, ddi_dma_sync() returns immediately passing back success.
4656 */
4657 /*ARGSUSED*/
4658 static int
4659 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4660 off_t off, size_t len, uint_t cache_flags)
4661 {
4662 #if defined(__amd64) && !defined(__xpv)
4663 if (IOMMU_USED(rdip)) {
4664 return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4665 cache_flags));
4666 }
4667 #endif
4668 return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4669 cache_flags));
4670 }
4671
4672 /*
4673 * rootnex_valid_sync_parms()
4674 * checks the parameters passed to sync to verify they are correct.
4675 */
4676 static int
4677 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4678 off_t offset, size_t size, uint_t cache_flags)
4679 {
4680 off_t woffset;
4681
4682
4683 /*
4684 * the first part of the test to make sure the offset passed in is
4685 * within the window.
4686 */
4687 if (offset < win->wd_offset) {
4688 return (DDI_FAILURE);
4689 }
4690
4691 /*
4692 * second and last part of the test to make sure the offset and length
4693 * passed in is within the window.
4694 */
4695 woffset = offset - win->wd_offset;
4696 if ((woffset + size) > win->wd_size) {
4697 return (DDI_FAILURE);
4698 }
4699
4700 /*
4701 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4702 * be set too.
4703 */
4704 if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4705 (hp->dmai_rflags & DDI_DMA_WRITE)) {
4706 return (DDI_SUCCESS);
4707 }
4708
4709 /*
4710 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4711 * should be set. Also DDI_DMA_READ should be set in the flags.
4712 */
4713 if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4714 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4715 (hp->dmai_rflags & DDI_DMA_READ)) {
4716 return (DDI_SUCCESS);
4717 }
4718
4719 return (DDI_FAILURE);
4720 }
4721
4722
4723 /*ARGSUSED*/
4724 static int
4725 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4726 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4727 uint_t *ccountp)
4728 {
4729 rootnex_window_t *window;
4730 rootnex_trim_t *trim;
4731 ddi_dma_impl_t *hp;
4732 rootnex_dma_t *dma;
4733 ddi_dma_obj_t *dmao;
4734 #if !defined(__amd64)
4735 rootnex_sglinfo_t *sinfo;
4736 rootnex_pgmap_t *pmap;
4737 uint_t pidx;
4738 uint_t pcnt;
4739 off_t poff;
4740 int i;
4741 #endif
4742
4743
4744 hp = (ddi_dma_impl_t *)handle;
4745 dma = (rootnex_dma_t *)hp->dmai_private;
4746 #if !defined(__amd64)
4747 sinfo = &dma->dp_sglinfo;
4748 #endif
4749
4750 /* If we try and get a window which doesn't exist, return failure */
4751 if (win >= hp->dmai_nwin) {
4752 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4753 return (DDI_FAILURE);
4754 }
4755
4756 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4757
4758 /*
4759 * if we don't have any windows, and they're asking for the first
4760 * window, setup the cookie pointer to the first cookie in the bind.
4761 * setup our return values, then increment the cookie since we return
4762 * the first cookie on the stack.
4763 */
4764 if (dma->dp_window == NULL) {
4765 if (win != 0) {
4766 ROOTNEX_DPROF_INC(
4767 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4768 return (DDI_FAILURE);
4769 }
4770 hp->dmai_cookie = dma->dp_cookies;
4771 *offp = 0;
4772 *lenp = dmao->dmao_size;
4773 *ccountp = dma->dp_sglinfo.si_sgl_size;
4774 *cookiep = hp->dmai_cookie[0];
4775 hp->dmai_cookie++;
4776 return (DDI_SUCCESS);
4777 }
4778
4779 /* sync the old window before moving on to the new one */
4780 window = &dma->dp_window[dma->dp_current_win];
4781 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4782 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4783 DDI_DMA_SYNC_FORCPU);
4784 }
4785
4786 #if !defined(__amd64)
4787 /*
4788 * before we move to the next window, if we need to re-map, unmap all
4789 * the pages in this window.
4790 */
4791 if (dma->dp_cb_remaping) {
4792 /*
4793 * If we switch to this window again, we'll need to map in
4794 * on the fly next time.
4795 */
4796 window->wd_remap_copybuf = B_TRUE;
4797
4798 /*
4799 * calculate the page index into the buffer where this window
4800 * starts, and the number of pages this window takes up.
4801 */
4802 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4803 MMU_PAGESHIFT;
4804 poff = (sinfo->si_buf_offset + window->wd_offset) &
4805 MMU_PAGEOFFSET;
4806 pcnt = mmu_btopr(window->wd_size + poff);
4807 ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4808
4809 /* unmap pages which are currently mapped in this window */
4810 for (i = 0; i < pcnt; i++) {
4811 if (dma->dp_pgmap[pidx].pm_mapped) {
4812 hat_unload(kas.a_hat,
4813 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4814 HAT_UNLOAD);
4815 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4816 }
4817 pidx++;
4818 }
4819 }
4820 #endif
4821
4822 /*
4823 * Move to the new window.
4824 * NOTE: current_win must be set for sync to work right
4825 */
4826 dma->dp_current_win = win;
4827 window = &dma->dp_window[win];
4828
4829 /* if needed, adjust the first and/or last cookies for trim */
4830 trim = &window->wd_trim;
4831 if (trim->tr_trim_first) {
4832 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4833 window->wd_first_cookie->dmac_size = trim->tr_first_size;
4834 #if !defined(__amd64)
4835 window->wd_first_cookie->dmac_type =
4836 (window->wd_first_cookie->dmac_type &
4837 ROOTNEX_USES_COPYBUF) + window->wd_offset;
4838 #endif
4839 if (trim->tr_first_copybuf_win) {
4840 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4841 trim->tr_first_cbaddr;
4842 #if !defined(__amd64)
4843 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4844 trim->tr_first_kaddr;
4845 #endif
4846 }
4847 }
4848 if (trim->tr_trim_last) {
4849 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4850 trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4851 if (trim->tr_last_copybuf_win) {
4852 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4853 trim->tr_last_cbaddr;
4854 #if !defined(__amd64)
4855 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4856 trim->tr_last_kaddr;
4857 #endif
4858 }
4859 }
4860
4861 /*
4862 * setup the cookie pointer to the first cookie in the window. setup
4863 * our return values, then increment the cookie since we return the
4864 * first cookie on the stack.
4865 */
4866 hp->dmai_cookie = window->wd_first_cookie;
4867 *offp = window->wd_offset;
4868 *lenp = window->wd_size;
4869 *ccountp = window->wd_cookie_cnt;
4870 *cookiep = hp->dmai_cookie[0];
4871 hp->dmai_cookie++;
4872
4873 #if !defined(__amd64)
4874 /* re-map copybuf if required for this window */
4875 if (dma->dp_cb_remaping) {
4876 /*
4877 * calculate the page index into the buffer where this
4878 * window starts.
4879 */
4880 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4881 MMU_PAGESHIFT;
4882 ASSERT(pidx < sinfo->si_max_pages);
4883
4884 /*
4885 * the first page can get unmapped if it's shared with the
4886 * previous window. Even if the rest of this window is already
4887 * mapped in, we need to still check this one.
4888 */
4889 pmap = &dma->dp_pgmap[pidx];
4890 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4891 if (pmap->pm_pp != NULL) {
4892 pmap->pm_mapped = B_TRUE;
4893 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4894 } else if (pmap->pm_vaddr != NULL) {
4895 pmap->pm_mapped = B_TRUE;
4896 i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4897 pmap->pm_kaddr);
4898 }
4899 }
4900 pidx++;
4901
4902 /* map in the rest of the pages if required */
4903 if (window->wd_remap_copybuf) {
4904 window->wd_remap_copybuf = B_FALSE;
4905
4906 /* figure out many pages this window takes up */
4907 poff = (sinfo->si_buf_offset + window->wd_offset) &
4908 MMU_PAGEOFFSET;
4909 pcnt = mmu_btopr(window->wd_size + poff);
4910 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4911
4912 /* map pages which require it */
4913 for (i = 1; i < pcnt; i++) {
4914 pmap = &dma->dp_pgmap[pidx];
4915 if (pmap->pm_uses_copybuf) {
4916 ASSERT(pmap->pm_mapped == B_FALSE);
4917 if (pmap->pm_pp != NULL) {
4918 pmap->pm_mapped = B_TRUE;
4919 i86_pp_map(pmap->pm_pp,
4920 pmap->pm_kaddr);
4921 } else if (pmap->pm_vaddr != NULL) {
4922 pmap->pm_mapped = B_TRUE;
4923 i86_va_map(pmap->pm_vaddr,
4924 sinfo->si_asp,
4925 pmap->pm_kaddr);
4926 }
4927 }
4928 pidx++;
4929 }
4930 }
4931 }
4932 #endif
4933
4934 /* if the new window uses the copy buffer, sync it for the device */
4935 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4936 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4937 DDI_DMA_SYNC_FORDEV);
4938 }
4939
4940 return (DDI_SUCCESS);
4941 }
4942
4943 /*
4944 * rootnex_dma_win()
4945 * called from ddi_dma_getwin()
4946 */
4947 /*ARGSUSED*/
4948 static int
4949 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4950 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4951 uint_t *ccountp)
4952 {
4953 #if defined(__amd64) && !defined(__xpv)
4954 if (IOMMU_USED(rdip)) {
4955 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4956 cookiep, ccountp));
4957 }
4958 #endif
4959
4960 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4961 cookiep, ccountp));
4962 }
4963
4964 #if defined(__amd64) && !defined(__xpv)
4965 /*ARGSUSED*/
4966 static int
4967 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4968 ddi_dma_handle_t handle, void *v)
4969 {
4970 ddi_dma_impl_t *hp;
4971 rootnex_dma_t *dma;
4972
4973 hp = (ddi_dma_impl_t *)handle;
4974 dma = (rootnex_dma_t *)hp->dmai_private;
4975 dma->dp_iommu_private = v;
4976
4977 return (DDI_SUCCESS);
4978 }
4979
4980 /*ARGSUSED*/
4981 static void *
4982 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4983 ddi_dma_handle_t handle)
4984 {
4985 ddi_dma_impl_t *hp;
4986 rootnex_dma_t *dma;
4987
4988 hp = (ddi_dma_impl_t *)handle;
4989 dma = (rootnex_dma_t *)hp->dmai_private;
4990
4991 return (dma->dp_iommu_private);
4992 }
4993 #endif
4994
4995 /*
4996 * ************************
4997 * obsoleted dma routines
4998 * ************************
4999 */
5000
5001 /*
5002 * rootnex_dma_mctl()
5003 *
5004 * We don't support this legacy interface any more on x86.
5005 */
5006 /* ARGSUSED */
5007 static int
5008 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5009 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5010 uint_t cache_flags)
5011 {
5012 /*
5013 * The only thing dma_mctl is usef for anymore is legacy SPARC
5014 * dvma and sbus-specific routines.
5015 */
5016 return (DDI_FAILURE);
5017 }
5018
5019 /*
5020 * *********
5021 * FMA Code
5022 * *********
5023 */
5024
5025 /*
5026 * rootnex_fm_init()
5027 * FMA init busop
5028 */
5029 /* ARGSUSED */
5030 static int
5031 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5032 ddi_iblock_cookie_t *ibc)
5033 {
5034 *ibc = rootnex_state->r_err_ibc;
5035
5036 return (ddi_system_fmcap);
5037 }
5038
5039 /*
5040 * rootnex_dma_check()
5041 * Function called after a dma fault occurred to find out whether the
5042 * fault address is associated with a driver that is able to handle faults
5043 * and recover from faults.
5044 */
5045 /* ARGSUSED */
5046 static int
5047 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5048 const void *not_used)
5049 {
5050 rootnex_window_t *window;
5051 uint64_t start_addr;
5052 uint64_t fault_addr;
5053 ddi_dma_impl_t *hp;
5054 rootnex_dma_t *dma;
5055 uint64_t end_addr;
5056 size_t csize;
5057 int i;
5058 int j;
5059
5060
5061 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5062 hp = (ddi_dma_impl_t *)handle;
5063 ASSERT(hp);
5064
5065 dma = (rootnex_dma_t *)hp->dmai_private;
5066
5067 /* Get the address that we need to search for */
5068 fault_addr = *(uint64_t *)addr;
5069
5070 /*
5071 * if we don't have any windows, we can just walk through all the
5072 * cookies.
5073 */
5074 if (dma->dp_window == NULL) {
5075 /* for each cookie */
5076 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5077 /*
5078 * if the faulted address is within the physical address
5079 * range of the cookie, return DDI_FM_NONFATAL.
5080 */
5081 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5082 (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5083 dma->dp_cookies[i].dmac_size))) {
5084 return (DDI_FM_NONFATAL);
5085 }
5086 }
5087
5088 /* fault_addr not within this DMA handle */
5089 return (DDI_FM_UNKNOWN);
5090 }
5091
5092 /* we have mutiple windows, walk through each window */
5093 for (i = 0; i < hp->dmai_nwin; i++) {
5094 window = &dma->dp_window[i];
5095
5096 /* Go through all the cookies in the window */
5097 for (j = 0; j < window->wd_cookie_cnt; j++) {
5098
5099 start_addr = window->wd_first_cookie[j].dmac_laddress;
5100 csize = window->wd_first_cookie[j].dmac_size;
5101
5102 /*
5103 * if we are trimming the first cookie in the window,
5104 * and this is the first cookie, adjust the start
5105 * address and size of the cookie to account for the
5106 * trim.
5107 */
5108 if (window->wd_trim.tr_trim_first && (j == 0)) {
5109 start_addr = window->wd_trim.tr_first_paddr;
5110 csize = window->wd_trim.tr_first_size;
5111 }
5112
5113 /*
5114 * if we are trimming the last cookie in the window,
5115 * and this is the last cookie, adjust the start
5116 * address and size of the cookie to account for the
5117 * trim.
5118 */
5119 if (window->wd_trim.tr_trim_last &&
5120 (j == (window->wd_cookie_cnt - 1))) {
5121 start_addr = window->wd_trim.tr_last_paddr;
5122 csize = window->wd_trim.tr_last_size;
5123 }
5124
5125 end_addr = start_addr + csize;
5126
5127 /*
5128 * if the faulted address is within the physical
5129 * address of the cookie, return DDI_FM_NONFATAL.
5130 */
5131 if ((fault_addr >= start_addr) &&
5132 (fault_addr <= end_addr)) {
5133 return (DDI_FM_NONFATAL);
5134 }
5135 }
5136 }
5137
5138 /* fault_addr not within this DMA handle */
5139 return (DDI_FM_UNKNOWN);
5140 }
5141
5142 /*ARGSUSED*/
5143 static int
5144 rootnex_quiesce(dev_info_t *dip)
5145 {
5146 #if defined(__amd64) && !defined(__xpv)
5147 return (immu_quiesce());
5148 #else
5149 return (DDI_SUCCESS);
5150 #endif
5151 }
5152
5153 #if defined(__xpv)
5154 void
5155 immu_init(void)
5156 {
5157 ;
5158 }
5159
5160 void
5161 immu_startup(void)
5162 {
5163 ;
5164 }
5165 /*ARGSUSED*/
5166 void
5167 immu_physmem_update(uint64_t addr, uint64_t size)
5168 {
5169 ;
5170 }
5171 #endif