1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved.
27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
28 */
29
30 /*
31 * x86 root nexus driver
32 */
33
34 #include <sys/sysmacros.h>
35 #include <sys/conf.h>
36 #include <sys/autoconf.h>
37 #include <sys/sysmacros.h>
38 #include <sys/debug.h>
39 #include <sys/psw.h>
40 #include <sys/ddidmareq.h>
41 #include <sys/promif.h>
42 #include <sys/devops.h>
43 #include <sys/kmem.h>
44 #include <sys/cmn_err.h>
45 #include <vm/seg.h>
46 #include <vm/seg_kmem.h>
47 #include <vm/seg_dev.h>
48 #include <sys/vmem.h>
49 #include <sys/mman.h>
50 #include <vm/hat.h>
51 #include <vm/as.h>
52 #include <vm/page.h>
53 #include <sys/avintr.h>
54 #include <sys/errno.h>
55 #include <sys/modctl.h>
56 #include <sys/ddi_impldefs.h>
57 #include <sys/sunddi.h>
58 #include <sys/sunndi.h>
59 #include <sys/mach_intr.h>
60 #include <sys/psm.h>
61 #include <sys/ontrap.h>
62 #include <sys/atomic.h>
63 #include <sys/sdt.h>
64 #include <sys/rootnex.h>
65 #include <vm/hat_i86.h>
66 #include <sys/ddifm.h>
67 #include <sys/ddi_isa.h>
68 #include <sys/apic.h>
69
70 #ifdef __xpv
71 #include <sys/bootinfo.h>
72 #include <sys/hypervisor.h>
73 #include <sys/bootconf.h>
74 #include <vm/kboot_mmu.h>
75 #endif
76
77 #if defined(__amd64) && !defined(__xpv)
78 #include <sys/immu.h>
79 #endif
80
81
82 /*
83 * enable/disable extra checking of function parameters. Useful for debugging
84 * drivers.
85 */
86 #ifdef DEBUG
87 int rootnex_alloc_check_parms = 1;
88 int rootnex_bind_check_parms = 1;
89 int rootnex_bind_check_inuse = 1;
90 int rootnex_unbind_verify_buffer = 0;
91 int rootnex_sync_check_parms = 1;
92 #else
93 int rootnex_alloc_check_parms = 0;
94 int rootnex_bind_check_parms = 0;
95 int rootnex_bind_check_inuse = 0;
96 int rootnex_unbind_verify_buffer = 0;
97 int rootnex_sync_check_parms = 0;
98 #endif
99
100 boolean_t rootnex_dmar_not_setup;
101
102 /* Master Abort and Target Abort panic flag */
103 int rootnex_fm_ma_ta_panic_flag = 0;
104
105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
106 int rootnex_bind_fail = 1;
107 int rootnex_bind_warn = 1;
108 uint8_t *rootnex_warn_list;
109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
110 #define ROOTNEX_BIND_WARNING (0x1 << 0)
111
112 /*
113 * revert back to old broken behavior of always sync'ing entire copy buffer.
114 * This is useful if be have a buggy driver which doesn't correctly pass in
115 * the offset and size into ddi_dma_sync().
116 */
117 int rootnex_sync_ignore_params = 0;
118
119 /*
120 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
121 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
122 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
123 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
124 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
125 * (< 8K). We will still need to allocate the copy buffer during bind though
126 * (if we need one). These can only be modified in /etc/system before rootnex
127 * attach.
128 */
129 #if defined(__amd64)
130 int rootnex_prealloc_cookies = 65;
131 int rootnex_prealloc_windows = 4;
132 int rootnex_prealloc_copybuf = 2;
133 #else
134 int rootnex_prealloc_cookies = 33;
135 int rootnex_prealloc_windows = 4;
136 int rootnex_prealloc_copybuf = 2;
137 #endif
138
139 /* driver global state */
140 static rootnex_state_t *rootnex_state;
141
142 #ifdef DEBUG
143 /* shortcut to rootnex counters */
144 static uint64_t *rootnex_cnt;
145 #endif
146
147 /*
148 * XXX - does x86 even need these or are they left over from the SPARC days?
149 */
150 /* statically defined integer/boolean properties for the root node */
151 static rootnex_intprop_t rootnex_intprp[] = {
152 { "PAGESIZE", PAGESIZE },
153 { "MMU_PAGESIZE", MMU_PAGESIZE },
154 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET },
155 { DDI_RELATIVE_ADDRESSING, 1 },
156 };
157 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
158
159 /*
160 * If we're dom0, we're using a real device so we need to load
161 * the cookies with MFNs instead of PFNs.
162 */
163 #ifdef __xpv
164 typedef maddr_t rootnex_addr_t;
165 #define ROOTNEX_PADDR_TO_RBASE(pa) \
166 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
167 #else
168 typedef paddr_t rootnex_addr_t;
169 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa)
170 #endif
171
172 static struct cb_ops rootnex_cb_ops = {
173 nodev, /* open */
174 nodev, /* close */
175 nodev, /* strategy */
176 nodev, /* print */
177 nodev, /* dump */
178 nodev, /* read */
179 nodev, /* write */
180 nodev, /* ioctl */
181 nodev, /* devmap */
182 nodev, /* mmap */
183 nodev, /* segmap */
184 nochpoll, /* chpoll */
185 ddi_prop_op, /* cb_prop_op */
186 NULL, /* struct streamtab */
187 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
188 CB_REV, /* Rev */
189 nodev, /* cb_aread */
190 nodev /* cb_awrite */
191 };
192
193 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
194 off_t offset, off_t len, caddr_t *vaddrp);
195 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
196 struct hat *hat, struct seg *seg, caddr_t addr,
197 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
198 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
199 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
200 ddi_dma_handle_t *handlep);
201 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
202 ddi_dma_handle_t handle);
203 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
204 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
205 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
206 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
207 ddi_dma_handle_t handle);
208 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
209 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
210 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
211 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
212 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
213 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
214 ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
215 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
216 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
217 ddi_ctl_enum_t ctlop, void *arg, void *result);
218 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
219 ddi_iblock_cookie_t *ibc);
220 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
221 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
222 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
223 void *);
224 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
225
226 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
227 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
228 ddi_dma_handle_t *handlep);
229 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
230 ddi_dma_handle_t handle);
231 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
232 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
233 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
234 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
235 ddi_dma_handle_t handle);
236 #if defined(__amd64) && !defined(__xpv)
237 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
238 ddi_dma_handle_t handle);
239 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
240 ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
241 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
242 ddi_dma_cookie_t *cookiep, uint_t ccount);
243 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
244 ddi_dma_handle_t handle);
245 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
246 #endif
247 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
248 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
249 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
250 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
251 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
252
253 #if defined(__amd64) && !defined(__xpv)
254 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
255 ddi_dma_handle_t handle, void *v);
256 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
257 ddi_dma_handle_t handle);
258 #endif
259
260
261 static struct bus_ops rootnex_bus_ops = {
262 BUSO_REV,
263 rootnex_map,
264 NULL,
265 NULL,
266 NULL,
267 rootnex_map_fault,
268 0,
269 rootnex_dma_allochdl,
270 rootnex_dma_freehdl,
271 rootnex_dma_bindhdl,
272 rootnex_dma_unbindhdl,
273 rootnex_dma_sync,
274 rootnex_dma_win,
275 rootnex_dma_mctl,
276 rootnex_ctlops,
277 ddi_bus_prop_op,
278 i_ddi_rootnex_get_eventcookie,
279 i_ddi_rootnex_add_eventcall,
280 i_ddi_rootnex_remove_eventcall,
281 i_ddi_rootnex_post_event,
282 0, /* bus_intr_ctl */
283 0, /* bus_config */
284 0, /* bus_unconfig */
285 rootnex_fm_init, /* bus_fm_init */
286 NULL, /* bus_fm_fini */
287 NULL, /* bus_fm_access_enter */
288 NULL, /* bus_fm_access_exit */
289 NULL, /* bus_powr */
290 rootnex_intr_ops /* bus_intr_op */
291 };
292
293 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
294 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
295 static int rootnex_quiesce(dev_info_t *dip);
296
297 static struct dev_ops rootnex_ops = {
298 DEVO_REV,
299 0,
300 ddi_no_info,
301 nulldev,
302 nulldev,
303 rootnex_attach,
304 rootnex_detach,
305 nulldev,
306 &rootnex_cb_ops,
307 &rootnex_bus_ops,
308 NULL,
309 rootnex_quiesce, /* quiesce */
310 };
311
312 static struct modldrv rootnex_modldrv = {
313 &mod_driverops,
314 "i86pc root nexus",
315 &rootnex_ops
316 };
317
318 static struct modlinkage rootnex_modlinkage = {
319 MODREV_1,
320 (void *)&rootnex_modldrv,
321 NULL
322 };
323
324 #if defined(__amd64) && !defined(__xpv)
325 static iommulib_nexops_t iommulib_nexops = {
326 IOMMU_NEXOPS_VERSION,
327 "Rootnex IOMMU ops Vers 1.1",
328 NULL,
329 rootnex_coredma_allochdl,
330 rootnex_coredma_freehdl,
331 rootnex_coredma_bindhdl,
332 rootnex_coredma_unbindhdl,
333 rootnex_coredma_reset_cookies,
334 rootnex_coredma_get_cookies,
335 rootnex_coredma_set_cookies,
336 rootnex_coredma_clear_cookies,
337 rootnex_coredma_get_sleep_flags,
338 rootnex_coredma_sync,
339 rootnex_coredma_win,
340 rootnex_coredma_hdl_setprivate,
341 rootnex_coredma_hdl_getprivate
342 };
343 #endif
344
345 /*
346 * extern hacks
347 */
348 extern struct seg_ops segdev_ops;
349 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
350 #ifdef DDI_MAP_DEBUG
351 extern int ddi_map_debug_flag;
352 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf
353 #endif
354 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
355 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
356 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
357 psm_intr_op_t, int *);
358 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
359 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
360
361 /*
362 * Use device arena to use for device control register mappings.
363 * Various kernel memory walkers (debugger, dtrace) need to know
364 * to avoid this address range to prevent undesired device activity.
365 */
366 extern void *device_arena_alloc(size_t size, int vm_flag);
367 extern void device_arena_free(void * vaddr, size_t size);
368
369
370 /*
371 * Internal functions
372 */
373 static int rootnex_dma_init();
374 static void rootnex_add_props(dev_info_t *);
375 static int rootnex_ctl_reportdev(dev_info_t *dip);
376 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
377 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
378 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
379 static int rootnex_map_handle(ddi_map_req_t *mp);
380 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
381 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
382 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
383 ddi_dma_attr_t *attr);
384 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
385 rootnex_sglinfo_t *sglinfo);
386 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
387 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
388 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
389 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
390 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
391 rootnex_dma_t *dma, ddi_dma_attr_t *attr);
392 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
393 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
394 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
395 static void rootnex_teardown_windows(rootnex_dma_t *dma);
396 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
397 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
398 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
399 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
400 size_t *copybuf_used, page_t **cur_pp);
401 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
402 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
403 ddi_dma_attr_t *attr, off_t cur_offset);
404 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
405 rootnex_dma_t *dma, rootnex_window_t **windowp,
406 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
407 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
408 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
409 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
410 off_t offset, size_t size, uint_t cache_flags);
411 static int rootnex_verify_buffer(rootnex_dma_t *dma);
412 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
413 const void *comp_addr, const void *not_used);
414 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
415 rootnex_sglinfo_t *sglinfo);
416 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
417
418 /*
419 * _init()
420 *
421 */
422 int
423 _init(void)
424 {
425
426 rootnex_state = NULL;
427 return (mod_install(&rootnex_modlinkage));
428 }
429
430
431 /*
432 * _info()
433 *
434 */
435 int
436 _info(struct modinfo *modinfop)
437 {
438 return (mod_info(&rootnex_modlinkage, modinfop));
439 }
440
441
442 /*
443 * _fini()
444 *
445 */
446 int
447 _fini(void)
448 {
449 return (EBUSY);
450 }
451
452
453 /*
454 * rootnex_attach()
455 *
456 */
457 static int
458 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
459 {
460 int fmcap;
461 int e;
462
463 switch (cmd) {
464 case DDI_ATTACH:
465 break;
466 case DDI_RESUME:
467 #if defined(__amd64) && !defined(__xpv)
468 return (immu_unquiesce());
469 #else
470 return (DDI_SUCCESS);
471 #endif
472 default:
473 return (DDI_FAILURE);
474 }
475
476 /*
477 * We should only have one instance of rootnex. Save it away since we
478 * don't have an easy way to get it back later.
479 */
480 ASSERT(rootnex_state == NULL);
481 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
482
483 rootnex_state->r_dip = dip;
484 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
485 rootnex_state->r_reserved_msg_printed = B_FALSE;
486 #ifdef DEBUG
487 rootnex_cnt = &rootnex_state->r_counters[0];
488 #endif
489
490 /*
491 * Set minimum fm capability level for i86pc platforms and then
492 * initialize error handling. Since we're the rootnex, we don't
493 * care what's returned in the fmcap field.
494 */
495 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
496 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
497 fmcap = ddi_system_fmcap;
498 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
499
500 /* initialize DMA related state */
501 e = rootnex_dma_init();
502 if (e != DDI_SUCCESS) {
503 kmem_free(rootnex_state, sizeof (rootnex_state_t));
504 return (DDI_FAILURE);
505 }
506
507 /* Add static root node properties */
508 rootnex_add_props(dip);
509
510 /* since we can't call ddi_report_dev() */
511 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
512
513 /* Initialize rootnex event handle */
514 i_ddi_rootnex_init_events(dip);
515
516 #if defined(__amd64) && !defined(__xpv)
517 e = iommulib_nexus_register(dip, &iommulib_nexops,
518 &rootnex_state->r_iommulib_handle);
519
520 ASSERT(e == DDI_SUCCESS);
521 #endif
522
523 return (DDI_SUCCESS);
524 }
525
526
527 /*
528 * rootnex_detach()
529 *
530 */
531 /*ARGSUSED*/
532 static int
533 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
534 {
535 switch (cmd) {
536 case DDI_SUSPEND:
537 #if defined(__amd64) && !defined(__xpv)
538 return (immu_quiesce());
539 #else
540 return (DDI_SUCCESS);
541 #endif
542 default:
543 return (DDI_FAILURE);
544 }
545 /*NOTREACHED*/
546
547 }
548
549
550 /*
551 * rootnex_dma_init()
552 *
553 */
554 /*ARGSUSED*/
555 static int
556 rootnex_dma_init()
557 {
558 size_t bufsize;
559
560
561 /*
562 * size of our cookie/window/copybuf state needed in dma bind that we
563 * pre-alloc in dma_alloc_handle
564 */
565 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
566 rootnex_state->r_prealloc_size =
567 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
568 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
569 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
570
571 /*
572 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
573 * allocate 16 extra bytes for struct pointer alignment
574 * (p->dmai_private & dma->dp_prealloc_buffer)
575 */
576 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
577 rootnex_state->r_prealloc_size + 0x10;
578 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
579 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
580 if (rootnex_state->r_dmahdl_cache == NULL) {
581 return (DDI_FAILURE);
582 }
583
584 /*
585 * allocate array to track which major numbers we have printed warnings
586 * for.
587 */
588 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
589 KM_SLEEP);
590
591 return (DDI_SUCCESS);
592 }
593
594
595 /*
596 * rootnex_add_props()
597 *
598 */
599 static void
600 rootnex_add_props(dev_info_t *dip)
601 {
602 rootnex_intprop_t *rpp;
603 int i;
604
605 /* Add static integer/boolean properties to the root node */
606 rpp = rootnex_intprp;
607 for (i = 0; i < NROOT_INTPROPS; i++) {
608 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
609 rpp[i].prop_name, rpp[i].prop_value);
610 }
611 }
612
613
614
615 /*
616 * *************************
617 * ctlops related routines
618 * *************************
619 */
620
621 /*
622 * rootnex_ctlops()
623 *
624 */
625 /*ARGSUSED*/
626 static int
627 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
628 void *arg, void *result)
629 {
630 int n, *ptr;
631 struct ddi_parent_private_data *pdp;
632
633 switch (ctlop) {
634 case DDI_CTLOPS_DMAPMAPC:
635 /*
636 * Return 'partial' to indicate that dma mapping
637 * has to be done in the main MMU.
638 */
639 return (DDI_DMA_PARTIAL);
640
641 case DDI_CTLOPS_BTOP:
642 /*
643 * Convert byte count input to physical page units.
644 * (byte counts that are not a page-size multiple
645 * are rounded down)
646 */
647 *(ulong_t *)result = btop(*(ulong_t *)arg);
648 return (DDI_SUCCESS);
649
650 case DDI_CTLOPS_PTOB:
651 /*
652 * Convert size in physical pages to bytes
653 */
654 *(ulong_t *)result = ptob(*(ulong_t *)arg);
655 return (DDI_SUCCESS);
656
657 case DDI_CTLOPS_BTOPR:
658 /*
659 * Convert byte count input to physical page units
660 * (byte counts that are not a page-size multiple
661 * are rounded up)
662 */
663 *(ulong_t *)result = btopr(*(ulong_t *)arg);
664 return (DDI_SUCCESS);
665
666 case DDI_CTLOPS_INITCHILD:
667 return (impl_ddi_sunbus_initchild(arg));
668
669 case DDI_CTLOPS_UNINITCHILD:
670 impl_ddi_sunbus_removechild(arg);
671 return (DDI_SUCCESS);
672
673 case DDI_CTLOPS_REPORTDEV:
674 return (rootnex_ctl_reportdev(rdip));
675
676 case DDI_CTLOPS_IOMIN:
677 /*
678 * Nothing to do here but reflect back..
679 */
680 return (DDI_SUCCESS);
681
682 case DDI_CTLOPS_REGSIZE:
683 case DDI_CTLOPS_NREGS:
684 break;
685
686 case DDI_CTLOPS_SIDDEV:
687 if (ndi_dev_is_prom_node(rdip))
688 return (DDI_SUCCESS);
689 if (ndi_dev_is_persistent_node(rdip))
690 return (DDI_SUCCESS);
691 return (DDI_FAILURE);
692
693 case DDI_CTLOPS_POWER:
694 return ((*pm_platform_power)((power_req_t *)arg));
695
696 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
697 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
698 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
699 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
700 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
701 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
702 if (!rootnex_state->r_reserved_msg_printed) {
703 rootnex_state->r_reserved_msg_printed = B_TRUE;
704 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
705 "1 or more reserved/obsolete operations.");
706 }
707 return (DDI_FAILURE);
708
709 default:
710 return (DDI_FAILURE);
711 }
712 /*
713 * The rest are for "hardware" properties
714 */
715 if ((pdp = ddi_get_parent_data(rdip)) == NULL)
716 return (DDI_FAILURE);
717
718 if (ctlop == DDI_CTLOPS_NREGS) {
719 ptr = (int *)result;
720 *ptr = pdp->par_nreg;
721 } else {
722 off_t *size = (off_t *)result;
723
724 ptr = (int *)arg;
725 n = *ptr;
726 if (n >= pdp->par_nreg) {
727 return (DDI_FAILURE);
728 }
729 *size = (off_t)pdp->par_reg[n].regspec_size;
730 }
731 return (DDI_SUCCESS);
732 }
733
734
735 /*
736 * rootnex_ctl_reportdev()
737 *
738 */
739 static int
740 rootnex_ctl_reportdev(dev_info_t *dev)
741 {
742 int i, n, len, f_len = 0;
743 char *buf;
744
745 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
746 f_len += snprintf(buf, REPORTDEV_BUFSIZE,
747 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
748 len = strlen(buf);
749
750 for (i = 0; i < sparc_pd_getnreg(dev); i++) {
751
752 struct regspec *rp = sparc_pd_getreg(dev, i);
753
754 if (i == 0)
755 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
756 ": ");
757 else
758 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
759 " and ");
760 len = strlen(buf);
761
762 switch (rp->regspec_bustype) {
763
764 case BTEISA:
765 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
766 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
767 break;
768
769 case BTISA:
770 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
771 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
772 break;
773
774 default:
775 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
776 "space %x offset %x",
777 rp->regspec_bustype, rp->regspec_addr);
778 break;
779 }
780 len = strlen(buf);
781 }
782 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
783 int pri;
784
785 if (i != 0) {
786 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
787 ",");
788 len = strlen(buf);
789 }
790 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
791 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
792 " sparc ipl %d", pri);
793 len = strlen(buf);
794 }
795 #ifdef DEBUG
796 if (f_len + 1 >= REPORTDEV_BUFSIZE) {
797 cmn_err(CE_NOTE, "next message is truncated: "
798 "printed length 1024, real length %d", f_len);
799 }
800 #endif /* DEBUG */
801 cmn_err(CE_CONT, "?%s\n", buf);
802 kmem_free(buf, REPORTDEV_BUFSIZE);
803 return (DDI_SUCCESS);
804 }
805
806
807 /*
808 * ******************
809 * map related code
810 * ******************
811 */
812
813 /*
814 * rootnex_map()
815 *
816 */
817 static int
818 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
819 off_t len, caddr_t *vaddrp)
820 {
821 struct regspec *rp, tmp_reg;
822 ddi_map_req_t mr = *mp; /* Get private copy of request */
823 int error;
824
825 mp = &mr;
826
827 switch (mp->map_op) {
828 case DDI_MO_MAP_LOCKED:
829 case DDI_MO_UNMAP:
830 case DDI_MO_MAP_HANDLE:
831 break;
832 default:
833 #ifdef DDI_MAP_DEBUG
834 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
835 mp->map_op);
836 #endif /* DDI_MAP_DEBUG */
837 return (DDI_ME_UNIMPLEMENTED);
838 }
839
840 if (mp->map_flags & DDI_MF_USER_MAPPING) {
841 #ifdef DDI_MAP_DEBUG
842 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
843 #endif /* DDI_MAP_DEBUG */
844 return (DDI_ME_UNIMPLEMENTED);
845 }
846
847 /*
848 * First, if given an rnumber, convert it to a regspec...
849 * (Presumably, this is on behalf of a child of the root node?)
850 */
851
852 if (mp->map_type == DDI_MT_RNUMBER) {
853
854 int rnumber = mp->map_obj.rnumber;
855 #ifdef DDI_MAP_DEBUG
856 static char *out_of_range =
857 "rootnex_map: Out of range rnumber <%d>, device <%s>";
858 #endif /* DDI_MAP_DEBUG */
859
860 rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
861 if (rp == NULL) {
862 #ifdef DDI_MAP_DEBUG
863 cmn_err(CE_WARN, out_of_range, rnumber,
864 ddi_get_name(rdip));
865 #endif /* DDI_MAP_DEBUG */
866 return (DDI_ME_RNUMBER_RANGE);
867 }
868
869 /*
870 * Convert the given ddi_map_req_t from rnumber to regspec...
871 */
872
873 mp->map_type = DDI_MT_REGSPEC;
874 mp->map_obj.rp = rp;
875 }
876
877 /*
878 * Adjust offset and length correspnding to called values...
879 * XXX: A non-zero length means override the one in the regspec
880 * XXX: (regardless of what's in the parent's range?)
881 */
882
883 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */
884 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */
885
886 #ifdef DDI_MAP_DEBUG
887 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
888 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
889 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
890 len, mp->map_handlep);
891 #endif /* DDI_MAP_DEBUG */
892
893 /*
894 * I/O or memory mapping:
895 *
896 * <bustype=0, addr=x, len=x>: memory
897 * <bustype=1, addr=x, len=x>: i/o
898 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
899 */
900
901 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
902 cmn_err(CE_WARN, "<%s,%s> invalid register spec"
903 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
904 ddi_get_name(rdip), rp->regspec_bustype,
905 rp->regspec_addr, rp->regspec_size);
906 return (DDI_ME_INVAL);
907 }
908
909 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
910 /*
911 * compatibility i/o mapping
912 */
913 rp->regspec_bustype += (uint_t)offset;
914 } else {
915 /*
916 * Normal memory or i/o mapping
917 */
918 rp->regspec_addr += (uint_t)offset;
919 }
920
921 if (len != 0)
922 rp->regspec_size = (uint_t)len;
923
924 #ifdef DDI_MAP_DEBUG
925 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
926 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
927 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
928 offset, len, mp->map_handlep);
929 #endif /* DDI_MAP_DEBUG */
930
931 /*
932 * Apply any parent ranges at this level, if applicable.
933 * (This is where nexus specific regspec translation takes place.
934 * Use of this function is implicit agreement that translation is
935 * provided via ddi_apply_range.)
936 */
937
938 #ifdef DDI_MAP_DEBUG
939 ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
940 ddi_get_name(dip), ddi_get_name(rdip));
941 #endif /* DDI_MAP_DEBUG */
942
943 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
944 return (error);
945
946 switch (mp->map_op) {
947 case DDI_MO_MAP_LOCKED:
948
949 /*
950 * Set up the locked down kernel mapping to the regspec...
951 */
952
953 return (rootnex_map_regspec(mp, vaddrp));
954
955 case DDI_MO_UNMAP:
956
957 /*
958 * Release mapping...
959 */
960
961 return (rootnex_unmap_regspec(mp, vaddrp));
962
963 case DDI_MO_MAP_HANDLE:
964
965 return (rootnex_map_handle(mp));
966
967 default:
968 return (DDI_ME_UNIMPLEMENTED);
969 }
970 }
971
972
973 /*
974 * rootnex_map_fault()
975 *
976 * fault in mappings for requestors
977 */
978 /*ARGSUSED*/
979 static int
980 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
981 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
982 uint_t lock)
983 {
984
985 #ifdef DDI_MAP_DEBUG
986 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
987 ddi_map_debug(" Seg <%s>\n",
988 seg->s_ops == &segdev_ops ? "segdev" :
989 seg == &kvseg ? "segkmem" : "NONE!");
990 #endif /* DDI_MAP_DEBUG */
991
992 /*
993 * This is all terribly broken, but it is a start
994 *
995 * XXX Note that this test means that segdev_ops
996 * must be exported from seg_dev.c.
997 * XXX What about devices with their own segment drivers?
998 */
999 if (seg->s_ops == &segdev_ops) {
1000 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1001
1002 if (hat == NULL) {
1003 /*
1004 * This is one plausible interpretation of
1005 * a null hat i.e. use the first hat on the
1006 * address space hat list which by convention is
1007 * the hat of the system MMU. At alternative
1008 * would be to panic .. this might well be better ..
1009 */
1010 ASSERT(AS_READ_HELD(seg->s_as));
1011 hat = seg->s_as->a_hat;
1012 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1013 }
1014 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1015 (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1016 } else if (seg == &kvseg && dp == NULL) {
1017 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1018 HAT_LOAD_LOCK);
1019 } else
1020 return (DDI_FAILURE);
1021 return (DDI_SUCCESS);
1022 }
1023
1024
1025 /*
1026 * rootnex_map_regspec()
1027 * we don't support mapping of I/O cards above 4Gb
1028 */
1029 static int
1030 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1031 {
1032 rootnex_addr_t rbase;
1033 void *cvaddr;
1034 uint_t npages, pgoffset;
1035 struct regspec *rp;
1036 ddi_acc_hdl_t *hp;
1037 ddi_acc_impl_t *ap;
1038 uint_t hat_acc_flags;
1039 paddr_t pbase;
1040
1041 rp = mp->map_obj.rp;
1042 hp = mp->map_handlep;
1043
1044 #ifdef DDI_MAP_DEBUG
1045 ddi_map_debug(
1046 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1047 rp->regspec_bustype, rp->regspec_addr,
1048 rp->regspec_size, mp->map_handlep);
1049 #endif /* DDI_MAP_DEBUG */
1050
1051 /*
1052 * I/O or memory mapping
1053 *
1054 * <bustype=0, addr=x, len=x>: memory
1055 * <bustype=1, addr=x, len=x>: i/o
1056 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1057 */
1058
1059 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1060 cmn_err(CE_WARN, "rootnex: invalid register spec"
1061 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
1062 rp->regspec_addr, rp->regspec_size);
1063 return (DDI_FAILURE);
1064 }
1065
1066 if (rp->regspec_bustype != 0) {
1067 /*
1068 * I/O space - needs a handle.
1069 */
1070 if (hp == NULL) {
1071 return (DDI_FAILURE);
1072 }
1073 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1074 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1075 impl_acc_hdl_init(hp);
1076
1077 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1078 #ifdef DDI_MAP_DEBUG
1079 ddi_map_debug("rootnex_map_regspec: mmap() "
1080 "to I/O space is not supported.\n");
1081 #endif /* DDI_MAP_DEBUG */
1082 return (DDI_ME_INVAL);
1083 } else {
1084 /*
1085 * 1275-compliant vs. compatibility i/o mapping
1086 */
1087 *vaddrp =
1088 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1089 ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1090 ((caddr_t)(uintptr_t)rp->regspec_addr);
1091 #ifdef __xpv
1092 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1093 hp->ah_pfn = xen_assign_pfn(
1094 mmu_btop((ulong_t)rp->regspec_addr &
1095 MMU_PAGEMASK));
1096 } else {
1097 hp->ah_pfn = mmu_btop(
1098 (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1099 }
1100 #else
1101 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1102 MMU_PAGEMASK);
1103 #endif
1104 hp->ah_pnum = mmu_btopr(rp->regspec_size +
1105 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1106 }
1107
1108 #ifdef DDI_MAP_DEBUG
1109 ddi_map_debug(
1110 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1111 rp->regspec_size, *vaddrp);
1112 #endif /* DDI_MAP_DEBUG */
1113 return (DDI_SUCCESS);
1114 }
1115
1116 /*
1117 * Memory space
1118 */
1119
1120 if (hp != NULL) {
1121 /*
1122 * hat layer ignores
1123 * hp->ah_acc.devacc_attr_endian_flags.
1124 */
1125 switch (hp->ah_acc.devacc_attr_dataorder) {
1126 case DDI_STRICTORDER_ACC:
1127 hat_acc_flags = HAT_STRICTORDER;
1128 break;
1129 case DDI_UNORDERED_OK_ACC:
1130 hat_acc_flags = HAT_UNORDERED_OK;
1131 break;
1132 case DDI_MERGING_OK_ACC:
1133 hat_acc_flags = HAT_MERGING_OK;
1134 break;
1135 case DDI_LOADCACHING_OK_ACC:
1136 hat_acc_flags = HAT_LOADCACHING_OK;
1137 break;
1138 case DDI_STORECACHING_OK_ACC:
1139 hat_acc_flags = HAT_STORECACHING_OK;
1140 break;
1141 }
1142 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1143 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1144 impl_acc_hdl_init(hp);
1145 hp->ah_hat_flags = hat_acc_flags;
1146 } else {
1147 hat_acc_flags = HAT_STRICTORDER;
1148 }
1149
1150 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1151 #ifdef __xpv
1152 /*
1153 * If we're dom0, we're using a real device so we need to translate
1154 * the MA to a PA.
1155 */
1156 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1157 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1158 } else {
1159 pbase = rbase;
1160 }
1161 #else
1162 pbase = rbase;
1163 #endif
1164 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1165
1166 if (rp->regspec_size == 0) {
1167 #ifdef DDI_MAP_DEBUG
1168 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1169 #endif /* DDI_MAP_DEBUG */
1170 return (DDI_ME_INVAL);
1171 }
1172
1173 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1174 /* extra cast to make gcc happy */
1175 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1176 } else {
1177 npages = mmu_btopr(rp->regspec_size + pgoffset);
1178
1179 #ifdef DDI_MAP_DEBUG
1180 ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1181 "physical %llx", npages, pbase);
1182 #endif /* DDI_MAP_DEBUG */
1183
1184 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1185 if (cvaddr == NULL)
1186 return (DDI_ME_NORESOURCES);
1187
1188 /*
1189 * Now map in the pages we've allocated...
1190 */
1191 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1192 mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1193 HAT_LOAD_LOCK);
1194 *vaddrp = (caddr_t)cvaddr + pgoffset;
1195
1196 /* save away pfn and npages for FMA */
1197 hp = mp->map_handlep;
1198 if (hp) {
1199 hp->ah_pfn = mmu_btop(pbase);
1200 hp->ah_pnum = npages;
1201 }
1202 }
1203
1204 #ifdef DDI_MAP_DEBUG
1205 ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1206 #endif /* DDI_MAP_DEBUG */
1207 return (DDI_SUCCESS);
1208 }
1209
1210
1211 /*
1212 * rootnex_unmap_regspec()
1213 *
1214 */
1215 static int
1216 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1217 {
1218 caddr_t addr = (caddr_t)*vaddrp;
1219 uint_t npages, pgoffset;
1220 struct regspec *rp;
1221
1222 if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1223 return (0);
1224
1225 rp = mp->map_obj.rp;
1226
1227 if (rp->regspec_size == 0) {
1228 #ifdef DDI_MAP_DEBUG
1229 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1230 #endif /* DDI_MAP_DEBUG */
1231 return (DDI_ME_INVAL);
1232 }
1233
1234 /*
1235 * I/O or memory mapping:
1236 *
1237 * <bustype=0, addr=x, len=x>: memory
1238 * <bustype=1, addr=x, len=x>: i/o
1239 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1240 */
1241 if (rp->regspec_bustype != 0) {
1242 /*
1243 * This is I/O space, which requires no particular
1244 * processing on unmap since it isn't mapped in the
1245 * first place.
1246 */
1247 return (DDI_SUCCESS);
1248 }
1249
1250 /*
1251 * Memory space
1252 */
1253 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1254 npages = mmu_btopr(rp->regspec_size + pgoffset);
1255 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1256 device_arena_free(addr - pgoffset, ptob(npages));
1257
1258 /*
1259 * Destroy the pointer - the mapping has logically gone
1260 */
1261 *vaddrp = NULL;
1262
1263 return (DDI_SUCCESS);
1264 }
1265
1266
1267 /*
1268 * rootnex_map_handle()
1269 *
1270 */
1271 static int
1272 rootnex_map_handle(ddi_map_req_t *mp)
1273 {
1274 rootnex_addr_t rbase;
1275 ddi_acc_hdl_t *hp;
1276 uint_t pgoffset;
1277 struct regspec *rp;
1278 paddr_t pbase;
1279
1280 rp = mp->map_obj.rp;
1281
1282 #ifdef DDI_MAP_DEBUG
1283 ddi_map_debug(
1284 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1285 rp->regspec_bustype, rp->regspec_addr,
1286 rp->regspec_size, mp->map_handlep);
1287 #endif /* DDI_MAP_DEBUG */
1288
1289 /*
1290 * I/O or memory mapping:
1291 *
1292 * <bustype=0, addr=x, len=x>: memory
1293 * <bustype=1, addr=x, len=x>: i/o
1294 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1295 */
1296 if (rp->regspec_bustype != 0) {
1297 /*
1298 * This refers to I/O space, and we don't support "mapping"
1299 * I/O space to a user.
1300 */
1301 return (DDI_FAILURE);
1302 }
1303
1304 /*
1305 * Set up the hat_flags for the mapping.
1306 */
1307 hp = mp->map_handlep;
1308
1309 switch (hp->ah_acc.devacc_attr_endian_flags) {
1310 case DDI_NEVERSWAP_ACC:
1311 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1312 break;
1313 case DDI_STRUCTURE_LE_ACC:
1314 hp->ah_hat_flags = HAT_STRUCTURE_LE;
1315 break;
1316 case DDI_STRUCTURE_BE_ACC:
1317 return (DDI_FAILURE);
1318 default:
1319 return (DDI_REGS_ACC_CONFLICT);
1320 }
1321
1322 switch (hp->ah_acc.devacc_attr_dataorder) {
1323 case DDI_STRICTORDER_ACC:
1324 break;
1325 case DDI_UNORDERED_OK_ACC:
1326 hp->ah_hat_flags |= HAT_UNORDERED_OK;
1327 break;
1328 case DDI_MERGING_OK_ACC:
1329 hp->ah_hat_flags |= HAT_MERGING_OK;
1330 break;
1331 case DDI_LOADCACHING_OK_ACC:
1332 hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1333 break;
1334 case DDI_STORECACHING_OK_ACC:
1335 hp->ah_hat_flags |= HAT_STORECACHING_OK;
1336 break;
1337 default:
1338 return (DDI_FAILURE);
1339 }
1340
1341 rbase = (rootnex_addr_t)rp->regspec_addr &
1342 (~(rootnex_addr_t)MMU_PAGEOFFSET);
1343 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1344
1345 if (rp->regspec_size == 0)
1346 return (DDI_ME_INVAL);
1347
1348 #ifdef __xpv
1349 /*
1350 * If we're dom0, we're using a real device so we need to translate
1351 * the MA to a PA.
1352 */
1353 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1354 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1355 (rbase & MMU_PAGEOFFSET);
1356 } else {
1357 pbase = rbase;
1358 }
1359 #else
1360 pbase = rbase;
1361 #endif
1362
1363 hp->ah_pfn = mmu_btop(pbase);
1364 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1365
1366 return (DDI_SUCCESS);
1367 }
1368
1369
1370
1371 /*
1372 * ************************
1373 * interrupt related code
1374 * ************************
1375 */
1376
1377 /*
1378 * rootnex_intr_ops()
1379 * bus_intr_op() function for interrupt support
1380 */
1381 /* ARGSUSED */
1382 static int
1383 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1384 ddi_intr_handle_impl_t *hdlp, void *result)
1385 {
1386 struct intrspec *ispec;
1387
1388 DDI_INTR_NEXDBG((CE_CONT,
1389 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1390 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1391
1392 /* Process the interrupt operation */
1393 switch (intr_op) {
1394 case DDI_INTROP_GETCAP:
1395 /* First check with pcplusmp */
1396 if (psm_intr_ops == NULL)
1397 return (DDI_FAILURE);
1398
1399 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1400 *(int *)result = 0;
1401 return (DDI_FAILURE);
1402 }
1403 break;
1404 case DDI_INTROP_SETCAP:
1405 if (psm_intr_ops == NULL)
1406 return (DDI_FAILURE);
1407
1408 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1409 return (DDI_FAILURE);
1410 break;
1411 case DDI_INTROP_ALLOC:
1412 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1413 return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1414 case DDI_INTROP_FREE:
1415 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1416 return (rootnex_free_intr_fixed(rdip, hdlp));
1417 case DDI_INTROP_GETPRI:
1418 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1419 return (DDI_FAILURE);
1420 *(int *)result = ispec->intrspec_pri;
1421 break;
1422 case DDI_INTROP_SETPRI:
1423 /* Validate the interrupt priority passed to us */
1424 if (*(int *)result > LOCK_LEVEL)
1425 return (DDI_FAILURE);
1426
1427 /* Ensure that PSM is all initialized and ispec is ok */
1428 if ((psm_intr_ops == NULL) ||
1429 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1430 return (DDI_FAILURE);
1431
1432 /* Change the priority */
1433 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1434 PSM_FAILURE)
1435 return (DDI_FAILURE);
1436
1437 /* update the ispec with the new priority */
1438 ispec->intrspec_pri = *(int *)result;
1439 break;
1440 case DDI_INTROP_ADDISR:
1441 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1442 return (DDI_FAILURE);
1443 ispec->intrspec_func = hdlp->ih_cb_func;
1444 break;
1445 case DDI_INTROP_REMISR:
1446 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1447 return (DDI_FAILURE);
1448 ispec->intrspec_func = (uint_t (*)()) 0;
1449 break;
1450 case DDI_INTROP_ENABLE:
1451 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1452 return (DDI_FAILURE);
1453
1454 /* Call psmi to translate irq with the dip */
1455 if (psm_intr_ops == NULL)
1456 return (DDI_FAILURE);
1457
1458 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1459 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1460 (int *)&hdlp->ih_vector) == PSM_FAILURE)
1461 return (DDI_FAILURE);
1462
1463 /* Add the interrupt handler */
1464 if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1465 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1466 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1467 return (DDI_FAILURE);
1468 break;
1469 case DDI_INTROP_DISABLE:
1470 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1471 return (DDI_FAILURE);
1472
1473 /* Call psm_ops() to translate irq with the dip */
1474 if (psm_intr_ops == NULL)
1475 return (DDI_FAILURE);
1476
1477 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1478 (void) (*psm_intr_ops)(rdip, hdlp,
1479 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1480
1481 /* Remove the interrupt handler */
1482 rem_avintr((void *)hdlp, ispec->intrspec_pri,
1483 hdlp->ih_cb_func, hdlp->ih_vector);
1484 break;
1485 case DDI_INTROP_SETMASK:
1486 if (psm_intr_ops == NULL)
1487 return (DDI_FAILURE);
1488
1489 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1490 return (DDI_FAILURE);
1491 break;
1492 case DDI_INTROP_CLRMASK:
1493 if (psm_intr_ops == NULL)
1494 return (DDI_FAILURE);
1495
1496 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1497 return (DDI_FAILURE);
1498 break;
1499 case DDI_INTROP_GETPENDING:
1500 if (psm_intr_ops == NULL)
1501 return (DDI_FAILURE);
1502
1503 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1504 result)) {
1505 *(int *)result = 0;
1506 return (DDI_FAILURE);
1507 }
1508 break;
1509 case DDI_INTROP_NAVAIL:
1510 case DDI_INTROP_NINTRS:
1511 *(int *)result = i_ddi_get_intx_nintrs(rdip);
1512 if (*(int *)result == 0) {
1513 /*
1514 * Special case for 'pcic' driver' only. This driver
1515 * driver is a child of 'isa' and 'rootnex' drivers.
1516 *
1517 * See detailed comments on this in the function
1518 * rootnex_get_ispec().
1519 *
1520 * Children of 'pcic' send 'NINITR' request all the
1521 * way to rootnex driver. But, the 'pdp->par_nintr'
1522 * field may not initialized. So, we fake it here
1523 * to return 1 (a la what PCMCIA nexus does).
1524 */
1525 if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1526 *(int *)result = 1;
1527 else
1528 return (DDI_FAILURE);
1529 }
1530 break;
1531 case DDI_INTROP_SUPPORTED_TYPES:
1532 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */
1533 break;
1534 default:
1535 return (DDI_FAILURE);
1536 }
1537
1538 return (DDI_SUCCESS);
1539 }
1540
1541
1542 /*
1543 * rootnex_get_ispec()
1544 * convert an interrupt number to an interrupt specification.
1545 * The interrupt number determines which interrupt spec will be
1546 * returned if more than one exists.
1547 *
1548 * Look into the parent private data area of the 'rdip' to find out
1549 * the interrupt specification. First check to make sure there is
1550 * one that matchs "inumber" and then return a pointer to it.
1551 *
1552 * Return NULL if one could not be found.
1553 *
1554 * NOTE: This is needed for rootnex_intr_ops()
1555 */
1556 static struct intrspec *
1557 rootnex_get_ispec(dev_info_t *rdip, int inum)
1558 {
1559 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1560
1561 /*
1562 * Special case handling for drivers that provide their own
1563 * intrspec structures instead of relying on the DDI framework.
1564 *
1565 * A broken hardware driver in ON could potentially provide its
1566 * own intrspec structure, instead of relying on the hardware.
1567 * If these drivers are children of 'rootnex' then we need to
1568 * continue to provide backward compatibility to them here.
1569 *
1570 * Following check is a special case for 'pcic' driver which
1571 * was found to have broken hardwre andby provides its own intrspec.
1572 *
1573 * Verbatim comments from this driver are shown here:
1574 * "Don't use the ddi_add_intr since we don't have a
1575 * default intrspec in all cases."
1576 *
1577 * Since an 'ispec' may not be always created for it,
1578 * check for that and create one if so.
1579 *
1580 * NOTE: Currently 'pcic' is the only driver found to do this.
1581 */
1582 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1583 pdp->par_nintr = 1;
1584 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1585 pdp->par_nintr, KM_SLEEP);
1586 }
1587
1588 /* Validate the interrupt number */
1589 if (inum >= pdp->par_nintr)
1590 return (NULL);
1591
1592 /* Get the interrupt structure pointer and return that */
1593 return ((struct intrspec *)&pdp->par_intr[inum]);
1594 }
1595
1596 /*
1597 * Allocate interrupt vector for FIXED (legacy) type.
1598 */
1599 static int
1600 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1601 void *result)
1602 {
1603 struct intrspec *ispec;
1604 ddi_intr_handle_impl_t info_hdl;
1605 int ret;
1606 int free_phdl = 0;
1607 apic_get_type_t type_info;
1608
1609 if (psm_intr_ops == NULL)
1610 return (DDI_FAILURE);
1611
1612 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1613 return (DDI_FAILURE);
1614
1615 /*
1616 * If the PSM module is "APIX" then pass the request for it
1617 * to allocate the vector now.
1618 */
1619 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1620 info_hdl.ih_private = &type_info;
1621 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1622 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1623 if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1624 free_phdl = 1;
1625 i_ddi_alloc_intr_phdl(hdlp);
1626 }
1627 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1628 ret = (*psm_intr_ops)(rdip, hdlp,
1629 PSM_INTR_OP_ALLOC_VECTORS, result);
1630 if (free_phdl) { /* free up the phdl structure */
1631 free_phdl = 0;
1632 i_ddi_free_intr_phdl(hdlp);
1633 hdlp->ih_private = NULL;
1634 }
1635 } else {
1636 /*
1637 * No APIX module; fall back to the old scheme where the
1638 * interrupt vector is allocated during ddi_enable_intr() call.
1639 */
1640 hdlp->ih_pri = ispec->intrspec_pri;
1641 *(int *)result = hdlp->ih_scratch1;
1642 ret = DDI_SUCCESS;
1643 }
1644
1645 return (ret);
1646 }
1647
1648 /*
1649 * Free up interrupt vector for FIXED (legacy) type.
1650 */
1651 static int
1652 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1653 {
1654 struct intrspec *ispec;
1655 struct ddi_parent_private_data *pdp;
1656 ddi_intr_handle_impl_t info_hdl;
1657 int ret;
1658 apic_get_type_t type_info;
1659
1660 if (psm_intr_ops == NULL)
1661 return (DDI_FAILURE);
1662
1663 /*
1664 * If the PSM module is "APIX" then pass the request for it
1665 * to free up the vector now.
1666 */
1667 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1668 info_hdl.ih_private = &type_info;
1669 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1670 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1671 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1672 return (DDI_FAILURE);
1673 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1674 ret = (*psm_intr_ops)(rdip, hdlp,
1675 PSM_INTR_OP_FREE_VECTORS, NULL);
1676 } else {
1677 /*
1678 * No APIX module; fall back to the old scheme where
1679 * the interrupt vector was already freed during
1680 * ddi_disable_intr() call.
1681 */
1682 ret = DDI_SUCCESS;
1683 }
1684
1685 pdp = ddi_get_parent_data(rdip);
1686
1687 /*
1688 * Special case for 'pcic' driver' only.
1689 * If an intrspec was created for it, clean it up here
1690 * See detailed comments on this in the function
1691 * rootnex_get_ispec().
1692 */
1693 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1694 kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1695 pdp->par_nintr);
1696 /*
1697 * Set it to zero; so that
1698 * DDI framework doesn't free it again
1699 */
1700 pdp->par_intr = NULL;
1701 pdp->par_nintr = 0;
1702 }
1703
1704 return (ret);
1705 }
1706
1707
1708 /*
1709 * ******************
1710 * dma related code
1711 * ******************
1712 */
1713
1714 /*ARGSUSED*/
1715 static int
1716 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1717 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1718 ddi_dma_handle_t *handlep)
1719 {
1720 uint64_t maxsegmentsize_ll;
1721 uint_t maxsegmentsize;
1722 ddi_dma_impl_t *hp;
1723 rootnex_dma_t *dma;
1724 uint64_t count_max;
1725 uint64_t seg;
1726 int kmflag;
1727 int e;
1728
1729
1730 /* convert our sleep flags */
1731 if (waitfp == DDI_DMA_SLEEP) {
1732 kmflag = KM_SLEEP;
1733 } else {
1734 kmflag = KM_NOSLEEP;
1735 }
1736
1737 /*
1738 * We try to do only one memory allocation here. We'll do a little
1739 * pointer manipulation later. If the bind ends up taking more than
1740 * our prealloc's space, we'll have to allocate more memory in the
1741 * bind operation. Not great, but much better than before and the
1742 * best we can do with the current bind interfaces.
1743 */
1744 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1745 if (hp == NULL)
1746 return (DDI_DMA_NORESOURCES);
1747
1748 /* Do our pointer manipulation now, align the structures */
1749 hp->dmai_private = (void *)(((uintptr_t)hp +
1750 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1751 dma = (rootnex_dma_t *)hp->dmai_private;
1752 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1753 sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1754
1755 /* setup the handle */
1756 rootnex_clean_dmahdl(hp);
1757 hp->dmai_error.err_fep = NULL;
1758 hp->dmai_error.err_cf = NULL;
1759 dma->dp_dip = rdip;
1760 dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1761 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1762
1763 /*
1764 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1765 * is being used. Set the upper limit to the seg value.
1766 * There will be enough DVMA space to always get addresses
1767 * that will match the constraints.
1768 */
1769 if (IOMMU_USED(rdip) &&
1770 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1771 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1772 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1773 } else
1774 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1775
1776 hp->dmai_minxfer = attr->dma_attr_minxfer;
1777 hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1778 hp->dmai_rdip = rdip;
1779 hp->dmai_attr = *attr;
1780
1781 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1782 dma->dp_sglinfo.si_cancross = B_FALSE;
1783 else
1784 dma->dp_sglinfo.si_cancross = B_TRUE;
1785
1786 /* we don't need to worry about the SPL since we do a tryenter */
1787 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1788
1789 /*
1790 * Figure out our maximum segment size. If the segment size is greater
1791 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1792 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1793 * dma_attr_count_max are size-1 type values.
1794 *
1795 * Maximum segment size is the largest physically contiguous chunk of
1796 * memory that we can return from a bind (i.e. the maximum size of a
1797 * single cookie).
1798 */
1799
1800 /* handle the rollover cases */
1801 seg = attr->dma_attr_seg + 1;
1802 if (seg < attr->dma_attr_seg) {
1803 seg = attr->dma_attr_seg;
1804 }
1805 count_max = attr->dma_attr_count_max + 1;
1806 if (count_max < attr->dma_attr_count_max) {
1807 count_max = attr->dma_attr_count_max;
1808 }
1809
1810 /*
1811 * granularity may or may not be a power of two. If it isn't, we can't
1812 * use a simple mask.
1813 */
1814 if (!ISP2(attr->dma_attr_granular)) {
1815 dma->dp_granularity_power_2 = B_FALSE;
1816 } else {
1817 dma->dp_granularity_power_2 = B_TRUE;
1818 }
1819
1820 /*
1821 * maxxfer should be a whole multiple of granularity. If we're going to
1822 * break up a window because we're greater than maxxfer, we might as
1823 * well make sure it's maxxfer is a whole multiple so we don't have to
1824 * worry about triming the window later on for this case.
1825 */
1826 if (attr->dma_attr_granular > 1) {
1827 if (dma->dp_granularity_power_2) {
1828 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1829 (attr->dma_attr_maxxfer &
1830 (attr->dma_attr_granular - 1));
1831 } else {
1832 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1833 (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1834 }
1835 } else {
1836 dma->dp_maxxfer = attr->dma_attr_maxxfer;
1837 }
1838
1839 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1840 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1841 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1842 maxsegmentsize = 0xFFFFFFFF;
1843 } else {
1844 maxsegmentsize = maxsegmentsize_ll;
1845 }
1846 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1847 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1848
1849 /* check the ddi_dma_attr arg to make sure it makes a little sense */
1850 if (rootnex_alloc_check_parms) {
1851 e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1852 if (e != DDI_SUCCESS) {
1853 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1854 (void) rootnex_dma_freehdl(dip, rdip,
1855 (ddi_dma_handle_t)hp);
1856 return (e);
1857 }
1858 }
1859
1860 *handlep = (ddi_dma_handle_t)hp;
1861
1862 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1863 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1864 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1865
1866 return (DDI_SUCCESS);
1867 }
1868
1869
1870 /*
1871 * rootnex_dma_allochdl()
1872 * called from ddi_dma_alloc_handle().
1873 */
1874 static int
1875 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1876 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1877 {
1878 int retval = DDI_SUCCESS;
1879 #if defined(__amd64) && !defined(__xpv)
1880
1881 if (IOMMU_UNITIALIZED(rdip)) {
1882 retval = iommulib_nex_open(dip, rdip);
1883
1884 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1885 return (retval);
1886 }
1887
1888 if (IOMMU_UNUSED(rdip)) {
1889 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1890 handlep);
1891 } else {
1892 retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1893 waitfp, arg, handlep);
1894 }
1895 #else
1896 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1897 handlep);
1898 #endif
1899 switch (retval) {
1900 case DDI_DMA_NORESOURCES:
1901 if (waitfp != DDI_DMA_DONTWAIT) {
1902 ddi_set_callback(waitfp, arg,
1903 &rootnex_state->r_dvma_call_list_id);
1904 }
1905 break;
1906 case DDI_SUCCESS:
1907 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1908 break;
1909 default:
1910 break;
1911 }
1912 return (retval);
1913 }
1914
1915 /*ARGSUSED*/
1916 static int
1917 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1918 ddi_dma_handle_t handle)
1919 {
1920 ddi_dma_impl_t *hp;
1921 rootnex_dma_t *dma;
1922
1923
1924 hp = (ddi_dma_impl_t *)handle;
1925 dma = (rootnex_dma_t *)hp->dmai_private;
1926
1927 /* unbind should have been called first */
1928 ASSERT(!dma->dp_inuse);
1929
1930 mutex_destroy(&dma->dp_mutex);
1931 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1932
1933 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1934 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1935 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1936
1937 return (DDI_SUCCESS);
1938 }
1939
1940 /*
1941 * rootnex_dma_freehdl()
1942 * called from ddi_dma_free_handle().
1943 */
1944 static int
1945 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1946 {
1947 int ret;
1948
1949 ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1950 #if defined(__amd64) && !defined(__xpv)
1951 if (IOMMU_USED(rdip))
1952 ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1953 else
1954 #endif
1955 ret = rootnex_coredma_freehdl(dip, rdip, handle);
1956
1957 if (rootnex_state->r_dvma_call_list_id)
1958 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1959
1960 return (ret);
1961 }
1962
1963 /*ARGSUSED*/
1964 static int
1965 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1966 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1967 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1968 {
1969 rootnex_sglinfo_t *sinfo;
1970 ddi_dma_obj_t *dmao;
1971 #if defined(__amd64) && !defined(__xpv)
1972 struct dvmaseg *dvs;
1973 ddi_dma_cookie_t *cookie;
1974 #endif
1975 ddi_dma_attr_t *attr;
1976 ddi_dma_impl_t *hp;
1977 rootnex_dma_t *dma;
1978 int kmflag;
1979 int e;
1980 uint_t ncookies;
1981
1982 hp = (ddi_dma_impl_t *)handle;
1983 dma = (rootnex_dma_t *)hp->dmai_private;
1984 dmao = &dma->dp_dma;
1985 sinfo = &dma->dp_sglinfo;
1986 attr = &hp->dmai_attr;
1987
1988 /* convert the sleep flags */
1989 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1990 dma->dp_sleep_flags = kmflag = KM_SLEEP;
1991 } else {
1992 dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1993 }
1994
1995 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1996
1997 /*
1998 * This is useful for debugging a driver. Not as useful in a production
1999 * system. The only time this will fail is if you have a driver bug.
2000 */
2001 if (rootnex_bind_check_inuse) {
2002 /*
2003 * No one else should ever have this lock unless someone else
2004 * is trying to use this handle. So contention on the lock
2005 * is the same as inuse being set.
2006 */
2007 e = mutex_tryenter(&dma->dp_mutex);
2008 if (e == 0) {
2009 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2010 return (DDI_DMA_INUSE);
2011 }
2012 if (dma->dp_inuse) {
2013 mutex_exit(&dma->dp_mutex);
2014 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2015 return (DDI_DMA_INUSE);
2016 }
2017 dma->dp_inuse = B_TRUE;
2018 mutex_exit(&dma->dp_mutex);
2019 }
2020
2021 /* check the ddi_dma_attr arg to make sure it makes a little sense */
2022 if (rootnex_bind_check_parms) {
2023 e = rootnex_valid_bind_parms(dmareq, attr);
2024 if (e != DDI_SUCCESS) {
2025 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2026 rootnex_clean_dmahdl(hp);
2027 return (e);
2028 }
2029 }
2030
2031 /* save away the original bind info */
2032 dma->dp_dma = dmareq->dmar_object;
2033
2034 #if defined(__amd64) && !defined(__xpv)
2035 if (IOMMU_USED(rdip)) {
2036 dmao = &dma->dp_dvma;
2037 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2038 switch (e) {
2039 case DDI_SUCCESS:
2040 if (sinfo->si_cancross ||
2041 dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2042 dmao->dmao_size > sinfo->si_max_cookie_size) {
2043 dma->dp_dvma_used = B_TRUE;
2044 break;
2045 }
2046 sinfo->si_sgl_size = 1;
2047 hp->dmai_rflags |= DMP_NOSYNC;
2048
2049 dma->dp_dvma_used = B_TRUE;
2050 dma->dp_need_to_free_cookie = B_FALSE;
2051
2052 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2053 cookie = hp->dmai_cookie = dma->dp_cookies =
2054 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2055 cookie->dmac_laddress = dvs->dvs_start +
2056 dmao->dmao_obj.dvma_obj.dv_off;
2057 cookie->dmac_size = dvs->dvs_len;
2058 cookie->dmac_type = 0;
2059
2060 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2061 rdip);
2062 goto fast;
2063 case DDI_ENOTSUP:
2064 break;
2065 default:
2066 rootnex_clean_dmahdl(hp);
2067 return (e);
2068 }
2069 }
2070 #endif
2071
2072 /*
2073 * Figure out a rough estimate of what maximum number of pages
2074 * this buffer could use (a high estimate of course).
2075 */
2076 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2077
2078 if (dma->dp_dvma_used) {
2079 /*
2080 * The number of physical pages is the worst case.
2081 *
2082 * For DVMA, the worst case is the length divided
2083 * by the maximum cookie length, plus 1. Add to that
2084 * the number of segment boundaries potentially crossed, and
2085 * the additional number of DVMA segments that was returned.
2086 *
2087 * In the normal case, for modern devices, si_cancross will
2088 * be false, and dv_nseg will be 1, and the fast path will
2089 * have been taken above.
2090 */
2091 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2092 + 1;
2093 if (sinfo->si_cancross)
2094 ncookies +=
2095 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2096 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2097
2098 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2099 }
2100
2101 /*
2102 * We'll use the pre-allocated cookies for any bind that will *always*
2103 * fit (more important to be consistent, we don't want to create
2104 * additional degenerate cases).
2105 */
2106 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2107 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2108 dma->dp_need_to_free_cookie = B_FALSE;
2109 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2110 uint_t, sinfo->si_max_pages);
2111
2112 /*
2113 * For anything larger than that, we'll go ahead and allocate the
2114 * maximum number of pages we expect to see. Hopefuly, we won't be
2115 * seeing this path in the fast path for high performance devices very
2116 * frequently.
2117 *
2118 * a ddi bind interface that allowed the driver to provide storage to
2119 * the bind interface would speed this case up.
2120 */
2121 } else {
2122 /*
2123 * Save away how much memory we allocated. If we're doing a
2124 * nosleep, the alloc could fail...
2125 */
2126 dma->dp_cookie_size = sinfo->si_max_pages *
2127 sizeof (ddi_dma_cookie_t);
2128 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2129 if (dma->dp_cookies == NULL) {
2130 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2131 rootnex_clean_dmahdl(hp);
2132 return (DDI_DMA_NORESOURCES);
2133 }
2134 dma->dp_need_to_free_cookie = B_TRUE;
2135 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2136 uint_t, sinfo->si_max_pages);
2137 }
2138 hp->dmai_cookie = dma->dp_cookies;
2139
2140 /*
2141 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2142 * looking at the constraints in the dma structure. It will then put
2143 * some additional state about the sgl in the dma struct (i.e. is
2144 * the sgl clean, or do we need to do some munging; how many pages
2145 * need to be copied, etc.)
2146 */
2147 if (dma->dp_dvma_used)
2148 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2149 else
2150 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2151
2152 out:
2153 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2154 /* if we don't need a copy buffer, we don't need to sync */
2155 if (sinfo->si_copybuf_req == 0) {
2156 hp->dmai_rflags |= DMP_NOSYNC;
2157 }
2158
2159 /*
2160 * if we don't need the copybuf and we don't need to do a partial, we
2161 * hit the fast path. All the high performance devices should be trying
2162 * to hit this path. To hit this path, a device should be able to reach
2163 * all of memory, shouldn't try to bind more than it can transfer, and
2164 * the buffer shouldn't require more cookies than the driver/device can
2165 * handle [sgllen]).
2166 *
2167 * Note that negative values of dma_attr_sgllen are supposed
2168 * to mean unlimited, but we just cast them to mean a
2169 * "ridiculous large limit". This saves some extra checks on
2170 * hot paths.
2171 */
2172 if ((sinfo->si_copybuf_req == 0) &&
2173 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2174 (dmao->dmao_size < dma->dp_maxxfer)) {
2175 fast:
2176 /*
2177 * If the driver supports FMA, insert the handle in the FMA DMA
2178 * handle cache.
2179 */
2180 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2181 hp->dmai_error.err_cf = rootnex_dma_check;
2182
2183 /*
2184 * copy out the first cookie and ccountp, set the cookie
2185 * pointer to the second cookie. The first cookie is passed
2186 * back on the stack. Additional cookies are accessed via
2187 * ddi_dma_nextcookie()
2188 */
2189 *cookiep = dma->dp_cookies[0];
2190 *ccountp = sinfo->si_sgl_size;
2191 hp->dmai_cookie++;
2192 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2193 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2194 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2195 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2196 uint_t, dmao->dmao_size, uint_t, *ccountp);
2197
2198
2199 return (DDI_DMA_MAPPED);
2200 }
2201
2202 /*
2203 * go to the slow path, we may need to alloc more memory, create
2204 * multiple windows, and munge up a sgl to make the device happy.
2205 */
2206
2207 /*
2208 * With the IOMMU mapobject method used, we should never hit
2209 * the slow path. If we do, something is seriously wrong.
2210 * Clean up and return an error.
2211 */
2212
2213 #if defined(__amd64) && !defined(__xpv)
2214
2215 if (dma->dp_dvma_used) {
2216 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2217 &dma->dp_dvma);
2218 e = DDI_DMA_NOMAPPING;
2219 } else {
2220 #endif
2221 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2222 kmflag);
2223 #if defined(__amd64) && !defined(__xpv)
2224 }
2225 #endif
2226 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2227 if (dma->dp_need_to_free_cookie) {
2228 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2229 }
2230 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2231 rootnex_clean_dmahdl(hp); /* must be after free cookie */
2232 return (e);
2233 }
2234
2235 /*
2236 * If the driver supports FMA, insert the handle in the FMA DMA handle
2237 * cache.
2238 */
2239 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2240 hp->dmai_error.err_cf = rootnex_dma_check;
2241
2242 /* if the first window uses the copy buffer, sync it for the device */
2243 if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2244 (hp->dmai_rflags & DDI_DMA_WRITE)) {
2245 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2246 DDI_DMA_SYNC_FORDEV);
2247 }
2248
2249 /*
2250 * copy out the first cookie and ccountp, set the cookie pointer to the
2251 * second cookie. Make sure the partial flag is set/cleared correctly.
2252 * If we have a partial map (i.e. multiple windows), the number of
2253 * cookies we return is the number of cookies in the first window.
2254 */
2255 if (e == DDI_DMA_MAPPED) {
2256 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2257 *ccountp = sinfo->si_sgl_size;
2258 hp->dmai_nwin = 1;
2259 } else {
2260 hp->dmai_rflags |= DDI_DMA_PARTIAL;
2261 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2262 ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2263 }
2264 *cookiep = dma->dp_cookies[0];
2265 hp->dmai_cookie++;
2266
2267 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2268 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2269 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2270 dmao->dmao_size, uint_t, *ccountp);
2271 return (e);
2272 }
2273
2274 /*
2275 * rootnex_dma_bindhdl()
2276 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2277 */
2278 static int
2279 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2280 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2281 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2282 {
2283 int ret;
2284 #if defined(__amd64) && !defined(__xpv)
2285 if (IOMMU_USED(rdip))
2286 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2287 cookiep, ccountp);
2288 else
2289 #endif
2290 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2291 cookiep, ccountp);
2292
2293 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2294 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2295 &rootnex_state->r_dvma_call_list_id);
2296 }
2297
2298 return (ret);
2299 }
2300
2301
2302
2303 /*ARGSUSED*/
2304 static int
2305 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2306 ddi_dma_handle_t handle)
2307 {
2308 ddi_dma_impl_t *hp;
2309 rootnex_dma_t *dma;
2310 int e;
2311
2312 hp = (ddi_dma_impl_t *)handle;
2313 dma = (rootnex_dma_t *)hp->dmai_private;
2314
2315 /* make sure the buffer wasn't free'd before calling unbind */
2316 if (rootnex_unbind_verify_buffer) {
2317 e = rootnex_verify_buffer(dma);
2318 if (e != DDI_SUCCESS) {
2319 ASSERT(0);
2320 return (DDI_FAILURE);
2321 }
2322 }
2323
2324 /* sync the current window before unbinding the buffer */
2325 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2326 (hp->dmai_rflags & DDI_DMA_READ)) {
2327 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2328 DDI_DMA_SYNC_FORCPU);
2329 }
2330
2331 /*
2332 * cleanup and copy buffer or window state. if we didn't use the copy
2333 * buffer or windows, there won't be much to do :-)
2334 */
2335 rootnex_teardown_copybuf(dma);
2336 rootnex_teardown_windows(dma);
2337
2338 #if defined(__amd64) && !defined(__xpv)
2339 if (IOMMU_USED(rdip))
2340 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2341 &dma->dp_dvma);
2342 #endif
2343
2344 /*
2345 * If we had to allocate space to for the worse case sgl (it didn't
2346 * fit into our pre-allocate buffer), free that up now
2347 */
2348 if (dma->dp_need_to_free_cookie) {
2349 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2350 }
2351
2352 /*
2353 * clean up the handle so it's ready for the next bind (i.e. if the
2354 * handle is reused).
2355 */
2356 rootnex_clean_dmahdl(hp);
2357 hp->dmai_error.err_cf = NULL;
2358
2359 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2360 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2361 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2362
2363 return (DDI_SUCCESS);
2364 }
2365
2366 /*
2367 * rootnex_dma_unbindhdl()
2368 * called from ddi_dma_unbind_handle()
2369 */
2370 /*ARGSUSED*/
2371 static int
2372 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2373 ddi_dma_handle_t handle)
2374 {
2375 int ret;
2376
2377 #if defined(__amd64) && !defined(__xpv)
2378 if (IOMMU_USED(rdip))
2379 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2380 else
2381 #endif
2382 ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2383
2384 if (rootnex_state->r_dvma_call_list_id)
2385 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2386
2387 return (ret);
2388 }
2389
2390 #if defined(__amd64) && !defined(__xpv)
2391
2392 static int
2393 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2394 {
2395 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2396 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2397
2398 if (dma->dp_sleep_flags != KM_SLEEP &&
2399 dma->dp_sleep_flags != KM_NOSLEEP)
2400 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2401 return (dma->dp_sleep_flags);
2402 }
2403 /*ARGSUSED*/
2404 static void
2405 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2406 {
2407 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2408 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2409 rootnex_window_t *window;
2410
2411 if (dma->dp_window) {
2412 window = &dma->dp_window[dma->dp_current_win];
2413 hp->dmai_cookie = window->wd_first_cookie;
2414 } else {
2415 hp->dmai_cookie = dma->dp_cookies;
2416 }
2417 hp->dmai_cookie++;
2418 }
2419
2420 /*ARGSUSED*/
2421 static int
2422 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2423 ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2424 {
2425 int i;
2426 int km_flags;
2427 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2428 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2429 rootnex_window_t *window;
2430 ddi_dma_cookie_t *cp;
2431 ddi_dma_cookie_t *cookie;
2432
2433 ASSERT(*cookiepp == NULL);
2434 ASSERT(*ccountp == 0);
2435
2436 if (dma->dp_window) {
2437 window = &dma->dp_window[dma->dp_current_win];
2438 cp = window->wd_first_cookie;
2439 *ccountp = window->wd_cookie_cnt;
2440 } else {
2441 cp = dma->dp_cookies;
2442 *ccountp = dma->dp_sglinfo.si_sgl_size;
2443 }
2444
2445 km_flags = rootnex_coredma_get_sleep_flags(handle);
2446 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2447 if (cookie == NULL) {
2448 return (DDI_DMA_NORESOURCES);
2449 }
2450
2451 for (i = 0; i < *ccountp; i++) {
2452 cookie[i].dmac_notused = cp[i].dmac_notused;
2453 cookie[i].dmac_type = cp[i].dmac_type;
2454 cookie[i].dmac_address = cp[i].dmac_address;
2455 cookie[i].dmac_size = cp[i].dmac_size;
2456 }
2457
2458 *cookiepp = cookie;
2459
2460 return (DDI_SUCCESS);
2461 }
2462
2463 /*ARGSUSED*/
2464 static int
2465 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2466 ddi_dma_cookie_t *cookiep, uint_t ccount)
2467 {
2468 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2469 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2470 rootnex_window_t *window;
2471 ddi_dma_cookie_t *cur_cookiep;
2472
2473 ASSERT(cookiep);
2474 ASSERT(ccount != 0);
2475 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2476
2477 if (dma->dp_window) {
2478 window = &dma->dp_window[dma->dp_current_win];
2479 dma->dp_saved_cookies = window->wd_first_cookie;
2480 window->wd_first_cookie = cookiep;
2481 ASSERT(ccount == window->wd_cookie_cnt);
2482 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2483 + window->wd_first_cookie;
2484 } else {
2485 dma->dp_saved_cookies = dma->dp_cookies;
2486 dma->dp_cookies = cookiep;
2487 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2488 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2489 + dma->dp_cookies;
2490 }
2491
2492 dma->dp_need_to_switch_cookies = B_TRUE;
2493 hp->dmai_cookie = cur_cookiep;
2494
2495 return (DDI_SUCCESS);
2496 }
2497
2498 /*ARGSUSED*/
2499 static int
2500 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2501 {
2502 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2503 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2504 rootnex_window_t *window;
2505 ddi_dma_cookie_t *cur_cookiep;
2506 ddi_dma_cookie_t *cookie_array;
2507 uint_t ccount;
2508
2509 /* check if cookies have not been switched */
2510 if (dma->dp_need_to_switch_cookies == B_FALSE)
2511 return (DDI_SUCCESS);
2512
2513 ASSERT(dma->dp_saved_cookies);
2514
2515 if (dma->dp_window) {
2516 window = &dma->dp_window[dma->dp_current_win];
2517 cookie_array = window->wd_first_cookie;
2518 window->wd_first_cookie = dma->dp_saved_cookies;
2519 dma->dp_saved_cookies = NULL;
2520 ccount = window->wd_cookie_cnt;
2521 cur_cookiep = (hp->dmai_cookie - cookie_array)
2522 + window->wd_first_cookie;
2523 } else {
2524 cookie_array = dma->dp_cookies;
2525 dma->dp_cookies = dma->dp_saved_cookies;
2526 dma->dp_saved_cookies = NULL;
2527 ccount = dma->dp_sglinfo.si_sgl_size;
2528 cur_cookiep = (hp->dmai_cookie - cookie_array)
2529 + dma->dp_cookies;
2530 }
2531
2532 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2533
2534 hp->dmai_cookie = cur_cookiep;
2535
2536 dma->dp_need_to_switch_cookies = B_FALSE;
2537
2538 return (DDI_SUCCESS);
2539 }
2540
2541 #endif
2542
2543 static struct as *
2544 rootnex_get_as(ddi_dma_obj_t *dmao)
2545 {
2546 struct as *asp;
2547
2548 switch (dmao->dmao_type) {
2549 case DMA_OTYP_VADDR:
2550 case DMA_OTYP_BUFVADDR:
2551 asp = dmao->dmao_obj.virt_obj.v_as;
2552 if (asp == NULL)
2553 asp = &kas;
2554 break;
2555 default:
2556 asp = NULL;
2557 break;
2558 }
2559 return (asp);
2560 }
2561
2562 /*
2563 * rootnex_verify_buffer()
2564 * verify buffer wasn't free'd
2565 */
2566 static int
2567 rootnex_verify_buffer(rootnex_dma_t *dma)
2568 {
2569 page_t **pplist;
2570 caddr_t vaddr;
2571 uint_t pcnt;
2572 uint_t poff;
2573 page_t *pp;
2574 char b;
2575 int i;
2576
2577 /* Figure out how many pages this buffer occupies */
2578 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2579 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2580 } else {
2581 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2582 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2583 }
2584 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2585
2586 switch (dma->dp_dma.dmao_type) {
2587 case DMA_OTYP_PAGES:
2588 /*
2589 * for a linked list of pp's walk through them to make sure
2590 * they're locked and not free.
2591 */
2592 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2593 for (i = 0; i < pcnt; i++) {
2594 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2595 return (DDI_FAILURE);
2596 }
2597 pp = pp->p_next;
2598 }
2599 break;
2600
2601 case DMA_OTYP_VADDR:
2602 case DMA_OTYP_BUFVADDR:
2603 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2604 /*
2605 * for an array of pp's walk through them to make sure they're
2606 * not free. It's possible that they may not be locked.
2607 */
2608 if (pplist) {
2609 for (i = 0; i < pcnt; i++) {
2610 if (PP_ISFREE(pplist[i])) {
2611 return (DDI_FAILURE);
2612 }
2613 }
2614
2615 /* For a virtual address, try to peek at each page */
2616 } else {
2617 if (rootnex_get_as(&dma->dp_dma) == &kas) {
2618 for (i = 0; i < pcnt; i++) {
2619 if (ddi_peek8(NULL, vaddr, &b) ==
2620 DDI_FAILURE)
2621 return (DDI_FAILURE);
2622 vaddr += MMU_PAGESIZE;
2623 }
2624 }
2625 }
2626 break;
2627
2628 default:
2629 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2630 break;
2631 }
2632
2633 return (DDI_SUCCESS);
2634 }
2635
2636
2637 /*
2638 * rootnex_clean_dmahdl()
2639 * Clean the dma handle. This should be called on a handle alloc and an
2640 * unbind handle. Set the handle state to the default settings.
2641 */
2642 static void
2643 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2644 {
2645 rootnex_dma_t *dma;
2646
2647
2648 dma = (rootnex_dma_t *)hp->dmai_private;
2649
2650 hp->dmai_nwin = 0;
2651 dma->dp_current_cookie = 0;
2652 dma->dp_copybuf_size = 0;
2653 dma->dp_window = NULL;
2654 dma->dp_cbaddr = NULL;
2655 dma->dp_inuse = B_FALSE;
2656 dma->dp_dvma_used = B_FALSE;
2657 dma->dp_need_to_free_cookie = B_FALSE;
2658 dma->dp_need_to_switch_cookies = B_FALSE;
2659 dma->dp_saved_cookies = NULL;
2660 dma->dp_sleep_flags = KM_PANIC;
2661 dma->dp_need_to_free_window = B_FALSE;
2662 dma->dp_partial_required = B_FALSE;
2663 dma->dp_trim_required = B_FALSE;
2664 dma->dp_sglinfo.si_copybuf_req = 0;
2665 #if !defined(__amd64)
2666 dma->dp_cb_remaping = B_FALSE;
2667 dma->dp_kva = NULL;
2668 #endif
2669
2670 /* FMA related initialization */
2671 hp->dmai_fault = 0;
2672 hp->dmai_fault_check = NULL;
2673 hp->dmai_fault_notify = NULL;
2674 hp->dmai_error.err_ena = 0;
2675 hp->dmai_error.err_status = DDI_FM_OK;
2676 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2677 hp->dmai_error.err_ontrap = NULL;
2678 }
2679
2680
2681 /*
2682 * rootnex_valid_alloc_parms()
2683 * Called in ddi_dma_alloc_handle path to validate its parameters.
2684 */
2685 static int
2686 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2687 {
2688 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2689 (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2690 (attr->dma_attr_granular > MMU_PAGESIZE) ||
2691 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2692 return (DDI_DMA_BADATTR);
2693 }
2694
2695 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2696 return (DDI_DMA_BADATTR);
2697 }
2698
2699 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2700 MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2701 attr->dma_attr_sgllen == 0) {
2702 return (DDI_DMA_BADATTR);
2703 }
2704
2705 /* We should be able to DMA into every byte offset in a page */
2706 if (maxsegmentsize < MMU_PAGESIZE) {
2707 return (DDI_DMA_BADATTR);
2708 }
2709
2710 /* if we're bouncing on seg, seg must be <= addr_hi */
2711 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2712 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2713 return (DDI_DMA_BADATTR);
2714 }
2715 return (DDI_SUCCESS);
2716 }
2717
2718 /*
2719 * rootnex_valid_bind_parms()
2720 * Called in ddi_dma_*_bind_handle path to validate its parameters.
2721 */
2722 /* ARGSUSED */
2723 static int
2724 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2725 {
2726 #if !defined(__amd64)
2727 /*
2728 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2729 * we can track the offset for the obsoleted interfaces.
2730 */
2731 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2732 return (DDI_DMA_TOOBIG);
2733 }
2734 #endif
2735
2736 return (DDI_SUCCESS);
2737 }
2738
2739
2740 /*
2741 * rootnex_need_bounce_seg()
2742 * check to see if the buffer lives on both side of the seg.
2743 */
2744 static boolean_t
2745 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2746 {
2747 ddi_dma_atyp_t buftype;
2748 rootnex_addr_t raddr;
2749 boolean_t lower_addr;
2750 boolean_t upper_addr;
2751 uint64_t offset;
2752 page_t **pplist;
2753 uint64_t paddr;
2754 uint32_t psize;
2755 uint32_t size;
2756 caddr_t vaddr;
2757 uint_t pcnt;
2758 page_t *pp;
2759
2760
2761 /* shortcuts */
2762 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2763 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2764 buftype = dmar_object->dmao_type;
2765 size = dmar_object->dmao_size;
2766
2767 lower_addr = B_FALSE;
2768 upper_addr = B_FALSE;
2769 pcnt = 0;
2770
2771 /*
2772 * Process the first page to handle the initial offset of the buffer.
2773 * We'll use the base address we get later when we loop through all
2774 * the pages.
2775 */
2776 if (buftype == DMA_OTYP_PAGES) {
2777 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2778 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2779 MMU_PAGEOFFSET;
2780 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2781 psize = MIN(size, (MMU_PAGESIZE - offset));
2782 pp = pp->p_next;
2783 sglinfo->si_asp = NULL;
2784 } else if (pplist != NULL) {
2785 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2786 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2787 if (sglinfo->si_asp == NULL) {
2788 sglinfo->si_asp = &kas;
2789 }
2790 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2791 paddr += offset;
2792 psize = MIN(size, (MMU_PAGESIZE - offset));
2793 pcnt++;
2794 } else {
2795 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2796 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2797 if (sglinfo->si_asp == NULL) {
2798 sglinfo->si_asp = &kas;
2799 }
2800 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2801 paddr += offset;
2802 psize = MIN(size, (MMU_PAGESIZE - offset));
2803 vaddr += psize;
2804 }
2805
2806 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2807
2808 if ((raddr + psize) > sglinfo->si_segmask) {
2809 upper_addr = B_TRUE;
2810 } else {
2811 lower_addr = B_TRUE;
2812 }
2813 size -= psize;
2814
2815 /*
2816 * Walk through the rest of the pages in the buffer. Track to see
2817 * if we have pages on both sides of the segment boundary.
2818 */
2819 while (size > 0) {
2820 /* partial or full page */
2821 psize = MIN(size, MMU_PAGESIZE);
2822
2823 if (buftype == DMA_OTYP_PAGES) {
2824 /* get the paddr from the page_t */
2825 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2826 paddr = pfn_to_pa(pp->p_pagenum);
2827 pp = pp->p_next;
2828 } else if (pplist != NULL) {
2829 /* index into the array of page_t's to get the paddr */
2830 ASSERT(!PP_ISFREE(pplist[pcnt]));
2831 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2832 pcnt++;
2833 } else {
2834 /* call into the VM to get the paddr */
2835 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2836 vaddr));
2837 vaddr += psize;
2838 }
2839
2840 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2841
2842 if ((raddr + psize) > sglinfo->si_segmask) {
2843 upper_addr = B_TRUE;
2844 } else {
2845 lower_addr = B_TRUE;
2846 }
2847 /*
2848 * if the buffer lives both above and below the segment
2849 * boundary, or the current page is the page immediately
2850 * after the segment, we will use a copy/bounce buffer for
2851 * all pages > seg.
2852 */
2853 if ((lower_addr && upper_addr) ||
2854 (raddr == (sglinfo->si_segmask + 1))) {
2855 return (B_TRUE);
2856 }
2857
2858 size -= psize;
2859 }
2860
2861 return (B_FALSE);
2862 }
2863
2864 /*
2865 * rootnex_get_sgl()
2866 * Called in bind fastpath to get the sgl. Most of this will be replaced
2867 * with a call to the vm layer when vm2.0 comes around...
2868 */
2869 static void
2870 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2871 rootnex_sglinfo_t *sglinfo)
2872 {
2873 ddi_dma_atyp_t buftype;
2874 rootnex_addr_t raddr;
2875 uint64_t last_page;
2876 uint64_t offset;
2877 uint64_t addrhi;
2878 uint64_t addrlo;
2879 uint64_t maxseg;
2880 page_t **pplist;
2881 uint64_t paddr;
2882 uint32_t psize;
2883 uint32_t size;
2884 caddr_t vaddr;
2885 uint_t pcnt;
2886 page_t *pp;
2887 uint_t cnt;
2888
2889
2890 /* shortcuts */
2891 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2892 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2893 maxseg = sglinfo->si_max_cookie_size;
2894 buftype = dmar_object->dmao_type;
2895 addrhi = sglinfo->si_max_addr;
2896 addrlo = sglinfo->si_min_addr;
2897 size = dmar_object->dmao_size;
2898
2899 pcnt = 0;
2900 cnt = 0;
2901
2902
2903 /*
2904 * check to see if we need to use the copy buffer for pages over
2905 * the segment attr.
2906 */
2907 sglinfo->si_bounce_on_seg = B_FALSE;
2908 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2909 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2910 dmar_object, sglinfo);
2911 }
2912
2913 /*
2914 * if we were passed down a linked list of pages, i.e. pointer to
2915 * page_t, use this to get our physical address and buf offset.
2916 */
2917 if (buftype == DMA_OTYP_PAGES) {
2918 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2919 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2920 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2921 MMU_PAGEOFFSET;
2922 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2923 psize = MIN(size, (MMU_PAGESIZE - offset));
2924 pp = pp->p_next;
2925 sglinfo->si_asp = NULL;
2926
2927 /*
2928 * We weren't passed down a linked list of pages, but if we were passed
2929 * down an array of pages, use this to get our physical address and buf
2930 * offset.
2931 */
2932 } else if (pplist != NULL) {
2933 ASSERT((buftype == DMA_OTYP_VADDR) ||
2934 (buftype == DMA_OTYP_BUFVADDR));
2935
2936 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2937 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2938 if (sglinfo->si_asp == NULL) {
2939 sglinfo->si_asp = &kas;
2940 }
2941
2942 ASSERT(!PP_ISFREE(pplist[pcnt]));
2943 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2944 paddr += offset;
2945 psize = MIN(size, (MMU_PAGESIZE - offset));
2946 pcnt++;
2947
2948 /*
2949 * All we have is a virtual address, we'll need to call into the VM
2950 * to get the physical address.
2951 */
2952 } else {
2953 ASSERT((buftype == DMA_OTYP_VADDR) ||
2954 (buftype == DMA_OTYP_BUFVADDR));
2955
2956 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2957 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2958 if (sglinfo->si_asp == NULL) {
2959 sglinfo->si_asp = &kas;
2960 }
2961
2962 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2963 paddr += offset;
2964 psize = MIN(size, (MMU_PAGESIZE - offset));
2965 vaddr += psize;
2966 }
2967
2968 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2969
2970 /*
2971 * Setup the first cookie with the physical address of the page and the
2972 * size of the page (which takes into account the initial offset into
2973 * the page.
2974 */
2975 sgl[cnt].dmac_laddress = raddr;
2976 sgl[cnt].dmac_size = psize;
2977 sgl[cnt].dmac_type = 0;
2978
2979 /*
2980 * Save away the buffer offset into the page. We'll need this later in
2981 * the copy buffer code to help figure out the page index within the
2982 * buffer and the offset into the current page.
2983 */
2984 sglinfo->si_buf_offset = offset;
2985
2986 /*
2987 * If we are using the copy buffer for anything over the segment
2988 * boundary, and this page is over the segment boundary.
2989 * OR
2990 * if the DMA engine can't reach the physical address.
2991 */
2992 if (((sglinfo->si_bounce_on_seg) &&
2993 ((raddr + psize) > sglinfo->si_segmask)) ||
2994 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2995 /*
2996 * Increase how much copy buffer we use. We always increase by
2997 * pagesize so we don't have to worry about converting offsets.
2998 * Set a flag in the cookies dmac_type to indicate that it uses
2999 * the copy buffer. If this isn't the last cookie, go to the
3000 * next cookie (since we separate each page which uses the copy
3001 * buffer in case the copy buffer is not physically contiguous.
3002 */
3003 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3004 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3005 if ((cnt + 1) < sglinfo->si_max_pages) {
3006 cnt++;
3007 sgl[cnt].dmac_laddress = 0;
3008 sgl[cnt].dmac_size = 0;
3009 sgl[cnt].dmac_type = 0;
3010 }
3011 }
3012
3013 /*
3014 * save this page's physical address so we can figure out if the next
3015 * page is physically contiguous. Keep decrementing size until we are
3016 * done with the buffer.
3017 */
3018 last_page = raddr & MMU_PAGEMASK;
3019 size -= psize;
3020
3021 while (size > 0) {
3022 /* Get the size for this page (i.e. partial or full page) */
3023 psize = MIN(size, MMU_PAGESIZE);
3024
3025 if (buftype == DMA_OTYP_PAGES) {
3026 /* get the paddr from the page_t */
3027 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3028 paddr = pfn_to_pa(pp->p_pagenum);
3029 pp = pp->p_next;
3030 } else if (pplist != NULL) {
3031 /* index into the array of page_t's to get the paddr */
3032 ASSERT(!PP_ISFREE(pplist[pcnt]));
3033 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3034 pcnt++;
3035 } else {
3036 /* call into the VM to get the paddr */
3037 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3038 vaddr));
3039 vaddr += psize;
3040 }
3041
3042 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3043
3044 /*
3045 * If we are using the copy buffer for anything over the
3046 * segment boundary, and this page is over the segment
3047 * boundary.
3048 * OR
3049 * if the DMA engine can't reach the physical address.
3050 */
3051 if (((sglinfo->si_bounce_on_seg) &&
3052 ((raddr + psize) > sglinfo->si_segmask)) ||
3053 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3054
3055 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3056
3057 /*
3058 * if there is something in the current cookie, go to
3059 * the next one. We only want one page in a cookie which
3060 * uses the copybuf since the copybuf doesn't have to
3061 * be physically contiguous.
3062 */
3063 if (sgl[cnt].dmac_size != 0) {
3064 cnt++;
3065 }
3066 sgl[cnt].dmac_laddress = raddr;
3067 sgl[cnt].dmac_size = psize;
3068 #if defined(__amd64)
3069 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3070 #else
3071 /*
3072 * save the buf offset for 32-bit kernel. used in the
3073 * obsoleted interfaces.
3074 */
3075 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3076 (dmar_object->dmao_size - size);
3077 #endif
3078 /* if this isn't the last cookie, go to the next one */
3079 if ((cnt + 1) < sglinfo->si_max_pages) {
3080 cnt++;
3081 sgl[cnt].dmac_laddress = 0;
3082 sgl[cnt].dmac_size = 0;
3083 sgl[cnt].dmac_type = 0;
3084 }
3085
3086 /*
3087 * this page didn't need the copy buffer, if it's not physically
3088 * contiguous, or it would put us over a segment boundary, or it
3089 * puts us over the max cookie size, or the current sgl doesn't
3090 * have anything in it.
3091 */
3092 } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3093 !(raddr & sglinfo->si_segmask) ||
3094 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3095 (sgl[cnt].dmac_size == 0)) {
3096 /*
3097 * if we're not already in a new cookie, go to the next
3098 * cookie.
3099 */
3100 if (sgl[cnt].dmac_size != 0) {
3101 cnt++;
3102 }
3103
3104 /* save the cookie information */
3105 sgl[cnt].dmac_laddress = raddr;
3106 sgl[cnt].dmac_size = psize;
3107 #if defined(__amd64)
3108 sgl[cnt].dmac_type = 0;
3109 #else
3110 /*
3111 * save the buf offset for 32-bit kernel. used in the
3112 * obsoleted interfaces.
3113 */
3114 sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3115 #endif
3116
3117 /*
3118 * this page didn't need the copy buffer, it is physically
3119 * contiguous with the last page, and it's <= the max cookie
3120 * size.
3121 */
3122 } else {
3123 sgl[cnt].dmac_size += psize;
3124
3125 /*
3126 * if this exactly == the maximum cookie size, and
3127 * it isn't the last cookie, go to the next cookie.
3128 */
3129 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3130 ((cnt + 1) < sglinfo->si_max_pages)) {
3131 cnt++;
3132 sgl[cnt].dmac_laddress = 0;
3133 sgl[cnt].dmac_size = 0;
3134 sgl[cnt].dmac_type = 0;
3135 }
3136 }
3137
3138 /*
3139 * save this page's physical address so we can figure out if the
3140 * next page is physically contiguous. Keep decrementing size
3141 * until we are done with the buffer.
3142 */
3143 last_page = raddr;
3144 size -= psize;
3145 }
3146
3147 /* we're done, save away how many cookies the sgl has */
3148 if (sgl[cnt].dmac_size == 0) {
3149 ASSERT(cnt < sglinfo->si_max_pages);
3150 sglinfo->si_sgl_size = cnt;
3151 } else {
3152 sglinfo->si_sgl_size = cnt + 1;
3153 }
3154 }
3155
3156 static void
3157 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3158 rootnex_sglinfo_t *sglinfo)
3159 {
3160 uint64_t offset;
3161 uint64_t maxseg;
3162 uint64_t dvaddr;
3163 struct dvmaseg *dvs;
3164 uint64_t paddr;
3165 uint32_t psize, ssize;
3166 uint32_t size;
3167 uint_t cnt;
3168 int physcontig;
3169
3170 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3171
3172 /* shortcuts */
3173 maxseg = sglinfo->si_max_cookie_size;
3174 size = dmar_object->dmao_size;
3175
3176 cnt = 0;
3177 sglinfo->si_bounce_on_seg = B_FALSE;
3178
3179 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3180 offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3181 ssize = dvs->dvs_len;
3182 paddr = dvs->dvs_start;
3183 paddr += offset;
3184 psize = MIN(ssize, (maxseg - offset));
3185 dvaddr = paddr + psize;
3186 ssize -= psize;
3187
3188 sgl[cnt].dmac_laddress = paddr;
3189 sgl[cnt].dmac_size = psize;
3190 sgl[cnt].dmac_type = 0;
3191
3192 size -= psize;
3193 while (size > 0) {
3194 if (ssize == 0) {
3195 dvs++;
3196 ssize = dvs->dvs_len;
3197 dvaddr = dvs->dvs_start;
3198 physcontig = 0;
3199 } else
3200 physcontig = 1;
3201
3202 paddr = dvaddr;
3203 psize = MIN(ssize, maxseg);
3204 dvaddr += psize;
3205 ssize -= psize;
3206
3207 if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3208 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3209 (sgl[cnt].dmac_size == 0)) {
3210 /*
3211 * if we're not already in a new cookie, go to the next
3212 * cookie.
3213 */
3214 if (sgl[cnt].dmac_size != 0) {
3215 cnt++;
3216 }
3217
3218 /* save the cookie information */
3219 sgl[cnt].dmac_laddress = paddr;
3220 sgl[cnt].dmac_size = psize;
3221 sgl[cnt].dmac_type = 0;
3222 } else {
3223 sgl[cnt].dmac_size += psize;
3224
3225 /*
3226 * if this exactly == the maximum cookie size, and
3227 * it isn't the last cookie, go to the next cookie.
3228 */
3229 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3230 ((cnt + 1) < sglinfo->si_max_pages)) {
3231 cnt++;
3232 sgl[cnt].dmac_laddress = 0;
3233 sgl[cnt].dmac_size = 0;
3234 sgl[cnt].dmac_type = 0;
3235 }
3236 }
3237 size -= psize;
3238 }
3239
3240 /* we're done, save away how many cookies the sgl has */
3241 if (sgl[cnt].dmac_size == 0) {
3242 sglinfo->si_sgl_size = cnt;
3243 } else {
3244 sglinfo->si_sgl_size = cnt + 1;
3245 }
3246 }
3247
3248 /*
3249 * rootnex_bind_slowpath()
3250 * Call in the bind path if the calling driver can't use the sgl without
3251 * modifying it. We either need to use the copy buffer and/or we will end up
3252 * with a partial bind.
3253 */
3254 static int
3255 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3256 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3257 {
3258 rootnex_sglinfo_t *sinfo;
3259 rootnex_window_t *window;
3260 ddi_dma_cookie_t *cookie;
3261 size_t copybuf_used;
3262 size_t dmac_size;
3263 boolean_t partial;
3264 off_t cur_offset;
3265 page_t *cur_pp;
3266 major_t mnum;
3267 int e;
3268 int i;
3269
3270
3271 sinfo = &dma->dp_sglinfo;
3272 copybuf_used = 0;
3273 partial = B_FALSE;
3274
3275 /*
3276 * If we're using the copybuf, set the copybuf state in dma struct.
3277 * Needs to be first since it sets the copy buffer size.
3278 */
3279 if (sinfo->si_copybuf_req != 0) {
3280 e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3281 if (e != DDI_SUCCESS) {
3282 return (e);
3283 }
3284 } else {
3285 dma->dp_copybuf_size = 0;
3286 }
3287
3288 /*
3289 * Figure out if we need to do a partial mapping. If so, figure out
3290 * if we need to trim the buffers when we munge the sgl.
3291 */
3292 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3293 (dmao->dmao_size > dma->dp_maxxfer) ||
3294 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3295 dma->dp_partial_required = B_TRUE;
3296 if (attr->dma_attr_granular != 1) {
3297 dma->dp_trim_required = B_TRUE;
3298 }
3299 } else {
3300 dma->dp_partial_required = B_FALSE;
3301 dma->dp_trim_required = B_FALSE;
3302 }
3303
3304 /* If we need to do a partial bind, make sure the driver supports it */
3305 if (dma->dp_partial_required &&
3306 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3307
3308 mnum = ddi_driver_major(dma->dp_dip);
3309 /*
3310 * patchable which allows us to print one warning per major
3311 * number.
3312 */
3313 if ((rootnex_bind_warn) &&
3314 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3315 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3316 cmn_err(CE_WARN, "!%s: coding error detected, the "
3317 "driver is using ddi_dma_attr(9S) incorrectly. "
3318 "There is a small risk of data corruption in "
3319 "particular with large I/Os. The driver should be "
3320 "replaced with a corrected version for proper "
3321 "system operation. To disable this warning, add "
3322 "'set rootnex:rootnex_bind_warn=0' to "
3323 "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3324 }
3325 return (DDI_DMA_TOOBIG);
3326 }
3327
3328 /*
3329 * we might need multiple windows, setup state to handle them. In this
3330 * code path, we will have at least one window.
3331 */
3332 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3333 if (e != DDI_SUCCESS) {
3334 rootnex_teardown_copybuf(dma);
3335 return (e);
3336 }
3337
3338 window = &dma->dp_window[0];
3339 cookie = &dma->dp_cookies[0];
3340 cur_offset = 0;
3341 rootnex_init_win(hp, dma, window, cookie, cur_offset);
3342 if (dmao->dmao_type == DMA_OTYP_PAGES) {
3343 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3344 }
3345
3346 /* loop though all the cookies we got back from get_sgl() */
3347 for (i = 0; i < sinfo->si_sgl_size; i++) {
3348 /*
3349 * If we're using the copy buffer, check this cookie and setup
3350 * its associated copy buffer state. If this cookie uses the
3351 * copy buffer, make sure we sync this window during dma_sync.
3352 */
3353 if (dma->dp_copybuf_size > 0) {
3354 rootnex_setup_cookie(dmao, dma, cookie,
3355 cur_offset, ©buf_used, &cur_pp);
3356 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3357 window->wd_dosync = B_TRUE;
3358 }
3359 }
3360
3361 /*
3362 * save away the cookie size, since it could be modified in
3363 * the windowing code.
3364 */
3365 dmac_size = cookie->dmac_size;
3366
3367 /* if we went over max copybuf size */
3368 if (dma->dp_copybuf_size &&
3369 (copybuf_used > dma->dp_copybuf_size)) {
3370 partial = B_TRUE;
3371 e = rootnex_copybuf_window_boundary(hp, dma, &window,
3372 cookie, cur_offset, ©buf_used);
3373 if (e != DDI_SUCCESS) {
3374 rootnex_teardown_copybuf(dma);
3375 rootnex_teardown_windows(dma);
3376 return (e);
3377 }
3378
3379 /*
3380 * if the coookie uses the copy buffer, make sure the
3381 * new window we just moved to is set to sync.
3382 */
3383 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3384 window->wd_dosync = B_TRUE;
3385 }
3386 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3387 dma->dp_dip);
3388
3389 /* if the cookie cnt == max sgllen, move to the next window */
3390 } else if (window->wd_cookie_cnt >=
3391 (unsigned)attr->dma_attr_sgllen) {
3392 partial = B_TRUE;
3393 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3394 e = rootnex_sgllen_window_boundary(hp, dma, &window,
3395 cookie, attr, cur_offset);
3396 if (e != DDI_SUCCESS) {
3397 rootnex_teardown_copybuf(dma);
3398 rootnex_teardown_windows(dma);
3399 return (e);
3400 }
3401
3402 /*
3403 * if the coookie uses the copy buffer, make sure the
3404 * new window we just moved to is set to sync.
3405 */
3406 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3407 window->wd_dosync = B_TRUE;
3408 }
3409 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3410 dma->dp_dip);
3411
3412 /* else if we will be over maxxfer */
3413 } else if ((window->wd_size + dmac_size) >
3414 dma->dp_maxxfer) {
3415 partial = B_TRUE;
3416 e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3417 cookie);
3418 if (e != DDI_SUCCESS) {
3419 rootnex_teardown_copybuf(dma);
3420 rootnex_teardown_windows(dma);
3421 return (e);
3422 }
3423
3424 /*
3425 * if the coookie uses the copy buffer, make sure the
3426 * new window we just moved to is set to sync.
3427 */
3428 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3429 window->wd_dosync = B_TRUE;
3430 }
3431 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3432 dma->dp_dip);
3433
3434 /* else this cookie fits in the current window */
3435 } else {
3436 window->wd_cookie_cnt++;
3437 window->wd_size += dmac_size;
3438 }
3439
3440 /* track our offset into the buffer, go to the next cookie */
3441 ASSERT(dmac_size <= dmao->dmao_size);
3442 ASSERT(cookie->dmac_size <= dmac_size);
3443 cur_offset += dmac_size;
3444 cookie++;
3445 }
3446
3447 /* if we ended up with a zero sized window in the end, clean it up */
3448 if (window->wd_size == 0) {
3449 hp->dmai_nwin--;
3450 window--;
3451 }
3452
3453 ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3454
3455 if (!partial) {
3456 return (DDI_DMA_MAPPED);
3457 }
3458
3459 ASSERT(dma->dp_partial_required);
3460 return (DDI_DMA_PARTIAL_MAP);
3461 }
3462
3463 /*
3464 * rootnex_setup_copybuf()
3465 * Called in bind slowpath. Figures out if we're going to use the copy
3466 * buffer, and if we do, sets up the basic state to handle it.
3467 */
3468 static int
3469 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3470 rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3471 {
3472 rootnex_sglinfo_t *sinfo;
3473 ddi_dma_attr_t lattr;
3474 size_t max_copybuf;
3475 int cansleep;
3476 int e;
3477 #if !defined(__amd64)
3478 int vmflag;
3479 #endif
3480
3481 ASSERT(!dma->dp_dvma_used);
3482
3483 sinfo = &dma->dp_sglinfo;
3484
3485 /* read this first so it's consistent through the routine */
3486 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3487
3488 /* We need to call into the rootnex on ddi_dma_sync() */
3489 hp->dmai_rflags &= ~DMP_NOSYNC;
3490
3491 /* make sure the copybuf size <= the max size */
3492 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3493 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3494
3495 #if !defined(__amd64)
3496 /*
3497 * if we don't have kva space to copy to/from, allocate the KVA space
3498 * now. We only do this for the 32-bit kernel. We use seg kpm space for
3499 * the 64-bit kernel.
3500 */
3501 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3502 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3503
3504 /* convert the sleep flags */
3505 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3506 vmflag = VM_SLEEP;
3507 } else {
3508 vmflag = VM_NOSLEEP;
3509 }
3510
3511 /* allocate Kernel VA space that we can bcopy to/from */
3512 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3513 vmflag);
3514 if (dma->dp_kva == NULL) {
3515 return (DDI_DMA_NORESOURCES);
3516 }
3517 }
3518 #endif
3519
3520 /* convert the sleep flags */
3521 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3522 cansleep = 1;
3523 } else {
3524 cansleep = 0;
3525 }
3526
3527 /*
3528 * Allocate the actual copy buffer. This needs to fit within the DMA
3529 * engine limits, so we can't use kmem_alloc... We don't need
3530 * contiguous memory (sgllen) since we will be forcing windows on
3531 * sgllen anyway.
3532 */
3533 lattr = *attr;
3534 lattr.dma_attr_align = MMU_PAGESIZE;
3535 lattr.dma_attr_sgllen = -1; /* no limit */
3536 /*
3537 * if we're using the copy buffer because of seg, use that for our
3538 * upper address limit.
3539 */
3540 if (sinfo->si_bounce_on_seg) {
3541 lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3542 }
3543 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3544 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3545 if (e != DDI_SUCCESS) {
3546 #if !defined(__amd64)
3547 if (dma->dp_kva != NULL) {
3548 vmem_free(heap_arena, dma->dp_kva,
3549 dma->dp_copybuf_size);
3550 }
3551 #endif
3552 return (DDI_DMA_NORESOURCES);
3553 }
3554
3555 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3556 size_t, dma->dp_copybuf_size);
3557
3558 return (DDI_SUCCESS);
3559 }
3560
3561
3562 /*
3563 * rootnex_setup_windows()
3564 * Called in bind slowpath to setup the window state. We always have windows
3565 * in the slowpath. Even if the window count = 1.
3566 */
3567 static int
3568 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3569 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3570 {
3571 rootnex_window_t *windowp;
3572 rootnex_sglinfo_t *sinfo;
3573 size_t copy_state_size;
3574 size_t win_state_size;
3575 size_t state_available;
3576 size_t space_needed;
3577 uint_t copybuf_win;
3578 uint_t maxxfer_win;
3579 size_t space_used;
3580 uint_t sglwin;
3581
3582
3583 sinfo = &dma->dp_sglinfo;
3584
3585 dma->dp_current_win = 0;
3586 hp->dmai_nwin = 0;
3587
3588 /* If we don't need to do a partial, we only have one window */
3589 if (!dma->dp_partial_required) {
3590 dma->dp_max_win = 1;
3591
3592 /*
3593 * we need multiple windows, need to figure out the worse case number
3594 * of windows.
3595 */
3596 } else {
3597 /*
3598 * if we need windows because we need more copy buffer that
3599 * we allow, the worse case number of windows we could need
3600 * here would be (copybuf space required / copybuf space that
3601 * we have) plus one for remainder, and plus 2 to handle the
3602 * extra pages on the trim for the first and last pages of the
3603 * buffer (a page is the minimum window size so under the right
3604 * attr settings, you could have a window for each page).
3605 * The last page will only be hit here if the size is not a
3606 * multiple of the granularity (which theoretically shouldn't
3607 * be the case but never has been enforced, so we could have
3608 * broken things without it).
3609 */
3610 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3611 ASSERT(dma->dp_copybuf_size > 0);
3612 copybuf_win = (sinfo->si_copybuf_req /
3613 dma->dp_copybuf_size) + 1 + 2;
3614 } else {
3615 copybuf_win = 0;
3616 }
3617
3618 /*
3619 * if we need windows because we have more cookies than the H/W
3620 * can handle, the number of windows we would need here would
3621 * be (cookie count / cookies count H/W supports minus 1[for
3622 * trim]) plus one for remainder.
3623 */
3624 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3625 sglwin = (sinfo->si_sgl_size /
3626 (attr->dma_attr_sgllen - 1)) + 1;
3627 } else {
3628 sglwin = 0;
3629 }
3630
3631 /*
3632 * if we need windows because we're binding more memory than the
3633 * H/W can transfer at once, the number of windows we would need
3634 * here would be (xfer count / max xfer H/W supports) plus one
3635 * for remainder, and plus 2 to handle the extra pages on the
3636 * trim (see above comment about trim)
3637 */
3638 if (dmao->dmao_size > dma->dp_maxxfer) {
3639 maxxfer_win = (dmao->dmao_size /
3640 dma->dp_maxxfer) + 1 + 2;
3641 } else {
3642 maxxfer_win = 0;
3643 }
3644 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win;
3645 ASSERT(dma->dp_max_win > 0);
3646 }
3647 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3648
3649 /*
3650 * Get space for window and potential copy buffer state. Before we
3651 * go and allocate memory, see if we can get away with using what's
3652 * left in the pre-allocted state or the dynamically allocated sgl.
3653 */
3654 space_used = (uintptr_t)(sinfo->si_sgl_size *
3655 sizeof (ddi_dma_cookie_t));
3656
3657 /* if we dynamically allocated space for the cookies */
3658 if (dma->dp_need_to_free_cookie) {
3659 /* if we have more space in the pre-allocted buffer, use it */
3660 ASSERT(space_used <= dma->dp_cookie_size);
3661 if ((dma->dp_cookie_size - space_used) <=
3662 rootnex_state->r_prealloc_size) {
3663 state_available = rootnex_state->r_prealloc_size;
3664 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3665
3666 /*
3667 * else, we have more free space in the dynamically allocated
3668 * buffer, i.e. the buffer wasn't worse case fragmented so we
3669 * didn't need a lot of cookies.
3670 */
3671 } else {
3672 state_available = dma->dp_cookie_size - space_used;
3673 windowp = (rootnex_window_t *)
3674 &dma->dp_cookies[sinfo->si_sgl_size];
3675 }
3676
3677 /* we used the pre-alloced buffer */
3678 } else {
3679 ASSERT(space_used <= rootnex_state->r_prealloc_size);
3680 state_available = rootnex_state->r_prealloc_size - space_used;
3681 windowp = (rootnex_window_t *)
3682 &dma->dp_cookies[sinfo->si_sgl_size];
3683 }
3684
3685 /*
3686 * figure out how much state we need to track the copy buffer. Add an
3687 * addition 8 bytes for pointer alignemnt later.
3688 */
3689 if (dma->dp_copybuf_size > 0) {
3690 copy_state_size = sinfo->si_max_pages *
3691 sizeof (rootnex_pgmap_t);
3692 } else {
3693 copy_state_size = 0;
3694 }
3695 /* add an additional 8 bytes for pointer alignment */
3696 space_needed = win_state_size + copy_state_size + 0x8;
3697
3698 /* if we have enough space already, use it */
3699 if (state_available >= space_needed) {
3700 dma->dp_window = windowp;
3701 dma->dp_need_to_free_window = B_FALSE;
3702
3703 /* not enough space, need to allocate more. */
3704 } else {
3705 dma->dp_window = kmem_alloc(space_needed, kmflag);
3706 if (dma->dp_window == NULL) {
3707 return (DDI_DMA_NORESOURCES);
3708 }
3709 dma->dp_need_to_free_window = B_TRUE;
3710 dma->dp_window_size = space_needed;
3711 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3712 dma->dp_dip, size_t, space_needed);
3713 }
3714
3715 /*
3716 * we allocate copy buffer state and window state at the same time.
3717 * setup our copy buffer state pointers. Make sure it's aligned.
3718 */
3719 if (dma->dp_copybuf_size > 0) {
3720 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3721 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3722
3723 #if !defined(__amd64)
3724 /*
3725 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3726 * false/NULL. Should be quicker to bzero vs loop and set.
3727 */
3728 bzero(dma->dp_pgmap, copy_state_size);
3729 #endif
3730 } else {
3731 dma->dp_pgmap = NULL;
3732 }
3733
3734 return (DDI_SUCCESS);
3735 }
3736
3737
3738 /*
3739 * rootnex_teardown_copybuf()
3740 * cleans up after rootnex_setup_copybuf()
3741 */
3742 static void
3743 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3744 {
3745 #if !defined(__amd64)
3746 int i;
3747
3748 /*
3749 * if we allocated kernel heap VMEM space, go through all the pages and
3750 * map out any of the ones that we're mapped into the kernel heap VMEM
3751 * arena. Then free the VMEM space.
3752 */
3753 if (dma->dp_kva != NULL) {
3754 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3755 if (dma->dp_pgmap[i].pm_mapped) {
3756 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3757 MMU_PAGESIZE, HAT_UNLOAD);
3758 dma->dp_pgmap[i].pm_mapped = B_FALSE;
3759 }
3760 }
3761
3762 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3763 }
3764
3765 #endif
3766
3767 /* if we allocated a copy buffer, free it */
3768 if (dma->dp_cbaddr != NULL) {
3769 i_ddi_mem_free(dma->dp_cbaddr, NULL);
3770 }
3771 }
3772
3773
3774 /*
3775 * rootnex_teardown_windows()
3776 * cleans up after rootnex_setup_windows()
3777 */
3778 static void
3779 rootnex_teardown_windows(rootnex_dma_t *dma)
3780 {
3781 /*
3782 * if we had to allocate window state on the last bind (because we
3783 * didn't have enough pre-allocated space in the handle), free it.
3784 */
3785 if (dma->dp_need_to_free_window) {
3786 kmem_free(dma->dp_window, dma->dp_window_size);
3787 }
3788 }
3789
3790
3791 /*
3792 * rootnex_init_win()
3793 * Called in bind slow path during creation of a new window. Initializes
3794 * window state to default values.
3795 */
3796 /*ARGSUSED*/
3797 static void
3798 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3799 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3800 {
3801 hp->dmai_nwin++;
3802 window->wd_dosync = B_FALSE;
3803 window->wd_offset = cur_offset;
3804 window->wd_size = 0;
3805 window->wd_first_cookie = cookie;
3806 window->wd_cookie_cnt = 0;
3807 window->wd_trim.tr_trim_first = B_FALSE;
3808 window->wd_trim.tr_trim_last = B_FALSE;
3809 window->wd_trim.tr_first_copybuf_win = B_FALSE;
3810 window->wd_trim.tr_last_copybuf_win = B_FALSE;
3811 #if !defined(__amd64)
3812 window->wd_remap_copybuf = dma->dp_cb_remaping;
3813 #endif
3814 }
3815
3816
3817 /*
3818 * rootnex_setup_cookie()
3819 * Called in the bind slow path when the sgl uses the copy buffer. If any of
3820 * the sgl uses the copy buffer, we need to go through each cookie, figure
3821 * out if it uses the copy buffer, and if it does, save away everything we'll
3822 * need during sync.
3823 */
3824 static void
3825 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3826 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3827 page_t **cur_pp)
3828 {
3829 boolean_t copybuf_sz_power_2;
3830 rootnex_sglinfo_t *sinfo;
3831 paddr_t paddr;
3832 uint_t pidx;
3833 uint_t pcnt;
3834 off_t poff;
3835 #if defined(__amd64)
3836 pfn_t pfn;
3837 #else
3838 page_t **pplist;
3839 #endif
3840
3841 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3842
3843 sinfo = &dma->dp_sglinfo;
3844
3845 /*
3846 * Calculate the page index relative to the start of the buffer. The
3847 * index to the current page for our buffer is the offset into the
3848 * first page of the buffer plus our current offset into the buffer
3849 * itself, shifted of course...
3850 */
3851 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3852 ASSERT(pidx < sinfo->si_max_pages);
3853
3854 /* if this cookie uses the copy buffer */
3855 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3856 /*
3857 * NOTE: we know that since this cookie uses the copy buffer, it
3858 * is <= MMU_PAGESIZE.
3859 */
3860
3861 /*
3862 * get the offset into the page. For the 64-bit kernel, get the
3863 * pfn which we'll use with seg kpm.
3864 */
3865 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3866 #if defined(__amd64)
3867 /* mfn_to_pfn() is a NOP on i86pc */
3868 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3869 #endif /* __amd64 */
3870
3871 /* figure out if the copybuf size is a power of 2 */
3872 if (!ISP2(dma->dp_copybuf_size)) {
3873 copybuf_sz_power_2 = B_FALSE;
3874 } else {
3875 copybuf_sz_power_2 = B_TRUE;
3876 }
3877
3878 /* This page uses the copy buffer */
3879 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3880
3881 /*
3882 * save the copy buffer KVA that we'll use with this page.
3883 * if we still fit within the copybuf, it's a simple add.
3884 * otherwise, we need to wrap over using & or % accordingly.
3885 */
3886 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3887 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3888 *copybuf_used;
3889 } else {
3890 if (copybuf_sz_power_2) {
3891 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3892 (uintptr_t)dma->dp_cbaddr +
3893 (*copybuf_used &
3894 (dma->dp_copybuf_size - 1)));
3895 } else {
3896 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3897 (uintptr_t)dma->dp_cbaddr +
3898 (*copybuf_used % dma->dp_copybuf_size));
3899 }
3900 }
3901
3902 /*
3903 * over write the cookie physical address with the address of
3904 * the physical address of the copy buffer page that we will
3905 * use.
3906 */
3907 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3908 dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3909
3910 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3911
3912 /* if we have a kernel VA, it's easy, just save that address */
3913 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3914 (sinfo->si_asp == &kas)) {
3915 /*
3916 * save away the page aligned virtual address of the
3917 * driver buffer. Offsets are handled in the sync code.
3918 */
3919 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3920 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3921 & MMU_PAGEMASK);
3922 #if !defined(__amd64)
3923 /*
3924 * we didn't need to, and will never need to map this
3925 * page.
3926 */
3927 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3928 #endif
3929
3930 /* we don't have a kernel VA. We need one for the bcopy. */
3931 } else {
3932 #if defined(__amd64)
3933 /*
3934 * for the 64-bit kernel, it's easy. We use seg kpm to
3935 * get a Kernel VA for the corresponding pfn.
3936 */
3937 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3938 #else
3939 /*
3940 * for the 32-bit kernel, this is a pain. First we'll
3941 * save away the page_t or user VA for this page. This
3942 * is needed in rootnex_dma_win() when we switch to a
3943 * new window which requires us to re-map the copy
3944 * buffer.
3945 */
3946 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3947 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3948 dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3949 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3950 } else if (pplist != NULL) {
3951 dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3952 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3953 } else {
3954 dma->dp_pgmap[pidx].pm_pp = NULL;
3955 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3956 (((uintptr_t)
3957 dmar_object->dmao_obj.virt_obj.v_addr +
3958 cur_offset) & MMU_PAGEMASK);
3959 }
3960
3961 /*
3962 * save away the page aligned virtual address which was
3963 * allocated from the kernel heap arena (taking into
3964 * account if we need more copy buffer than we alloced
3965 * and use multiple windows to handle this, i.e. &,%).
3966 * NOTE: there isn't and physical memory backing up this
3967 * virtual address space currently.
3968 */
3969 if ((*copybuf_used + MMU_PAGESIZE) <=
3970 dma->dp_copybuf_size) {
3971 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3972 (((uintptr_t)dma->dp_kva + *copybuf_used) &
3973 MMU_PAGEMASK);
3974 } else {
3975 if (copybuf_sz_power_2) {
3976 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3977 (((uintptr_t)dma->dp_kva +
3978 (*copybuf_used &
3979 (dma->dp_copybuf_size - 1))) &
3980 MMU_PAGEMASK);
3981 } else {
3982 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3983 (((uintptr_t)dma->dp_kva +
3984 (*copybuf_used %
3985 dma->dp_copybuf_size)) &
3986 MMU_PAGEMASK);
3987 }
3988 }
3989
3990 /*
3991 * if we haven't used up the available copy buffer yet,
3992 * map the kva to the physical page.
3993 */
3994 if (!dma->dp_cb_remaping && ((*copybuf_used +
3995 MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3996 dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3997 if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3998 i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3999 dma->dp_pgmap[pidx].pm_kaddr);
4000 } else {
4001 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4002 sinfo->si_asp,
4003 dma->dp_pgmap[pidx].pm_kaddr);
4004 }
4005
4006 /*
4007 * we've used up the available copy buffer, this page
4008 * will have to be mapped during rootnex_dma_win() when
4009 * we switch to a new window which requires a re-map
4010 * the copy buffer. (32-bit kernel only)
4011 */
4012 } else {
4013 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4014 }
4015 #endif
4016 /* go to the next page_t */
4017 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4018 *cur_pp = (*cur_pp)->p_next;
4019 }
4020 }
4021
4022 /* add to the copy buffer count */
4023 *copybuf_used += MMU_PAGESIZE;
4024
4025 /*
4026 * This cookie doesn't use the copy buffer. Walk through the pages this
4027 * cookie occupies to reflect this.
4028 */
4029 } else {
4030 /*
4031 * figure out how many pages the cookie occupies. We need to
4032 * use the original page offset of the buffer and the cookies
4033 * offset in the buffer to do this.
4034 */
4035 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4036 pcnt = mmu_btopr(cookie->dmac_size + poff);
4037
4038 while (pcnt > 0) {
4039 #if !defined(__amd64)
4040 /*
4041 * the 32-bit kernel doesn't have seg kpm, so we need
4042 * to map in the driver buffer (if it didn't come down
4043 * with a kernel VA) on the fly. Since this page doesn't
4044 * use the copy buffer, it's not, or will it ever, have
4045 * to be mapped in.
4046 */
4047 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4048 #endif
4049 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4050
4051 /*
4052 * we need to update pidx and cur_pp or we'll loose
4053 * track of where we are.
4054 */
4055 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4056 *cur_pp = (*cur_pp)->p_next;
4057 }
4058 pidx++;
4059 pcnt--;
4060 }
4061 }
4062 }
4063
4064
4065 /*
4066 * rootnex_sgllen_window_boundary()
4067 * Called in the bind slow path when the next cookie causes us to exceed (in
4068 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4069 * length supported by the DMA H/W.
4070 */
4071 static int
4072 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4073 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4074 off_t cur_offset)
4075 {
4076 off_t new_offset;
4077 size_t trim_sz;
4078 off_t coffset;
4079
4080
4081 /*
4082 * if we know we'll never have to trim, it's pretty easy. Just move to
4083 * the next window and init it. We're done.
4084 */
4085 if (!dma->dp_trim_required) {
4086 (*windowp)++;
4087 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4088 (*windowp)->wd_cookie_cnt++;
4089 (*windowp)->wd_size = cookie->dmac_size;
4090 return (DDI_SUCCESS);
4091 }
4092
4093 /* figure out how much we need to trim from the window */
4094 ASSERT(attr->dma_attr_granular != 0);
4095 if (dma->dp_granularity_power_2) {
4096 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4097 } else {
4098 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4099 }
4100
4101 /* The window's a whole multiple of granularity. We're done */
4102 if (trim_sz == 0) {
4103 (*windowp)++;
4104 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4105 (*windowp)->wd_cookie_cnt++;
4106 (*windowp)->wd_size = cookie->dmac_size;
4107 return (DDI_SUCCESS);
4108 }
4109
4110 /*
4111 * The window's not a whole multiple of granularity, since we know this
4112 * is due to the sgllen, we need to go back to the last cookie and trim
4113 * that one, add the left over part of the old cookie into the new
4114 * window, and then add in the new cookie into the new window.
4115 */
4116
4117 /*
4118 * make sure the driver isn't making us do something bad... Trimming and
4119 * sgllen == 1 don't go together.
4120 */
4121 if (attr->dma_attr_sgllen == 1) {
4122 return (DDI_DMA_NOMAPPING);
4123 }
4124
4125 /*
4126 * first, setup the current window to account for the trim. Need to go
4127 * back to the last cookie for this.
4128 */
4129 cookie--;
4130 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4131 (*windowp)->wd_trim.tr_last_cookie = cookie;
4132 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4133 ASSERT(cookie->dmac_size > trim_sz);
4134 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4135 (*windowp)->wd_size -= trim_sz;
4136
4137 /* save the buffer offsets for the next window */
4138 coffset = cookie->dmac_size - trim_sz;
4139 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4140
4141 /*
4142 * set this now in case this is the first window. all other cases are
4143 * set in dma_win()
4144 */
4145 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4146
4147 /*
4148 * initialize the next window using what's left over in the previous
4149 * cookie.
4150 */
4151 (*windowp)++;
4152 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4153 (*windowp)->wd_cookie_cnt++;
4154 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4155 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4156 (*windowp)->wd_trim.tr_first_size = trim_sz;
4157 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4158 (*windowp)->wd_dosync = B_TRUE;
4159 }
4160
4161 /*
4162 * now go back to the current cookie and add it to the new window. set
4163 * the new window size to the what was left over from the previous
4164 * cookie and what's in the current cookie.
4165 */
4166 cookie++;
4167 (*windowp)->wd_cookie_cnt++;
4168 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4169
4170 /*
4171 * trim plus the next cookie could put us over maxxfer (a cookie can be
4172 * a max size of maxxfer). Handle that case.
4173 */
4174 if ((*windowp)->wd_size > dma->dp_maxxfer) {
4175 /*
4176 * maxxfer is already a whole multiple of granularity, and this
4177 * trim will be <= the previous trim (since a cookie can't be
4178 * larger than maxxfer). Make things simple here.
4179 */
4180 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4181 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4182 (*windowp)->wd_trim.tr_last_cookie = cookie;
4183 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4184 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4185 (*windowp)->wd_size -= trim_sz;
4186 ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4187
4188 /* save the buffer offsets for the next window */
4189 coffset = cookie->dmac_size - trim_sz;
4190 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4191
4192 /* setup the next window */
4193 (*windowp)++;
4194 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4195 (*windowp)->wd_cookie_cnt++;
4196 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4197 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4198 coffset;
4199 (*windowp)->wd_trim.tr_first_size = trim_sz;
4200 }
4201
4202 return (DDI_SUCCESS);
4203 }
4204
4205
4206 /*
4207 * rootnex_copybuf_window_boundary()
4208 * Called in bind slowpath when we get to a window boundary because we used
4209 * up all the copy buffer that we have.
4210 */
4211 static int
4212 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4213 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4214 size_t *copybuf_used)
4215 {
4216 rootnex_sglinfo_t *sinfo;
4217 off_t new_offset;
4218 size_t trim_sz;
4219 paddr_t paddr;
4220 off_t coffset;
4221 uint_t pidx;
4222 off_t poff;
4223
4224
4225 sinfo = &dma->dp_sglinfo;
4226
4227 /*
4228 * the copy buffer should be a whole multiple of page size. We know that
4229 * this cookie is <= MMU_PAGESIZE.
4230 */
4231 ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4232
4233 /*
4234 * from now on, all new windows in this bind need to be re-mapped during
4235 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4236 * space...
4237 */
4238 #if !defined(__amd64)
4239 dma->dp_cb_remaping = B_TRUE;
4240 #endif
4241
4242 /* reset copybuf used */
4243 *copybuf_used = 0;
4244
4245 /*
4246 * if we don't have to trim (since granularity is set to 1), go to the
4247 * next window and add the current cookie to it. We know the current
4248 * cookie uses the copy buffer since we're in this code path.
4249 */
4250 if (!dma->dp_trim_required) {
4251 (*windowp)++;
4252 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4253
4254 /* Add this cookie to the new window */
4255 (*windowp)->wd_cookie_cnt++;
4256 (*windowp)->wd_size += cookie->dmac_size;
4257 *copybuf_used += MMU_PAGESIZE;
4258 return (DDI_SUCCESS);
4259 }
4260
4261 /*
4262 * *** may need to trim, figure it out.
4263 */
4264
4265 /* figure out how much we need to trim from the window */
4266 if (dma->dp_granularity_power_2) {
4267 trim_sz = (*windowp)->wd_size &
4268 (hp->dmai_attr.dma_attr_granular - 1);
4269 } else {
4270 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4271 }
4272
4273 /*
4274 * if the window's a whole multiple of granularity, go to the next
4275 * window, init it, then add in the current cookie. We know the current
4276 * cookie uses the copy buffer since we're in this code path.
4277 */
4278 if (trim_sz == 0) {
4279 (*windowp)++;
4280 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4281
4282 /* Add this cookie to the new window */
4283 (*windowp)->wd_cookie_cnt++;
4284 (*windowp)->wd_size += cookie->dmac_size;
4285 *copybuf_used += MMU_PAGESIZE;
4286 return (DDI_SUCCESS);
4287 }
4288
4289 /*
4290 * *** We figured it out, we definitly need to trim
4291 */
4292
4293 /*
4294 * make sure the driver isn't making us do something bad...
4295 * Trimming and sgllen == 1 don't go together.
4296 */
4297 if (hp->dmai_attr.dma_attr_sgllen == 1) {
4298 return (DDI_DMA_NOMAPPING);
4299 }
4300
4301 /*
4302 * first, setup the current window to account for the trim. Need to go
4303 * back to the last cookie for this. Some of the last cookie will be in
4304 * the current window, and some of the last cookie will be in the new
4305 * window. All of the current cookie will be in the new window.
4306 */
4307 cookie--;
4308 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4309 (*windowp)->wd_trim.tr_last_cookie = cookie;
4310 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4311 ASSERT(cookie->dmac_size > trim_sz);
4312 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4313 (*windowp)->wd_size -= trim_sz;
4314
4315 /*
4316 * we're trimming the last cookie (not the current cookie). So that
4317 * last cookie may have or may not have been using the copy buffer (
4318 * we know the cookie passed in uses the copy buffer since we're in
4319 * this code path).
4320 *
4321 * If the last cookie doesn't use the copy buffer, nothing special to
4322 * do. However, if it does uses the copy buffer, it will be both the
4323 * last page in the current window and the first page in the next
4324 * window. Since we are reusing the copy buffer (and KVA space on the
4325 * 32-bit kernel), this page will use the end of the copy buffer in the
4326 * current window, and the start of the copy buffer in the next window.
4327 * Track that info... The cookie physical address was already set to
4328 * the copy buffer physical address in setup_cookie..
4329 */
4330 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4331 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4332 (*windowp)->wd_size) >> MMU_PAGESHIFT;
4333 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4334 (*windowp)->wd_trim.tr_last_pidx = pidx;
4335 (*windowp)->wd_trim.tr_last_cbaddr =
4336 dma->dp_pgmap[pidx].pm_cbaddr;
4337 #if !defined(__amd64)
4338 (*windowp)->wd_trim.tr_last_kaddr =
4339 dma->dp_pgmap[pidx].pm_kaddr;
4340 #endif
4341 }
4342
4343 /* save the buffer offsets for the next window */
4344 coffset = cookie->dmac_size - trim_sz;
4345 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4346
4347 /*
4348 * set this now in case this is the first window. all other cases are
4349 * set in dma_win()
4350 */
4351 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4352
4353 /*
4354 * initialize the next window using what's left over in the previous
4355 * cookie.
4356 */
4357 (*windowp)++;
4358 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4359 (*windowp)->wd_cookie_cnt++;
4360 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4361 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4362 (*windowp)->wd_trim.tr_first_size = trim_sz;
4363
4364 /*
4365 * again, we're tracking if the last cookie uses the copy buffer.
4366 * read the comment above for more info on why we need to track
4367 * additional state.
4368 *
4369 * For the first cookie in the new window, we need reset the physical
4370 * address to DMA into to the start of the copy buffer plus any
4371 * initial page offset which may be present.
4372 */
4373 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4374 (*windowp)->wd_dosync = B_TRUE;
4375 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4376 (*windowp)->wd_trim.tr_first_pidx = pidx;
4377 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4378 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4379
4380 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4381 poff;
4382 (*windowp)->wd_trim.tr_first_paddr =
4383 ROOTNEX_PADDR_TO_RBASE(paddr);
4384
4385 #if !defined(__amd64)
4386 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4387 #endif
4388 /* account for the cookie copybuf usage in the new window */
4389 *copybuf_used += MMU_PAGESIZE;
4390
4391 /*
4392 * every piece of code has to have a hack, and here is this
4393 * ones :-)
4394 *
4395 * There is a complex interaction between setup_cookie and the
4396 * copybuf window boundary. The complexity had to be in either
4397 * the maxxfer window, or the copybuf window, and I chose the
4398 * copybuf code.
4399 *
4400 * So in this code path, we have taken the last cookie,
4401 * virtually broken it in half due to the trim, and it happens
4402 * to use the copybuf which further complicates life. At the
4403 * same time, we have already setup the current cookie, which
4404 * is now wrong. More background info: the current cookie uses
4405 * the copybuf, so it is only a page long max. So we need to
4406 * fix the current cookies copy buffer address, physical
4407 * address, and kva for the 32-bit kernel. We due this by
4408 * bumping them by page size (of course, we can't due this on
4409 * the physical address since the copy buffer may not be
4410 * physically contiguous).
4411 */
4412 cookie++;
4413 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4414 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4415
4416 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4417 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4418 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4419
4420 #if !defined(__amd64)
4421 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4422 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4423 #endif
4424 } else {
4425 /* go back to the current cookie */
4426 cookie++;
4427 }
4428
4429 /*
4430 * add the current cookie to the new window. set the new window size to
4431 * the what was left over from the previous cookie and what's in the
4432 * current cookie.
4433 */
4434 (*windowp)->wd_cookie_cnt++;
4435 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4436 ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4437
4438 /*
4439 * we know that the cookie passed in always uses the copy buffer. We
4440 * wouldn't be here if it didn't.
4441 */
4442 *copybuf_used += MMU_PAGESIZE;
4443
4444 return (DDI_SUCCESS);
4445 }
4446
4447
4448 /*
4449 * rootnex_maxxfer_window_boundary()
4450 * Called in bind slowpath when we get to a window boundary because we will
4451 * go over maxxfer.
4452 */
4453 static int
4454 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4455 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4456 {
4457 size_t dmac_size;
4458 off_t new_offset;
4459 size_t trim_sz;
4460 off_t coffset;
4461
4462
4463 /*
4464 * calculate how much we have to trim off of the current cookie to equal
4465 * maxxfer. We don't have to account for granularity here since our
4466 * maxxfer already takes that into account.
4467 */
4468 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4469 ASSERT(trim_sz <= cookie->dmac_size);
4470 ASSERT(trim_sz <= dma->dp_maxxfer);
4471
4472 /* save cookie size since we need it later and we might change it */
4473 dmac_size = cookie->dmac_size;
4474
4475 /*
4476 * if we're not trimming the entire cookie, setup the current window to
4477 * account for the trim.
4478 */
4479 if (trim_sz < cookie->dmac_size) {
4480 (*windowp)->wd_cookie_cnt++;
4481 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4482 (*windowp)->wd_trim.tr_last_cookie = cookie;
4483 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4484 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4485 (*windowp)->wd_size = dma->dp_maxxfer;
4486
4487 /*
4488 * set the adjusted cookie size now in case this is the first
4489 * window. All other windows are taken care of in get win
4490 */
4491 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4492 }
4493
4494 /*
4495 * coffset is the current offset within the cookie, new_offset is the
4496 * current offset with the entire buffer.
4497 */
4498 coffset = dmac_size - trim_sz;
4499 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4500
4501 /* initialize the next window */
4502 (*windowp)++;
4503 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4504 (*windowp)->wd_cookie_cnt++;
4505 (*windowp)->wd_size = trim_sz;
4506 if (trim_sz < dmac_size) {
4507 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4508 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4509 coffset;
4510 (*windowp)->wd_trim.tr_first_size = trim_sz;
4511 }
4512
4513 return (DDI_SUCCESS);
4514 }
4515
4516
4517 /*ARGSUSED*/
4518 static int
4519 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4520 off_t off, size_t len, uint_t cache_flags)
4521 {
4522 rootnex_sglinfo_t *sinfo;
4523 rootnex_pgmap_t *cbpage;
4524 rootnex_window_t *win;
4525 ddi_dma_impl_t *hp;
4526 rootnex_dma_t *dma;
4527 caddr_t fromaddr;
4528 caddr_t toaddr;
4529 uint_t psize;
4530 off_t offset;
4531 uint_t pidx;
4532 size_t size;
4533 off_t poff;
4534 int e;
4535
4536
4537 hp = (ddi_dma_impl_t *)handle;
4538 dma = (rootnex_dma_t *)hp->dmai_private;
4539 sinfo = &dma->dp_sglinfo;
4540
4541 /*
4542 * if we don't have any windows, we don't need to sync. A copybuf
4543 * will cause us to have at least one window.
4544 */
4545 if (dma->dp_window == NULL) {
4546 return (DDI_SUCCESS);
4547 }
4548
4549 /* This window may not need to be sync'd */
4550 win = &dma->dp_window[dma->dp_current_win];
4551 if (!win->wd_dosync) {
4552 return (DDI_SUCCESS);
4553 }
4554
4555 /* handle off and len special cases */
4556 if ((off == 0) || (rootnex_sync_ignore_params)) {
4557 offset = win->wd_offset;
4558 } else {
4559 offset = off;
4560 }
4561 if ((len == 0) || (rootnex_sync_ignore_params)) {
4562 size = win->wd_size;
4563 } else {
4564 size = len;
4565 }
4566
4567 /* check the sync args to make sure they make a little sense */
4568 if (rootnex_sync_check_parms) {
4569 e = rootnex_valid_sync_parms(hp, win, offset, size,
4570 cache_flags);
4571 if (e != DDI_SUCCESS) {
4572 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4573 return (DDI_FAILURE);
4574 }
4575 }
4576
4577 /*
4578 * special case the first page to handle the offset into the page. The
4579 * offset to the current page for our buffer is the offset into the
4580 * first page of the buffer plus our current offset into the buffer
4581 * itself, masked of course.
4582 */
4583 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4584 psize = MIN((MMU_PAGESIZE - poff), size);
4585
4586 /* go through all the pages that we want to sync */
4587 while (size > 0) {
4588 /*
4589 * Calculate the page index relative to the start of the buffer.
4590 * The index to the current page for our buffer is the offset
4591 * into the first page of the buffer plus our current offset
4592 * into the buffer itself, shifted of course...
4593 */
4594 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4595 ASSERT(pidx < sinfo->si_max_pages);
4596
4597 /*
4598 * if this page uses the copy buffer, we need to sync it,
4599 * otherwise, go on to the next page.
4600 */
4601 cbpage = &dma->dp_pgmap[pidx];
4602 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4603 (cbpage->pm_uses_copybuf == B_FALSE));
4604 if (cbpage->pm_uses_copybuf) {
4605 /* cbaddr and kaddr should be page aligned */
4606 ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4607 MMU_PAGEOFFSET) == 0);
4608 ASSERT(((uintptr_t)cbpage->pm_kaddr &
4609 MMU_PAGEOFFSET) == 0);
4610
4611 /*
4612 * if we're copying for the device, we are going to
4613 * copy from the drivers buffer and to the rootnex
4614 * allocated copy buffer.
4615 */
4616 if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4617 fromaddr = cbpage->pm_kaddr + poff;
4618 toaddr = cbpage->pm_cbaddr + poff;
4619 ROOTNEX_DPROBE2(rootnex__sync__dev,
4620 dev_info_t *, dma->dp_dip, size_t, psize);
4621
4622 /*
4623 * if we're copying for the cpu/kernel, we are going to
4624 * copy from the rootnex allocated copy buffer to the
4625 * drivers buffer.
4626 */
4627 } else {
4628 fromaddr = cbpage->pm_cbaddr + poff;
4629 toaddr = cbpage->pm_kaddr + poff;
4630 ROOTNEX_DPROBE2(rootnex__sync__cpu,
4631 dev_info_t *, dma->dp_dip, size_t, psize);
4632 }
4633
4634 bcopy(fromaddr, toaddr, psize);
4635 }
4636
4637 /*
4638 * decrement size until we're done, update our offset into the
4639 * buffer, and get the next page size.
4640 */
4641 size -= psize;
4642 offset += psize;
4643 psize = MIN(MMU_PAGESIZE, size);
4644
4645 /* page offset is zero for the rest of this loop */
4646 poff = 0;
4647 }
4648
4649 return (DDI_SUCCESS);
4650 }
4651
4652 /*
4653 * rootnex_dma_sync()
4654 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4655 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4656 * is set, ddi_dma_sync() returns immediately passing back success.
4657 */
4658 /*ARGSUSED*/
4659 static int
4660 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4661 off_t off, size_t len, uint_t cache_flags)
4662 {
4663 #if defined(__amd64) && !defined(__xpv)
4664 if (IOMMU_USED(rdip)) {
4665 return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4666 cache_flags));
4667 }
4668 #endif
4669 return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4670 cache_flags));
4671 }
4672
4673 /*
4674 * rootnex_valid_sync_parms()
4675 * checks the parameters passed to sync to verify they are correct.
4676 */
4677 static int
4678 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4679 off_t offset, size_t size, uint_t cache_flags)
4680 {
4681 off_t woffset;
4682
4683
4684 /*
4685 * the first part of the test to make sure the offset passed in is
4686 * within the window.
4687 */
4688 if (offset < win->wd_offset) {
4689 return (DDI_FAILURE);
4690 }
4691
4692 /*
4693 * second and last part of the test to make sure the offset and length
4694 * passed in is within the window.
4695 */
4696 woffset = offset - win->wd_offset;
4697 if ((woffset + size) > win->wd_size) {
4698 return (DDI_FAILURE);
4699 }
4700
4701 /*
4702 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4703 * be set too.
4704 */
4705 if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4706 (hp->dmai_rflags & DDI_DMA_WRITE)) {
4707 return (DDI_SUCCESS);
4708 }
4709
4710 /*
4711 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4712 * should be set. Also DDI_DMA_READ should be set in the flags.
4713 */
4714 if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4715 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4716 (hp->dmai_rflags & DDI_DMA_READ)) {
4717 return (DDI_SUCCESS);
4718 }
4719
4720 return (DDI_FAILURE);
4721 }
4722
4723
4724 /*ARGSUSED*/
4725 static int
4726 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4727 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4728 uint_t *ccountp)
4729 {
4730 rootnex_window_t *window;
4731 rootnex_trim_t *trim;
4732 ddi_dma_impl_t *hp;
4733 rootnex_dma_t *dma;
4734 ddi_dma_obj_t *dmao;
4735 #if !defined(__amd64)
4736 rootnex_sglinfo_t *sinfo;
4737 rootnex_pgmap_t *pmap;
4738 uint_t pidx;
4739 uint_t pcnt;
4740 off_t poff;
4741 int i;
4742 #endif
4743
4744
4745 hp = (ddi_dma_impl_t *)handle;
4746 dma = (rootnex_dma_t *)hp->dmai_private;
4747 #if !defined(__amd64)
4748 sinfo = &dma->dp_sglinfo;
4749 #endif
4750
4751 /* If we try and get a window which doesn't exist, return failure */
4752 if (win >= hp->dmai_nwin) {
4753 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4754 return (DDI_FAILURE);
4755 }
4756
4757 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4758
4759 /*
4760 * if we don't have any windows, and they're asking for the first
4761 * window, setup the cookie pointer to the first cookie in the bind.
4762 * setup our return values, then increment the cookie since we return
4763 * the first cookie on the stack.
4764 */
4765 if (dma->dp_window == NULL) {
4766 if (win != 0) {
4767 ROOTNEX_DPROF_INC(
4768 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4769 return (DDI_FAILURE);
4770 }
4771 hp->dmai_cookie = dma->dp_cookies;
4772 *offp = 0;
4773 *lenp = dmao->dmao_size;
4774 *ccountp = dma->dp_sglinfo.si_sgl_size;
4775 *cookiep = hp->dmai_cookie[0];
4776 hp->dmai_cookie++;
4777 return (DDI_SUCCESS);
4778 }
4779
4780 /* sync the old window before moving on to the new one */
4781 window = &dma->dp_window[dma->dp_current_win];
4782 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4783 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4784 DDI_DMA_SYNC_FORCPU);
4785 }
4786
4787 #if !defined(__amd64)
4788 /*
4789 * before we move to the next window, if we need to re-map, unmap all
4790 * the pages in this window.
4791 */
4792 if (dma->dp_cb_remaping) {
4793 /*
4794 * If we switch to this window again, we'll need to map in
4795 * on the fly next time.
4796 */
4797 window->wd_remap_copybuf = B_TRUE;
4798
4799 /*
4800 * calculate the page index into the buffer where this window
4801 * starts, and the number of pages this window takes up.
4802 */
4803 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4804 MMU_PAGESHIFT;
4805 poff = (sinfo->si_buf_offset + window->wd_offset) &
4806 MMU_PAGEOFFSET;
4807 pcnt = mmu_btopr(window->wd_size + poff);
4808 ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4809
4810 /* unmap pages which are currently mapped in this window */
4811 for (i = 0; i < pcnt; i++) {
4812 if (dma->dp_pgmap[pidx].pm_mapped) {
4813 hat_unload(kas.a_hat,
4814 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4815 HAT_UNLOAD);
4816 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4817 }
4818 pidx++;
4819 }
4820 }
4821 #endif
4822
4823 /*
4824 * Move to the new window.
4825 * NOTE: current_win must be set for sync to work right
4826 */
4827 dma->dp_current_win = win;
4828 window = &dma->dp_window[win];
4829
4830 /* if needed, adjust the first and/or last cookies for trim */
4831 trim = &window->wd_trim;
4832 if (trim->tr_trim_first) {
4833 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4834 window->wd_first_cookie->dmac_size = trim->tr_first_size;
4835 #if !defined(__amd64)
4836 window->wd_first_cookie->dmac_type =
4837 (window->wd_first_cookie->dmac_type &
4838 ROOTNEX_USES_COPYBUF) + window->wd_offset;
4839 #endif
4840 if (trim->tr_first_copybuf_win) {
4841 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4842 trim->tr_first_cbaddr;
4843 #if !defined(__amd64)
4844 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4845 trim->tr_first_kaddr;
4846 #endif
4847 }
4848 }
4849 if (trim->tr_trim_last) {
4850 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4851 trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4852 if (trim->tr_last_copybuf_win) {
4853 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4854 trim->tr_last_cbaddr;
4855 #if !defined(__amd64)
4856 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4857 trim->tr_last_kaddr;
4858 #endif
4859 }
4860 }
4861
4862 /*
4863 * setup the cookie pointer to the first cookie in the window. setup
4864 * our return values, then increment the cookie since we return the
4865 * first cookie on the stack.
4866 */
4867 hp->dmai_cookie = window->wd_first_cookie;
4868 *offp = window->wd_offset;
4869 *lenp = window->wd_size;
4870 *ccountp = window->wd_cookie_cnt;
4871 *cookiep = hp->dmai_cookie[0];
4872 hp->dmai_cookie++;
4873
4874 #if !defined(__amd64)
4875 /* re-map copybuf if required for this window */
4876 if (dma->dp_cb_remaping) {
4877 /*
4878 * calculate the page index into the buffer where this
4879 * window starts.
4880 */
4881 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4882 MMU_PAGESHIFT;
4883 ASSERT(pidx < sinfo->si_max_pages);
4884
4885 /*
4886 * the first page can get unmapped if it's shared with the
4887 * previous window. Even if the rest of this window is already
4888 * mapped in, we need to still check this one.
4889 */
4890 pmap = &dma->dp_pgmap[pidx];
4891 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4892 if (pmap->pm_pp != NULL) {
4893 pmap->pm_mapped = B_TRUE;
4894 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4895 } else if (pmap->pm_vaddr != NULL) {
4896 pmap->pm_mapped = B_TRUE;
4897 i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4898 pmap->pm_kaddr);
4899 }
4900 }
4901 pidx++;
4902
4903 /* map in the rest of the pages if required */
4904 if (window->wd_remap_copybuf) {
4905 window->wd_remap_copybuf = B_FALSE;
4906
4907 /* figure out many pages this window takes up */
4908 poff = (sinfo->si_buf_offset + window->wd_offset) &
4909 MMU_PAGEOFFSET;
4910 pcnt = mmu_btopr(window->wd_size + poff);
4911 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4912
4913 /* map pages which require it */
4914 for (i = 1; i < pcnt; i++) {
4915 pmap = &dma->dp_pgmap[pidx];
4916 if (pmap->pm_uses_copybuf) {
4917 ASSERT(pmap->pm_mapped == B_FALSE);
4918 if (pmap->pm_pp != NULL) {
4919 pmap->pm_mapped = B_TRUE;
4920 i86_pp_map(pmap->pm_pp,
4921 pmap->pm_kaddr);
4922 } else if (pmap->pm_vaddr != NULL) {
4923 pmap->pm_mapped = B_TRUE;
4924 i86_va_map(pmap->pm_vaddr,
4925 sinfo->si_asp,
4926 pmap->pm_kaddr);
4927 }
4928 }
4929 pidx++;
4930 }
4931 }
4932 }
4933 #endif
4934
4935 /* if the new window uses the copy buffer, sync it for the device */
4936 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4937 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4938 DDI_DMA_SYNC_FORDEV);
4939 }
4940
4941 return (DDI_SUCCESS);
4942 }
4943
4944 /*
4945 * rootnex_dma_win()
4946 * called from ddi_dma_getwin()
4947 */
4948 /*ARGSUSED*/
4949 static int
4950 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4951 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4952 uint_t *ccountp)
4953 {
4954 #if defined(__amd64) && !defined(__xpv)
4955 if (IOMMU_USED(rdip)) {
4956 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4957 cookiep, ccountp));
4958 }
4959 #endif
4960
4961 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4962 cookiep, ccountp));
4963 }
4964
4965 #if defined(__amd64) && !defined(__xpv)
4966 /*ARGSUSED*/
4967 static int
4968 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4969 ddi_dma_handle_t handle, void *v)
4970 {
4971 ddi_dma_impl_t *hp;
4972 rootnex_dma_t *dma;
4973
4974 hp = (ddi_dma_impl_t *)handle;
4975 dma = (rootnex_dma_t *)hp->dmai_private;
4976 dma->dp_iommu_private = v;
4977
4978 return (DDI_SUCCESS);
4979 }
4980
4981 /*ARGSUSED*/
4982 static void *
4983 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4984 ddi_dma_handle_t handle)
4985 {
4986 ddi_dma_impl_t *hp;
4987 rootnex_dma_t *dma;
4988
4989 hp = (ddi_dma_impl_t *)handle;
4990 dma = (rootnex_dma_t *)hp->dmai_private;
4991
4992 return (dma->dp_iommu_private);
4993 }
4994 #endif
4995
4996 /*
4997 * ************************
4998 * obsoleted dma routines
4999 * ************************
5000 */
5001
5002 /*
5003 * rootnex_dma_mctl()
5004 *
5005 * We don't support this legacy interface any more on x86.
5006 */
5007 /* ARGSUSED */
5008 static int
5009 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5010 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5011 uint_t cache_flags)
5012 {
5013 /*
5014 * The only thing dma_mctl is usef for anymore is legacy SPARC
5015 * dvma and sbus-specific routines.
5016 */
5017 return (DDI_FAILURE);
5018 }
5019
5020 /*
5021 * *********
5022 * FMA Code
5023 * *********
5024 */
5025
5026 /*
5027 * rootnex_fm_init()
5028 * FMA init busop
5029 */
5030 /* ARGSUSED */
5031 static int
5032 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5033 ddi_iblock_cookie_t *ibc)
5034 {
5035 *ibc = rootnex_state->r_err_ibc;
5036
5037 return (ddi_system_fmcap);
5038 }
5039
5040 /*
5041 * rootnex_dma_check()
5042 * Function called after a dma fault occurred to find out whether the
5043 * fault address is associated with a driver that is able to handle faults
5044 * and recover from faults.
5045 */
5046 /* ARGSUSED */
5047 static int
5048 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5049 const void *not_used)
5050 {
5051 rootnex_window_t *window;
5052 uint64_t start_addr;
5053 uint64_t fault_addr;
5054 ddi_dma_impl_t *hp;
5055 rootnex_dma_t *dma;
5056 uint64_t end_addr;
5057 size_t csize;
5058 int i;
5059 int j;
5060
5061
5062 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5063 hp = (ddi_dma_impl_t *)handle;
5064 ASSERT(hp);
5065
5066 dma = (rootnex_dma_t *)hp->dmai_private;
5067
5068 /* Get the address that we need to search for */
5069 fault_addr = *(uint64_t *)addr;
5070
5071 /*
5072 * if we don't have any windows, we can just walk through all the
5073 * cookies.
5074 */
5075 if (dma->dp_window == NULL) {
5076 /* for each cookie */
5077 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5078 /*
5079 * if the faulted address is within the physical address
5080 * range of the cookie, return DDI_FM_NONFATAL.
5081 */
5082 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5083 (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5084 dma->dp_cookies[i].dmac_size))) {
5085 return (DDI_FM_NONFATAL);
5086 }
5087 }
5088
5089 /* fault_addr not within this DMA handle */
5090 return (DDI_FM_UNKNOWN);
5091 }
5092
5093 /* we have mutiple windows, walk through each window */
5094 for (i = 0; i < hp->dmai_nwin; i++) {
5095 window = &dma->dp_window[i];
5096
5097 /* Go through all the cookies in the window */
5098 for (j = 0; j < window->wd_cookie_cnt; j++) {
5099
5100 start_addr = window->wd_first_cookie[j].dmac_laddress;
5101 csize = window->wd_first_cookie[j].dmac_size;
5102
5103 /*
5104 * if we are trimming the first cookie in the window,
5105 * and this is the first cookie, adjust the start
5106 * address and size of the cookie to account for the
5107 * trim.
5108 */
5109 if (window->wd_trim.tr_trim_first && (j == 0)) {
5110 start_addr = window->wd_trim.tr_first_paddr;
5111 csize = window->wd_trim.tr_first_size;
5112 }
5113
5114 /*
5115 * if we are trimming the last cookie in the window,
5116 * and this is the last cookie, adjust the start
5117 * address and size of the cookie to account for the
5118 * trim.
5119 */
5120 if (window->wd_trim.tr_trim_last &&
5121 (j == (window->wd_cookie_cnt - 1))) {
5122 start_addr = window->wd_trim.tr_last_paddr;
5123 csize = window->wd_trim.tr_last_size;
5124 }
5125
5126 end_addr = start_addr + csize;
5127
5128 /*
5129 * if the faulted address is within the physical
5130 * address of the cookie, return DDI_FM_NONFATAL.
5131 */
5132 if ((fault_addr >= start_addr) &&
5133 (fault_addr <= end_addr)) {
5134 return (DDI_FM_NONFATAL);
5135 }
5136 }
5137 }
5138
5139 /* fault_addr not within this DMA handle */
5140 return (DDI_FM_UNKNOWN);
5141 }
5142
5143 /*ARGSUSED*/
5144 static int
5145 rootnex_quiesce(dev_info_t *dip)
5146 {
5147 #if defined(__amd64) && !defined(__xpv)
5148 return (immu_quiesce());
5149 #else
5150 return (DDI_SUCCESS);
5151 #endif
5152 }
5153
5154 #if defined(__xpv)
5155 void
5156 immu_init(void)
5157 {
5158 ;
5159 }
5160
5161 void
5162 immu_startup(void)
5163 {
5164 ;
5165 }
5166 /*ARGSUSED*/
5167 void
5168 immu_physmem_update(uint64_t addr, uint64_t size)
5169 {
5170 ;
5171 }
5172 #endif