Print this page
7127 remove -Wno-missing-braces from Makefile.uts
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/io/rootnex.c
+++ new/usr/src/uts/i86pc/io/rootnex.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24 /*
25 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved.
27 27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
28 28 */
29 29
30 30 /*
31 31 * x86 root nexus driver
32 32 */
33 33
34 34 #include <sys/sysmacros.h>
35 35 #include <sys/conf.h>
36 36 #include <sys/autoconf.h>
37 37 #include <sys/sysmacros.h>
38 38 #include <sys/debug.h>
39 39 #include <sys/psw.h>
40 40 #include <sys/ddidmareq.h>
41 41 #include <sys/promif.h>
42 42 #include <sys/devops.h>
43 43 #include <sys/kmem.h>
44 44 #include <sys/cmn_err.h>
45 45 #include <vm/seg.h>
46 46 #include <vm/seg_kmem.h>
47 47 #include <vm/seg_dev.h>
48 48 #include <sys/vmem.h>
49 49 #include <sys/mman.h>
50 50 #include <vm/hat.h>
51 51 #include <vm/as.h>
52 52 #include <vm/page.h>
53 53 #include <sys/avintr.h>
54 54 #include <sys/errno.h>
55 55 #include <sys/modctl.h>
56 56 #include <sys/ddi_impldefs.h>
57 57 #include <sys/sunddi.h>
58 58 #include <sys/sunndi.h>
59 59 #include <sys/mach_intr.h>
60 60 #include <sys/psm.h>
61 61 #include <sys/ontrap.h>
62 62 #include <sys/atomic.h>
63 63 #include <sys/sdt.h>
64 64 #include <sys/rootnex.h>
65 65 #include <vm/hat_i86.h>
66 66 #include <sys/ddifm.h>
67 67 #include <sys/ddi_isa.h>
68 68 #include <sys/apic.h>
69 69
70 70 #ifdef __xpv
71 71 #include <sys/bootinfo.h>
72 72 #include <sys/hypervisor.h>
73 73 #include <sys/bootconf.h>
74 74 #include <vm/kboot_mmu.h>
75 75 #endif
76 76
77 77 #if defined(__amd64) && !defined(__xpv)
78 78 #include <sys/immu.h>
79 79 #endif
80 80
81 81
82 82 /*
83 83 * enable/disable extra checking of function parameters. Useful for debugging
84 84 * drivers.
85 85 */
86 86 #ifdef DEBUG
87 87 int rootnex_alloc_check_parms = 1;
88 88 int rootnex_bind_check_parms = 1;
89 89 int rootnex_bind_check_inuse = 1;
90 90 int rootnex_unbind_verify_buffer = 0;
91 91 int rootnex_sync_check_parms = 1;
92 92 #else
93 93 int rootnex_alloc_check_parms = 0;
94 94 int rootnex_bind_check_parms = 0;
95 95 int rootnex_bind_check_inuse = 0;
96 96 int rootnex_unbind_verify_buffer = 0;
97 97 int rootnex_sync_check_parms = 0;
98 98 #endif
99 99
100 100 boolean_t rootnex_dmar_not_setup;
101 101
102 102 /* Master Abort and Target Abort panic flag */
103 103 int rootnex_fm_ma_ta_panic_flag = 0;
104 104
105 105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
106 106 int rootnex_bind_fail = 1;
107 107 int rootnex_bind_warn = 1;
108 108 uint8_t *rootnex_warn_list;
109 109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
110 110 #define ROOTNEX_BIND_WARNING (0x1 << 0)
111 111
112 112 /*
113 113 * revert back to old broken behavior of always sync'ing entire copy buffer.
114 114 * This is useful if be have a buggy driver which doesn't correctly pass in
115 115 * the offset and size into ddi_dma_sync().
116 116 */
117 117 int rootnex_sync_ignore_params = 0;
118 118
119 119 /*
120 120 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
121 121 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
122 122 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
123 123 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
124 124 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
125 125 * (< 8K). We will still need to allocate the copy buffer during bind though
126 126 * (if we need one). These can only be modified in /etc/system before rootnex
127 127 * attach.
128 128 */
129 129 #if defined(__amd64)
130 130 int rootnex_prealloc_cookies = 65;
131 131 int rootnex_prealloc_windows = 4;
132 132 int rootnex_prealloc_copybuf = 2;
133 133 #else
134 134 int rootnex_prealloc_cookies = 33;
135 135 int rootnex_prealloc_windows = 4;
136 136 int rootnex_prealloc_copybuf = 2;
137 137 #endif
138 138
139 139 /* driver global state */
140 140 static rootnex_state_t *rootnex_state;
141 141
142 142 #ifdef DEBUG
143 143 /* shortcut to rootnex counters */
144 144 static uint64_t *rootnex_cnt;
145 145 #endif
146 146
147 147 /*
148 148 * XXX - does x86 even need these or are they left over from the SPARC days?
149 149 */
150 150 /* statically defined integer/boolean properties for the root node */
151 151 static rootnex_intprop_t rootnex_intprp[] = {
152 152 { "PAGESIZE", PAGESIZE },
153 153 { "MMU_PAGESIZE", MMU_PAGESIZE },
154 154 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET },
155 155 { DDI_RELATIVE_ADDRESSING, 1 },
156 156 };
157 157 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
158 158
159 159 /*
160 160 * If we're dom0, we're using a real device so we need to load
161 161 * the cookies with MFNs instead of PFNs.
162 162 */
163 163 #ifdef __xpv
164 164 typedef maddr_t rootnex_addr_t;
165 165 #define ROOTNEX_PADDR_TO_RBASE(pa) \
166 166 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
167 167 #else
168 168 typedef paddr_t rootnex_addr_t;
169 169 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa)
170 170 #endif
171 171
172 172 static struct cb_ops rootnex_cb_ops = {
173 173 nodev, /* open */
174 174 nodev, /* close */
175 175 nodev, /* strategy */
176 176 nodev, /* print */
177 177 nodev, /* dump */
178 178 nodev, /* read */
179 179 nodev, /* write */
180 180 nodev, /* ioctl */
181 181 nodev, /* devmap */
182 182 nodev, /* mmap */
183 183 nodev, /* segmap */
184 184 nochpoll, /* chpoll */
185 185 ddi_prop_op, /* cb_prop_op */
186 186 NULL, /* struct streamtab */
187 187 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
188 188 CB_REV, /* Rev */
189 189 nodev, /* cb_aread */
190 190 nodev /* cb_awrite */
191 191 };
192 192
193 193 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
194 194 off_t offset, off_t len, caddr_t *vaddrp);
195 195 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
196 196 struct hat *hat, struct seg *seg, caddr_t addr,
197 197 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
198 198 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
199 199 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
200 200 ddi_dma_handle_t *handlep);
201 201 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
202 202 ddi_dma_handle_t handle);
203 203 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
204 204 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
205 205 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
206 206 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
207 207 ddi_dma_handle_t handle);
208 208 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
209 209 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
210 210 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
211 211 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
212 212 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
213 213 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
214 214 ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
215 215 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
216 216 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
217 217 ddi_ctl_enum_t ctlop, void *arg, void *result);
218 218 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
219 219 ddi_iblock_cookie_t *ibc);
220 220 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
221 221 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
222 222 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
223 223 void *);
224 224 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
225 225
226 226 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
227 227 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
228 228 ddi_dma_handle_t *handlep);
229 229 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
230 230 ddi_dma_handle_t handle);
231 231 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
232 232 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
233 233 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
234 234 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
235 235 ddi_dma_handle_t handle);
236 236 #if defined(__amd64) && !defined(__xpv)
237 237 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
238 238 ddi_dma_handle_t handle);
239 239 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
240 240 ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
241 241 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
242 242 ddi_dma_cookie_t *cookiep, uint_t ccount);
243 243 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
244 244 ddi_dma_handle_t handle);
245 245 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
246 246 #endif
247 247 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
248 248 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
249 249 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
250 250 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
251 251 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
252 252
253 253 #if defined(__amd64) && !defined(__xpv)
254 254 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
255 255 ddi_dma_handle_t handle, void *v);
256 256 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
257 257 ddi_dma_handle_t handle);
258 258 #endif
259 259
260 260
261 261 static struct bus_ops rootnex_bus_ops = {
262 262 BUSO_REV,
263 263 rootnex_map,
264 264 NULL,
265 265 NULL,
266 266 NULL,
267 267 rootnex_map_fault,
268 268 0,
269 269 rootnex_dma_allochdl,
270 270 rootnex_dma_freehdl,
271 271 rootnex_dma_bindhdl,
272 272 rootnex_dma_unbindhdl,
273 273 rootnex_dma_sync,
274 274 rootnex_dma_win,
275 275 rootnex_dma_mctl,
276 276 rootnex_ctlops,
277 277 ddi_bus_prop_op,
278 278 i_ddi_rootnex_get_eventcookie,
279 279 i_ddi_rootnex_add_eventcall,
280 280 i_ddi_rootnex_remove_eventcall,
281 281 i_ddi_rootnex_post_event,
282 282 0, /* bus_intr_ctl */
283 283 0, /* bus_config */
284 284 0, /* bus_unconfig */
285 285 rootnex_fm_init, /* bus_fm_init */
286 286 NULL, /* bus_fm_fini */
287 287 NULL, /* bus_fm_access_enter */
288 288 NULL, /* bus_fm_access_exit */
289 289 NULL, /* bus_powr */
290 290 rootnex_intr_ops /* bus_intr_op */
291 291 };
292 292
293 293 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
294 294 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
295 295 static int rootnex_quiesce(dev_info_t *dip);
296 296
297 297 static struct dev_ops rootnex_ops = {
298 298 DEVO_REV,
299 299 0,
300 300 ddi_no_info,
301 301 nulldev,
302 302 nulldev,
303 303 rootnex_attach,
304 304 rootnex_detach,
305 305 nulldev,
306 306 &rootnex_cb_ops,
307 307 &rootnex_bus_ops,
308 308 NULL,
309 309 rootnex_quiesce, /* quiesce */
|
↓ open down ↓ |
309 lines elided |
↑ open up ↑ |
310 310 };
311 311
312 312 static struct modldrv rootnex_modldrv = {
313 313 &mod_driverops,
314 314 "i86pc root nexus",
315 315 &rootnex_ops
316 316 };
317 317
318 318 static struct modlinkage rootnex_modlinkage = {
319 319 MODREV_1,
320 - (void *)&rootnex_modldrv,
321 - NULL
320 + { (void *)&rootnex_modldrv, NULL }
322 321 };
323 322
324 323 #if defined(__amd64) && !defined(__xpv)
325 324 static iommulib_nexops_t iommulib_nexops = {
326 325 IOMMU_NEXOPS_VERSION,
327 326 "Rootnex IOMMU ops Vers 1.1",
328 327 NULL,
329 328 rootnex_coredma_allochdl,
330 329 rootnex_coredma_freehdl,
331 330 rootnex_coredma_bindhdl,
332 331 rootnex_coredma_unbindhdl,
333 332 rootnex_coredma_reset_cookies,
334 333 rootnex_coredma_get_cookies,
335 334 rootnex_coredma_set_cookies,
336 335 rootnex_coredma_clear_cookies,
337 336 rootnex_coredma_get_sleep_flags,
338 337 rootnex_coredma_sync,
339 338 rootnex_coredma_win,
340 339 rootnex_coredma_hdl_setprivate,
341 340 rootnex_coredma_hdl_getprivate
342 341 };
343 342 #endif
344 343
345 344 /*
346 345 * extern hacks
347 346 */
348 347 extern struct seg_ops segdev_ops;
349 348 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
350 349 #ifdef DDI_MAP_DEBUG
351 350 extern int ddi_map_debug_flag;
352 351 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf
353 352 #endif
354 353 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
355 354 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
356 355 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
357 356 psm_intr_op_t, int *);
358 357 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
359 358 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
360 359
361 360 /*
362 361 * Use device arena to use for device control register mappings.
363 362 * Various kernel memory walkers (debugger, dtrace) need to know
364 363 * to avoid this address range to prevent undesired device activity.
365 364 */
366 365 extern void *device_arena_alloc(size_t size, int vm_flag);
367 366 extern void device_arena_free(void * vaddr, size_t size);
368 367
369 368
370 369 /*
371 370 * Internal functions
372 371 */
373 372 static int rootnex_dma_init();
374 373 static void rootnex_add_props(dev_info_t *);
375 374 static int rootnex_ctl_reportdev(dev_info_t *dip);
376 375 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
377 376 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
378 377 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
379 378 static int rootnex_map_handle(ddi_map_req_t *mp);
380 379 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
381 380 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
382 381 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
383 382 ddi_dma_attr_t *attr);
384 383 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
385 384 rootnex_sglinfo_t *sglinfo);
386 385 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
387 386 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
388 387 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
389 388 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
390 389 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
391 390 rootnex_dma_t *dma, ddi_dma_attr_t *attr);
392 391 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
393 392 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
394 393 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
395 394 static void rootnex_teardown_windows(rootnex_dma_t *dma);
396 395 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
397 396 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
398 397 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
399 398 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
400 399 size_t *copybuf_used, page_t **cur_pp);
401 400 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
402 401 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
403 402 ddi_dma_attr_t *attr, off_t cur_offset);
404 403 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
405 404 rootnex_dma_t *dma, rootnex_window_t **windowp,
406 405 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
407 406 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
408 407 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
409 408 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
410 409 off_t offset, size_t size, uint_t cache_flags);
411 410 static int rootnex_verify_buffer(rootnex_dma_t *dma);
412 411 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
413 412 const void *comp_addr, const void *not_used);
414 413 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
415 414 rootnex_sglinfo_t *sglinfo);
416 415 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
417 416
418 417 /*
419 418 * _init()
420 419 *
421 420 */
422 421 int
423 422 _init(void)
424 423 {
425 424
426 425 rootnex_state = NULL;
427 426 return (mod_install(&rootnex_modlinkage));
428 427 }
429 428
430 429
431 430 /*
432 431 * _info()
433 432 *
434 433 */
435 434 int
436 435 _info(struct modinfo *modinfop)
437 436 {
438 437 return (mod_info(&rootnex_modlinkage, modinfop));
439 438 }
440 439
441 440
442 441 /*
443 442 * _fini()
444 443 *
445 444 */
446 445 int
447 446 _fini(void)
448 447 {
449 448 return (EBUSY);
450 449 }
451 450
452 451
453 452 /*
454 453 * rootnex_attach()
455 454 *
456 455 */
457 456 static int
458 457 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
459 458 {
460 459 int fmcap;
461 460 int e;
462 461
463 462 switch (cmd) {
464 463 case DDI_ATTACH:
465 464 break;
466 465 case DDI_RESUME:
467 466 #if defined(__amd64) && !defined(__xpv)
468 467 return (immu_unquiesce());
469 468 #else
470 469 return (DDI_SUCCESS);
471 470 #endif
472 471 default:
473 472 return (DDI_FAILURE);
474 473 }
475 474
476 475 /*
477 476 * We should only have one instance of rootnex. Save it away since we
478 477 * don't have an easy way to get it back later.
479 478 */
480 479 ASSERT(rootnex_state == NULL);
481 480 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
482 481
483 482 rootnex_state->r_dip = dip;
484 483 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
485 484 rootnex_state->r_reserved_msg_printed = B_FALSE;
486 485 #ifdef DEBUG
487 486 rootnex_cnt = &rootnex_state->r_counters[0];
488 487 #endif
489 488
490 489 /*
491 490 * Set minimum fm capability level for i86pc platforms and then
492 491 * initialize error handling. Since we're the rootnex, we don't
493 492 * care what's returned in the fmcap field.
494 493 */
495 494 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
496 495 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
497 496 fmcap = ddi_system_fmcap;
498 497 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
499 498
500 499 /* initialize DMA related state */
501 500 e = rootnex_dma_init();
502 501 if (e != DDI_SUCCESS) {
503 502 kmem_free(rootnex_state, sizeof (rootnex_state_t));
504 503 return (DDI_FAILURE);
505 504 }
506 505
507 506 /* Add static root node properties */
508 507 rootnex_add_props(dip);
509 508
510 509 /* since we can't call ddi_report_dev() */
511 510 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
512 511
513 512 /* Initialize rootnex event handle */
514 513 i_ddi_rootnex_init_events(dip);
515 514
516 515 #if defined(__amd64) && !defined(__xpv)
517 516 e = iommulib_nexus_register(dip, &iommulib_nexops,
518 517 &rootnex_state->r_iommulib_handle);
519 518
520 519 ASSERT(e == DDI_SUCCESS);
521 520 #endif
522 521
523 522 return (DDI_SUCCESS);
524 523 }
525 524
526 525
527 526 /*
528 527 * rootnex_detach()
529 528 *
530 529 */
531 530 /*ARGSUSED*/
532 531 static int
533 532 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
534 533 {
535 534 switch (cmd) {
536 535 case DDI_SUSPEND:
537 536 #if defined(__amd64) && !defined(__xpv)
538 537 return (immu_quiesce());
539 538 #else
540 539 return (DDI_SUCCESS);
541 540 #endif
542 541 default:
543 542 return (DDI_FAILURE);
544 543 }
545 544 /*NOTREACHED*/
546 545
547 546 }
548 547
549 548
550 549 /*
551 550 * rootnex_dma_init()
552 551 *
553 552 */
554 553 /*ARGSUSED*/
555 554 static int
556 555 rootnex_dma_init()
557 556 {
558 557 size_t bufsize;
559 558
560 559
561 560 /*
562 561 * size of our cookie/window/copybuf state needed in dma bind that we
563 562 * pre-alloc in dma_alloc_handle
564 563 */
565 564 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
566 565 rootnex_state->r_prealloc_size =
567 566 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
568 567 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
569 568 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
570 569
571 570 /*
572 571 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
573 572 * allocate 16 extra bytes for struct pointer alignment
574 573 * (p->dmai_private & dma->dp_prealloc_buffer)
575 574 */
576 575 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
577 576 rootnex_state->r_prealloc_size + 0x10;
578 577 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
579 578 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
580 579 if (rootnex_state->r_dmahdl_cache == NULL) {
581 580 return (DDI_FAILURE);
582 581 }
583 582
584 583 /*
585 584 * allocate array to track which major numbers we have printed warnings
586 585 * for.
587 586 */
588 587 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
589 588 KM_SLEEP);
590 589
591 590 return (DDI_SUCCESS);
592 591 }
593 592
594 593
595 594 /*
596 595 * rootnex_add_props()
597 596 *
598 597 */
599 598 static void
600 599 rootnex_add_props(dev_info_t *dip)
601 600 {
602 601 rootnex_intprop_t *rpp;
603 602 int i;
604 603
605 604 /* Add static integer/boolean properties to the root node */
606 605 rpp = rootnex_intprp;
607 606 for (i = 0; i < NROOT_INTPROPS; i++) {
608 607 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
609 608 rpp[i].prop_name, rpp[i].prop_value);
610 609 }
611 610 }
612 611
613 612
614 613
615 614 /*
616 615 * *************************
617 616 * ctlops related routines
618 617 * *************************
619 618 */
620 619
621 620 /*
622 621 * rootnex_ctlops()
623 622 *
624 623 */
625 624 /*ARGSUSED*/
626 625 static int
627 626 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
628 627 void *arg, void *result)
629 628 {
630 629 int n, *ptr;
631 630 struct ddi_parent_private_data *pdp;
632 631
633 632 switch (ctlop) {
634 633 case DDI_CTLOPS_DMAPMAPC:
635 634 /*
636 635 * Return 'partial' to indicate that dma mapping
637 636 * has to be done in the main MMU.
638 637 */
639 638 return (DDI_DMA_PARTIAL);
640 639
641 640 case DDI_CTLOPS_BTOP:
642 641 /*
643 642 * Convert byte count input to physical page units.
644 643 * (byte counts that are not a page-size multiple
645 644 * are rounded down)
646 645 */
647 646 *(ulong_t *)result = btop(*(ulong_t *)arg);
648 647 return (DDI_SUCCESS);
649 648
650 649 case DDI_CTLOPS_PTOB:
651 650 /*
652 651 * Convert size in physical pages to bytes
653 652 */
654 653 *(ulong_t *)result = ptob(*(ulong_t *)arg);
655 654 return (DDI_SUCCESS);
656 655
657 656 case DDI_CTLOPS_BTOPR:
658 657 /*
659 658 * Convert byte count input to physical page units
660 659 * (byte counts that are not a page-size multiple
661 660 * are rounded up)
662 661 */
663 662 *(ulong_t *)result = btopr(*(ulong_t *)arg);
664 663 return (DDI_SUCCESS);
665 664
666 665 case DDI_CTLOPS_INITCHILD:
667 666 return (impl_ddi_sunbus_initchild(arg));
668 667
669 668 case DDI_CTLOPS_UNINITCHILD:
670 669 impl_ddi_sunbus_removechild(arg);
671 670 return (DDI_SUCCESS);
672 671
673 672 case DDI_CTLOPS_REPORTDEV:
674 673 return (rootnex_ctl_reportdev(rdip));
675 674
676 675 case DDI_CTLOPS_IOMIN:
677 676 /*
678 677 * Nothing to do here but reflect back..
679 678 */
680 679 return (DDI_SUCCESS);
681 680
682 681 case DDI_CTLOPS_REGSIZE:
683 682 case DDI_CTLOPS_NREGS:
684 683 break;
685 684
686 685 case DDI_CTLOPS_SIDDEV:
687 686 if (ndi_dev_is_prom_node(rdip))
688 687 return (DDI_SUCCESS);
689 688 if (ndi_dev_is_persistent_node(rdip))
690 689 return (DDI_SUCCESS);
691 690 return (DDI_FAILURE);
692 691
693 692 case DDI_CTLOPS_POWER:
694 693 return ((*pm_platform_power)((power_req_t *)arg));
695 694
696 695 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
697 696 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
698 697 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
699 698 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
700 699 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
701 700 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
702 701 if (!rootnex_state->r_reserved_msg_printed) {
703 702 rootnex_state->r_reserved_msg_printed = B_TRUE;
704 703 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
705 704 "1 or more reserved/obsolete operations.");
706 705 }
707 706 return (DDI_FAILURE);
708 707
709 708 default:
710 709 return (DDI_FAILURE);
711 710 }
712 711 /*
713 712 * The rest are for "hardware" properties
714 713 */
715 714 if ((pdp = ddi_get_parent_data(rdip)) == NULL)
716 715 return (DDI_FAILURE);
717 716
718 717 if (ctlop == DDI_CTLOPS_NREGS) {
719 718 ptr = (int *)result;
720 719 *ptr = pdp->par_nreg;
721 720 } else {
722 721 off_t *size = (off_t *)result;
723 722
724 723 ptr = (int *)arg;
725 724 n = *ptr;
726 725 if (n >= pdp->par_nreg) {
727 726 return (DDI_FAILURE);
728 727 }
729 728 *size = (off_t)pdp->par_reg[n].regspec_size;
730 729 }
731 730 return (DDI_SUCCESS);
732 731 }
733 732
734 733
735 734 /*
736 735 * rootnex_ctl_reportdev()
737 736 *
738 737 */
739 738 static int
740 739 rootnex_ctl_reportdev(dev_info_t *dev)
741 740 {
742 741 int i, n, len, f_len = 0;
743 742 char *buf;
744 743
745 744 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
746 745 f_len += snprintf(buf, REPORTDEV_BUFSIZE,
747 746 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
748 747 len = strlen(buf);
749 748
750 749 for (i = 0; i < sparc_pd_getnreg(dev); i++) {
751 750
752 751 struct regspec *rp = sparc_pd_getreg(dev, i);
753 752
754 753 if (i == 0)
755 754 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
756 755 ": ");
757 756 else
758 757 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
759 758 " and ");
760 759 len = strlen(buf);
761 760
762 761 switch (rp->regspec_bustype) {
763 762
764 763 case BTEISA:
765 764 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
766 765 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
767 766 break;
768 767
769 768 case BTISA:
770 769 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
771 770 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
772 771 break;
773 772
774 773 default:
775 774 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
776 775 "space %x offset %x",
777 776 rp->regspec_bustype, rp->regspec_addr);
778 777 break;
779 778 }
780 779 len = strlen(buf);
781 780 }
782 781 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
783 782 int pri;
784 783
785 784 if (i != 0) {
786 785 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
787 786 ",");
788 787 len = strlen(buf);
789 788 }
790 789 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
791 790 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
792 791 " sparc ipl %d", pri);
793 792 len = strlen(buf);
794 793 }
795 794 #ifdef DEBUG
796 795 if (f_len + 1 >= REPORTDEV_BUFSIZE) {
797 796 cmn_err(CE_NOTE, "next message is truncated: "
798 797 "printed length 1024, real length %d", f_len);
799 798 }
800 799 #endif /* DEBUG */
801 800 cmn_err(CE_CONT, "?%s\n", buf);
802 801 kmem_free(buf, REPORTDEV_BUFSIZE);
803 802 return (DDI_SUCCESS);
804 803 }
805 804
806 805
807 806 /*
808 807 * ******************
809 808 * map related code
810 809 * ******************
811 810 */
812 811
813 812 /*
814 813 * rootnex_map()
815 814 *
816 815 */
817 816 static int
818 817 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
819 818 off_t len, caddr_t *vaddrp)
820 819 {
821 820 struct regspec *rp, tmp_reg;
822 821 ddi_map_req_t mr = *mp; /* Get private copy of request */
823 822 int error;
824 823
825 824 mp = &mr;
826 825
827 826 switch (mp->map_op) {
828 827 case DDI_MO_MAP_LOCKED:
829 828 case DDI_MO_UNMAP:
830 829 case DDI_MO_MAP_HANDLE:
831 830 break;
832 831 default:
833 832 #ifdef DDI_MAP_DEBUG
834 833 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
835 834 mp->map_op);
836 835 #endif /* DDI_MAP_DEBUG */
837 836 return (DDI_ME_UNIMPLEMENTED);
838 837 }
839 838
840 839 if (mp->map_flags & DDI_MF_USER_MAPPING) {
841 840 #ifdef DDI_MAP_DEBUG
842 841 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
843 842 #endif /* DDI_MAP_DEBUG */
844 843 return (DDI_ME_UNIMPLEMENTED);
845 844 }
846 845
847 846 /*
848 847 * First, if given an rnumber, convert it to a regspec...
849 848 * (Presumably, this is on behalf of a child of the root node?)
850 849 */
851 850
852 851 if (mp->map_type == DDI_MT_RNUMBER) {
853 852
854 853 int rnumber = mp->map_obj.rnumber;
855 854 #ifdef DDI_MAP_DEBUG
856 855 static char *out_of_range =
857 856 "rootnex_map: Out of range rnumber <%d>, device <%s>";
858 857 #endif /* DDI_MAP_DEBUG */
859 858
860 859 rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
861 860 if (rp == NULL) {
862 861 #ifdef DDI_MAP_DEBUG
863 862 cmn_err(CE_WARN, out_of_range, rnumber,
864 863 ddi_get_name(rdip));
865 864 #endif /* DDI_MAP_DEBUG */
866 865 return (DDI_ME_RNUMBER_RANGE);
867 866 }
868 867
869 868 /*
870 869 * Convert the given ddi_map_req_t from rnumber to regspec...
871 870 */
872 871
873 872 mp->map_type = DDI_MT_REGSPEC;
874 873 mp->map_obj.rp = rp;
875 874 }
876 875
877 876 /*
878 877 * Adjust offset and length correspnding to called values...
879 878 * XXX: A non-zero length means override the one in the regspec
880 879 * XXX: (regardless of what's in the parent's range?)
881 880 */
882 881
883 882 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */
884 883 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */
885 884
886 885 #ifdef DDI_MAP_DEBUG
887 886 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
888 887 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
889 888 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
890 889 len, mp->map_handlep);
891 890 #endif /* DDI_MAP_DEBUG */
892 891
893 892 /*
894 893 * I/O or memory mapping:
895 894 *
896 895 * <bustype=0, addr=x, len=x>: memory
897 896 * <bustype=1, addr=x, len=x>: i/o
898 897 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
899 898 */
900 899
901 900 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
902 901 cmn_err(CE_WARN, "<%s,%s> invalid register spec"
903 902 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
904 903 ddi_get_name(rdip), rp->regspec_bustype,
905 904 rp->regspec_addr, rp->regspec_size);
906 905 return (DDI_ME_INVAL);
907 906 }
908 907
909 908 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
910 909 /*
911 910 * compatibility i/o mapping
912 911 */
913 912 rp->regspec_bustype += (uint_t)offset;
914 913 } else {
915 914 /*
916 915 * Normal memory or i/o mapping
917 916 */
918 917 rp->regspec_addr += (uint_t)offset;
919 918 }
920 919
921 920 if (len != 0)
922 921 rp->regspec_size = (uint_t)len;
923 922
924 923 #ifdef DDI_MAP_DEBUG
925 924 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
926 925 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
927 926 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
928 927 offset, len, mp->map_handlep);
929 928 #endif /* DDI_MAP_DEBUG */
930 929
931 930 /*
932 931 * Apply any parent ranges at this level, if applicable.
933 932 * (This is where nexus specific regspec translation takes place.
934 933 * Use of this function is implicit agreement that translation is
935 934 * provided via ddi_apply_range.)
936 935 */
937 936
938 937 #ifdef DDI_MAP_DEBUG
939 938 ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
940 939 ddi_get_name(dip), ddi_get_name(rdip));
941 940 #endif /* DDI_MAP_DEBUG */
942 941
943 942 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
944 943 return (error);
945 944
946 945 switch (mp->map_op) {
947 946 case DDI_MO_MAP_LOCKED:
948 947
949 948 /*
950 949 * Set up the locked down kernel mapping to the regspec...
951 950 */
952 951
953 952 return (rootnex_map_regspec(mp, vaddrp));
954 953
955 954 case DDI_MO_UNMAP:
956 955
957 956 /*
958 957 * Release mapping...
959 958 */
960 959
961 960 return (rootnex_unmap_regspec(mp, vaddrp));
962 961
963 962 case DDI_MO_MAP_HANDLE:
964 963
965 964 return (rootnex_map_handle(mp));
966 965
967 966 default:
968 967 return (DDI_ME_UNIMPLEMENTED);
969 968 }
970 969 }
971 970
972 971
973 972 /*
974 973 * rootnex_map_fault()
975 974 *
976 975 * fault in mappings for requestors
977 976 */
978 977 /*ARGSUSED*/
979 978 static int
980 979 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
981 980 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
982 981 uint_t lock)
983 982 {
984 983
985 984 #ifdef DDI_MAP_DEBUG
986 985 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
987 986 ddi_map_debug(" Seg <%s>\n",
988 987 seg->s_ops == &segdev_ops ? "segdev" :
989 988 seg == &kvseg ? "segkmem" : "NONE!");
990 989 #endif /* DDI_MAP_DEBUG */
991 990
992 991 /*
993 992 * This is all terribly broken, but it is a start
994 993 *
995 994 * XXX Note that this test means that segdev_ops
996 995 * must be exported from seg_dev.c.
997 996 * XXX What about devices with their own segment drivers?
998 997 */
999 998 if (seg->s_ops == &segdev_ops) {
1000 999 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1001 1000
1002 1001 if (hat == NULL) {
1003 1002 /*
1004 1003 * This is one plausible interpretation of
1005 1004 * a null hat i.e. use the first hat on the
1006 1005 * address space hat list which by convention is
1007 1006 * the hat of the system MMU. At alternative
1008 1007 * would be to panic .. this might well be better ..
1009 1008 */
1010 1009 ASSERT(AS_READ_HELD(seg->s_as));
1011 1010 hat = seg->s_as->a_hat;
1012 1011 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1013 1012 }
1014 1013 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1015 1014 (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1016 1015 } else if (seg == &kvseg && dp == NULL) {
1017 1016 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1018 1017 HAT_LOAD_LOCK);
1019 1018 } else
1020 1019 return (DDI_FAILURE);
1021 1020 return (DDI_SUCCESS);
1022 1021 }
1023 1022
1024 1023
1025 1024 /*
1026 1025 * rootnex_map_regspec()
1027 1026 * we don't support mapping of I/O cards above 4Gb
1028 1027 */
1029 1028 static int
1030 1029 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1031 1030 {
1032 1031 rootnex_addr_t rbase;
1033 1032 void *cvaddr;
1034 1033 uint_t npages, pgoffset;
1035 1034 struct regspec *rp;
1036 1035 ddi_acc_hdl_t *hp;
1037 1036 ddi_acc_impl_t *ap;
1038 1037 uint_t hat_acc_flags;
1039 1038 paddr_t pbase;
1040 1039
1041 1040 rp = mp->map_obj.rp;
1042 1041 hp = mp->map_handlep;
1043 1042
1044 1043 #ifdef DDI_MAP_DEBUG
1045 1044 ddi_map_debug(
1046 1045 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1047 1046 rp->regspec_bustype, rp->regspec_addr,
1048 1047 rp->regspec_size, mp->map_handlep);
1049 1048 #endif /* DDI_MAP_DEBUG */
1050 1049
1051 1050 /*
1052 1051 * I/O or memory mapping
1053 1052 *
1054 1053 * <bustype=0, addr=x, len=x>: memory
1055 1054 * <bustype=1, addr=x, len=x>: i/o
1056 1055 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1057 1056 */
1058 1057
1059 1058 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1060 1059 cmn_err(CE_WARN, "rootnex: invalid register spec"
1061 1060 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
1062 1061 rp->regspec_addr, rp->regspec_size);
1063 1062 return (DDI_FAILURE);
1064 1063 }
1065 1064
1066 1065 if (rp->regspec_bustype != 0) {
1067 1066 /*
1068 1067 * I/O space - needs a handle.
1069 1068 */
1070 1069 if (hp == NULL) {
1071 1070 return (DDI_FAILURE);
1072 1071 }
1073 1072 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1074 1073 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1075 1074 impl_acc_hdl_init(hp);
1076 1075
1077 1076 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1078 1077 #ifdef DDI_MAP_DEBUG
1079 1078 ddi_map_debug("rootnex_map_regspec: mmap() "
1080 1079 "to I/O space is not supported.\n");
1081 1080 #endif /* DDI_MAP_DEBUG */
1082 1081 return (DDI_ME_INVAL);
1083 1082 } else {
1084 1083 /*
1085 1084 * 1275-compliant vs. compatibility i/o mapping
1086 1085 */
1087 1086 *vaddrp =
1088 1087 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1089 1088 ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1090 1089 ((caddr_t)(uintptr_t)rp->regspec_addr);
1091 1090 #ifdef __xpv
1092 1091 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1093 1092 hp->ah_pfn = xen_assign_pfn(
1094 1093 mmu_btop((ulong_t)rp->regspec_addr &
1095 1094 MMU_PAGEMASK));
1096 1095 } else {
1097 1096 hp->ah_pfn = mmu_btop(
1098 1097 (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1099 1098 }
1100 1099 #else
1101 1100 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1102 1101 MMU_PAGEMASK);
1103 1102 #endif
1104 1103 hp->ah_pnum = mmu_btopr(rp->regspec_size +
1105 1104 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1106 1105 }
1107 1106
1108 1107 #ifdef DDI_MAP_DEBUG
1109 1108 ddi_map_debug(
1110 1109 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1111 1110 rp->regspec_size, *vaddrp);
1112 1111 #endif /* DDI_MAP_DEBUG */
1113 1112 return (DDI_SUCCESS);
1114 1113 }
1115 1114
1116 1115 /*
1117 1116 * Memory space
1118 1117 */
1119 1118
1120 1119 if (hp != NULL) {
1121 1120 /*
1122 1121 * hat layer ignores
1123 1122 * hp->ah_acc.devacc_attr_endian_flags.
1124 1123 */
1125 1124 switch (hp->ah_acc.devacc_attr_dataorder) {
1126 1125 case DDI_STRICTORDER_ACC:
1127 1126 hat_acc_flags = HAT_STRICTORDER;
1128 1127 break;
1129 1128 case DDI_UNORDERED_OK_ACC:
1130 1129 hat_acc_flags = HAT_UNORDERED_OK;
1131 1130 break;
1132 1131 case DDI_MERGING_OK_ACC:
1133 1132 hat_acc_flags = HAT_MERGING_OK;
1134 1133 break;
1135 1134 case DDI_LOADCACHING_OK_ACC:
1136 1135 hat_acc_flags = HAT_LOADCACHING_OK;
1137 1136 break;
1138 1137 case DDI_STORECACHING_OK_ACC:
1139 1138 hat_acc_flags = HAT_STORECACHING_OK;
1140 1139 break;
1141 1140 }
1142 1141 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1143 1142 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1144 1143 impl_acc_hdl_init(hp);
1145 1144 hp->ah_hat_flags = hat_acc_flags;
1146 1145 } else {
1147 1146 hat_acc_flags = HAT_STRICTORDER;
1148 1147 }
1149 1148
1150 1149 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1151 1150 #ifdef __xpv
1152 1151 /*
1153 1152 * If we're dom0, we're using a real device so we need to translate
1154 1153 * the MA to a PA.
1155 1154 */
1156 1155 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1157 1156 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1158 1157 } else {
1159 1158 pbase = rbase;
1160 1159 }
1161 1160 #else
1162 1161 pbase = rbase;
1163 1162 #endif
1164 1163 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1165 1164
1166 1165 if (rp->regspec_size == 0) {
1167 1166 #ifdef DDI_MAP_DEBUG
1168 1167 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1169 1168 #endif /* DDI_MAP_DEBUG */
1170 1169 return (DDI_ME_INVAL);
1171 1170 }
1172 1171
1173 1172 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1174 1173 /* extra cast to make gcc happy */
1175 1174 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1176 1175 } else {
1177 1176 npages = mmu_btopr(rp->regspec_size + pgoffset);
1178 1177
1179 1178 #ifdef DDI_MAP_DEBUG
1180 1179 ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1181 1180 "physical %llx", npages, pbase);
1182 1181 #endif /* DDI_MAP_DEBUG */
1183 1182
1184 1183 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1185 1184 if (cvaddr == NULL)
1186 1185 return (DDI_ME_NORESOURCES);
1187 1186
1188 1187 /*
1189 1188 * Now map in the pages we've allocated...
1190 1189 */
1191 1190 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1192 1191 mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1193 1192 HAT_LOAD_LOCK);
1194 1193 *vaddrp = (caddr_t)cvaddr + pgoffset;
1195 1194
1196 1195 /* save away pfn and npages for FMA */
1197 1196 hp = mp->map_handlep;
1198 1197 if (hp) {
1199 1198 hp->ah_pfn = mmu_btop(pbase);
1200 1199 hp->ah_pnum = npages;
1201 1200 }
1202 1201 }
1203 1202
1204 1203 #ifdef DDI_MAP_DEBUG
1205 1204 ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1206 1205 #endif /* DDI_MAP_DEBUG */
1207 1206 return (DDI_SUCCESS);
1208 1207 }
1209 1208
1210 1209
1211 1210 /*
1212 1211 * rootnex_unmap_regspec()
1213 1212 *
1214 1213 */
1215 1214 static int
1216 1215 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1217 1216 {
1218 1217 caddr_t addr = (caddr_t)*vaddrp;
1219 1218 uint_t npages, pgoffset;
1220 1219 struct regspec *rp;
1221 1220
1222 1221 if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1223 1222 return (0);
1224 1223
1225 1224 rp = mp->map_obj.rp;
1226 1225
1227 1226 if (rp->regspec_size == 0) {
1228 1227 #ifdef DDI_MAP_DEBUG
1229 1228 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1230 1229 #endif /* DDI_MAP_DEBUG */
1231 1230 return (DDI_ME_INVAL);
1232 1231 }
1233 1232
1234 1233 /*
1235 1234 * I/O or memory mapping:
1236 1235 *
1237 1236 * <bustype=0, addr=x, len=x>: memory
1238 1237 * <bustype=1, addr=x, len=x>: i/o
1239 1238 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1240 1239 */
1241 1240 if (rp->regspec_bustype != 0) {
1242 1241 /*
1243 1242 * This is I/O space, which requires no particular
1244 1243 * processing on unmap since it isn't mapped in the
1245 1244 * first place.
1246 1245 */
1247 1246 return (DDI_SUCCESS);
1248 1247 }
1249 1248
1250 1249 /*
1251 1250 * Memory space
1252 1251 */
1253 1252 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1254 1253 npages = mmu_btopr(rp->regspec_size + pgoffset);
1255 1254 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1256 1255 device_arena_free(addr - pgoffset, ptob(npages));
1257 1256
1258 1257 /*
1259 1258 * Destroy the pointer - the mapping has logically gone
1260 1259 */
1261 1260 *vaddrp = NULL;
1262 1261
1263 1262 return (DDI_SUCCESS);
1264 1263 }
1265 1264
1266 1265
1267 1266 /*
1268 1267 * rootnex_map_handle()
1269 1268 *
1270 1269 */
1271 1270 static int
1272 1271 rootnex_map_handle(ddi_map_req_t *mp)
1273 1272 {
1274 1273 rootnex_addr_t rbase;
1275 1274 ddi_acc_hdl_t *hp;
1276 1275 uint_t pgoffset;
1277 1276 struct regspec *rp;
1278 1277 paddr_t pbase;
1279 1278
1280 1279 rp = mp->map_obj.rp;
1281 1280
1282 1281 #ifdef DDI_MAP_DEBUG
1283 1282 ddi_map_debug(
1284 1283 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1285 1284 rp->regspec_bustype, rp->regspec_addr,
1286 1285 rp->regspec_size, mp->map_handlep);
1287 1286 #endif /* DDI_MAP_DEBUG */
1288 1287
1289 1288 /*
1290 1289 * I/O or memory mapping:
1291 1290 *
1292 1291 * <bustype=0, addr=x, len=x>: memory
1293 1292 * <bustype=1, addr=x, len=x>: i/o
1294 1293 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1295 1294 */
1296 1295 if (rp->regspec_bustype != 0) {
1297 1296 /*
1298 1297 * This refers to I/O space, and we don't support "mapping"
1299 1298 * I/O space to a user.
1300 1299 */
1301 1300 return (DDI_FAILURE);
1302 1301 }
1303 1302
1304 1303 /*
1305 1304 * Set up the hat_flags for the mapping.
1306 1305 */
1307 1306 hp = mp->map_handlep;
1308 1307
1309 1308 switch (hp->ah_acc.devacc_attr_endian_flags) {
1310 1309 case DDI_NEVERSWAP_ACC:
1311 1310 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1312 1311 break;
1313 1312 case DDI_STRUCTURE_LE_ACC:
1314 1313 hp->ah_hat_flags = HAT_STRUCTURE_LE;
1315 1314 break;
1316 1315 case DDI_STRUCTURE_BE_ACC:
1317 1316 return (DDI_FAILURE);
1318 1317 default:
1319 1318 return (DDI_REGS_ACC_CONFLICT);
1320 1319 }
1321 1320
1322 1321 switch (hp->ah_acc.devacc_attr_dataorder) {
1323 1322 case DDI_STRICTORDER_ACC:
1324 1323 break;
1325 1324 case DDI_UNORDERED_OK_ACC:
1326 1325 hp->ah_hat_flags |= HAT_UNORDERED_OK;
1327 1326 break;
1328 1327 case DDI_MERGING_OK_ACC:
1329 1328 hp->ah_hat_flags |= HAT_MERGING_OK;
1330 1329 break;
1331 1330 case DDI_LOADCACHING_OK_ACC:
1332 1331 hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1333 1332 break;
1334 1333 case DDI_STORECACHING_OK_ACC:
1335 1334 hp->ah_hat_flags |= HAT_STORECACHING_OK;
1336 1335 break;
1337 1336 default:
1338 1337 return (DDI_FAILURE);
1339 1338 }
1340 1339
1341 1340 rbase = (rootnex_addr_t)rp->regspec_addr &
1342 1341 (~(rootnex_addr_t)MMU_PAGEOFFSET);
1343 1342 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1344 1343
1345 1344 if (rp->regspec_size == 0)
1346 1345 return (DDI_ME_INVAL);
1347 1346
1348 1347 #ifdef __xpv
1349 1348 /*
1350 1349 * If we're dom0, we're using a real device so we need to translate
1351 1350 * the MA to a PA.
1352 1351 */
1353 1352 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1354 1353 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1355 1354 (rbase & MMU_PAGEOFFSET);
1356 1355 } else {
1357 1356 pbase = rbase;
1358 1357 }
1359 1358 #else
1360 1359 pbase = rbase;
1361 1360 #endif
1362 1361
1363 1362 hp->ah_pfn = mmu_btop(pbase);
1364 1363 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1365 1364
1366 1365 return (DDI_SUCCESS);
1367 1366 }
1368 1367
1369 1368
1370 1369
1371 1370 /*
1372 1371 * ************************
1373 1372 * interrupt related code
1374 1373 * ************************
1375 1374 */
1376 1375
1377 1376 /*
1378 1377 * rootnex_intr_ops()
1379 1378 * bus_intr_op() function for interrupt support
1380 1379 */
1381 1380 /* ARGSUSED */
1382 1381 static int
1383 1382 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1384 1383 ddi_intr_handle_impl_t *hdlp, void *result)
1385 1384 {
1386 1385 struct intrspec *ispec;
1387 1386
1388 1387 DDI_INTR_NEXDBG((CE_CONT,
1389 1388 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1390 1389 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1391 1390
1392 1391 /* Process the interrupt operation */
1393 1392 switch (intr_op) {
1394 1393 case DDI_INTROP_GETCAP:
1395 1394 /* First check with pcplusmp */
1396 1395 if (psm_intr_ops == NULL)
1397 1396 return (DDI_FAILURE);
1398 1397
1399 1398 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1400 1399 *(int *)result = 0;
1401 1400 return (DDI_FAILURE);
1402 1401 }
1403 1402 break;
1404 1403 case DDI_INTROP_SETCAP:
1405 1404 if (psm_intr_ops == NULL)
1406 1405 return (DDI_FAILURE);
1407 1406
1408 1407 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1409 1408 return (DDI_FAILURE);
1410 1409 break;
1411 1410 case DDI_INTROP_ALLOC:
1412 1411 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1413 1412 return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1414 1413 case DDI_INTROP_FREE:
1415 1414 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1416 1415 return (rootnex_free_intr_fixed(rdip, hdlp));
1417 1416 case DDI_INTROP_GETPRI:
1418 1417 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1419 1418 return (DDI_FAILURE);
1420 1419 *(int *)result = ispec->intrspec_pri;
1421 1420 break;
1422 1421 case DDI_INTROP_SETPRI:
1423 1422 /* Validate the interrupt priority passed to us */
1424 1423 if (*(int *)result > LOCK_LEVEL)
1425 1424 return (DDI_FAILURE);
1426 1425
1427 1426 /* Ensure that PSM is all initialized and ispec is ok */
1428 1427 if ((psm_intr_ops == NULL) ||
1429 1428 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1430 1429 return (DDI_FAILURE);
1431 1430
1432 1431 /* Change the priority */
1433 1432 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1434 1433 PSM_FAILURE)
1435 1434 return (DDI_FAILURE);
1436 1435
1437 1436 /* update the ispec with the new priority */
1438 1437 ispec->intrspec_pri = *(int *)result;
1439 1438 break;
1440 1439 case DDI_INTROP_ADDISR:
1441 1440 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1442 1441 return (DDI_FAILURE);
1443 1442 ispec->intrspec_func = hdlp->ih_cb_func;
1444 1443 break;
1445 1444 case DDI_INTROP_REMISR:
1446 1445 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1447 1446 return (DDI_FAILURE);
1448 1447 ispec->intrspec_func = (uint_t (*)()) 0;
1449 1448 break;
1450 1449 case DDI_INTROP_ENABLE:
1451 1450 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1452 1451 return (DDI_FAILURE);
1453 1452
1454 1453 /* Call psmi to translate irq with the dip */
1455 1454 if (psm_intr_ops == NULL)
1456 1455 return (DDI_FAILURE);
1457 1456
1458 1457 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1459 1458 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1460 1459 (int *)&hdlp->ih_vector) == PSM_FAILURE)
1461 1460 return (DDI_FAILURE);
1462 1461
1463 1462 /* Add the interrupt handler */
1464 1463 if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1465 1464 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1466 1465 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1467 1466 return (DDI_FAILURE);
1468 1467 break;
1469 1468 case DDI_INTROP_DISABLE:
1470 1469 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1471 1470 return (DDI_FAILURE);
1472 1471
1473 1472 /* Call psm_ops() to translate irq with the dip */
1474 1473 if (psm_intr_ops == NULL)
1475 1474 return (DDI_FAILURE);
1476 1475
1477 1476 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1478 1477 (void) (*psm_intr_ops)(rdip, hdlp,
1479 1478 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1480 1479
1481 1480 /* Remove the interrupt handler */
1482 1481 rem_avintr((void *)hdlp, ispec->intrspec_pri,
1483 1482 hdlp->ih_cb_func, hdlp->ih_vector);
1484 1483 break;
1485 1484 case DDI_INTROP_SETMASK:
1486 1485 if (psm_intr_ops == NULL)
1487 1486 return (DDI_FAILURE);
1488 1487
1489 1488 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1490 1489 return (DDI_FAILURE);
1491 1490 break;
1492 1491 case DDI_INTROP_CLRMASK:
1493 1492 if (psm_intr_ops == NULL)
1494 1493 return (DDI_FAILURE);
1495 1494
1496 1495 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1497 1496 return (DDI_FAILURE);
1498 1497 break;
1499 1498 case DDI_INTROP_GETPENDING:
1500 1499 if (psm_intr_ops == NULL)
1501 1500 return (DDI_FAILURE);
1502 1501
1503 1502 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1504 1503 result)) {
1505 1504 *(int *)result = 0;
1506 1505 return (DDI_FAILURE);
1507 1506 }
1508 1507 break;
1509 1508 case DDI_INTROP_NAVAIL:
1510 1509 case DDI_INTROP_NINTRS:
1511 1510 *(int *)result = i_ddi_get_intx_nintrs(rdip);
1512 1511 if (*(int *)result == 0) {
1513 1512 /*
1514 1513 * Special case for 'pcic' driver' only. This driver
1515 1514 * driver is a child of 'isa' and 'rootnex' drivers.
1516 1515 *
1517 1516 * See detailed comments on this in the function
1518 1517 * rootnex_get_ispec().
1519 1518 *
1520 1519 * Children of 'pcic' send 'NINITR' request all the
1521 1520 * way to rootnex driver. But, the 'pdp->par_nintr'
1522 1521 * field may not initialized. So, we fake it here
1523 1522 * to return 1 (a la what PCMCIA nexus does).
1524 1523 */
1525 1524 if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1526 1525 *(int *)result = 1;
1527 1526 else
1528 1527 return (DDI_FAILURE);
1529 1528 }
1530 1529 break;
1531 1530 case DDI_INTROP_SUPPORTED_TYPES:
1532 1531 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */
1533 1532 break;
1534 1533 default:
1535 1534 return (DDI_FAILURE);
1536 1535 }
1537 1536
1538 1537 return (DDI_SUCCESS);
1539 1538 }
1540 1539
1541 1540
1542 1541 /*
1543 1542 * rootnex_get_ispec()
1544 1543 * convert an interrupt number to an interrupt specification.
1545 1544 * The interrupt number determines which interrupt spec will be
1546 1545 * returned if more than one exists.
1547 1546 *
1548 1547 * Look into the parent private data area of the 'rdip' to find out
1549 1548 * the interrupt specification. First check to make sure there is
1550 1549 * one that matchs "inumber" and then return a pointer to it.
1551 1550 *
1552 1551 * Return NULL if one could not be found.
1553 1552 *
1554 1553 * NOTE: This is needed for rootnex_intr_ops()
1555 1554 */
1556 1555 static struct intrspec *
1557 1556 rootnex_get_ispec(dev_info_t *rdip, int inum)
1558 1557 {
1559 1558 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1560 1559
1561 1560 /*
1562 1561 * Special case handling for drivers that provide their own
1563 1562 * intrspec structures instead of relying on the DDI framework.
1564 1563 *
1565 1564 * A broken hardware driver in ON could potentially provide its
1566 1565 * own intrspec structure, instead of relying on the hardware.
1567 1566 * If these drivers are children of 'rootnex' then we need to
1568 1567 * continue to provide backward compatibility to them here.
1569 1568 *
1570 1569 * Following check is a special case for 'pcic' driver which
1571 1570 * was found to have broken hardwre andby provides its own intrspec.
1572 1571 *
1573 1572 * Verbatim comments from this driver are shown here:
1574 1573 * "Don't use the ddi_add_intr since we don't have a
1575 1574 * default intrspec in all cases."
1576 1575 *
1577 1576 * Since an 'ispec' may not be always created for it,
1578 1577 * check for that and create one if so.
1579 1578 *
1580 1579 * NOTE: Currently 'pcic' is the only driver found to do this.
1581 1580 */
1582 1581 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1583 1582 pdp->par_nintr = 1;
1584 1583 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1585 1584 pdp->par_nintr, KM_SLEEP);
1586 1585 }
1587 1586
1588 1587 /* Validate the interrupt number */
1589 1588 if (inum >= pdp->par_nintr)
1590 1589 return (NULL);
1591 1590
1592 1591 /* Get the interrupt structure pointer and return that */
1593 1592 return ((struct intrspec *)&pdp->par_intr[inum]);
1594 1593 }
1595 1594
1596 1595 /*
1597 1596 * Allocate interrupt vector for FIXED (legacy) type.
1598 1597 */
1599 1598 static int
1600 1599 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1601 1600 void *result)
1602 1601 {
1603 1602 struct intrspec *ispec;
1604 1603 ddi_intr_handle_impl_t info_hdl;
1605 1604 int ret;
1606 1605 int free_phdl = 0;
1607 1606 apic_get_type_t type_info;
1608 1607
1609 1608 if (psm_intr_ops == NULL)
1610 1609 return (DDI_FAILURE);
1611 1610
1612 1611 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1613 1612 return (DDI_FAILURE);
1614 1613
1615 1614 /*
1616 1615 * If the PSM module is "APIX" then pass the request for it
1617 1616 * to allocate the vector now.
1618 1617 */
1619 1618 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1620 1619 info_hdl.ih_private = &type_info;
1621 1620 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1622 1621 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1623 1622 if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1624 1623 free_phdl = 1;
1625 1624 i_ddi_alloc_intr_phdl(hdlp);
1626 1625 }
1627 1626 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1628 1627 ret = (*psm_intr_ops)(rdip, hdlp,
1629 1628 PSM_INTR_OP_ALLOC_VECTORS, result);
1630 1629 if (free_phdl) { /* free up the phdl structure */
1631 1630 free_phdl = 0;
1632 1631 i_ddi_free_intr_phdl(hdlp);
1633 1632 hdlp->ih_private = NULL;
1634 1633 }
1635 1634 } else {
1636 1635 /*
1637 1636 * No APIX module; fall back to the old scheme where the
1638 1637 * interrupt vector is allocated during ddi_enable_intr() call.
1639 1638 */
1640 1639 hdlp->ih_pri = ispec->intrspec_pri;
1641 1640 *(int *)result = hdlp->ih_scratch1;
1642 1641 ret = DDI_SUCCESS;
1643 1642 }
1644 1643
1645 1644 return (ret);
1646 1645 }
1647 1646
1648 1647 /*
1649 1648 * Free up interrupt vector for FIXED (legacy) type.
1650 1649 */
1651 1650 static int
1652 1651 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1653 1652 {
1654 1653 struct intrspec *ispec;
1655 1654 struct ddi_parent_private_data *pdp;
1656 1655 ddi_intr_handle_impl_t info_hdl;
1657 1656 int ret;
1658 1657 apic_get_type_t type_info;
1659 1658
1660 1659 if (psm_intr_ops == NULL)
1661 1660 return (DDI_FAILURE);
1662 1661
1663 1662 /*
1664 1663 * If the PSM module is "APIX" then pass the request for it
1665 1664 * to free up the vector now.
1666 1665 */
1667 1666 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1668 1667 info_hdl.ih_private = &type_info;
1669 1668 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1670 1669 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1671 1670 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1672 1671 return (DDI_FAILURE);
1673 1672 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1674 1673 ret = (*psm_intr_ops)(rdip, hdlp,
1675 1674 PSM_INTR_OP_FREE_VECTORS, NULL);
1676 1675 } else {
1677 1676 /*
1678 1677 * No APIX module; fall back to the old scheme where
1679 1678 * the interrupt vector was already freed during
1680 1679 * ddi_disable_intr() call.
1681 1680 */
1682 1681 ret = DDI_SUCCESS;
1683 1682 }
1684 1683
1685 1684 pdp = ddi_get_parent_data(rdip);
1686 1685
1687 1686 /*
1688 1687 * Special case for 'pcic' driver' only.
1689 1688 * If an intrspec was created for it, clean it up here
1690 1689 * See detailed comments on this in the function
1691 1690 * rootnex_get_ispec().
1692 1691 */
1693 1692 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1694 1693 kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1695 1694 pdp->par_nintr);
1696 1695 /*
1697 1696 * Set it to zero; so that
1698 1697 * DDI framework doesn't free it again
1699 1698 */
1700 1699 pdp->par_intr = NULL;
1701 1700 pdp->par_nintr = 0;
1702 1701 }
1703 1702
1704 1703 return (ret);
1705 1704 }
1706 1705
1707 1706
1708 1707 /*
1709 1708 * ******************
1710 1709 * dma related code
1711 1710 * ******************
1712 1711 */
1713 1712
1714 1713 /*ARGSUSED*/
1715 1714 static int
1716 1715 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1717 1716 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1718 1717 ddi_dma_handle_t *handlep)
1719 1718 {
1720 1719 uint64_t maxsegmentsize_ll;
1721 1720 uint_t maxsegmentsize;
1722 1721 ddi_dma_impl_t *hp;
1723 1722 rootnex_dma_t *dma;
1724 1723 uint64_t count_max;
1725 1724 uint64_t seg;
1726 1725 int kmflag;
1727 1726 int e;
1728 1727
1729 1728
1730 1729 /* convert our sleep flags */
1731 1730 if (waitfp == DDI_DMA_SLEEP) {
1732 1731 kmflag = KM_SLEEP;
1733 1732 } else {
1734 1733 kmflag = KM_NOSLEEP;
1735 1734 }
1736 1735
1737 1736 /*
1738 1737 * We try to do only one memory allocation here. We'll do a little
1739 1738 * pointer manipulation later. If the bind ends up taking more than
1740 1739 * our prealloc's space, we'll have to allocate more memory in the
1741 1740 * bind operation. Not great, but much better than before and the
1742 1741 * best we can do with the current bind interfaces.
1743 1742 */
1744 1743 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1745 1744 if (hp == NULL)
1746 1745 return (DDI_DMA_NORESOURCES);
1747 1746
1748 1747 /* Do our pointer manipulation now, align the structures */
1749 1748 hp->dmai_private = (void *)(((uintptr_t)hp +
1750 1749 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1751 1750 dma = (rootnex_dma_t *)hp->dmai_private;
1752 1751 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1753 1752 sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1754 1753
1755 1754 /* setup the handle */
1756 1755 rootnex_clean_dmahdl(hp);
1757 1756 hp->dmai_error.err_fep = NULL;
1758 1757 hp->dmai_error.err_cf = NULL;
1759 1758 dma->dp_dip = rdip;
1760 1759 dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1761 1760 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1762 1761
1763 1762 /*
1764 1763 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1765 1764 * is being used. Set the upper limit to the seg value.
1766 1765 * There will be enough DVMA space to always get addresses
1767 1766 * that will match the constraints.
1768 1767 */
1769 1768 if (IOMMU_USED(rdip) &&
1770 1769 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1771 1770 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1772 1771 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1773 1772 } else
1774 1773 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1775 1774
1776 1775 hp->dmai_minxfer = attr->dma_attr_minxfer;
1777 1776 hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1778 1777 hp->dmai_rdip = rdip;
1779 1778 hp->dmai_attr = *attr;
1780 1779
1781 1780 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1782 1781 dma->dp_sglinfo.si_cancross = B_FALSE;
1783 1782 else
1784 1783 dma->dp_sglinfo.si_cancross = B_TRUE;
1785 1784
1786 1785 /* we don't need to worry about the SPL since we do a tryenter */
1787 1786 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1788 1787
1789 1788 /*
1790 1789 * Figure out our maximum segment size. If the segment size is greater
1791 1790 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1792 1791 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1793 1792 * dma_attr_count_max are size-1 type values.
1794 1793 *
1795 1794 * Maximum segment size is the largest physically contiguous chunk of
1796 1795 * memory that we can return from a bind (i.e. the maximum size of a
1797 1796 * single cookie).
1798 1797 */
1799 1798
1800 1799 /* handle the rollover cases */
1801 1800 seg = attr->dma_attr_seg + 1;
1802 1801 if (seg < attr->dma_attr_seg) {
1803 1802 seg = attr->dma_attr_seg;
1804 1803 }
1805 1804 count_max = attr->dma_attr_count_max + 1;
1806 1805 if (count_max < attr->dma_attr_count_max) {
1807 1806 count_max = attr->dma_attr_count_max;
1808 1807 }
1809 1808
1810 1809 /*
1811 1810 * granularity may or may not be a power of two. If it isn't, we can't
1812 1811 * use a simple mask.
1813 1812 */
1814 1813 if (!ISP2(attr->dma_attr_granular)) {
1815 1814 dma->dp_granularity_power_2 = B_FALSE;
1816 1815 } else {
1817 1816 dma->dp_granularity_power_2 = B_TRUE;
1818 1817 }
1819 1818
1820 1819 /*
1821 1820 * maxxfer should be a whole multiple of granularity. If we're going to
1822 1821 * break up a window because we're greater than maxxfer, we might as
1823 1822 * well make sure it's maxxfer is a whole multiple so we don't have to
1824 1823 * worry about triming the window later on for this case.
1825 1824 */
1826 1825 if (attr->dma_attr_granular > 1) {
1827 1826 if (dma->dp_granularity_power_2) {
1828 1827 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1829 1828 (attr->dma_attr_maxxfer &
1830 1829 (attr->dma_attr_granular - 1));
1831 1830 } else {
1832 1831 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1833 1832 (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1834 1833 }
1835 1834 } else {
1836 1835 dma->dp_maxxfer = attr->dma_attr_maxxfer;
1837 1836 }
1838 1837
1839 1838 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1840 1839 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1841 1840 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1842 1841 maxsegmentsize = 0xFFFFFFFF;
1843 1842 } else {
1844 1843 maxsegmentsize = maxsegmentsize_ll;
1845 1844 }
1846 1845 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1847 1846 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1848 1847
1849 1848 /* check the ddi_dma_attr arg to make sure it makes a little sense */
1850 1849 if (rootnex_alloc_check_parms) {
1851 1850 e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1852 1851 if (e != DDI_SUCCESS) {
1853 1852 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1854 1853 (void) rootnex_dma_freehdl(dip, rdip,
1855 1854 (ddi_dma_handle_t)hp);
1856 1855 return (e);
1857 1856 }
1858 1857 }
1859 1858
1860 1859 *handlep = (ddi_dma_handle_t)hp;
1861 1860
1862 1861 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1863 1862 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1864 1863 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1865 1864
1866 1865 return (DDI_SUCCESS);
1867 1866 }
1868 1867
1869 1868
1870 1869 /*
1871 1870 * rootnex_dma_allochdl()
1872 1871 * called from ddi_dma_alloc_handle().
1873 1872 */
1874 1873 static int
1875 1874 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1876 1875 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1877 1876 {
1878 1877 int retval = DDI_SUCCESS;
1879 1878 #if defined(__amd64) && !defined(__xpv)
1880 1879
1881 1880 if (IOMMU_UNITIALIZED(rdip)) {
1882 1881 retval = iommulib_nex_open(dip, rdip);
1883 1882
1884 1883 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1885 1884 return (retval);
1886 1885 }
1887 1886
1888 1887 if (IOMMU_UNUSED(rdip)) {
1889 1888 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1890 1889 handlep);
1891 1890 } else {
1892 1891 retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1893 1892 waitfp, arg, handlep);
1894 1893 }
1895 1894 #else
1896 1895 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1897 1896 handlep);
1898 1897 #endif
1899 1898 switch (retval) {
1900 1899 case DDI_DMA_NORESOURCES:
1901 1900 if (waitfp != DDI_DMA_DONTWAIT) {
1902 1901 ddi_set_callback(waitfp, arg,
1903 1902 &rootnex_state->r_dvma_call_list_id);
1904 1903 }
1905 1904 break;
1906 1905 case DDI_SUCCESS:
1907 1906 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1908 1907 break;
1909 1908 default:
1910 1909 break;
1911 1910 }
1912 1911 return (retval);
1913 1912 }
1914 1913
1915 1914 /*ARGSUSED*/
1916 1915 static int
1917 1916 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1918 1917 ddi_dma_handle_t handle)
1919 1918 {
1920 1919 ddi_dma_impl_t *hp;
1921 1920 rootnex_dma_t *dma;
1922 1921
1923 1922
1924 1923 hp = (ddi_dma_impl_t *)handle;
1925 1924 dma = (rootnex_dma_t *)hp->dmai_private;
1926 1925
1927 1926 /* unbind should have been called first */
1928 1927 ASSERT(!dma->dp_inuse);
1929 1928
1930 1929 mutex_destroy(&dma->dp_mutex);
1931 1930 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1932 1931
1933 1932 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1934 1933 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1935 1934 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1936 1935
1937 1936 return (DDI_SUCCESS);
1938 1937 }
1939 1938
1940 1939 /*
1941 1940 * rootnex_dma_freehdl()
1942 1941 * called from ddi_dma_free_handle().
1943 1942 */
1944 1943 static int
1945 1944 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1946 1945 {
1947 1946 int ret;
1948 1947
1949 1948 ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1950 1949 #if defined(__amd64) && !defined(__xpv)
1951 1950 if (IOMMU_USED(rdip))
1952 1951 ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1953 1952 else
1954 1953 #endif
1955 1954 ret = rootnex_coredma_freehdl(dip, rdip, handle);
1956 1955
1957 1956 if (rootnex_state->r_dvma_call_list_id)
1958 1957 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1959 1958
1960 1959 return (ret);
1961 1960 }
1962 1961
1963 1962 /*ARGSUSED*/
1964 1963 static int
1965 1964 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1966 1965 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1967 1966 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1968 1967 {
1969 1968 rootnex_sglinfo_t *sinfo;
1970 1969 ddi_dma_obj_t *dmao;
1971 1970 #if defined(__amd64) && !defined(__xpv)
1972 1971 struct dvmaseg *dvs;
1973 1972 ddi_dma_cookie_t *cookie;
1974 1973 #endif
1975 1974 ddi_dma_attr_t *attr;
1976 1975 ddi_dma_impl_t *hp;
1977 1976 rootnex_dma_t *dma;
1978 1977 int kmflag;
1979 1978 int e;
1980 1979 uint_t ncookies;
1981 1980
1982 1981 hp = (ddi_dma_impl_t *)handle;
1983 1982 dma = (rootnex_dma_t *)hp->dmai_private;
1984 1983 dmao = &dma->dp_dma;
1985 1984 sinfo = &dma->dp_sglinfo;
1986 1985 attr = &hp->dmai_attr;
1987 1986
1988 1987 /* convert the sleep flags */
1989 1988 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1990 1989 dma->dp_sleep_flags = kmflag = KM_SLEEP;
1991 1990 } else {
1992 1991 dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1993 1992 }
1994 1993
1995 1994 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1996 1995
1997 1996 /*
1998 1997 * This is useful for debugging a driver. Not as useful in a production
1999 1998 * system. The only time this will fail is if you have a driver bug.
2000 1999 */
2001 2000 if (rootnex_bind_check_inuse) {
2002 2001 /*
2003 2002 * No one else should ever have this lock unless someone else
2004 2003 * is trying to use this handle. So contention on the lock
2005 2004 * is the same as inuse being set.
2006 2005 */
2007 2006 e = mutex_tryenter(&dma->dp_mutex);
2008 2007 if (e == 0) {
2009 2008 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2010 2009 return (DDI_DMA_INUSE);
2011 2010 }
2012 2011 if (dma->dp_inuse) {
2013 2012 mutex_exit(&dma->dp_mutex);
2014 2013 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2015 2014 return (DDI_DMA_INUSE);
2016 2015 }
2017 2016 dma->dp_inuse = B_TRUE;
2018 2017 mutex_exit(&dma->dp_mutex);
2019 2018 }
2020 2019
2021 2020 /* check the ddi_dma_attr arg to make sure it makes a little sense */
2022 2021 if (rootnex_bind_check_parms) {
2023 2022 e = rootnex_valid_bind_parms(dmareq, attr);
2024 2023 if (e != DDI_SUCCESS) {
2025 2024 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2026 2025 rootnex_clean_dmahdl(hp);
2027 2026 return (e);
2028 2027 }
2029 2028 }
2030 2029
2031 2030 /* save away the original bind info */
2032 2031 dma->dp_dma = dmareq->dmar_object;
2033 2032
2034 2033 #if defined(__amd64) && !defined(__xpv)
2035 2034 if (IOMMU_USED(rdip)) {
2036 2035 dmao = &dma->dp_dvma;
2037 2036 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2038 2037 switch (e) {
2039 2038 case DDI_SUCCESS:
2040 2039 if (sinfo->si_cancross ||
2041 2040 dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2042 2041 dmao->dmao_size > sinfo->si_max_cookie_size) {
2043 2042 dma->dp_dvma_used = B_TRUE;
2044 2043 break;
2045 2044 }
2046 2045 sinfo->si_sgl_size = 1;
2047 2046 hp->dmai_rflags |= DMP_NOSYNC;
2048 2047
2049 2048 dma->dp_dvma_used = B_TRUE;
2050 2049 dma->dp_need_to_free_cookie = B_FALSE;
2051 2050
2052 2051 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2053 2052 cookie = hp->dmai_cookie = dma->dp_cookies =
2054 2053 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2055 2054 cookie->dmac_laddress = dvs->dvs_start +
2056 2055 dmao->dmao_obj.dvma_obj.dv_off;
2057 2056 cookie->dmac_size = dvs->dvs_len;
2058 2057 cookie->dmac_type = 0;
2059 2058
2060 2059 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2061 2060 rdip);
2062 2061 goto fast;
2063 2062 case DDI_ENOTSUP:
2064 2063 break;
2065 2064 default:
2066 2065 rootnex_clean_dmahdl(hp);
2067 2066 return (e);
2068 2067 }
2069 2068 }
2070 2069 #endif
2071 2070
2072 2071 /*
2073 2072 * Figure out a rough estimate of what maximum number of pages
2074 2073 * this buffer could use (a high estimate of course).
2075 2074 */
2076 2075 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2077 2076
2078 2077 if (dma->dp_dvma_used) {
2079 2078 /*
2080 2079 * The number of physical pages is the worst case.
2081 2080 *
2082 2081 * For DVMA, the worst case is the length divided
2083 2082 * by the maximum cookie length, plus 1. Add to that
2084 2083 * the number of segment boundaries potentially crossed, and
2085 2084 * the additional number of DVMA segments that was returned.
2086 2085 *
2087 2086 * In the normal case, for modern devices, si_cancross will
2088 2087 * be false, and dv_nseg will be 1, and the fast path will
2089 2088 * have been taken above.
2090 2089 */
2091 2090 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2092 2091 + 1;
2093 2092 if (sinfo->si_cancross)
2094 2093 ncookies +=
2095 2094 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2096 2095 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2097 2096
2098 2097 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2099 2098 }
2100 2099
2101 2100 /*
2102 2101 * We'll use the pre-allocated cookies for any bind that will *always*
2103 2102 * fit (more important to be consistent, we don't want to create
2104 2103 * additional degenerate cases).
2105 2104 */
2106 2105 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2107 2106 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2108 2107 dma->dp_need_to_free_cookie = B_FALSE;
2109 2108 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2110 2109 uint_t, sinfo->si_max_pages);
2111 2110
2112 2111 /*
2113 2112 * For anything larger than that, we'll go ahead and allocate the
2114 2113 * maximum number of pages we expect to see. Hopefuly, we won't be
2115 2114 * seeing this path in the fast path for high performance devices very
2116 2115 * frequently.
2117 2116 *
2118 2117 * a ddi bind interface that allowed the driver to provide storage to
2119 2118 * the bind interface would speed this case up.
2120 2119 */
2121 2120 } else {
2122 2121 /*
2123 2122 * Save away how much memory we allocated. If we're doing a
2124 2123 * nosleep, the alloc could fail...
2125 2124 */
2126 2125 dma->dp_cookie_size = sinfo->si_max_pages *
2127 2126 sizeof (ddi_dma_cookie_t);
2128 2127 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2129 2128 if (dma->dp_cookies == NULL) {
2130 2129 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2131 2130 rootnex_clean_dmahdl(hp);
2132 2131 return (DDI_DMA_NORESOURCES);
2133 2132 }
2134 2133 dma->dp_need_to_free_cookie = B_TRUE;
2135 2134 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2136 2135 uint_t, sinfo->si_max_pages);
2137 2136 }
2138 2137 hp->dmai_cookie = dma->dp_cookies;
2139 2138
2140 2139 /*
2141 2140 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2142 2141 * looking at the constraints in the dma structure. It will then put
2143 2142 * some additional state about the sgl in the dma struct (i.e. is
2144 2143 * the sgl clean, or do we need to do some munging; how many pages
2145 2144 * need to be copied, etc.)
2146 2145 */
2147 2146 if (dma->dp_dvma_used)
2148 2147 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2149 2148 else
2150 2149 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2151 2150
2152 2151 out:
2153 2152 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2154 2153 /* if we don't need a copy buffer, we don't need to sync */
2155 2154 if (sinfo->si_copybuf_req == 0) {
2156 2155 hp->dmai_rflags |= DMP_NOSYNC;
2157 2156 }
2158 2157
2159 2158 /*
2160 2159 * if we don't need the copybuf and we don't need to do a partial, we
2161 2160 * hit the fast path. All the high performance devices should be trying
2162 2161 * to hit this path. To hit this path, a device should be able to reach
2163 2162 * all of memory, shouldn't try to bind more than it can transfer, and
2164 2163 * the buffer shouldn't require more cookies than the driver/device can
2165 2164 * handle [sgllen]).
2166 2165 *
2167 2166 * Note that negative values of dma_attr_sgllen are supposed
2168 2167 * to mean unlimited, but we just cast them to mean a
2169 2168 * "ridiculous large limit". This saves some extra checks on
2170 2169 * hot paths.
2171 2170 */
2172 2171 if ((sinfo->si_copybuf_req == 0) &&
2173 2172 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2174 2173 (dmao->dmao_size < dma->dp_maxxfer)) {
2175 2174 fast:
2176 2175 /*
2177 2176 * If the driver supports FMA, insert the handle in the FMA DMA
2178 2177 * handle cache.
2179 2178 */
2180 2179 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2181 2180 hp->dmai_error.err_cf = rootnex_dma_check;
2182 2181
2183 2182 /*
2184 2183 * copy out the first cookie and ccountp, set the cookie
2185 2184 * pointer to the second cookie. The first cookie is passed
2186 2185 * back on the stack. Additional cookies are accessed via
2187 2186 * ddi_dma_nextcookie()
2188 2187 */
2189 2188 *cookiep = dma->dp_cookies[0];
2190 2189 *ccountp = sinfo->si_sgl_size;
2191 2190 hp->dmai_cookie++;
2192 2191 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2193 2192 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2194 2193 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2195 2194 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2196 2195 uint_t, dmao->dmao_size, uint_t, *ccountp);
2197 2196
2198 2197
2199 2198 return (DDI_DMA_MAPPED);
2200 2199 }
2201 2200
2202 2201 /*
2203 2202 * go to the slow path, we may need to alloc more memory, create
2204 2203 * multiple windows, and munge up a sgl to make the device happy.
2205 2204 */
2206 2205
2207 2206 /*
2208 2207 * With the IOMMU mapobject method used, we should never hit
2209 2208 * the slow path. If we do, something is seriously wrong.
2210 2209 * Clean up and return an error.
2211 2210 */
2212 2211
2213 2212 #if defined(__amd64) && !defined(__xpv)
2214 2213
2215 2214 if (dma->dp_dvma_used) {
2216 2215 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2217 2216 &dma->dp_dvma);
2218 2217 e = DDI_DMA_NOMAPPING;
2219 2218 } else {
2220 2219 #endif
2221 2220 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2222 2221 kmflag);
2223 2222 #if defined(__amd64) && !defined(__xpv)
2224 2223 }
2225 2224 #endif
2226 2225 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2227 2226 if (dma->dp_need_to_free_cookie) {
2228 2227 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2229 2228 }
2230 2229 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2231 2230 rootnex_clean_dmahdl(hp); /* must be after free cookie */
2232 2231 return (e);
2233 2232 }
2234 2233
2235 2234 /*
2236 2235 * If the driver supports FMA, insert the handle in the FMA DMA handle
2237 2236 * cache.
2238 2237 */
2239 2238 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2240 2239 hp->dmai_error.err_cf = rootnex_dma_check;
2241 2240
2242 2241 /* if the first window uses the copy buffer, sync it for the device */
2243 2242 if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2244 2243 (hp->dmai_rflags & DDI_DMA_WRITE)) {
2245 2244 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2246 2245 DDI_DMA_SYNC_FORDEV);
2247 2246 }
2248 2247
2249 2248 /*
2250 2249 * copy out the first cookie and ccountp, set the cookie pointer to the
2251 2250 * second cookie. Make sure the partial flag is set/cleared correctly.
2252 2251 * If we have a partial map (i.e. multiple windows), the number of
2253 2252 * cookies we return is the number of cookies in the first window.
2254 2253 */
2255 2254 if (e == DDI_DMA_MAPPED) {
2256 2255 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2257 2256 *ccountp = sinfo->si_sgl_size;
2258 2257 hp->dmai_nwin = 1;
2259 2258 } else {
2260 2259 hp->dmai_rflags |= DDI_DMA_PARTIAL;
2261 2260 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2262 2261 ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2263 2262 }
2264 2263 *cookiep = dma->dp_cookies[0];
2265 2264 hp->dmai_cookie++;
2266 2265
2267 2266 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2268 2267 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2269 2268 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2270 2269 dmao->dmao_size, uint_t, *ccountp);
2271 2270 return (e);
2272 2271 }
2273 2272
2274 2273 /*
2275 2274 * rootnex_dma_bindhdl()
2276 2275 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2277 2276 */
2278 2277 static int
2279 2278 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2280 2279 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2281 2280 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2282 2281 {
2283 2282 int ret;
2284 2283 #if defined(__amd64) && !defined(__xpv)
2285 2284 if (IOMMU_USED(rdip))
2286 2285 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2287 2286 cookiep, ccountp);
2288 2287 else
2289 2288 #endif
2290 2289 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2291 2290 cookiep, ccountp);
2292 2291
2293 2292 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2294 2293 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2295 2294 &rootnex_state->r_dvma_call_list_id);
2296 2295 }
2297 2296
2298 2297 return (ret);
2299 2298 }
2300 2299
2301 2300
2302 2301
2303 2302 /*ARGSUSED*/
2304 2303 static int
2305 2304 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2306 2305 ddi_dma_handle_t handle)
2307 2306 {
2308 2307 ddi_dma_impl_t *hp;
2309 2308 rootnex_dma_t *dma;
2310 2309 int e;
2311 2310
2312 2311 hp = (ddi_dma_impl_t *)handle;
2313 2312 dma = (rootnex_dma_t *)hp->dmai_private;
2314 2313
2315 2314 /* make sure the buffer wasn't free'd before calling unbind */
2316 2315 if (rootnex_unbind_verify_buffer) {
2317 2316 e = rootnex_verify_buffer(dma);
2318 2317 if (e != DDI_SUCCESS) {
2319 2318 ASSERT(0);
2320 2319 return (DDI_FAILURE);
2321 2320 }
2322 2321 }
2323 2322
2324 2323 /* sync the current window before unbinding the buffer */
2325 2324 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2326 2325 (hp->dmai_rflags & DDI_DMA_READ)) {
2327 2326 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2328 2327 DDI_DMA_SYNC_FORCPU);
2329 2328 }
2330 2329
2331 2330 /*
2332 2331 * cleanup and copy buffer or window state. if we didn't use the copy
2333 2332 * buffer or windows, there won't be much to do :-)
2334 2333 */
2335 2334 rootnex_teardown_copybuf(dma);
2336 2335 rootnex_teardown_windows(dma);
2337 2336
2338 2337 #if defined(__amd64) && !defined(__xpv)
2339 2338 if (IOMMU_USED(rdip))
2340 2339 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2341 2340 &dma->dp_dvma);
2342 2341 #endif
2343 2342
2344 2343 /*
2345 2344 * If we had to allocate space to for the worse case sgl (it didn't
2346 2345 * fit into our pre-allocate buffer), free that up now
2347 2346 */
2348 2347 if (dma->dp_need_to_free_cookie) {
2349 2348 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2350 2349 }
2351 2350
2352 2351 /*
2353 2352 * clean up the handle so it's ready for the next bind (i.e. if the
2354 2353 * handle is reused).
2355 2354 */
2356 2355 rootnex_clean_dmahdl(hp);
2357 2356 hp->dmai_error.err_cf = NULL;
2358 2357
2359 2358 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2360 2359 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2361 2360 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2362 2361
2363 2362 return (DDI_SUCCESS);
2364 2363 }
2365 2364
2366 2365 /*
2367 2366 * rootnex_dma_unbindhdl()
2368 2367 * called from ddi_dma_unbind_handle()
2369 2368 */
2370 2369 /*ARGSUSED*/
2371 2370 static int
2372 2371 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2373 2372 ddi_dma_handle_t handle)
2374 2373 {
2375 2374 int ret;
2376 2375
2377 2376 #if defined(__amd64) && !defined(__xpv)
2378 2377 if (IOMMU_USED(rdip))
2379 2378 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2380 2379 else
2381 2380 #endif
2382 2381 ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2383 2382
2384 2383 if (rootnex_state->r_dvma_call_list_id)
2385 2384 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2386 2385
2387 2386 return (ret);
2388 2387 }
2389 2388
2390 2389 #if defined(__amd64) && !defined(__xpv)
2391 2390
2392 2391 static int
2393 2392 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2394 2393 {
2395 2394 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2396 2395 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2397 2396
2398 2397 if (dma->dp_sleep_flags != KM_SLEEP &&
2399 2398 dma->dp_sleep_flags != KM_NOSLEEP)
2400 2399 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2401 2400 return (dma->dp_sleep_flags);
2402 2401 }
2403 2402 /*ARGSUSED*/
2404 2403 static void
2405 2404 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2406 2405 {
2407 2406 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2408 2407 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2409 2408 rootnex_window_t *window;
2410 2409
2411 2410 if (dma->dp_window) {
2412 2411 window = &dma->dp_window[dma->dp_current_win];
2413 2412 hp->dmai_cookie = window->wd_first_cookie;
2414 2413 } else {
2415 2414 hp->dmai_cookie = dma->dp_cookies;
2416 2415 }
2417 2416 hp->dmai_cookie++;
2418 2417 }
2419 2418
2420 2419 /*ARGSUSED*/
2421 2420 static int
2422 2421 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2423 2422 ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2424 2423 {
2425 2424 int i;
2426 2425 int km_flags;
2427 2426 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2428 2427 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2429 2428 rootnex_window_t *window;
2430 2429 ddi_dma_cookie_t *cp;
2431 2430 ddi_dma_cookie_t *cookie;
2432 2431
2433 2432 ASSERT(*cookiepp == NULL);
2434 2433 ASSERT(*ccountp == 0);
2435 2434
2436 2435 if (dma->dp_window) {
2437 2436 window = &dma->dp_window[dma->dp_current_win];
2438 2437 cp = window->wd_first_cookie;
2439 2438 *ccountp = window->wd_cookie_cnt;
2440 2439 } else {
2441 2440 cp = dma->dp_cookies;
2442 2441 *ccountp = dma->dp_sglinfo.si_sgl_size;
2443 2442 }
2444 2443
2445 2444 km_flags = rootnex_coredma_get_sleep_flags(handle);
2446 2445 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2447 2446 if (cookie == NULL) {
2448 2447 return (DDI_DMA_NORESOURCES);
2449 2448 }
2450 2449
2451 2450 for (i = 0; i < *ccountp; i++) {
2452 2451 cookie[i].dmac_notused = cp[i].dmac_notused;
2453 2452 cookie[i].dmac_type = cp[i].dmac_type;
2454 2453 cookie[i].dmac_address = cp[i].dmac_address;
2455 2454 cookie[i].dmac_size = cp[i].dmac_size;
2456 2455 }
2457 2456
2458 2457 *cookiepp = cookie;
2459 2458
2460 2459 return (DDI_SUCCESS);
2461 2460 }
2462 2461
2463 2462 /*ARGSUSED*/
2464 2463 static int
2465 2464 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2466 2465 ddi_dma_cookie_t *cookiep, uint_t ccount)
2467 2466 {
2468 2467 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2469 2468 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2470 2469 rootnex_window_t *window;
2471 2470 ddi_dma_cookie_t *cur_cookiep;
2472 2471
2473 2472 ASSERT(cookiep);
2474 2473 ASSERT(ccount != 0);
2475 2474 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2476 2475
2477 2476 if (dma->dp_window) {
2478 2477 window = &dma->dp_window[dma->dp_current_win];
2479 2478 dma->dp_saved_cookies = window->wd_first_cookie;
2480 2479 window->wd_first_cookie = cookiep;
2481 2480 ASSERT(ccount == window->wd_cookie_cnt);
2482 2481 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2483 2482 + window->wd_first_cookie;
2484 2483 } else {
2485 2484 dma->dp_saved_cookies = dma->dp_cookies;
2486 2485 dma->dp_cookies = cookiep;
2487 2486 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2488 2487 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2489 2488 + dma->dp_cookies;
2490 2489 }
2491 2490
2492 2491 dma->dp_need_to_switch_cookies = B_TRUE;
2493 2492 hp->dmai_cookie = cur_cookiep;
2494 2493
2495 2494 return (DDI_SUCCESS);
2496 2495 }
2497 2496
2498 2497 /*ARGSUSED*/
2499 2498 static int
2500 2499 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2501 2500 {
2502 2501 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2503 2502 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2504 2503 rootnex_window_t *window;
2505 2504 ddi_dma_cookie_t *cur_cookiep;
2506 2505 ddi_dma_cookie_t *cookie_array;
2507 2506 uint_t ccount;
2508 2507
2509 2508 /* check if cookies have not been switched */
2510 2509 if (dma->dp_need_to_switch_cookies == B_FALSE)
2511 2510 return (DDI_SUCCESS);
2512 2511
2513 2512 ASSERT(dma->dp_saved_cookies);
2514 2513
2515 2514 if (dma->dp_window) {
2516 2515 window = &dma->dp_window[dma->dp_current_win];
2517 2516 cookie_array = window->wd_first_cookie;
2518 2517 window->wd_first_cookie = dma->dp_saved_cookies;
2519 2518 dma->dp_saved_cookies = NULL;
2520 2519 ccount = window->wd_cookie_cnt;
2521 2520 cur_cookiep = (hp->dmai_cookie - cookie_array)
2522 2521 + window->wd_first_cookie;
2523 2522 } else {
2524 2523 cookie_array = dma->dp_cookies;
2525 2524 dma->dp_cookies = dma->dp_saved_cookies;
2526 2525 dma->dp_saved_cookies = NULL;
2527 2526 ccount = dma->dp_sglinfo.si_sgl_size;
2528 2527 cur_cookiep = (hp->dmai_cookie - cookie_array)
2529 2528 + dma->dp_cookies;
2530 2529 }
2531 2530
2532 2531 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2533 2532
2534 2533 hp->dmai_cookie = cur_cookiep;
2535 2534
2536 2535 dma->dp_need_to_switch_cookies = B_FALSE;
2537 2536
2538 2537 return (DDI_SUCCESS);
2539 2538 }
2540 2539
2541 2540 #endif
2542 2541
2543 2542 static struct as *
2544 2543 rootnex_get_as(ddi_dma_obj_t *dmao)
2545 2544 {
2546 2545 struct as *asp;
2547 2546
2548 2547 switch (dmao->dmao_type) {
2549 2548 case DMA_OTYP_VADDR:
2550 2549 case DMA_OTYP_BUFVADDR:
2551 2550 asp = dmao->dmao_obj.virt_obj.v_as;
2552 2551 if (asp == NULL)
2553 2552 asp = &kas;
2554 2553 break;
2555 2554 default:
2556 2555 asp = NULL;
2557 2556 break;
2558 2557 }
2559 2558 return (asp);
2560 2559 }
2561 2560
2562 2561 /*
2563 2562 * rootnex_verify_buffer()
2564 2563 * verify buffer wasn't free'd
2565 2564 */
2566 2565 static int
2567 2566 rootnex_verify_buffer(rootnex_dma_t *dma)
2568 2567 {
2569 2568 page_t **pplist;
2570 2569 caddr_t vaddr;
2571 2570 uint_t pcnt;
2572 2571 uint_t poff;
2573 2572 page_t *pp;
2574 2573 char b;
2575 2574 int i;
2576 2575
2577 2576 /* Figure out how many pages this buffer occupies */
2578 2577 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2579 2578 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2580 2579 } else {
2581 2580 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2582 2581 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2583 2582 }
2584 2583 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2585 2584
2586 2585 switch (dma->dp_dma.dmao_type) {
2587 2586 case DMA_OTYP_PAGES:
2588 2587 /*
2589 2588 * for a linked list of pp's walk through them to make sure
2590 2589 * they're locked and not free.
2591 2590 */
2592 2591 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2593 2592 for (i = 0; i < pcnt; i++) {
2594 2593 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2595 2594 return (DDI_FAILURE);
2596 2595 }
2597 2596 pp = pp->p_next;
2598 2597 }
2599 2598 break;
2600 2599
2601 2600 case DMA_OTYP_VADDR:
2602 2601 case DMA_OTYP_BUFVADDR:
2603 2602 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2604 2603 /*
2605 2604 * for an array of pp's walk through them to make sure they're
2606 2605 * not free. It's possible that they may not be locked.
2607 2606 */
2608 2607 if (pplist) {
2609 2608 for (i = 0; i < pcnt; i++) {
2610 2609 if (PP_ISFREE(pplist[i])) {
2611 2610 return (DDI_FAILURE);
2612 2611 }
2613 2612 }
2614 2613
2615 2614 /* For a virtual address, try to peek at each page */
2616 2615 } else {
2617 2616 if (rootnex_get_as(&dma->dp_dma) == &kas) {
2618 2617 for (i = 0; i < pcnt; i++) {
2619 2618 if (ddi_peek8(NULL, vaddr, &b) ==
2620 2619 DDI_FAILURE)
2621 2620 return (DDI_FAILURE);
2622 2621 vaddr += MMU_PAGESIZE;
2623 2622 }
2624 2623 }
2625 2624 }
2626 2625 break;
2627 2626
2628 2627 default:
2629 2628 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2630 2629 break;
2631 2630 }
2632 2631
2633 2632 return (DDI_SUCCESS);
2634 2633 }
2635 2634
2636 2635
2637 2636 /*
2638 2637 * rootnex_clean_dmahdl()
2639 2638 * Clean the dma handle. This should be called on a handle alloc and an
2640 2639 * unbind handle. Set the handle state to the default settings.
2641 2640 */
2642 2641 static void
2643 2642 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2644 2643 {
2645 2644 rootnex_dma_t *dma;
2646 2645
2647 2646
2648 2647 dma = (rootnex_dma_t *)hp->dmai_private;
2649 2648
2650 2649 hp->dmai_nwin = 0;
2651 2650 dma->dp_current_cookie = 0;
2652 2651 dma->dp_copybuf_size = 0;
2653 2652 dma->dp_window = NULL;
2654 2653 dma->dp_cbaddr = NULL;
2655 2654 dma->dp_inuse = B_FALSE;
2656 2655 dma->dp_dvma_used = B_FALSE;
2657 2656 dma->dp_need_to_free_cookie = B_FALSE;
2658 2657 dma->dp_need_to_switch_cookies = B_FALSE;
2659 2658 dma->dp_saved_cookies = NULL;
2660 2659 dma->dp_sleep_flags = KM_PANIC;
2661 2660 dma->dp_need_to_free_window = B_FALSE;
2662 2661 dma->dp_partial_required = B_FALSE;
2663 2662 dma->dp_trim_required = B_FALSE;
2664 2663 dma->dp_sglinfo.si_copybuf_req = 0;
2665 2664 #if !defined(__amd64)
2666 2665 dma->dp_cb_remaping = B_FALSE;
2667 2666 dma->dp_kva = NULL;
2668 2667 #endif
2669 2668
2670 2669 /* FMA related initialization */
2671 2670 hp->dmai_fault = 0;
2672 2671 hp->dmai_fault_check = NULL;
2673 2672 hp->dmai_fault_notify = NULL;
2674 2673 hp->dmai_error.err_ena = 0;
2675 2674 hp->dmai_error.err_status = DDI_FM_OK;
2676 2675 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2677 2676 hp->dmai_error.err_ontrap = NULL;
2678 2677 }
2679 2678
2680 2679
2681 2680 /*
2682 2681 * rootnex_valid_alloc_parms()
2683 2682 * Called in ddi_dma_alloc_handle path to validate its parameters.
2684 2683 */
2685 2684 static int
2686 2685 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2687 2686 {
2688 2687 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2689 2688 (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2690 2689 (attr->dma_attr_granular > MMU_PAGESIZE) ||
2691 2690 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2692 2691 return (DDI_DMA_BADATTR);
2693 2692 }
2694 2693
2695 2694 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2696 2695 return (DDI_DMA_BADATTR);
2697 2696 }
2698 2697
2699 2698 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2700 2699 MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2701 2700 attr->dma_attr_sgllen == 0) {
2702 2701 return (DDI_DMA_BADATTR);
2703 2702 }
2704 2703
2705 2704 /* We should be able to DMA into every byte offset in a page */
2706 2705 if (maxsegmentsize < MMU_PAGESIZE) {
2707 2706 return (DDI_DMA_BADATTR);
2708 2707 }
2709 2708
2710 2709 /* if we're bouncing on seg, seg must be <= addr_hi */
2711 2710 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2712 2711 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2713 2712 return (DDI_DMA_BADATTR);
2714 2713 }
2715 2714 return (DDI_SUCCESS);
2716 2715 }
2717 2716
2718 2717 /*
2719 2718 * rootnex_valid_bind_parms()
2720 2719 * Called in ddi_dma_*_bind_handle path to validate its parameters.
2721 2720 */
2722 2721 /* ARGSUSED */
2723 2722 static int
2724 2723 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2725 2724 {
2726 2725 #if !defined(__amd64)
2727 2726 /*
2728 2727 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2729 2728 * we can track the offset for the obsoleted interfaces.
2730 2729 */
2731 2730 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2732 2731 return (DDI_DMA_TOOBIG);
2733 2732 }
2734 2733 #endif
2735 2734
2736 2735 return (DDI_SUCCESS);
2737 2736 }
2738 2737
2739 2738
2740 2739 /*
2741 2740 * rootnex_need_bounce_seg()
2742 2741 * check to see if the buffer lives on both side of the seg.
2743 2742 */
2744 2743 static boolean_t
2745 2744 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2746 2745 {
2747 2746 ddi_dma_atyp_t buftype;
2748 2747 rootnex_addr_t raddr;
2749 2748 boolean_t lower_addr;
2750 2749 boolean_t upper_addr;
2751 2750 uint64_t offset;
2752 2751 page_t **pplist;
2753 2752 uint64_t paddr;
2754 2753 uint32_t psize;
2755 2754 uint32_t size;
2756 2755 caddr_t vaddr;
2757 2756 uint_t pcnt;
2758 2757 page_t *pp;
2759 2758
2760 2759
2761 2760 /* shortcuts */
2762 2761 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2763 2762 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2764 2763 buftype = dmar_object->dmao_type;
2765 2764 size = dmar_object->dmao_size;
2766 2765
2767 2766 lower_addr = B_FALSE;
2768 2767 upper_addr = B_FALSE;
2769 2768 pcnt = 0;
2770 2769
2771 2770 /*
2772 2771 * Process the first page to handle the initial offset of the buffer.
2773 2772 * We'll use the base address we get later when we loop through all
2774 2773 * the pages.
2775 2774 */
2776 2775 if (buftype == DMA_OTYP_PAGES) {
2777 2776 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2778 2777 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2779 2778 MMU_PAGEOFFSET;
2780 2779 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2781 2780 psize = MIN(size, (MMU_PAGESIZE - offset));
2782 2781 pp = pp->p_next;
2783 2782 sglinfo->si_asp = NULL;
2784 2783 } else if (pplist != NULL) {
2785 2784 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2786 2785 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2787 2786 if (sglinfo->si_asp == NULL) {
2788 2787 sglinfo->si_asp = &kas;
2789 2788 }
2790 2789 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2791 2790 paddr += offset;
2792 2791 psize = MIN(size, (MMU_PAGESIZE - offset));
2793 2792 pcnt++;
2794 2793 } else {
2795 2794 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2796 2795 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2797 2796 if (sglinfo->si_asp == NULL) {
2798 2797 sglinfo->si_asp = &kas;
2799 2798 }
2800 2799 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2801 2800 paddr += offset;
2802 2801 psize = MIN(size, (MMU_PAGESIZE - offset));
2803 2802 vaddr += psize;
2804 2803 }
2805 2804
2806 2805 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2807 2806
2808 2807 if ((raddr + psize) > sglinfo->si_segmask) {
2809 2808 upper_addr = B_TRUE;
2810 2809 } else {
2811 2810 lower_addr = B_TRUE;
2812 2811 }
2813 2812 size -= psize;
2814 2813
2815 2814 /*
2816 2815 * Walk through the rest of the pages in the buffer. Track to see
2817 2816 * if we have pages on both sides of the segment boundary.
2818 2817 */
2819 2818 while (size > 0) {
2820 2819 /* partial or full page */
2821 2820 psize = MIN(size, MMU_PAGESIZE);
2822 2821
2823 2822 if (buftype == DMA_OTYP_PAGES) {
2824 2823 /* get the paddr from the page_t */
2825 2824 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2826 2825 paddr = pfn_to_pa(pp->p_pagenum);
2827 2826 pp = pp->p_next;
2828 2827 } else if (pplist != NULL) {
2829 2828 /* index into the array of page_t's to get the paddr */
2830 2829 ASSERT(!PP_ISFREE(pplist[pcnt]));
2831 2830 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2832 2831 pcnt++;
2833 2832 } else {
2834 2833 /* call into the VM to get the paddr */
2835 2834 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2836 2835 vaddr));
2837 2836 vaddr += psize;
2838 2837 }
2839 2838
2840 2839 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2841 2840
2842 2841 if ((raddr + psize) > sglinfo->si_segmask) {
2843 2842 upper_addr = B_TRUE;
2844 2843 } else {
2845 2844 lower_addr = B_TRUE;
2846 2845 }
2847 2846 /*
2848 2847 * if the buffer lives both above and below the segment
2849 2848 * boundary, or the current page is the page immediately
2850 2849 * after the segment, we will use a copy/bounce buffer for
2851 2850 * all pages > seg.
2852 2851 */
2853 2852 if ((lower_addr && upper_addr) ||
2854 2853 (raddr == (sglinfo->si_segmask + 1))) {
2855 2854 return (B_TRUE);
2856 2855 }
2857 2856
2858 2857 size -= psize;
2859 2858 }
2860 2859
2861 2860 return (B_FALSE);
2862 2861 }
2863 2862
2864 2863 /*
2865 2864 * rootnex_get_sgl()
2866 2865 * Called in bind fastpath to get the sgl. Most of this will be replaced
2867 2866 * with a call to the vm layer when vm2.0 comes around...
2868 2867 */
2869 2868 static void
2870 2869 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2871 2870 rootnex_sglinfo_t *sglinfo)
2872 2871 {
2873 2872 ddi_dma_atyp_t buftype;
2874 2873 rootnex_addr_t raddr;
2875 2874 uint64_t last_page;
2876 2875 uint64_t offset;
2877 2876 uint64_t addrhi;
2878 2877 uint64_t addrlo;
2879 2878 uint64_t maxseg;
2880 2879 page_t **pplist;
2881 2880 uint64_t paddr;
2882 2881 uint32_t psize;
2883 2882 uint32_t size;
2884 2883 caddr_t vaddr;
2885 2884 uint_t pcnt;
2886 2885 page_t *pp;
2887 2886 uint_t cnt;
2888 2887
2889 2888
2890 2889 /* shortcuts */
2891 2890 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2892 2891 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2893 2892 maxseg = sglinfo->si_max_cookie_size;
2894 2893 buftype = dmar_object->dmao_type;
2895 2894 addrhi = sglinfo->si_max_addr;
2896 2895 addrlo = sglinfo->si_min_addr;
2897 2896 size = dmar_object->dmao_size;
2898 2897
2899 2898 pcnt = 0;
2900 2899 cnt = 0;
2901 2900
2902 2901
2903 2902 /*
2904 2903 * check to see if we need to use the copy buffer for pages over
2905 2904 * the segment attr.
2906 2905 */
2907 2906 sglinfo->si_bounce_on_seg = B_FALSE;
2908 2907 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2909 2908 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2910 2909 dmar_object, sglinfo);
2911 2910 }
2912 2911
2913 2912 /*
2914 2913 * if we were passed down a linked list of pages, i.e. pointer to
2915 2914 * page_t, use this to get our physical address and buf offset.
2916 2915 */
2917 2916 if (buftype == DMA_OTYP_PAGES) {
2918 2917 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2919 2918 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2920 2919 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2921 2920 MMU_PAGEOFFSET;
2922 2921 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2923 2922 psize = MIN(size, (MMU_PAGESIZE - offset));
2924 2923 pp = pp->p_next;
2925 2924 sglinfo->si_asp = NULL;
2926 2925
2927 2926 /*
2928 2927 * We weren't passed down a linked list of pages, but if we were passed
2929 2928 * down an array of pages, use this to get our physical address and buf
2930 2929 * offset.
2931 2930 */
2932 2931 } else if (pplist != NULL) {
2933 2932 ASSERT((buftype == DMA_OTYP_VADDR) ||
2934 2933 (buftype == DMA_OTYP_BUFVADDR));
2935 2934
2936 2935 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2937 2936 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2938 2937 if (sglinfo->si_asp == NULL) {
2939 2938 sglinfo->si_asp = &kas;
2940 2939 }
2941 2940
2942 2941 ASSERT(!PP_ISFREE(pplist[pcnt]));
2943 2942 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2944 2943 paddr += offset;
2945 2944 psize = MIN(size, (MMU_PAGESIZE - offset));
2946 2945 pcnt++;
2947 2946
2948 2947 /*
2949 2948 * All we have is a virtual address, we'll need to call into the VM
2950 2949 * to get the physical address.
2951 2950 */
2952 2951 } else {
2953 2952 ASSERT((buftype == DMA_OTYP_VADDR) ||
2954 2953 (buftype == DMA_OTYP_BUFVADDR));
2955 2954
2956 2955 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2957 2956 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2958 2957 if (sglinfo->si_asp == NULL) {
2959 2958 sglinfo->si_asp = &kas;
2960 2959 }
2961 2960
2962 2961 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2963 2962 paddr += offset;
2964 2963 psize = MIN(size, (MMU_PAGESIZE - offset));
2965 2964 vaddr += psize;
2966 2965 }
2967 2966
2968 2967 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2969 2968
2970 2969 /*
2971 2970 * Setup the first cookie with the physical address of the page and the
2972 2971 * size of the page (which takes into account the initial offset into
2973 2972 * the page.
2974 2973 */
2975 2974 sgl[cnt].dmac_laddress = raddr;
2976 2975 sgl[cnt].dmac_size = psize;
2977 2976 sgl[cnt].dmac_type = 0;
2978 2977
2979 2978 /*
2980 2979 * Save away the buffer offset into the page. We'll need this later in
2981 2980 * the copy buffer code to help figure out the page index within the
2982 2981 * buffer and the offset into the current page.
2983 2982 */
2984 2983 sglinfo->si_buf_offset = offset;
2985 2984
2986 2985 /*
2987 2986 * If we are using the copy buffer for anything over the segment
2988 2987 * boundary, and this page is over the segment boundary.
2989 2988 * OR
2990 2989 * if the DMA engine can't reach the physical address.
2991 2990 */
2992 2991 if (((sglinfo->si_bounce_on_seg) &&
2993 2992 ((raddr + psize) > sglinfo->si_segmask)) ||
2994 2993 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2995 2994 /*
2996 2995 * Increase how much copy buffer we use. We always increase by
2997 2996 * pagesize so we don't have to worry about converting offsets.
2998 2997 * Set a flag in the cookies dmac_type to indicate that it uses
2999 2998 * the copy buffer. If this isn't the last cookie, go to the
3000 2999 * next cookie (since we separate each page which uses the copy
3001 3000 * buffer in case the copy buffer is not physically contiguous.
3002 3001 */
3003 3002 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3004 3003 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3005 3004 if ((cnt + 1) < sglinfo->si_max_pages) {
3006 3005 cnt++;
3007 3006 sgl[cnt].dmac_laddress = 0;
3008 3007 sgl[cnt].dmac_size = 0;
3009 3008 sgl[cnt].dmac_type = 0;
3010 3009 }
3011 3010 }
3012 3011
3013 3012 /*
3014 3013 * save this page's physical address so we can figure out if the next
3015 3014 * page is physically contiguous. Keep decrementing size until we are
3016 3015 * done with the buffer.
3017 3016 */
3018 3017 last_page = raddr & MMU_PAGEMASK;
3019 3018 size -= psize;
3020 3019
3021 3020 while (size > 0) {
3022 3021 /* Get the size for this page (i.e. partial or full page) */
3023 3022 psize = MIN(size, MMU_PAGESIZE);
3024 3023
3025 3024 if (buftype == DMA_OTYP_PAGES) {
3026 3025 /* get the paddr from the page_t */
3027 3026 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3028 3027 paddr = pfn_to_pa(pp->p_pagenum);
3029 3028 pp = pp->p_next;
3030 3029 } else if (pplist != NULL) {
3031 3030 /* index into the array of page_t's to get the paddr */
3032 3031 ASSERT(!PP_ISFREE(pplist[pcnt]));
3033 3032 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3034 3033 pcnt++;
3035 3034 } else {
3036 3035 /* call into the VM to get the paddr */
3037 3036 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3038 3037 vaddr));
3039 3038 vaddr += psize;
3040 3039 }
3041 3040
3042 3041 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3043 3042
3044 3043 /*
3045 3044 * If we are using the copy buffer for anything over the
3046 3045 * segment boundary, and this page is over the segment
3047 3046 * boundary.
3048 3047 * OR
3049 3048 * if the DMA engine can't reach the physical address.
3050 3049 */
3051 3050 if (((sglinfo->si_bounce_on_seg) &&
3052 3051 ((raddr + psize) > sglinfo->si_segmask)) ||
3053 3052 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3054 3053
3055 3054 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3056 3055
3057 3056 /*
3058 3057 * if there is something in the current cookie, go to
3059 3058 * the next one. We only want one page in a cookie which
3060 3059 * uses the copybuf since the copybuf doesn't have to
3061 3060 * be physically contiguous.
3062 3061 */
3063 3062 if (sgl[cnt].dmac_size != 0) {
3064 3063 cnt++;
3065 3064 }
3066 3065 sgl[cnt].dmac_laddress = raddr;
3067 3066 sgl[cnt].dmac_size = psize;
3068 3067 #if defined(__amd64)
3069 3068 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3070 3069 #else
3071 3070 /*
3072 3071 * save the buf offset for 32-bit kernel. used in the
3073 3072 * obsoleted interfaces.
3074 3073 */
3075 3074 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3076 3075 (dmar_object->dmao_size - size);
3077 3076 #endif
3078 3077 /* if this isn't the last cookie, go to the next one */
3079 3078 if ((cnt + 1) < sglinfo->si_max_pages) {
3080 3079 cnt++;
3081 3080 sgl[cnt].dmac_laddress = 0;
3082 3081 sgl[cnt].dmac_size = 0;
3083 3082 sgl[cnt].dmac_type = 0;
3084 3083 }
3085 3084
3086 3085 /*
3087 3086 * this page didn't need the copy buffer, if it's not physically
3088 3087 * contiguous, or it would put us over a segment boundary, or it
3089 3088 * puts us over the max cookie size, or the current sgl doesn't
3090 3089 * have anything in it.
3091 3090 */
3092 3091 } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3093 3092 !(raddr & sglinfo->si_segmask) ||
3094 3093 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3095 3094 (sgl[cnt].dmac_size == 0)) {
3096 3095 /*
3097 3096 * if we're not already in a new cookie, go to the next
3098 3097 * cookie.
3099 3098 */
3100 3099 if (sgl[cnt].dmac_size != 0) {
3101 3100 cnt++;
3102 3101 }
3103 3102
3104 3103 /* save the cookie information */
3105 3104 sgl[cnt].dmac_laddress = raddr;
3106 3105 sgl[cnt].dmac_size = psize;
3107 3106 #if defined(__amd64)
3108 3107 sgl[cnt].dmac_type = 0;
3109 3108 #else
3110 3109 /*
3111 3110 * save the buf offset for 32-bit kernel. used in the
3112 3111 * obsoleted interfaces.
3113 3112 */
3114 3113 sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3115 3114 #endif
3116 3115
3117 3116 /*
3118 3117 * this page didn't need the copy buffer, it is physically
3119 3118 * contiguous with the last page, and it's <= the max cookie
3120 3119 * size.
3121 3120 */
3122 3121 } else {
3123 3122 sgl[cnt].dmac_size += psize;
3124 3123
3125 3124 /*
3126 3125 * if this exactly == the maximum cookie size, and
3127 3126 * it isn't the last cookie, go to the next cookie.
3128 3127 */
3129 3128 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3130 3129 ((cnt + 1) < sglinfo->si_max_pages)) {
3131 3130 cnt++;
3132 3131 sgl[cnt].dmac_laddress = 0;
3133 3132 sgl[cnt].dmac_size = 0;
3134 3133 sgl[cnt].dmac_type = 0;
3135 3134 }
3136 3135 }
3137 3136
3138 3137 /*
3139 3138 * save this page's physical address so we can figure out if the
3140 3139 * next page is physically contiguous. Keep decrementing size
3141 3140 * until we are done with the buffer.
3142 3141 */
3143 3142 last_page = raddr;
3144 3143 size -= psize;
3145 3144 }
3146 3145
3147 3146 /* we're done, save away how many cookies the sgl has */
3148 3147 if (sgl[cnt].dmac_size == 0) {
3149 3148 ASSERT(cnt < sglinfo->si_max_pages);
3150 3149 sglinfo->si_sgl_size = cnt;
3151 3150 } else {
3152 3151 sglinfo->si_sgl_size = cnt + 1;
3153 3152 }
3154 3153 }
3155 3154
3156 3155 static void
3157 3156 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3158 3157 rootnex_sglinfo_t *sglinfo)
3159 3158 {
3160 3159 uint64_t offset;
3161 3160 uint64_t maxseg;
3162 3161 uint64_t dvaddr;
3163 3162 struct dvmaseg *dvs;
3164 3163 uint64_t paddr;
3165 3164 uint32_t psize, ssize;
3166 3165 uint32_t size;
3167 3166 uint_t cnt;
3168 3167 int physcontig;
3169 3168
3170 3169 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3171 3170
3172 3171 /* shortcuts */
3173 3172 maxseg = sglinfo->si_max_cookie_size;
3174 3173 size = dmar_object->dmao_size;
3175 3174
3176 3175 cnt = 0;
3177 3176 sglinfo->si_bounce_on_seg = B_FALSE;
3178 3177
3179 3178 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3180 3179 offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3181 3180 ssize = dvs->dvs_len;
3182 3181 paddr = dvs->dvs_start;
3183 3182 paddr += offset;
3184 3183 psize = MIN(ssize, (maxseg - offset));
3185 3184 dvaddr = paddr + psize;
3186 3185 ssize -= psize;
3187 3186
3188 3187 sgl[cnt].dmac_laddress = paddr;
3189 3188 sgl[cnt].dmac_size = psize;
3190 3189 sgl[cnt].dmac_type = 0;
3191 3190
3192 3191 size -= psize;
3193 3192 while (size > 0) {
3194 3193 if (ssize == 0) {
3195 3194 dvs++;
3196 3195 ssize = dvs->dvs_len;
3197 3196 dvaddr = dvs->dvs_start;
3198 3197 physcontig = 0;
3199 3198 } else
3200 3199 physcontig = 1;
3201 3200
3202 3201 paddr = dvaddr;
3203 3202 psize = MIN(ssize, maxseg);
3204 3203 dvaddr += psize;
3205 3204 ssize -= psize;
3206 3205
3207 3206 if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3208 3207 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3209 3208 (sgl[cnt].dmac_size == 0)) {
3210 3209 /*
3211 3210 * if we're not already in a new cookie, go to the next
3212 3211 * cookie.
3213 3212 */
3214 3213 if (sgl[cnt].dmac_size != 0) {
3215 3214 cnt++;
3216 3215 }
3217 3216
3218 3217 /* save the cookie information */
3219 3218 sgl[cnt].dmac_laddress = paddr;
3220 3219 sgl[cnt].dmac_size = psize;
3221 3220 sgl[cnt].dmac_type = 0;
3222 3221 } else {
3223 3222 sgl[cnt].dmac_size += psize;
3224 3223
3225 3224 /*
3226 3225 * if this exactly == the maximum cookie size, and
3227 3226 * it isn't the last cookie, go to the next cookie.
3228 3227 */
3229 3228 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3230 3229 ((cnt + 1) < sglinfo->si_max_pages)) {
3231 3230 cnt++;
3232 3231 sgl[cnt].dmac_laddress = 0;
3233 3232 sgl[cnt].dmac_size = 0;
3234 3233 sgl[cnt].dmac_type = 0;
3235 3234 }
3236 3235 }
3237 3236 size -= psize;
3238 3237 }
3239 3238
3240 3239 /* we're done, save away how many cookies the sgl has */
3241 3240 if (sgl[cnt].dmac_size == 0) {
3242 3241 sglinfo->si_sgl_size = cnt;
3243 3242 } else {
3244 3243 sglinfo->si_sgl_size = cnt + 1;
3245 3244 }
3246 3245 }
3247 3246
3248 3247 /*
3249 3248 * rootnex_bind_slowpath()
3250 3249 * Call in the bind path if the calling driver can't use the sgl without
3251 3250 * modifying it. We either need to use the copy buffer and/or we will end up
3252 3251 * with a partial bind.
3253 3252 */
3254 3253 static int
3255 3254 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3256 3255 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3257 3256 {
3258 3257 rootnex_sglinfo_t *sinfo;
3259 3258 rootnex_window_t *window;
3260 3259 ddi_dma_cookie_t *cookie;
3261 3260 size_t copybuf_used;
3262 3261 size_t dmac_size;
3263 3262 boolean_t partial;
3264 3263 off_t cur_offset;
3265 3264 page_t *cur_pp;
3266 3265 major_t mnum;
3267 3266 int e;
3268 3267 int i;
3269 3268
3270 3269
3271 3270 sinfo = &dma->dp_sglinfo;
3272 3271 copybuf_used = 0;
3273 3272 partial = B_FALSE;
3274 3273
3275 3274 /*
3276 3275 * If we're using the copybuf, set the copybuf state in dma struct.
3277 3276 * Needs to be first since it sets the copy buffer size.
3278 3277 */
3279 3278 if (sinfo->si_copybuf_req != 0) {
3280 3279 e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3281 3280 if (e != DDI_SUCCESS) {
3282 3281 return (e);
3283 3282 }
3284 3283 } else {
3285 3284 dma->dp_copybuf_size = 0;
3286 3285 }
3287 3286
3288 3287 /*
3289 3288 * Figure out if we need to do a partial mapping. If so, figure out
3290 3289 * if we need to trim the buffers when we munge the sgl.
3291 3290 */
3292 3291 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3293 3292 (dmao->dmao_size > dma->dp_maxxfer) ||
3294 3293 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3295 3294 dma->dp_partial_required = B_TRUE;
3296 3295 if (attr->dma_attr_granular != 1) {
3297 3296 dma->dp_trim_required = B_TRUE;
3298 3297 }
3299 3298 } else {
3300 3299 dma->dp_partial_required = B_FALSE;
3301 3300 dma->dp_trim_required = B_FALSE;
3302 3301 }
3303 3302
3304 3303 /* If we need to do a partial bind, make sure the driver supports it */
3305 3304 if (dma->dp_partial_required &&
3306 3305 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3307 3306
3308 3307 mnum = ddi_driver_major(dma->dp_dip);
3309 3308 /*
3310 3309 * patchable which allows us to print one warning per major
3311 3310 * number.
3312 3311 */
3313 3312 if ((rootnex_bind_warn) &&
3314 3313 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3315 3314 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3316 3315 cmn_err(CE_WARN, "!%s: coding error detected, the "
3317 3316 "driver is using ddi_dma_attr(9S) incorrectly. "
3318 3317 "There is a small risk of data corruption in "
3319 3318 "particular with large I/Os. The driver should be "
3320 3319 "replaced with a corrected version for proper "
3321 3320 "system operation. To disable this warning, add "
3322 3321 "'set rootnex:rootnex_bind_warn=0' to "
3323 3322 "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3324 3323 }
3325 3324 return (DDI_DMA_TOOBIG);
3326 3325 }
3327 3326
3328 3327 /*
3329 3328 * we might need multiple windows, setup state to handle them. In this
3330 3329 * code path, we will have at least one window.
3331 3330 */
3332 3331 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3333 3332 if (e != DDI_SUCCESS) {
3334 3333 rootnex_teardown_copybuf(dma);
3335 3334 return (e);
3336 3335 }
3337 3336
3338 3337 window = &dma->dp_window[0];
3339 3338 cookie = &dma->dp_cookies[0];
3340 3339 cur_offset = 0;
3341 3340 rootnex_init_win(hp, dma, window, cookie, cur_offset);
3342 3341 if (dmao->dmao_type == DMA_OTYP_PAGES) {
3343 3342 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3344 3343 }
3345 3344
3346 3345 /* loop though all the cookies we got back from get_sgl() */
3347 3346 for (i = 0; i < sinfo->si_sgl_size; i++) {
3348 3347 /*
3349 3348 * If we're using the copy buffer, check this cookie and setup
3350 3349 * its associated copy buffer state. If this cookie uses the
3351 3350 * copy buffer, make sure we sync this window during dma_sync.
3352 3351 */
3353 3352 if (dma->dp_copybuf_size > 0) {
3354 3353 rootnex_setup_cookie(dmao, dma, cookie,
3355 3354 cur_offset, ©buf_used, &cur_pp);
3356 3355 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3357 3356 window->wd_dosync = B_TRUE;
3358 3357 }
3359 3358 }
3360 3359
3361 3360 /*
3362 3361 * save away the cookie size, since it could be modified in
3363 3362 * the windowing code.
3364 3363 */
3365 3364 dmac_size = cookie->dmac_size;
3366 3365
3367 3366 /* if we went over max copybuf size */
3368 3367 if (dma->dp_copybuf_size &&
3369 3368 (copybuf_used > dma->dp_copybuf_size)) {
3370 3369 partial = B_TRUE;
3371 3370 e = rootnex_copybuf_window_boundary(hp, dma, &window,
3372 3371 cookie, cur_offset, ©buf_used);
3373 3372 if (e != DDI_SUCCESS) {
3374 3373 rootnex_teardown_copybuf(dma);
3375 3374 rootnex_teardown_windows(dma);
3376 3375 return (e);
3377 3376 }
3378 3377
3379 3378 /*
3380 3379 * if the coookie uses the copy buffer, make sure the
3381 3380 * new window we just moved to is set to sync.
3382 3381 */
3383 3382 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3384 3383 window->wd_dosync = B_TRUE;
3385 3384 }
3386 3385 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3387 3386 dma->dp_dip);
3388 3387
3389 3388 /* if the cookie cnt == max sgllen, move to the next window */
3390 3389 } else if (window->wd_cookie_cnt >=
3391 3390 (unsigned)attr->dma_attr_sgllen) {
3392 3391 partial = B_TRUE;
3393 3392 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3394 3393 e = rootnex_sgllen_window_boundary(hp, dma, &window,
3395 3394 cookie, attr, cur_offset);
3396 3395 if (e != DDI_SUCCESS) {
3397 3396 rootnex_teardown_copybuf(dma);
3398 3397 rootnex_teardown_windows(dma);
3399 3398 return (e);
3400 3399 }
3401 3400
3402 3401 /*
3403 3402 * if the coookie uses the copy buffer, make sure the
3404 3403 * new window we just moved to is set to sync.
3405 3404 */
3406 3405 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3407 3406 window->wd_dosync = B_TRUE;
3408 3407 }
3409 3408 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3410 3409 dma->dp_dip);
3411 3410
3412 3411 /* else if we will be over maxxfer */
3413 3412 } else if ((window->wd_size + dmac_size) >
3414 3413 dma->dp_maxxfer) {
3415 3414 partial = B_TRUE;
3416 3415 e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3417 3416 cookie);
3418 3417 if (e != DDI_SUCCESS) {
3419 3418 rootnex_teardown_copybuf(dma);
3420 3419 rootnex_teardown_windows(dma);
3421 3420 return (e);
3422 3421 }
3423 3422
3424 3423 /*
3425 3424 * if the coookie uses the copy buffer, make sure the
3426 3425 * new window we just moved to is set to sync.
3427 3426 */
3428 3427 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3429 3428 window->wd_dosync = B_TRUE;
3430 3429 }
3431 3430 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3432 3431 dma->dp_dip);
3433 3432
3434 3433 /* else this cookie fits in the current window */
3435 3434 } else {
3436 3435 window->wd_cookie_cnt++;
3437 3436 window->wd_size += dmac_size;
3438 3437 }
3439 3438
3440 3439 /* track our offset into the buffer, go to the next cookie */
3441 3440 ASSERT(dmac_size <= dmao->dmao_size);
3442 3441 ASSERT(cookie->dmac_size <= dmac_size);
3443 3442 cur_offset += dmac_size;
3444 3443 cookie++;
3445 3444 }
3446 3445
3447 3446 /* if we ended up with a zero sized window in the end, clean it up */
3448 3447 if (window->wd_size == 0) {
3449 3448 hp->dmai_nwin--;
3450 3449 window--;
3451 3450 }
3452 3451
3453 3452 ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3454 3453
3455 3454 if (!partial) {
3456 3455 return (DDI_DMA_MAPPED);
3457 3456 }
3458 3457
3459 3458 ASSERT(dma->dp_partial_required);
3460 3459 return (DDI_DMA_PARTIAL_MAP);
3461 3460 }
3462 3461
3463 3462 /*
3464 3463 * rootnex_setup_copybuf()
3465 3464 * Called in bind slowpath. Figures out if we're going to use the copy
3466 3465 * buffer, and if we do, sets up the basic state to handle it.
3467 3466 */
3468 3467 static int
3469 3468 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3470 3469 rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3471 3470 {
3472 3471 rootnex_sglinfo_t *sinfo;
3473 3472 ddi_dma_attr_t lattr;
3474 3473 size_t max_copybuf;
3475 3474 int cansleep;
3476 3475 int e;
3477 3476 #if !defined(__amd64)
3478 3477 int vmflag;
3479 3478 #endif
3480 3479
3481 3480 ASSERT(!dma->dp_dvma_used);
3482 3481
3483 3482 sinfo = &dma->dp_sglinfo;
3484 3483
3485 3484 /* read this first so it's consistent through the routine */
3486 3485 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3487 3486
3488 3487 /* We need to call into the rootnex on ddi_dma_sync() */
3489 3488 hp->dmai_rflags &= ~DMP_NOSYNC;
3490 3489
3491 3490 /* make sure the copybuf size <= the max size */
3492 3491 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3493 3492 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3494 3493
3495 3494 #if !defined(__amd64)
3496 3495 /*
3497 3496 * if we don't have kva space to copy to/from, allocate the KVA space
3498 3497 * now. We only do this for the 32-bit kernel. We use seg kpm space for
3499 3498 * the 64-bit kernel.
3500 3499 */
3501 3500 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3502 3501 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3503 3502
3504 3503 /* convert the sleep flags */
3505 3504 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3506 3505 vmflag = VM_SLEEP;
3507 3506 } else {
3508 3507 vmflag = VM_NOSLEEP;
3509 3508 }
3510 3509
3511 3510 /* allocate Kernel VA space that we can bcopy to/from */
3512 3511 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3513 3512 vmflag);
3514 3513 if (dma->dp_kva == NULL) {
3515 3514 return (DDI_DMA_NORESOURCES);
3516 3515 }
3517 3516 }
3518 3517 #endif
3519 3518
3520 3519 /* convert the sleep flags */
3521 3520 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3522 3521 cansleep = 1;
3523 3522 } else {
3524 3523 cansleep = 0;
3525 3524 }
3526 3525
3527 3526 /*
3528 3527 * Allocate the actual copy buffer. This needs to fit within the DMA
3529 3528 * engine limits, so we can't use kmem_alloc... We don't need
3530 3529 * contiguous memory (sgllen) since we will be forcing windows on
3531 3530 * sgllen anyway.
3532 3531 */
3533 3532 lattr = *attr;
3534 3533 lattr.dma_attr_align = MMU_PAGESIZE;
3535 3534 lattr.dma_attr_sgllen = -1; /* no limit */
3536 3535 /*
3537 3536 * if we're using the copy buffer because of seg, use that for our
3538 3537 * upper address limit.
3539 3538 */
3540 3539 if (sinfo->si_bounce_on_seg) {
3541 3540 lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3542 3541 }
3543 3542 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3544 3543 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3545 3544 if (e != DDI_SUCCESS) {
3546 3545 #if !defined(__amd64)
3547 3546 if (dma->dp_kva != NULL) {
3548 3547 vmem_free(heap_arena, dma->dp_kva,
3549 3548 dma->dp_copybuf_size);
3550 3549 }
3551 3550 #endif
3552 3551 return (DDI_DMA_NORESOURCES);
3553 3552 }
3554 3553
3555 3554 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3556 3555 size_t, dma->dp_copybuf_size);
3557 3556
3558 3557 return (DDI_SUCCESS);
3559 3558 }
3560 3559
3561 3560
3562 3561 /*
3563 3562 * rootnex_setup_windows()
3564 3563 * Called in bind slowpath to setup the window state. We always have windows
3565 3564 * in the slowpath. Even if the window count = 1.
3566 3565 */
3567 3566 static int
3568 3567 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3569 3568 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3570 3569 {
3571 3570 rootnex_window_t *windowp;
3572 3571 rootnex_sglinfo_t *sinfo;
3573 3572 size_t copy_state_size;
3574 3573 size_t win_state_size;
3575 3574 size_t state_available;
3576 3575 size_t space_needed;
3577 3576 uint_t copybuf_win;
3578 3577 uint_t maxxfer_win;
3579 3578 size_t space_used;
3580 3579 uint_t sglwin;
3581 3580
3582 3581
3583 3582 sinfo = &dma->dp_sglinfo;
3584 3583
3585 3584 dma->dp_current_win = 0;
3586 3585 hp->dmai_nwin = 0;
3587 3586
3588 3587 /* If we don't need to do a partial, we only have one window */
3589 3588 if (!dma->dp_partial_required) {
3590 3589 dma->dp_max_win = 1;
3591 3590
3592 3591 /*
3593 3592 * we need multiple windows, need to figure out the worse case number
3594 3593 * of windows.
3595 3594 */
3596 3595 } else {
3597 3596 /*
3598 3597 * if we need windows because we need more copy buffer that
3599 3598 * we allow, the worse case number of windows we could need
3600 3599 * here would be (copybuf space required / copybuf space that
3601 3600 * we have) plus one for remainder, and plus 2 to handle the
3602 3601 * extra pages on the trim for the first and last pages of the
3603 3602 * buffer (a page is the minimum window size so under the right
3604 3603 * attr settings, you could have a window for each page).
3605 3604 * The last page will only be hit here if the size is not a
3606 3605 * multiple of the granularity (which theoretically shouldn't
3607 3606 * be the case but never has been enforced, so we could have
3608 3607 * broken things without it).
3609 3608 */
3610 3609 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3611 3610 ASSERT(dma->dp_copybuf_size > 0);
3612 3611 copybuf_win = (sinfo->si_copybuf_req /
3613 3612 dma->dp_copybuf_size) + 1 + 2;
3614 3613 } else {
3615 3614 copybuf_win = 0;
3616 3615 }
3617 3616
3618 3617 /*
3619 3618 * if we need windows because we have more cookies than the H/W
3620 3619 * can handle, the number of windows we would need here would
3621 3620 * be (cookie count / cookies count H/W supports minus 1[for
3622 3621 * trim]) plus one for remainder.
3623 3622 */
3624 3623 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3625 3624 sglwin = (sinfo->si_sgl_size /
3626 3625 (attr->dma_attr_sgllen - 1)) + 1;
3627 3626 } else {
3628 3627 sglwin = 0;
3629 3628 }
3630 3629
3631 3630 /*
3632 3631 * if we need windows because we're binding more memory than the
3633 3632 * H/W can transfer at once, the number of windows we would need
3634 3633 * here would be (xfer count / max xfer H/W supports) plus one
3635 3634 * for remainder, and plus 2 to handle the extra pages on the
3636 3635 * trim (see above comment about trim)
3637 3636 */
3638 3637 if (dmao->dmao_size > dma->dp_maxxfer) {
3639 3638 maxxfer_win = (dmao->dmao_size /
3640 3639 dma->dp_maxxfer) + 1 + 2;
3641 3640 } else {
3642 3641 maxxfer_win = 0;
3643 3642 }
3644 3643 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win;
3645 3644 ASSERT(dma->dp_max_win > 0);
3646 3645 }
3647 3646 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3648 3647
3649 3648 /*
3650 3649 * Get space for window and potential copy buffer state. Before we
3651 3650 * go and allocate memory, see if we can get away with using what's
3652 3651 * left in the pre-allocted state or the dynamically allocated sgl.
3653 3652 */
3654 3653 space_used = (uintptr_t)(sinfo->si_sgl_size *
3655 3654 sizeof (ddi_dma_cookie_t));
3656 3655
3657 3656 /* if we dynamically allocated space for the cookies */
3658 3657 if (dma->dp_need_to_free_cookie) {
3659 3658 /* if we have more space in the pre-allocted buffer, use it */
3660 3659 ASSERT(space_used <= dma->dp_cookie_size);
3661 3660 if ((dma->dp_cookie_size - space_used) <=
3662 3661 rootnex_state->r_prealloc_size) {
3663 3662 state_available = rootnex_state->r_prealloc_size;
3664 3663 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3665 3664
3666 3665 /*
3667 3666 * else, we have more free space in the dynamically allocated
3668 3667 * buffer, i.e. the buffer wasn't worse case fragmented so we
3669 3668 * didn't need a lot of cookies.
3670 3669 */
3671 3670 } else {
3672 3671 state_available = dma->dp_cookie_size - space_used;
3673 3672 windowp = (rootnex_window_t *)
3674 3673 &dma->dp_cookies[sinfo->si_sgl_size];
3675 3674 }
3676 3675
3677 3676 /* we used the pre-alloced buffer */
3678 3677 } else {
3679 3678 ASSERT(space_used <= rootnex_state->r_prealloc_size);
3680 3679 state_available = rootnex_state->r_prealloc_size - space_used;
3681 3680 windowp = (rootnex_window_t *)
3682 3681 &dma->dp_cookies[sinfo->si_sgl_size];
3683 3682 }
3684 3683
3685 3684 /*
3686 3685 * figure out how much state we need to track the copy buffer. Add an
3687 3686 * addition 8 bytes for pointer alignemnt later.
3688 3687 */
3689 3688 if (dma->dp_copybuf_size > 0) {
3690 3689 copy_state_size = sinfo->si_max_pages *
3691 3690 sizeof (rootnex_pgmap_t);
3692 3691 } else {
3693 3692 copy_state_size = 0;
3694 3693 }
3695 3694 /* add an additional 8 bytes for pointer alignment */
3696 3695 space_needed = win_state_size + copy_state_size + 0x8;
3697 3696
3698 3697 /* if we have enough space already, use it */
3699 3698 if (state_available >= space_needed) {
3700 3699 dma->dp_window = windowp;
3701 3700 dma->dp_need_to_free_window = B_FALSE;
3702 3701
3703 3702 /* not enough space, need to allocate more. */
3704 3703 } else {
3705 3704 dma->dp_window = kmem_alloc(space_needed, kmflag);
3706 3705 if (dma->dp_window == NULL) {
3707 3706 return (DDI_DMA_NORESOURCES);
3708 3707 }
3709 3708 dma->dp_need_to_free_window = B_TRUE;
3710 3709 dma->dp_window_size = space_needed;
3711 3710 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3712 3711 dma->dp_dip, size_t, space_needed);
3713 3712 }
3714 3713
3715 3714 /*
3716 3715 * we allocate copy buffer state and window state at the same time.
3717 3716 * setup our copy buffer state pointers. Make sure it's aligned.
3718 3717 */
3719 3718 if (dma->dp_copybuf_size > 0) {
3720 3719 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3721 3720 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3722 3721
3723 3722 #if !defined(__amd64)
3724 3723 /*
3725 3724 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3726 3725 * false/NULL. Should be quicker to bzero vs loop and set.
3727 3726 */
3728 3727 bzero(dma->dp_pgmap, copy_state_size);
3729 3728 #endif
3730 3729 } else {
3731 3730 dma->dp_pgmap = NULL;
3732 3731 }
3733 3732
3734 3733 return (DDI_SUCCESS);
3735 3734 }
3736 3735
3737 3736
3738 3737 /*
3739 3738 * rootnex_teardown_copybuf()
3740 3739 * cleans up after rootnex_setup_copybuf()
3741 3740 */
3742 3741 static void
3743 3742 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3744 3743 {
3745 3744 #if !defined(__amd64)
3746 3745 int i;
3747 3746
3748 3747 /*
3749 3748 * if we allocated kernel heap VMEM space, go through all the pages and
3750 3749 * map out any of the ones that we're mapped into the kernel heap VMEM
3751 3750 * arena. Then free the VMEM space.
3752 3751 */
3753 3752 if (dma->dp_kva != NULL) {
3754 3753 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3755 3754 if (dma->dp_pgmap[i].pm_mapped) {
3756 3755 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3757 3756 MMU_PAGESIZE, HAT_UNLOAD);
3758 3757 dma->dp_pgmap[i].pm_mapped = B_FALSE;
3759 3758 }
3760 3759 }
3761 3760
3762 3761 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3763 3762 }
3764 3763
3765 3764 #endif
3766 3765
3767 3766 /* if we allocated a copy buffer, free it */
3768 3767 if (dma->dp_cbaddr != NULL) {
3769 3768 i_ddi_mem_free(dma->dp_cbaddr, NULL);
3770 3769 }
3771 3770 }
3772 3771
3773 3772
3774 3773 /*
3775 3774 * rootnex_teardown_windows()
3776 3775 * cleans up after rootnex_setup_windows()
3777 3776 */
3778 3777 static void
3779 3778 rootnex_teardown_windows(rootnex_dma_t *dma)
3780 3779 {
3781 3780 /*
3782 3781 * if we had to allocate window state on the last bind (because we
3783 3782 * didn't have enough pre-allocated space in the handle), free it.
3784 3783 */
3785 3784 if (dma->dp_need_to_free_window) {
3786 3785 kmem_free(dma->dp_window, dma->dp_window_size);
3787 3786 }
3788 3787 }
3789 3788
3790 3789
3791 3790 /*
3792 3791 * rootnex_init_win()
3793 3792 * Called in bind slow path during creation of a new window. Initializes
3794 3793 * window state to default values.
3795 3794 */
3796 3795 /*ARGSUSED*/
3797 3796 static void
3798 3797 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3799 3798 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3800 3799 {
3801 3800 hp->dmai_nwin++;
3802 3801 window->wd_dosync = B_FALSE;
3803 3802 window->wd_offset = cur_offset;
3804 3803 window->wd_size = 0;
3805 3804 window->wd_first_cookie = cookie;
3806 3805 window->wd_cookie_cnt = 0;
3807 3806 window->wd_trim.tr_trim_first = B_FALSE;
3808 3807 window->wd_trim.tr_trim_last = B_FALSE;
3809 3808 window->wd_trim.tr_first_copybuf_win = B_FALSE;
3810 3809 window->wd_trim.tr_last_copybuf_win = B_FALSE;
3811 3810 #if !defined(__amd64)
3812 3811 window->wd_remap_copybuf = dma->dp_cb_remaping;
3813 3812 #endif
3814 3813 }
3815 3814
3816 3815
3817 3816 /*
3818 3817 * rootnex_setup_cookie()
3819 3818 * Called in the bind slow path when the sgl uses the copy buffer. If any of
3820 3819 * the sgl uses the copy buffer, we need to go through each cookie, figure
3821 3820 * out if it uses the copy buffer, and if it does, save away everything we'll
3822 3821 * need during sync.
3823 3822 */
3824 3823 static void
3825 3824 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3826 3825 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3827 3826 page_t **cur_pp)
3828 3827 {
3829 3828 boolean_t copybuf_sz_power_2;
3830 3829 rootnex_sglinfo_t *sinfo;
3831 3830 paddr_t paddr;
3832 3831 uint_t pidx;
3833 3832 uint_t pcnt;
3834 3833 off_t poff;
3835 3834 #if defined(__amd64)
3836 3835 pfn_t pfn;
3837 3836 #else
3838 3837 page_t **pplist;
3839 3838 #endif
3840 3839
3841 3840 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3842 3841
3843 3842 sinfo = &dma->dp_sglinfo;
3844 3843
3845 3844 /*
3846 3845 * Calculate the page index relative to the start of the buffer. The
3847 3846 * index to the current page for our buffer is the offset into the
3848 3847 * first page of the buffer plus our current offset into the buffer
3849 3848 * itself, shifted of course...
3850 3849 */
3851 3850 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3852 3851 ASSERT(pidx < sinfo->si_max_pages);
3853 3852
3854 3853 /* if this cookie uses the copy buffer */
3855 3854 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3856 3855 /*
3857 3856 * NOTE: we know that since this cookie uses the copy buffer, it
3858 3857 * is <= MMU_PAGESIZE.
3859 3858 */
3860 3859
3861 3860 /*
3862 3861 * get the offset into the page. For the 64-bit kernel, get the
3863 3862 * pfn which we'll use with seg kpm.
3864 3863 */
3865 3864 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3866 3865 #if defined(__amd64)
3867 3866 /* mfn_to_pfn() is a NOP on i86pc */
3868 3867 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3869 3868 #endif /* __amd64 */
3870 3869
3871 3870 /* figure out if the copybuf size is a power of 2 */
3872 3871 if (!ISP2(dma->dp_copybuf_size)) {
3873 3872 copybuf_sz_power_2 = B_FALSE;
3874 3873 } else {
3875 3874 copybuf_sz_power_2 = B_TRUE;
3876 3875 }
3877 3876
3878 3877 /* This page uses the copy buffer */
3879 3878 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3880 3879
3881 3880 /*
3882 3881 * save the copy buffer KVA that we'll use with this page.
3883 3882 * if we still fit within the copybuf, it's a simple add.
3884 3883 * otherwise, we need to wrap over using & or % accordingly.
3885 3884 */
3886 3885 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3887 3886 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3888 3887 *copybuf_used;
3889 3888 } else {
3890 3889 if (copybuf_sz_power_2) {
3891 3890 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3892 3891 (uintptr_t)dma->dp_cbaddr +
3893 3892 (*copybuf_used &
3894 3893 (dma->dp_copybuf_size - 1)));
3895 3894 } else {
3896 3895 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3897 3896 (uintptr_t)dma->dp_cbaddr +
3898 3897 (*copybuf_used % dma->dp_copybuf_size));
3899 3898 }
3900 3899 }
3901 3900
3902 3901 /*
3903 3902 * over write the cookie physical address with the address of
3904 3903 * the physical address of the copy buffer page that we will
3905 3904 * use.
3906 3905 */
3907 3906 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3908 3907 dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3909 3908
3910 3909 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3911 3910
3912 3911 /* if we have a kernel VA, it's easy, just save that address */
3913 3912 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3914 3913 (sinfo->si_asp == &kas)) {
3915 3914 /*
3916 3915 * save away the page aligned virtual address of the
3917 3916 * driver buffer. Offsets are handled in the sync code.
3918 3917 */
3919 3918 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3920 3919 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3921 3920 & MMU_PAGEMASK);
3922 3921 #if !defined(__amd64)
3923 3922 /*
3924 3923 * we didn't need to, and will never need to map this
3925 3924 * page.
3926 3925 */
3927 3926 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3928 3927 #endif
3929 3928
3930 3929 /* we don't have a kernel VA. We need one for the bcopy. */
3931 3930 } else {
3932 3931 #if defined(__amd64)
3933 3932 /*
3934 3933 * for the 64-bit kernel, it's easy. We use seg kpm to
3935 3934 * get a Kernel VA for the corresponding pfn.
3936 3935 */
3937 3936 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3938 3937 #else
3939 3938 /*
3940 3939 * for the 32-bit kernel, this is a pain. First we'll
3941 3940 * save away the page_t or user VA for this page. This
3942 3941 * is needed in rootnex_dma_win() when we switch to a
3943 3942 * new window which requires us to re-map the copy
3944 3943 * buffer.
3945 3944 */
3946 3945 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3947 3946 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3948 3947 dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3949 3948 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3950 3949 } else if (pplist != NULL) {
3951 3950 dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3952 3951 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3953 3952 } else {
3954 3953 dma->dp_pgmap[pidx].pm_pp = NULL;
3955 3954 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3956 3955 (((uintptr_t)
3957 3956 dmar_object->dmao_obj.virt_obj.v_addr +
3958 3957 cur_offset) & MMU_PAGEMASK);
3959 3958 }
3960 3959
3961 3960 /*
3962 3961 * save away the page aligned virtual address which was
3963 3962 * allocated from the kernel heap arena (taking into
3964 3963 * account if we need more copy buffer than we alloced
3965 3964 * and use multiple windows to handle this, i.e. &,%).
3966 3965 * NOTE: there isn't and physical memory backing up this
3967 3966 * virtual address space currently.
3968 3967 */
3969 3968 if ((*copybuf_used + MMU_PAGESIZE) <=
3970 3969 dma->dp_copybuf_size) {
3971 3970 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3972 3971 (((uintptr_t)dma->dp_kva + *copybuf_used) &
3973 3972 MMU_PAGEMASK);
3974 3973 } else {
3975 3974 if (copybuf_sz_power_2) {
3976 3975 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3977 3976 (((uintptr_t)dma->dp_kva +
3978 3977 (*copybuf_used &
3979 3978 (dma->dp_copybuf_size - 1))) &
3980 3979 MMU_PAGEMASK);
3981 3980 } else {
3982 3981 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3983 3982 (((uintptr_t)dma->dp_kva +
3984 3983 (*copybuf_used %
3985 3984 dma->dp_copybuf_size)) &
3986 3985 MMU_PAGEMASK);
3987 3986 }
3988 3987 }
3989 3988
3990 3989 /*
3991 3990 * if we haven't used up the available copy buffer yet,
3992 3991 * map the kva to the physical page.
3993 3992 */
3994 3993 if (!dma->dp_cb_remaping && ((*copybuf_used +
3995 3994 MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3996 3995 dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3997 3996 if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3998 3997 i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3999 3998 dma->dp_pgmap[pidx].pm_kaddr);
4000 3999 } else {
4001 4000 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4002 4001 sinfo->si_asp,
4003 4002 dma->dp_pgmap[pidx].pm_kaddr);
4004 4003 }
4005 4004
4006 4005 /*
4007 4006 * we've used up the available copy buffer, this page
4008 4007 * will have to be mapped during rootnex_dma_win() when
4009 4008 * we switch to a new window which requires a re-map
4010 4009 * the copy buffer. (32-bit kernel only)
4011 4010 */
4012 4011 } else {
4013 4012 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4014 4013 }
4015 4014 #endif
4016 4015 /* go to the next page_t */
4017 4016 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4018 4017 *cur_pp = (*cur_pp)->p_next;
4019 4018 }
4020 4019 }
4021 4020
4022 4021 /* add to the copy buffer count */
4023 4022 *copybuf_used += MMU_PAGESIZE;
4024 4023
4025 4024 /*
4026 4025 * This cookie doesn't use the copy buffer. Walk through the pages this
4027 4026 * cookie occupies to reflect this.
4028 4027 */
4029 4028 } else {
4030 4029 /*
4031 4030 * figure out how many pages the cookie occupies. We need to
4032 4031 * use the original page offset of the buffer and the cookies
4033 4032 * offset in the buffer to do this.
4034 4033 */
4035 4034 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4036 4035 pcnt = mmu_btopr(cookie->dmac_size + poff);
4037 4036
4038 4037 while (pcnt > 0) {
4039 4038 #if !defined(__amd64)
4040 4039 /*
4041 4040 * the 32-bit kernel doesn't have seg kpm, so we need
4042 4041 * to map in the driver buffer (if it didn't come down
4043 4042 * with a kernel VA) on the fly. Since this page doesn't
4044 4043 * use the copy buffer, it's not, or will it ever, have
4045 4044 * to be mapped in.
4046 4045 */
4047 4046 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4048 4047 #endif
4049 4048 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4050 4049
4051 4050 /*
4052 4051 * we need to update pidx and cur_pp or we'll loose
4053 4052 * track of where we are.
4054 4053 */
4055 4054 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4056 4055 *cur_pp = (*cur_pp)->p_next;
4057 4056 }
4058 4057 pidx++;
4059 4058 pcnt--;
4060 4059 }
4061 4060 }
4062 4061 }
4063 4062
4064 4063
4065 4064 /*
4066 4065 * rootnex_sgllen_window_boundary()
4067 4066 * Called in the bind slow path when the next cookie causes us to exceed (in
4068 4067 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4069 4068 * length supported by the DMA H/W.
4070 4069 */
4071 4070 static int
4072 4071 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4073 4072 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4074 4073 off_t cur_offset)
4075 4074 {
4076 4075 off_t new_offset;
4077 4076 size_t trim_sz;
4078 4077 off_t coffset;
4079 4078
4080 4079
4081 4080 /*
4082 4081 * if we know we'll never have to trim, it's pretty easy. Just move to
4083 4082 * the next window and init it. We're done.
4084 4083 */
4085 4084 if (!dma->dp_trim_required) {
4086 4085 (*windowp)++;
4087 4086 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4088 4087 (*windowp)->wd_cookie_cnt++;
4089 4088 (*windowp)->wd_size = cookie->dmac_size;
4090 4089 return (DDI_SUCCESS);
4091 4090 }
4092 4091
4093 4092 /* figure out how much we need to trim from the window */
4094 4093 ASSERT(attr->dma_attr_granular != 0);
4095 4094 if (dma->dp_granularity_power_2) {
4096 4095 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4097 4096 } else {
4098 4097 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4099 4098 }
4100 4099
4101 4100 /* The window's a whole multiple of granularity. We're done */
4102 4101 if (trim_sz == 0) {
4103 4102 (*windowp)++;
4104 4103 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4105 4104 (*windowp)->wd_cookie_cnt++;
4106 4105 (*windowp)->wd_size = cookie->dmac_size;
4107 4106 return (DDI_SUCCESS);
4108 4107 }
4109 4108
4110 4109 /*
4111 4110 * The window's not a whole multiple of granularity, since we know this
4112 4111 * is due to the sgllen, we need to go back to the last cookie and trim
4113 4112 * that one, add the left over part of the old cookie into the new
4114 4113 * window, and then add in the new cookie into the new window.
4115 4114 */
4116 4115
4117 4116 /*
4118 4117 * make sure the driver isn't making us do something bad... Trimming and
4119 4118 * sgllen == 1 don't go together.
4120 4119 */
4121 4120 if (attr->dma_attr_sgllen == 1) {
4122 4121 return (DDI_DMA_NOMAPPING);
4123 4122 }
4124 4123
4125 4124 /*
4126 4125 * first, setup the current window to account for the trim. Need to go
4127 4126 * back to the last cookie for this.
4128 4127 */
4129 4128 cookie--;
4130 4129 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4131 4130 (*windowp)->wd_trim.tr_last_cookie = cookie;
4132 4131 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4133 4132 ASSERT(cookie->dmac_size > trim_sz);
4134 4133 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4135 4134 (*windowp)->wd_size -= trim_sz;
4136 4135
4137 4136 /* save the buffer offsets for the next window */
4138 4137 coffset = cookie->dmac_size - trim_sz;
4139 4138 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4140 4139
4141 4140 /*
4142 4141 * set this now in case this is the first window. all other cases are
4143 4142 * set in dma_win()
4144 4143 */
4145 4144 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4146 4145
4147 4146 /*
4148 4147 * initialize the next window using what's left over in the previous
4149 4148 * cookie.
4150 4149 */
4151 4150 (*windowp)++;
4152 4151 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4153 4152 (*windowp)->wd_cookie_cnt++;
4154 4153 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4155 4154 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4156 4155 (*windowp)->wd_trim.tr_first_size = trim_sz;
4157 4156 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4158 4157 (*windowp)->wd_dosync = B_TRUE;
4159 4158 }
4160 4159
4161 4160 /*
4162 4161 * now go back to the current cookie and add it to the new window. set
4163 4162 * the new window size to the what was left over from the previous
4164 4163 * cookie and what's in the current cookie.
4165 4164 */
4166 4165 cookie++;
4167 4166 (*windowp)->wd_cookie_cnt++;
4168 4167 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4169 4168
4170 4169 /*
4171 4170 * trim plus the next cookie could put us over maxxfer (a cookie can be
4172 4171 * a max size of maxxfer). Handle that case.
4173 4172 */
4174 4173 if ((*windowp)->wd_size > dma->dp_maxxfer) {
4175 4174 /*
4176 4175 * maxxfer is already a whole multiple of granularity, and this
4177 4176 * trim will be <= the previous trim (since a cookie can't be
4178 4177 * larger than maxxfer). Make things simple here.
4179 4178 */
4180 4179 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4181 4180 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4182 4181 (*windowp)->wd_trim.tr_last_cookie = cookie;
4183 4182 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4184 4183 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4185 4184 (*windowp)->wd_size -= trim_sz;
4186 4185 ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4187 4186
4188 4187 /* save the buffer offsets for the next window */
4189 4188 coffset = cookie->dmac_size - trim_sz;
4190 4189 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4191 4190
4192 4191 /* setup the next window */
4193 4192 (*windowp)++;
4194 4193 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4195 4194 (*windowp)->wd_cookie_cnt++;
4196 4195 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4197 4196 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4198 4197 coffset;
4199 4198 (*windowp)->wd_trim.tr_first_size = trim_sz;
4200 4199 }
4201 4200
4202 4201 return (DDI_SUCCESS);
4203 4202 }
4204 4203
4205 4204
4206 4205 /*
4207 4206 * rootnex_copybuf_window_boundary()
4208 4207 * Called in bind slowpath when we get to a window boundary because we used
4209 4208 * up all the copy buffer that we have.
4210 4209 */
4211 4210 static int
4212 4211 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4213 4212 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4214 4213 size_t *copybuf_used)
4215 4214 {
4216 4215 rootnex_sglinfo_t *sinfo;
4217 4216 off_t new_offset;
4218 4217 size_t trim_sz;
4219 4218 paddr_t paddr;
4220 4219 off_t coffset;
4221 4220 uint_t pidx;
4222 4221 off_t poff;
4223 4222
4224 4223
4225 4224 sinfo = &dma->dp_sglinfo;
4226 4225
4227 4226 /*
4228 4227 * the copy buffer should be a whole multiple of page size. We know that
4229 4228 * this cookie is <= MMU_PAGESIZE.
4230 4229 */
4231 4230 ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4232 4231
4233 4232 /*
4234 4233 * from now on, all new windows in this bind need to be re-mapped during
4235 4234 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4236 4235 * space...
4237 4236 */
4238 4237 #if !defined(__amd64)
4239 4238 dma->dp_cb_remaping = B_TRUE;
4240 4239 #endif
4241 4240
4242 4241 /* reset copybuf used */
4243 4242 *copybuf_used = 0;
4244 4243
4245 4244 /*
4246 4245 * if we don't have to trim (since granularity is set to 1), go to the
4247 4246 * next window and add the current cookie to it. We know the current
4248 4247 * cookie uses the copy buffer since we're in this code path.
4249 4248 */
4250 4249 if (!dma->dp_trim_required) {
4251 4250 (*windowp)++;
4252 4251 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4253 4252
4254 4253 /* Add this cookie to the new window */
4255 4254 (*windowp)->wd_cookie_cnt++;
4256 4255 (*windowp)->wd_size += cookie->dmac_size;
4257 4256 *copybuf_used += MMU_PAGESIZE;
4258 4257 return (DDI_SUCCESS);
4259 4258 }
4260 4259
4261 4260 /*
4262 4261 * *** may need to trim, figure it out.
4263 4262 */
4264 4263
4265 4264 /* figure out how much we need to trim from the window */
4266 4265 if (dma->dp_granularity_power_2) {
4267 4266 trim_sz = (*windowp)->wd_size &
4268 4267 (hp->dmai_attr.dma_attr_granular - 1);
4269 4268 } else {
4270 4269 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4271 4270 }
4272 4271
4273 4272 /*
4274 4273 * if the window's a whole multiple of granularity, go to the next
4275 4274 * window, init it, then add in the current cookie. We know the current
4276 4275 * cookie uses the copy buffer since we're in this code path.
4277 4276 */
4278 4277 if (trim_sz == 0) {
4279 4278 (*windowp)++;
4280 4279 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4281 4280
4282 4281 /* Add this cookie to the new window */
4283 4282 (*windowp)->wd_cookie_cnt++;
4284 4283 (*windowp)->wd_size += cookie->dmac_size;
4285 4284 *copybuf_used += MMU_PAGESIZE;
4286 4285 return (DDI_SUCCESS);
4287 4286 }
4288 4287
4289 4288 /*
4290 4289 * *** We figured it out, we definitly need to trim
4291 4290 */
4292 4291
4293 4292 /*
4294 4293 * make sure the driver isn't making us do something bad...
4295 4294 * Trimming and sgllen == 1 don't go together.
4296 4295 */
4297 4296 if (hp->dmai_attr.dma_attr_sgllen == 1) {
4298 4297 return (DDI_DMA_NOMAPPING);
4299 4298 }
4300 4299
4301 4300 /*
4302 4301 * first, setup the current window to account for the trim. Need to go
4303 4302 * back to the last cookie for this. Some of the last cookie will be in
4304 4303 * the current window, and some of the last cookie will be in the new
4305 4304 * window. All of the current cookie will be in the new window.
4306 4305 */
4307 4306 cookie--;
4308 4307 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4309 4308 (*windowp)->wd_trim.tr_last_cookie = cookie;
4310 4309 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4311 4310 ASSERT(cookie->dmac_size > trim_sz);
4312 4311 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4313 4312 (*windowp)->wd_size -= trim_sz;
4314 4313
4315 4314 /*
4316 4315 * we're trimming the last cookie (not the current cookie). So that
4317 4316 * last cookie may have or may not have been using the copy buffer (
4318 4317 * we know the cookie passed in uses the copy buffer since we're in
4319 4318 * this code path).
4320 4319 *
4321 4320 * If the last cookie doesn't use the copy buffer, nothing special to
4322 4321 * do. However, if it does uses the copy buffer, it will be both the
4323 4322 * last page in the current window and the first page in the next
4324 4323 * window. Since we are reusing the copy buffer (and KVA space on the
4325 4324 * 32-bit kernel), this page will use the end of the copy buffer in the
4326 4325 * current window, and the start of the copy buffer in the next window.
4327 4326 * Track that info... The cookie physical address was already set to
4328 4327 * the copy buffer physical address in setup_cookie..
4329 4328 */
4330 4329 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4331 4330 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4332 4331 (*windowp)->wd_size) >> MMU_PAGESHIFT;
4333 4332 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4334 4333 (*windowp)->wd_trim.tr_last_pidx = pidx;
4335 4334 (*windowp)->wd_trim.tr_last_cbaddr =
4336 4335 dma->dp_pgmap[pidx].pm_cbaddr;
4337 4336 #if !defined(__amd64)
4338 4337 (*windowp)->wd_trim.tr_last_kaddr =
4339 4338 dma->dp_pgmap[pidx].pm_kaddr;
4340 4339 #endif
4341 4340 }
4342 4341
4343 4342 /* save the buffer offsets for the next window */
4344 4343 coffset = cookie->dmac_size - trim_sz;
4345 4344 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4346 4345
4347 4346 /*
4348 4347 * set this now in case this is the first window. all other cases are
4349 4348 * set in dma_win()
4350 4349 */
4351 4350 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4352 4351
4353 4352 /*
4354 4353 * initialize the next window using what's left over in the previous
4355 4354 * cookie.
4356 4355 */
4357 4356 (*windowp)++;
4358 4357 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4359 4358 (*windowp)->wd_cookie_cnt++;
4360 4359 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4361 4360 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4362 4361 (*windowp)->wd_trim.tr_first_size = trim_sz;
4363 4362
4364 4363 /*
4365 4364 * again, we're tracking if the last cookie uses the copy buffer.
4366 4365 * read the comment above for more info on why we need to track
4367 4366 * additional state.
4368 4367 *
4369 4368 * For the first cookie in the new window, we need reset the physical
4370 4369 * address to DMA into to the start of the copy buffer plus any
4371 4370 * initial page offset which may be present.
4372 4371 */
4373 4372 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4374 4373 (*windowp)->wd_dosync = B_TRUE;
4375 4374 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4376 4375 (*windowp)->wd_trim.tr_first_pidx = pidx;
4377 4376 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4378 4377 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4379 4378
4380 4379 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4381 4380 poff;
4382 4381 (*windowp)->wd_trim.tr_first_paddr =
4383 4382 ROOTNEX_PADDR_TO_RBASE(paddr);
4384 4383
4385 4384 #if !defined(__amd64)
4386 4385 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4387 4386 #endif
4388 4387 /* account for the cookie copybuf usage in the new window */
4389 4388 *copybuf_used += MMU_PAGESIZE;
4390 4389
4391 4390 /*
4392 4391 * every piece of code has to have a hack, and here is this
4393 4392 * ones :-)
4394 4393 *
4395 4394 * There is a complex interaction between setup_cookie and the
4396 4395 * copybuf window boundary. The complexity had to be in either
4397 4396 * the maxxfer window, or the copybuf window, and I chose the
4398 4397 * copybuf code.
4399 4398 *
4400 4399 * So in this code path, we have taken the last cookie,
4401 4400 * virtually broken it in half due to the trim, and it happens
4402 4401 * to use the copybuf which further complicates life. At the
4403 4402 * same time, we have already setup the current cookie, which
4404 4403 * is now wrong. More background info: the current cookie uses
4405 4404 * the copybuf, so it is only a page long max. So we need to
4406 4405 * fix the current cookies copy buffer address, physical
4407 4406 * address, and kva for the 32-bit kernel. We due this by
4408 4407 * bumping them by page size (of course, we can't due this on
4409 4408 * the physical address since the copy buffer may not be
4410 4409 * physically contiguous).
4411 4410 */
4412 4411 cookie++;
4413 4412 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4414 4413 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4415 4414
4416 4415 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4417 4416 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4418 4417 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4419 4418
4420 4419 #if !defined(__amd64)
4421 4420 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4422 4421 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4423 4422 #endif
4424 4423 } else {
4425 4424 /* go back to the current cookie */
4426 4425 cookie++;
4427 4426 }
4428 4427
4429 4428 /*
4430 4429 * add the current cookie to the new window. set the new window size to
4431 4430 * the what was left over from the previous cookie and what's in the
4432 4431 * current cookie.
4433 4432 */
4434 4433 (*windowp)->wd_cookie_cnt++;
4435 4434 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4436 4435 ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4437 4436
4438 4437 /*
4439 4438 * we know that the cookie passed in always uses the copy buffer. We
4440 4439 * wouldn't be here if it didn't.
4441 4440 */
4442 4441 *copybuf_used += MMU_PAGESIZE;
4443 4442
4444 4443 return (DDI_SUCCESS);
4445 4444 }
4446 4445
4447 4446
4448 4447 /*
4449 4448 * rootnex_maxxfer_window_boundary()
4450 4449 * Called in bind slowpath when we get to a window boundary because we will
4451 4450 * go over maxxfer.
4452 4451 */
4453 4452 static int
4454 4453 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4455 4454 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4456 4455 {
4457 4456 size_t dmac_size;
4458 4457 off_t new_offset;
4459 4458 size_t trim_sz;
4460 4459 off_t coffset;
4461 4460
4462 4461
4463 4462 /*
4464 4463 * calculate how much we have to trim off of the current cookie to equal
4465 4464 * maxxfer. We don't have to account for granularity here since our
4466 4465 * maxxfer already takes that into account.
4467 4466 */
4468 4467 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4469 4468 ASSERT(trim_sz <= cookie->dmac_size);
4470 4469 ASSERT(trim_sz <= dma->dp_maxxfer);
4471 4470
4472 4471 /* save cookie size since we need it later and we might change it */
4473 4472 dmac_size = cookie->dmac_size;
4474 4473
4475 4474 /*
4476 4475 * if we're not trimming the entire cookie, setup the current window to
4477 4476 * account for the trim.
4478 4477 */
4479 4478 if (trim_sz < cookie->dmac_size) {
4480 4479 (*windowp)->wd_cookie_cnt++;
4481 4480 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4482 4481 (*windowp)->wd_trim.tr_last_cookie = cookie;
4483 4482 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4484 4483 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4485 4484 (*windowp)->wd_size = dma->dp_maxxfer;
4486 4485
4487 4486 /*
4488 4487 * set the adjusted cookie size now in case this is the first
4489 4488 * window. All other windows are taken care of in get win
4490 4489 */
4491 4490 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4492 4491 }
4493 4492
4494 4493 /*
4495 4494 * coffset is the current offset within the cookie, new_offset is the
4496 4495 * current offset with the entire buffer.
4497 4496 */
4498 4497 coffset = dmac_size - trim_sz;
4499 4498 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4500 4499
4501 4500 /* initialize the next window */
4502 4501 (*windowp)++;
4503 4502 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4504 4503 (*windowp)->wd_cookie_cnt++;
4505 4504 (*windowp)->wd_size = trim_sz;
4506 4505 if (trim_sz < dmac_size) {
4507 4506 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4508 4507 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4509 4508 coffset;
4510 4509 (*windowp)->wd_trim.tr_first_size = trim_sz;
4511 4510 }
4512 4511
4513 4512 return (DDI_SUCCESS);
4514 4513 }
4515 4514
4516 4515
4517 4516 /*ARGSUSED*/
4518 4517 static int
4519 4518 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4520 4519 off_t off, size_t len, uint_t cache_flags)
4521 4520 {
4522 4521 rootnex_sglinfo_t *sinfo;
4523 4522 rootnex_pgmap_t *cbpage;
4524 4523 rootnex_window_t *win;
4525 4524 ddi_dma_impl_t *hp;
4526 4525 rootnex_dma_t *dma;
4527 4526 caddr_t fromaddr;
4528 4527 caddr_t toaddr;
4529 4528 uint_t psize;
4530 4529 off_t offset;
4531 4530 uint_t pidx;
4532 4531 size_t size;
4533 4532 off_t poff;
4534 4533 int e;
4535 4534
4536 4535
4537 4536 hp = (ddi_dma_impl_t *)handle;
4538 4537 dma = (rootnex_dma_t *)hp->dmai_private;
4539 4538 sinfo = &dma->dp_sglinfo;
4540 4539
4541 4540 /*
4542 4541 * if we don't have any windows, we don't need to sync. A copybuf
4543 4542 * will cause us to have at least one window.
4544 4543 */
4545 4544 if (dma->dp_window == NULL) {
4546 4545 return (DDI_SUCCESS);
4547 4546 }
4548 4547
4549 4548 /* This window may not need to be sync'd */
4550 4549 win = &dma->dp_window[dma->dp_current_win];
4551 4550 if (!win->wd_dosync) {
4552 4551 return (DDI_SUCCESS);
4553 4552 }
4554 4553
4555 4554 /* handle off and len special cases */
4556 4555 if ((off == 0) || (rootnex_sync_ignore_params)) {
4557 4556 offset = win->wd_offset;
4558 4557 } else {
4559 4558 offset = off;
4560 4559 }
4561 4560 if ((len == 0) || (rootnex_sync_ignore_params)) {
4562 4561 size = win->wd_size;
4563 4562 } else {
4564 4563 size = len;
4565 4564 }
4566 4565
4567 4566 /* check the sync args to make sure they make a little sense */
4568 4567 if (rootnex_sync_check_parms) {
4569 4568 e = rootnex_valid_sync_parms(hp, win, offset, size,
4570 4569 cache_flags);
4571 4570 if (e != DDI_SUCCESS) {
4572 4571 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4573 4572 return (DDI_FAILURE);
4574 4573 }
4575 4574 }
4576 4575
4577 4576 /*
4578 4577 * special case the first page to handle the offset into the page. The
4579 4578 * offset to the current page for our buffer is the offset into the
4580 4579 * first page of the buffer plus our current offset into the buffer
4581 4580 * itself, masked of course.
4582 4581 */
4583 4582 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4584 4583 psize = MIN((MMU_PAGESIZE - poff), size);
4585 4584
4586 4585 /* go through all the pages that we want to sync */
4587 4586 while (size > 0) {
4588 4587 /*
4589 4588 * Calculate the page index relative to the start of the buffer.
4590 4589 * The index to the current page for our buffer is the offset
4591 4590 * into the first page of the buffer plus our current offset
4592 4591 * into the buffer itself, shifted of course...
4593 4592 */
4594 4593 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4595 4594 ASSERT(pidx < sinfo->si_max_pages);
4596 4595
4597 4596 /*
4598 4597 * if this page uses the copy buffer, we need to sync it,
4599 4598 * otherwise, go on to the next page.
4600 4599 */
4601 4600 cbpage = &dma->dp_pgmap[pidx];
4602 4601 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4603 4602 (cbpage->pm_uses_copybuf == B_FALSE));
4604 4603 if (cbpage->pm_uses_copybuf) {
4605 4604 /* cbaddr and kaddr should be page aligned */
4606 4605 ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4607 4606 MMU_PAGEOFFSET) == 0);
4608 4607 ASSERT(((uintptr_t)cbpage->pm_kaddr &
4609 4608 MMU_PAGEOFFSET) == 0);
4610 4609
4611 4610 /*
4612 4611 * if we're copying for the device, we are going to
4613 4612 * copy from the drivers buffer and to the rootnex
4614 4613 * allocated copy buffer.
4615 4614 */
4616 4615 if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4617 4616 fromaddr = cbpage->pm_kaddr + poff;
4618 4617 toaddr = cbpage->pm_cbaddr + poff;
4619 4618 ROOTNEX_DPROBE2(rootnex__sync__dev,
4620 4619 dev_info_t *, dma->dp_dip, size_t, psize);
4621 4620
4622 4621 /*
4623 4622 * if we're copying for the cpu/kernel, we are going to
4624 4623 * copy from the rootnex allocated copy buffer to the
4625 4624 * drivers buffer.
4626 4625 */
4627 4626 } else {
4628 4627 fromaddr = cbpage->pm_cbaddr + poff;
4629 4628 toaddr = cbpage->pm_kaddr + poff;
4630 4629 ROOTNEX_DPROBE2(rootnex__sync__cpu,
4631 4630 dev_info_t *, dma->dp_dip, size_t, psize);
4632 4631 }
4633 4632
4634 4633 bcopy(fromaddr, toaddr, psize);
4635 4634 }
4636 4635
4637 4636 /*
4638 4637 * decrement size until we're done, update our offset into the
4639 4638 * buffer, and get the next page size.
4640 4639 */
4641 4640 size -= psize;
4642 4641 offset += psize;
4643 4642 psize = MIN(MMU_PAGESIZE, size);
4644 4643
4645 4644 /* page offset is zero for the rest of this loop */
4646 4645 poff = 0;
4647 4646 }
4648 4647
4649 4648 return (DDI_SUCCESS);
4650 4649 }
4651 4650
4652 4651 /*
4653 4652 * rootnex_dma_sync()
4654 4653 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4655 4654 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4656 4655 * is set, ddi_dma_sync() returns immediately passing back success.
4657 4656 */
4658 4657 /*ARGSUSED*/
4659 4658 static int
4660 4659 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4661 4660 off_t off, size_t len, uint_t cache_flags)
4662 4661 {
4663 4662 #if defined(__amd64) && !defined(__xpv)
4664 4663 if (IOMMU_USED(rdip)) {
4665 4664 return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4666 4665 cache_flags));
4667 4666 }
4668 4667 #endif
4669 4668 return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4670 4669 cache_flags));
4671 4670 }
4672 4671
4673 4672 /*
4674 4673 * rootnex_valid_sync_parms()
4675 4674 * checks the parameters passed to sync to verify they are correct.
4676 4675 */
4677 4676 static int
4678 4677 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4679 4678 off_t offset, size_t size, uint_t cache_flags)
4680 4679 {
4681 4680 off_t woffset;
4682 4681
4683 4682
4684 4683 /*
4685 4684 * the first part of the test to make sure the offset passed in is
4686 4685 * within the window.
4687 4686 */
4688 4687 if (offset < win->wd_offset) {
4689 4688 return (DDI_FAILURE);
4690 4689 }
4691 4690
4692 4691 /*
4693 4692 * second and last part of the test to make sure the offset and length
4694 4693 * passed in is within the window.
4695 4694 */
4696 4695 woffset = offset - win->wd_offset;
4697 4696 if ((woffset + size) > win->wd_size) {
4698 4697 return (DDI_FAILURE);
4699 4698 }
4700 4699
4701 4700 /*
4702 4701 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4703 4702 * be set too.
4704 4703 */
4705 4704 if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4706 4705 (hp->dmai_rflags & DDI_DMA_WRITE)) {
4707 4706 return (DDI_SUCCESS);
4708 4707 }
4709 4708
4710 4709 /*
4711 4710 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4712 4711 * should be set. Also DDI_DMA_READ should be set in the flags.
4713 4712 */
4714 4713 if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4715 4714 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4716 4715 (hp->dmai_rflags & DDI_DMA_READ)) {
4717 4716 return (DDI_SUCCESS);
4718 4717 }
4719 4718
4720 4719 return (DDI_FAILURE);
4721 4720 }
4722 4721
4723 4722
4724 4723 /*ARGSUSED*/
4725 4724 static int
4726 4725 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4727 4726 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4728 4727 uint_t *ccountp)
4729 4728 {
4730 4729 rootnex_window_t *window;
4731 4730 rootnex_trim_t *trim;
4732 4731 ddi_dma_impl_t *hp;
4733 4732 rootnex_dma_t *dma;
4734 4733 ddi_dma_obj_t *dmao;
4735 4734 #if !defined(__amd64)
4736 4735 rootnex_sglinfo_t *sinfo;
4737 4736 rootnex_pgmap_t *pmap;
4738 4737 uint_t pidx;
4739 4738 uint_t pcnt;
4740 4739 off_t poff;
4741 4740 int i;
4742 4741 #endif
4743 4742
4744 4743
4745 4744 hp = (ddi_dma_impl_t *)handle;
4746 4745 dma = (rootnex_dma_t *)hp->dmai_private;
4747 4746 #if !defined(__amd64)
4748 4747 sinfo = &dma->dp_sglinfo;
4749 4748 #endif
4750 4749
4751 4750 /* If we try and get a window which doesn't exist, return failure */
4752 4751 if (win >= hp->dmai_nwin) {
4753 4752 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4754 4753 return (DDI_FAILURE);
4755 4754 }
4756 4755
4757 4756 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4758 4757
4759 4758 /*
4760 4759 * if we don't have any windows, and they're asking for the first
4761 4760 * window, setup the cookie pointer to the first cookie in the bind.
4762 4761 * setup our return values, then increment the cookie since we return
4763 4762 * the first cookie on the stack.
4764 4763 */
4765 4764 if (dma->dp_window == NULL) {
4766 4765 if (win != 0) {
4767 4766 ROOTNEX_DPROF_INC(
4768 4767 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4769 4768 return (DDI_FAILURE);
4770 4769 }
4771 4770 hp->dmai_cookie = dma->dp_cookies;
4772 4771 *offp = 0;
4773 4772 *lenp = dmao->dmao_size;
4774 4773 *ccountp = dma->dp_sglinfo.si_sgl_size;
4775 4774 *cookiep = hp->dmai_cookie[0];
4776 4775 hp->dmai_cookie++;
4777 4776 return (DDI_SUCCESS);
4778 4777 }
4779 4778
4780 4779 /* sync the old window before moving on to the new one */
4781 4780 window = &dma->dp_window[dma->dp_current_win];
4782 4781 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4783 4782 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4784 4783 DDI_DMA_SYNC_FORCPU);
4785 4784 }
4786 4785
4787 4786 #if !defined(__amd64)
4788 4787 /*
4789 4788 * before we move to the next window, if we need to re-map, unmap all
4790 4789 * the pages in this window.
4791 4790 */
4792 4791 if (dma->dp_cb_remaping) {
4793 4792 /*
4794 4793 * If we switch to this window again, we'll need to map in
4795 4794 * on the fly next time.
4796 4795 */
4797 4796 window->wd_remap_copybuf = B_TRUE;
4798 4797
4799 4798 /*
4800 4799 * calculate the page index into the buffer where this window
4801 4800 * starts, and the number of pages this window takes up.
4802 4801 */
4803 4802 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4804 4803 MMU_PAGESHIFT;
4805 4804 poff = (sinfo->si_buf_offset + window->wd_offset) &
4806 4805 MMU_PAGEOFFSET;
4807 4806 pcnt = mmu_btopr(window->wd_size + poff);
4808 4807 ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4809 4808
4810 4809 /* unmap pages which are currently mapped in this window */
4811 4810 for (i = 0; i < pcnt; i++) {
4812 4811 if (dma->dp_pgmap[pidx].pm_mapped) {
4813 4812 hat_unload(kas.a_hat,
4814 4813 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4815 4814 HAT_UNLOAD);
4816 4815 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4817 4816 }
4818 4817 pidx++;
4819 4818 }
4820 4819 }
4821 4820 #endif
4822 4821
4823 4822 /*
4824 4823 * Move to the new window.
4825 4824 * NOTE: current_win must be set for sync to work right
4826 4825 */
4827 4826 dma->dp_current_win = win;
4828 4827 window = &dma->dp_window[win];
4829 4828
4830 4829 /* if needed, adjust the first and/or last cookies for trim */
4831 4830 trim = &window->wd_trim;
4832 4831 if (trim->tr_trim_first) {
4833 4832 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4834 4833 window->wd_first_cookie->dmac_size = trim->tr_first_size;
4835 4834 #if !defined(__amd64)
4836 4835 window->wd_first_cookie->dmac_type =
4837 4836 (window->wd_first_cookie->dmac_type &
4838 4837 ROOTNEX_USES_COPYBUF) + window->wd_offset;
4839 4838 #endif
4840 4839 if (trim->tr_first_copybuf_win) {
4841 4840 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4842 4841 trim->tr_first_cbaddr;
4843 4842 #if !defined(__amd64)
4844 4843 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4845 4844 trim->tr_first_kaddr;
4846 4845 #endif
4847 4846 }
4848 4847 }
4849 4848 if (trim->tr_trim_last) {
4850 4849 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4851 4850 trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4852 4851 if (trim->tr_last_copybuf_win) {
4853 4852 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4854 4853 trim->tr_last_cbaddr;
4855 4854 #if !defined(__amd64)
4856 4855 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4857 4856 trim->tr_last_kaddr;
4858 4857 #endif
4859 4858 }
4860 4859 }
4861 4860
4862 4861 /*
4863 4862 * setup the cookie pointer to the first cookie in the window. setup
4864 4863 * our return values, then increment the cookie since we return the
4865 4864 * first cookie on the stack.
4866 4865 */
4867 4866 hp->dmai_cookie = window->wd_first_cookie;
4868 4867 *offp = window->wd_offset;
4869 4868 *lenp = window->wd_size;
4870 4869 *ccountp = window->wd_cookie_cnt;
4871 4870 *cookiep = hp->dmai_cookie[0];
4872 4871 hp->dmai_cookie++;
4873 4872
4874 4873 #if !defined(__amd64)
4875 4874 /* re-map copybuf if required for this window */
4876 4875 if (dma->dp_cb_remaping) {
4877 4876 /*
4878 4877 * calculate the page index into the buffer where this
4879 4878 * window starts.
4880 4879 */
4881 4880 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4882 4881 MMU_PAGESHIFT;
4883 4882 ASSERT(pidx < sinfo->si_max_pages);
4884 4883
4885 4884 /*
4886 4885 * the first page can get unmapped if it's shared with the
4887 4886 * previous window. Even if the rest of this window is already
4888 4887 * mapped in, we need to still check this one.
4889 4888 */
4890 4889 pmap = &dma->dp_pgmap[pidx];
4891 4890 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4892 4891 if (pmap->pm_pp != NULL) {
4893 4892 pmap->pm_mapped = B_TRUE;
4894 4893 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4895 4894 } else if (pmap->pm_vaddr != NULL) {
4896 4895 pmap->pm_mapped = B_TRUE;
4897 4896 i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4898 4897 pmap->pm_kaddr);
4899 4898 }
4900 4899 }
4901 4900 pidx++;
4902 4901
4903 4902 /* map in the rest of the pages if required */
4904 4903 if (window->wd_remap_copybuf) {
4905 4904 window->wd_remap_copybuf = B_FALSE;
4906 4905
4907 4906 /* figure out many pages this window takes up */
4908 4907 poff = (sinfo->si_buf_offset + window->wd_offset) &
4909 4908 MMU_PAGEOFFSET;
4910 4909 pcnt = mmu_btopr(window->wd_size + poff);
4911 4910 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4912 4911
4913 4912 /* map pages which require it */
4914 4913 for (i = 1; i < pcnt; i++) {
4915 4914 pmap = &dma->dp_pgmap[pidx];
4916 4915 if (pmap->pm_uses_copybuf) {
4917 4916 ASSERT(pmap->pm_mapped == B_FALSE);
4918 4917 if (pmap->pm_pp != NULL) {
4919 4918 pmap->pm_mapped = B_TRUE;
4920 4919 i86_pp_map(pmap->pm_pp,
4921 4920 pmap->pm_kaddr);
4922 4921 } else if (pmap->pm_vaddr != NULL) {
4923 4922 pmap->pm_mapped = B_TRUE;
4924 4923 i86_va_map(pmap->pm_vaddr,
4925 4924 sinfo->si_asp,
4926 4925 pmap->pm_kaddr);
4927 4926 }
4928 4927 }
4929 4928 pidx++;
4930 4929 }
4931 4930 }
4932 4931 }
4933 4932 #endif
4934 4933
4935 4934 /* if the new window uses the copy buffer, sync it for the device */
4936 4935 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4937 4936 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4938 4937 DDI_DMA_SYNC_FORDEV);
4939 4938 }
4940 4939
4941 4940 return (DDI_SUCCESS);
4942 4941 }
4943 4942
4944 4943 /*
4945 4944 * rootnex_dma_win()
4946 4945 * called from ddi_dma_getwin()
4947 4946 */
4948 4947 /*ARGSUSED*/
4949 4948 static int
4950 4949 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4951 4950 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4952 4951 uint_t *ccountp)
4953 4952 {
4954 4953 #if defined(__amd64) && !defined(__xpv)
4955 4954 if (IOMMU_USED(rdip)) {
4956 4955 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4957 4956 cookiep, ccountp));
4958 4957 }
4959 4958 #endif
4960 4959
4961 4960 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4962 4961 cookiep, ccountp));
4963 4962 }
4964 4963
4965 4964 #if defined(__amd64) && !defined(__xpv)
4966 4965 /*ARGSUSED*/
4967 4966 static int
4968 4967 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4969 4968 ddi_dma_handle_t handle, void *v)
4970 4969 {
4971 4970 ddi_dma_impl_t *hp;
4972 4971 rootnex_dma_t *dma;
4973 4972
4974 4973 hp = (ddi_dma_impl_t *)handle;
4975 4974 dma = (rootnex_dma_t *)hp->dmai_private;
4976 4975 dma->dp_iommu_private = v;
4977 4976
4978 4977 return (DDI_SUCCESS);
4979 4978 }
4980 4979
4981 4980 /*ARGSUSED*/
4982 4981 static void *
4983 4982 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4984 4983 ddi_dma_handle_t handle)
4985 4984 {
4986 4985 ddi_dma_impl_t *hp;
4987 4986 rootnex_dma_t *dma;
4988 4987
4989 4988 hp = (ddi_dma_impl_t *)handle;
4990 4989 dma = (rootnex_dma_t *)hp->dmai_private;
4991 4990
4992 4991 return (dma->dp_iommu_private);
4993 4992 }
4994 4993 #endif
4995 4994
4996 4995 /*
4997 4996 * ************************
4998 4997 * obsoleted dma routines
4999 4998 * ************************
5000 4999 */
5001 5000
5002 5001 /*
5003 5002 * rootnex_dma_mctl()
5004 5003 *
5005 5004 * We don't support this legacy interface any more on x86.
5006 5005 */
5007 5006 /* ARGSUSED */
5008 5007 static int
5009 5008 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5010 5009 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5011 5010 uint_t cache_flags)
5012 5011 {
5013 5012 /*
5014 5013 * The only thing dma_mctl is usef for anymore is legacy SPARC
5015 5014 * dvma and sbus-specific routines.
5016 5015 */
5017 5016 return (DDI_FAILURE);
5018 5017 }
5019 5018
5020 5019 /*
5021 5020 * *********
5022 5021 * FMA Code
5023 5022 * *********
5024 5023 */
5025 5024
5026 5025 /*
5027 5026 * rootnex_fm_init()
5028 5027 * FMA init busop
5029 5028 */
5030 5029 /* ARGSUSED */
5031 5030 static int
5032 5031 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5033 5032 ddi_iblock_cookie_t *ibc)
5034 5033 {
5035 5034 *ibc = rootnex_state->r_err_ibc;
5036 5035
5037 5036 return (ddi_system_fmcap);
5038 5037 }
5039 5038
5040 5039 /*
5041 5040 * rootnex_dma_check()
5042 5041 * Function called after a dma fault occurred to find out whether the
5043 5042 * fault address is associated with a driver that is able to handle faults
5044 5043 * and recover from faults.
5045 5044 */
5046 5045 /* ARGSUSED */
5047 5046 static int
5048 5047 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5049 5048 const void *not_used)
5050 5049 {
5051 5050 rootnex_window_t *window;
5052 5051 uint64_t start_addr;
5053 5052 uint64_t fault_addr;
5054 5053 ddi_dma_impl_t *hp;
5055 5054 rootnex_dma_t *dma;
5056 5055 uint64_t end_addr;
5057 5056 size_t csize;
5058 5057 int i;
5059 5058 int j;
5060 5059
5061 5060
5062 5061 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5063 5062 hp = (ddi_dma_impl_t *)handle;
5064 5063 ASSERT(hp);
5065 5064
5066 5065 dma = (rootnex_dma_t *)hp->dmai_private;
5067 5066
5068 5067 /* Get the address that we need to search for */
5069 5068 fault_addr = *(uint64_t *)addr;
5070 5069
5071 5070 /*
5072 5071 * if we don't have any windows, we can just walk through all the
5073 5072 * cookies.
5074 5073 */
5075 5074 if (dma->dp_window == NULL) {
5076 5075 /* for each cookie */
5077 5076 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5078 5077 /*
5079 5078 * if the faulted address is within the physical address
5080 5079 * range of the cookie, return DDI_FM_NONFATAL.
5081 5080 */
5082 5081 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5083 5082 (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5084 5083 dma->dp_cookies[i].dmac_size))) {
5085 5084 return (DDI_FM_NONFATAL);
5086 5085 }
5087 5086 }
5088 5087
5089 5088 /* fault_addr not within this DMA handle */
5090 5089 return (DDI_FM_UNKNOWN);
5091 5090 }
5092 5091
5093 5092 /* we have mutiple windows, walk through each window */
5094 5093 for (i = 0; i < hp->dmai_nwin; i++) {
5095 5094 window = &dma->dp_window[i];
5096 5095
5097 5096 /* Go through all the cookies in the window */
5098 5097 for (j = 0; j < window->wd_cookie_cnt; j++) {
5099 5098
5100 5099 start_addr = window->wd_first_cookie[j].dmac_laddress;
5101 5100 csize = window->wd_first_cookie[j].dmac_size;
5102 5101
5103 5102 /*
5104 5103 * if we are trimming the first cookie in the window,
5105 5104 * and this is the first cookie, adjust the start
5106 5105 * address and size of the cookie to account for the
5107 5106 * trim.
5108 5107 */
5109 5108 if (window->wd_trim.tr_trim_first && (j == 0)) {
5110 5109 start_addr = window->wd_trim.tr_first_paddr;
5111 5110 csize = window->wd_trim.tr_first_size;
5112 5111 }
5113 5112
5114 5113 /*
5115 5114 * if we are trimming the last cookie in the window,
5116 5115 * and this is the last cookie, adjust the start
5117 5116 * address and size of the cookie to account for the
5118 5117 * trim.
5119 5118 */
5120 5119 if (window->wd_trim.tr_trim_last &&
5121 5120 (j == (window->wd_cookie_cnt - 1))) {
5122 5121 start_addr = window->wd_trim.tr_last_paddr;
5123 5122 csize = window->wd_trim.tr_last_size;
5124 5123 }
5125 5124
5126 5125 end_addr = start_addr + csize;
5127 5126
5128 5127 /*
5129 5128 * if the faulted address is within the physical
5130 5129 * address of the cookie, return DDI_FM_NONFATAL.
5131 5130 */
5132 5131 if ((fault_addr >= start_addr) &&
5133 5132 (fault_addr <= end_addr)) {
5134 5133 return (DDI_FM_NONFATAL);
5135 5134 }
5136 5135 }
5137 5136 }
5138 5137
5139 5138 /* fault_addr not within this DMA handle */
5140 5139 return (DDI_FM_UNKNOWN);
5141 5140 }
5142 5141
5143 5142 /*ARGSUSED*/
5144 5143 static int
5145 5144 rootnex_quiesce(dev_info_t *dip)
5146 5145 {
5147 5146 #if defined(__amd64) && !defined(__xpv)
5148 5147 return (immu_quiesce());
5149 5148 #else
5150 5149 return (DDI_SUCCESS);
5151 5150 #endif
5152 5151 }
5153 5152
5154 5153 #if defined(__xpv)
5155 5154 void
5156 5155 immu_init(void)
5157 5156 {
5158 5157 ;
5159 5158 }
5160 5159
5161 5160 void
5162 5161 immu_startup(void)
5163 5162 {
5164 5163 ;
5165 5164 }
5166 5165 /*ARGSUSED*/
5167 5166 void
5168 5167 immu_physmem_update(uint64_t addr, uint64_t size)
5169 5168 {
5170 5169 ;
5171 5170 }
5172 5171 #endif
|
↓ open down ↓ |
4841 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX