1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
24 */
25
26 #pragma ident "@(#)iommulib.c 1.6 08/09/07 SMI"
27
28 #include <sys/sunddi.h>
29 #include <sys/sunndi.h>
30 #include <sys/errno.h>
31 #include <sys/modctl.h>
32 #include <sys/iommulib.h>
33
34 /* ******** Type definitions private to this file ********************** */
35
36 /* 1 per IOMMU unit. There may be more than one per dip */
37 typedef struct iommulib_unit {
38 kmutex_t ilu_lock;
39 uint64_t ilu_ref;
40 uint32_t ilu_unitid;
41 dev_info_t *ilu_dip;
42 iommulib_ops_t *ilu_ops;
43 void* ilu_data;
44 struct iommulib_unit *ilu_next;
45 struct iommulib_unit *ilu_prev;
46 iommulib_nexhandle_t ilu_nex;
47 } iommulib_unit_t;
48
49 typedef struct iommulib_nex {
50 dev_info_t *nex_dip;
51 iommulib_nexops_t nex_ops;
52 struct iommulib_nex *nex_next;
53 struct iommulib_nex *nex_prev;
54 uint_t nex_ref;
55 } iommulib_nex_t;
56
57 /* ********* Globals ************************ */
58
59 /* For IOMMU drivers */
60 smbios_hdl_t *iommulib_smbios;
61
62 /* IOMMU side: Following data protected by lock */
63 static kmutex_t iommulib_lock;
64 static iommulib_unit_t *iommulib_list;
65 static uint64_t iommulib_unit_ids = 0;
66 static uint64_t iommulib_num_units = 0;
67
68 /* rootnex side data */
69
70 static kmutex_t iommulib_nexus_lock;
71 static iommulib_nex_t *iommulib_nexus_list;
72
73 /* can be set atomically without lock */
74 static volatile uint32_t iommulib_fini;
75
76 /* debug flag */
77 static int iommulib_debug;
78
79 /*
80 * Module linkage information for the kernel.
81 */
82 static struct modlmisc modlmisc = {
83 &mod_miscops, "IOMMU library module"
84 };
85
86 static struct modlinkage modlinkage = {
87 MODREV_1, { (void *)&modlmisc, NULL }
88 };
89
90 int
91 _init(void)
92 {
93 return (mod_install(&modlinkage));
94 }
95
96 int
97 _fini(void)
98 {
99 mutex_enter(&iommulib_lock);
100 if (iommulib_list != NULL || iommulib_nexus_list != NULL) {
101 mutex_exit(&iommulib_lock);
102 return (EBUSY);
103 }
104 iommulib_fini = 1;
105
106 mutex_exit(&iommulib_lock);
107 return (mod_remove(&modlinkage));
108 }
109
110 int
111 _info(struct modinfo *modinfop)
112 {
113 return (mod_info(&modlinkage, modinfop));
114 }
115
116 /*
117 * Routines with iommulib_iommu_* are invoked from the
118 * IOMMU driver.
119 * Routines with iommulib_nex* are invoked from the
120 * nexus driver (typically rootnex)
121 */
122
123 int
124 iommulib_nexus_register(dev_info_t *dip, iommulib_nexops_t *nexops,
125 iommulib_nexhandle_t *handle)
126 {
127 iommulib_nex_t *nexp;
128 int instance = ddi_get_instance(dip);
129 const char *driver = ddi_driver_name(dip);
130 dev_info_t *pdip = ddi_get_parent(dip);
131 const char *f = "iommulib_nexus_register";
132
133 ASSERT(nexops);
134 ASSERT(handle);
135
136 *handle = NULL;
137
138 /*
139 * Root node is never busy held
140 */
141 if (dip != ddi_root_node() && (i_ddi_node_state(dip) < DS_PROBED ||
142 !DEVI_BUSY_OWNED(pdip))) {
143 cmn_err(CE_WARN, "%s: NEXUS devinfo node not in DS_PROBED "
144 "or busy held for nexops vector (%p). Failing registration",
145 f, (void *)nexops);
146 return (DDI_FAILURE);
147 }
148
149 if (nexops->nops_vers != IOMMU_NEXOPS_VERSION) {
150 cmn_err(CE_WARN, "%s: %s%d: Invalid IOMMULIB nexops version "
151 "in nexops vector (%p). Failing NEXUS registration",
152 f, driver, instance, (void *)nexops);
153 return (DDI_FAILURE);
154 }
155
156 ASSERT(nexops->nops_data == NULL);
157
158 if (nexops->nops_id == NULL) {
159 cmn_err(CE_WARN, "%s: %s%d: NULL ID field. "
160 "Failing registration for nexops vector: %p",
161 f, driver, instance, (void *)nexops);
162 return (DDI_FAILURE);
163 }
164
165 if (nexops->nops_dma_allochdl == NULL) {
166 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_allochdl op. "
167 "Failing registration for ops vector: %p", f,
168 driver, instance, (void *)nexops);
169 return (DDI_FAILURE);
170 }
171
172 if (nexops->nops_dma_freehdl == NULL) {
173 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_freehdl op. "
174 "Failing registration for ops vector: %p", f,
175 driver, instance, (void *)nexops);
176 return (DDI_FAILURE);
177 }
178
179 if (nexops->nops_dma_bindhdl == NULL) {
180 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_bindhdl op. "
181 "Failing registration for ops vector: %p", f,
182 driver, instance, (void *)nexops);
183 return (DDI_FAILURE);
184 }
185
186 if (nexops->nops_dma_sync == NULL) {
187 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_sync op. "
188 "Failing registration for ops vector: %p", f,
189 driver, instance, (void *)nexops);
190 return (DDI_FAILURE);
191 }
192
193 if (nexops->nops_dma_reset_cookies == NULL) {
194 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_reset_cookies op. "
195 "Failing registration for ops vector: %p", f,
196 driver, instance, (void *)nexops);
197 return (DDI_FAILURE);
198 }
199
200 if (nexops->nops_dma_get_cookies == NULL) {
201 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_get_cookies op. "
202 "Failing registration for ops vector: %p", f,
203 driver, instance, (void *)nexops);
204 return (DDI_FAILURE);
205 }
206
207 if (nexops->nops_dma_set_cookies == NULL) {
208 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_set_cookies op. "
209 "Failing registration for ops vector: %p", f,
210 driver, instance, (void *)nexops);
211 return (DDI_FAILURE);
212 }
213
214 if (nexops->nops_dma_clear_cookies == NULL) {
215 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_clear_cookies op. "
216 "Failing registration for ops vector: %p", f,
217 driver, instance, (void *)nexops);
218 return (DDI_FAILURE);
219 }
220
221 if (nexops->nops_dma_get_sleep_flags == NULL) {
222 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_get_sleep_flags op. "
223 "Failing registration for ops vector: %p", f,
224 driver, instance, (void *)nexops);
225 return (DDI_FAILURE);
226 }
227
228 if (nexops->nops_dma_win == NULL) {
229 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dma_win op. "
230 "Failing registration for ops vector: %p", f,
231 driver, instance, (void *)nexops);
232 return (DDI_FAILURE);
233 }
234
235 if (nexops->nops_dmahdl_setprivate == NULL) {
236 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dmahdl_setprivate op. "
237 "Failing registration for ops vector: %p", f,
238 driver, instance, (void *)nexops);
239 return (DDI_FAILURE);
240 }
241
242 if (nexops->nops_dmahdl_getprivate == NULL) {
243 cmn_err(CE_WARN, "%s: %s%d: NULL nops_dmahdl_getprivate op. "
244 "Failing registration for ops vector: %p", f,
245 driver, instance, (void *)nexops);
246 return (DDI_FAILURE);
247 }
248
249 nexp = kmem_zalloc(sizeof (iommulib_nex_t), KM_SLEEP);
250
251 mutex_enter(&iommulib_lock);
252 if (iommulib_fini == 1) {
253 mutex_exit(&iommulib_lock);
254 cmn_err(CE_WARN, "%s: IOMMULIB unloading. "
255 "Failing NEXUS register.", f);
256 kmem_free(nexp, sizeof (iommulib_nex_t));
257 return (DDI_FAILURE);
258 }
259
260 /*
261 * fini/register race conditions have been handled. Now create the
262 * nexus struct
263 */
264 ndi_hold_devi(dip);
265 nexp->nex_dip = dip;
266 nexp->nex_ops = *nexops;
267
268 mutex_enter(&iommulib_nexus_lock);
269 nexp->nex_next = iommulib_nexus_list;
270 iommulib_nexus_list = nexp;
271 nexp->nex_prev = NULL;
272
273 if (nexp->nex_next != NULL)
274 nexp->nex_next->nex_prev = nexp;
275
276 nexp->nex_ref = 0;
277
278 /*
279 * The nexus device won't be controlled by an IOMMU.
280 */
281 DEVI(dip)->devi_iommulib_handle = IOMMU_HANDLE_UNUSED;
282
283 DEVI(dip)->devi_iommulib_nex_handle = nexp;
284
285 mutex_exit(&iommulib_nexus_lock);
286 mutex_exit(&iommulib_lock);
287
288 cmn_err(CE_NOTE, "!%s: %s%d: Succesfully registered NEXUS %s "
289 "nexops=%p", f, driver, instance, ddi_node_name(dip),
290 (void *)nexops);
291
292 *handle = nexp;
293
294 return (DDI_SUCCESS);
295 }
296
297 int
298 iommulib_nexus_unregister(iommulib_nexhandle_t handle)
299 {
300 dev_info_t *dip;
301 int instance;
302 const char *driver;
303 iommulib_nex_t *nexp = (iommulib_nex_t *)handle;
304 const char *f = "iommulib_nexus_unregister";
305
306 ASSERT(nexp);
307
308 if (nexp->nex_ref != 0)
309 return (DDI_FAILURE);
310
311 mutex_enter(&iommulib_nexus_lock);
312
313 dip = nexp->nex_dip;
314 driver = ddi_driver_name(dip);
315 instance = ddi_get_instance(dip);
316
317 /* A future enhancement would be to add ref-counts */
318
319 if (nexp->nex_prev == NULL) {
320 iommulib_nexus_list = nexp->nex_next;
321 } else {
322 nexp->nex_prev->nex_next = nexp->nex_next;
323 }
324
325 if (nexp->nex_next != NULL)
326 nexp->nex_next->nex_prev = nexp->nex_prev;
327
328 mutex_exit(&iommulib_nexus_lock);
329
330 kmem_free(nexp, sizeof (iommulib_nex_t));
331
332 cmn_err(CE_NOTE, "!%s: %s%d: NEXUS (%s) handle successfully "
333 "unregistered from IOMMULIB", f, driver, instance,
334 ddi_node_name(dip));
335
336 ndi_rele_devi(dip);
337
338 return (DDI_SUCCESS);
339 }
340
341 int
342 iommulib_iommu_register(dev_info_t *dip, iommulib_ops_t *ops,
343 iommulib_handle_t *handle)
344 {
345 const char *vendor;
346 iommulib_unit_t *unitp;
347 int instance = ddi_get_instance(dip);
348 const char *driver = ddi_driver_name(dip);
349 const char *f = "iommulib_register";
350
351 ASSERT(ops);
352 ASSERT(handle);
353
354 if (ops->ilops_vers != IOMMU_OPS_VERSION) {
355 cmn_err(CE_WARN, "%s: %s%d: Invalid IOMMULIB ops version "
356 "in ops vector (%p). Failing registration", f, driver,
357 instance, (void *)ops);
358 return (DDI_FAILURE);
359 }
360
361 switch (ops->ilops_vendor) {
362 case AMD_IOMMU:
363 vendor = "AMD";
364 break;
365 case INTEL_IOMMU:
366 vendor = "Intel";
367 break;
368 case INVALID_VENDOR:
369 cmn_err(CE_WARN, "%s: %s%d: vendor field (%x) not initialized. "
370 "Failing registration for ops vector: %p", f,
371 driver, instance, ops->ilops_vendor, (void *)ops);
372 return (DDI_FAILURE);
373 default:
374 cmn_err(CE_WARN, "%s: %s%d: Invalid vendor field (%x). "
375 "Failing registration for ops vector: %p", f,
376 driver, instance, ops->ilops_vendor, (void *)ops);
377 return (DDI_FAILURE);
378 }
379
380 cmn_err(CE_NOTE, "!%s: %s%d: Detected IOMMU registration from vendor"
381 " %s", f, driver, instance, vendor);
382
383 if (ops->ilops_data == NULL) {
384 cmn_err(CE_WARN, "%s: %s%d: NULL IOMMU data field. "
385 "Failing registration for ops vector: %p", f,
386 driver, instance, (void *)ops);
387 return (DDI_FAILURE);
388 }
389
390 if (ops->ilops_id == NULL) {
391 cmn_err(CE_WARN, "%s: %s%d: NULL ID field. "
392 "Failing registration for ops vector: %p", f,
393 driver, instance, (void *)ops);
394 return (DDI_FAILURE);
395 }
396
397 if (ops->ilops_probe == NULL) {
398 cmn_err(CE_WARN, "%s: %s%d: NULL probe op. "
399 "Failing registration for ops vector: %p", f,
400 driver, instance, (void *)ops);
401 return (DDI_FAILURE);
402 }
403
404 if (ops->ilops_dma_allochdl == NULL) {
405 cmn_err(CE_WARN, "%s: %s%d: NULL dma_allochdl op. "
406 "Failing registration for ops vector: %p", f,
407 driver, instance, (void *)ops);
408 return (DDI_FAILURE);
409 }
410
411 if (ops->ilops_dma_freehdl == NULL) {
412 cmn_err(CE_WARN, "%s: %s%d: NULL dma_freehdl op. "
413 "Failing registration for ops vector: %p", f,
414 driver, instance, (void *)ops);
415 return (DDI_FAILURE);
416 }
417
418 if (ops->ilops_dma_bindhdl == NULL) {
419 cmn_err(CE_WARN, "%s: %s%d: NULL dma_bindhdl op. "
420 "Failing registration for ops vector: %p", f,
421 driver, instance, (void *)ops);
422 return (DDI_FAILURE);
423 }
424
425 if (ops->ilops_dma_sync == NULL) {
426 cmn_err(CE_WARN, "%s: %s%d: NULL dma_sync op. "
427 "Failing registration for ops vector: %p", f,
428 driver, instance, (void *)ops);
429 return (DDI_FAILURE);
430 }
431
432 if (ops->ilops_dma_win == NULL) {
433 cmn_err(CE_WARN, "%s: %s%d: NULL dma_win op. "
434 "Failing registration for ops vector: %p", f,
435 driver, instance, (void *)ops);
436 return (DDI_FAILURE);
437 }
438
439 unitp = kmem_zalloc(sizeof (iommulib_unit_t), KM_SLEEP);
440 mutex_enter(&iommulib_lock);
441 if (iommulib_fini == 1) {
442 mutex_exit(&iommulib_lock);
443 cmn_err(CE_WARN, "%s: IOMMULIB unloading. Failing register.",
444 f);
445 kmem_free(unitp, sizeof (iommulib_unit_t));
446 return (DDI_FAILURE);
447 }
448
449 /*
450 * fini/register race conditions have been handled. Now create the
451 * IOMMU unit
452 */
453 mutex_init(&unitp->ilu_lock, NULL, MUTEX_DEFAULT, NULL);
454
455 mutex_enter(&unitp->ilu_lock);
456 unitp->ilu_unitid = ++iommulib_unit_ids;
457 unitp->ilu_ref = 0;
458 ndi_hold_devi(dip);
459 unitp->ilu_dip = dip;
460 unitp->ilu_ops = ops;
461 unitp->ilu_data = ops->ilops_data;
462
463 unitp->ilu_next = iommulib_list;
464 iommulib_list = unitp;
465 unitp->ilu_prev = NULL;
466 if (unitp->ilu_next)
467 unitp->ilu_next->ilu_prev = unitp;
468
469 /*
470 * The IOMMU device itself is not controlled by an IOMMU.
471 */
472 DEVI(dip)->devi_iommulib_handle = IOMMU_HANDLE_UNUSED;
473
474 mutex_exit(&unitp->ilu_lock);
475
476 iommulib_num_units++;
477
478 *handle = unitp;
479
480 mutex_exit(&iommulib_lock);
481
482 cmn_err(CE_NOTE, "!%s: %s%d: Succesfully registered IOMMU unit "
483 "from vendor=%s, ops=%p, data=%p, IOMMULIB unitid=%u",
484 f, driver, instance, vendor, (void *)ops, (void *)unitp->ilu_data,
485 unitp->ilu_unitid);
486
487 return (DDI_SUCCESS);
488 }
489
490 int
491 iommulib_iommu_unregister(iommulib_handle_t handle)
492 {
493 uint32_t unitid;
494 dev_info_t *dip;
495 int instance;
496 const char *driver;
497 iommulib_unit_t *unitp = (iommulib_unit_t *)handle;
498 const char *f = "iommulib_unregister";
499
500 ASSERT(unitp);
501
502 mutex_enter(&iommulib_lock);
503 mutex_enter(&unitp->ilu_lock);
504
505 unitid = unitp->ilu_unitid;
506 dip = unitp->ilu_dip;
507 driver = ddi_driver_name(dip);
508 instance = ddi_get_instance(dip);
509
510 if (unitp->ilu_ref != 0) {
511 mutex_exit(&unitp->ilu_lock);
512 mutex_exit(&iommulib_lock);
513 cmn_err(CE_WARN, "%s: %s%d: IOMMULIB handle is busy. Cannot "
514 "unregister IOMMULIB unitid %u",
515 f, driver, instance, unitid);
516 return (DDI_FAILURE);
517 }
518 unitp->ilu_unitid = 0;
519 ASSERT(unitp->ilu_ref == 0);
520
521 if (unitp->ilu_prev == NULL) {
522 iommulib_list = unitp->ilu_next;
523 unitp->ilu_next->ilu_prev = NULL;
524 } else {
525 unitp->ilu_prev->ilu_next = unitp->ilu_next;
526 unitp->ilu_next->ilu_prev = unitp->ilu_prev;
527 }
528
529 iommulib_num_units--;
530
531 mutex_exit(&unitp->ilu_lock);
532
533 mutex_destroy(&unitp->ilu_lock);
534 kmem_free(unitp, sizeof (iommulib_unit_t));
535
536 mutex_exit(&iommulib_lock);
537
538 cmn_err(CE_WARN, "%s: %s%d: IOMMULIB handle (unitid=%u) successfully "
539 "unregistered", f, driver, instance, unitid);
540
541 ndi_rele_devi(dip);
542
543 return (DDI_SUCCESS);
544 }
545
546 int
547 iommulib_nex_open(dev_info_t *dip, dev_info_t *rdip)
548 {
549 iommulib_unit_t *unitp;
550 int instance = ddi_get_instance(rdip);
551 const char *driver = ddi_driver_name(rdip);
552 const char *f = "iommulib_nex_open";
553
554 ASSERT(DEVI(dip)->devi_iommulib_nex_handle != NULL);
555 ASSERT(DEVI(rdip)->devi_iommulib_handle == NULL);
556
557 /* prevent use of IOMMU for AMD IOMMU's DMA */
558 if (strcmp(driver, "amd_iommu") == 0) {
559 DEVI(rdip)->devi_iommulib_handle = IOMMU_HANDLE_UNUSED;
560 return (DDI_ENOTSUP);
561 }
562
563 /*
564 * Use the probe entry point to determine in a hardware specific
565 * manner whether this dip is controlled by an IOMMU. If yes,
566 * return the handle corresponding to the IOMMU unit.
567 */
568
569 mutex_enter(&iommulib_lock);
570 for (unitp = iommulib_list; unitp; unitp = unitp->ilu_next) {
571 if (unitp->ilu_ops->ilops_probe(unitp, rdip) == DDI_SUCCESS)
572 break;
573 }
574
575 if (unitp == NULL) {
576 mutex_exit(&iommulib_lock);
577 if (iommulib_debug) {
578 char *buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
579 cmn_err(CE_WARN, "%s: %s%d: devinfo node (%p): is not "
580 "controlled by an IOMMU: path=%s", f, driver,
581 instance, (void *)rdip, ddi_pathname(rdip, buf));
582 kmem_free(buf, MAXPATHLEN);
583 }
584 DEVI(rdip)->devi_iommulib_handle = IOMMU_HANDLE_UNUSED;
585 return (DDI_ENOTSUP);
586 }
587
588 mutex_enter(&unitp->ilu_lock);
589 unitp->ilu_nex = DEVI(dip)->devi_iommulib_nex_handle;
590 unitp->ilu_ref++;
591 DEVI(rdip)->devi_iommulib_handle = unitp;
592 mutex_exit(&unitp->ilu_lock);
593 mutex_exit(&iommulib_lock);
594
595 atomic_inc_uint(&DEVI(dip)->devi_iommulib_nex_handle->nex_ref);
596
597 return (DDI_SUCCESS);
598 }
599
600 void
601 iommulib_nex_close(dev_info_t *rdip)
602 {
603 iommulib_unit_t *unitp;
604 const char *driver;
605 int instance;
606 uint32_t unitid;
607 iommulib_nex_t *nexp;
608 const char *f = "iommulib_nex_close";
609
610 ASSERT(IOMMU_USED(rdip));
611
612 unitp = DEVI(rdip)->devi_iommulib_handle;
613
614 mutex_enter(&iommulib_lock);
615 mutex_enter(&unitp->ilu_lock);
616
617 nexp = (iommulib_nex_t *)unitp->ilu_nex;
618 DEVI(rdip)->devi_iommulib_handle = NULL;
619
620 unitid = unitp->ilu_unitid;
621 driver = ddi_driver_name(unitp->ilu_dip);
622 instance = ddi_get_instance(unitp->ilu_dip);
623
624 unitp->ilu_ref--;
625 mutex_exit(&unitp->ilu_lock);
626 mutex_exit(&iommulib_lock);
627
628 atomic_dec_uint(&nexp->nex_ref);
629
630 if (iommulib_debug) {
631 char *buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
632 (void) ddi_pathname(rdip, buf);
633 cmn_err(CE_NOTE, "%s: %s%d: closing IOMMU for dip (%p), "
634 "unitid=%u rdip path = %s", f, driver, instance,
635 (void *)rdip, unitid, buf);
636 kmem_free(buf, MAXPATHLEN);
637 }
638 }
639
640 int
641 iommulib_nexdma_allochdl(dev_info_t *dip, dev_info_t *rdip,
642 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t),
643 caddr_t arg, ddi_dma_handle_t *dma_handlep)
644 {
645 iommulib_handle_t handle = DEVI(rdip)->devi_iommulib_handle;
646 iommulib_unit_t *unitp = (iommulib_unit_t *)handle;
647
648 ASSERT(unitp);
649
650 /* No need to grab lock - the handle is reference counted */
651 return (unitp->ilu_ops->ilops_dma_allochdl(handle, dip, rdip,
652 attr, waitfp, arg, dma_handlep));
653 }
654
655 int
656 iommulib_nexdma_freehdl(dev_info_t *dip, dev_info_t *rdip,
657 ddi_dma_handle_t dma_handle)
658 {
659 int error;
660 iommulib_handle_t handle = DEVI(rdip)->devi_iommulib_handle;
661 iommulib_unit_t *unitp = (iommulib_unit_t *)handle;
662
663 ASSERT(unitp);
664
665 /* No need to grab lock - the handle is reference counted */
666 error = unitp->ilu_ops->ilops_dma_freehdl(handle, dip,
667 rdip, dma_handle);
668
669 return (error);
670 }
671
672 int
673 iommulib_nexdma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
674 ddi_dma_handle_t dma_handle, struct ddi_dma_req *dmareq,
675 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
676 {
677 iommulib_handle_t handle = DEVI(rdip)->devi_iommulib_handle;
678 iommulib_unit_t *unitp = (iommulib_unit_t *)handle;
679
680 ASSERT(unitp);
681
682 /* No need to grab lock - the handle is reference counted */
683 return (unitp->ilu_ops->ilops_dma_bindhdl(handle, dip, rdip, dma_handle,
684 dmareq, cookiep, ccountp));
685 }
686
687 int
688 iommulib_nexdma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
689 ddi_dma_handle_t dma_handle)
690 {
691 iommulib_handle_t handle = DEVI(rdip)->devi_iommulib_handle;
692 iommulib_unit_t *unitp = (iommulib_unit_t *)handle;
693
694 ASSERT(unitp);
695
696 /* No need to grab lock - the handle is reference counted */
697 return (unitp->ilu_ops->ilops_dma_unbindhdl(handle, dip, rdip,
698 dma_handle));
699 }
700
701 int
702 iommulib_nexdma_sync(dev_info_t *dip, dev_info_t *rdip,
703 ddi_dma_handle_t dma_handle, off_t off, size_t len,
704 uint_t cache_flags)
705 {
706 iommulib_handle_t handle = DEVI(rdip)->devi_iommulib_handle;
707 iommulib_unit_t *unitp = (iommulib_unit_t *)handle;
708
709 ASSERT(unitp);
710
711 /* No need to grab lock - the handle is reference counted */
712 return (unitp->ilu_ops->ilops_dma_sync(handle, dip, rdip, dma_handle,
713 off, len, cache_flags));
714 }
715
716 int
717 iommulib_nexdma_win(dev_info_t *dip, dev_info_t *rdip,
718 ddi_dma_handle_t dma_handle, uint_t win, off_t *offp, size_t *lenp,
719 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
720 {
721 iommulib_handle_t handle = DEVI(rdip)->devi_iommulib_handle;
722 iommulib_unit_t *unitp = (iommulib_unit_t *)handle;
723
724 ASSERT(unitp);
725
726 /* No need to grab lock - the handle is reference counted */
727 return (unitp->ilu_ops->ilops_dma_win(handle, dip, rdip, dma_handle,
728 win, offp, lenp, cookiep, ccountp));
729 }
730
731 int
732 iommulib_nexdma_mapobject(dev_info_t *dip, dev_info_t *rdip,
733 ddi_dma_handle_t dma_handle, struct ddi_dma_req *dmareq,
734 ddi_dma_obj_t *dmao)
735 {
736 iommulib_handle_t handle = DEVI(rdip)->devi_iommulib_handle;
737 iommulib_unit_t *unitp = (iommulib_unit_t *)handle;
738
739 return (unitp->ilu_ops->ilops_dma_mapobject(handle, dip, rdip,
740 dma_handle, dmareq, dmao));
741 }
742
743 int
744 iommulib_nexdma_unmapobject(dev_info_t *dip, dev_info_t *rdip,
745 ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao)
746 {
747 iommulib_handle_t handle = DEVI(rdip)->devi_iommulib_handle;
748 iommulib_unit_t *unitp = (iommulib_unit_t *)handle;
749
750 return (unitp->ilu_ops->ilops_dma_unmapobject(handle, dip, rdip,
751 dma_handle, dmao));
752 }
753
754 /* Utility routines invoked by IOMMU drivers */
755 int
756 iommulib_iommu_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
757 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
758 ddi_dma_handle_t *handlep)
759 {
760 iommulib_nexops_t *nexops;
761
762 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
763 return (nexops->nops_dma_allochdl(dip, rdip, attr, waitfp, arg,
764 handlep));
765 }
766
767 int
768 iommulib_iommu_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
769 ddi_dma_handle_t handle)
770 {
771 iommulib_nexops_t *nexops;
772
773 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
774 ASSERT(nexops);
775 return (nexops->nops_dma_freehdl(dip, rdip, handle));
776 }
777
778 int
779 iommulib_iommu_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
780 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
781 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
782 {
783 iommulib_nexops_t *nexops;
784
785 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
786 return (nexops->nops_dma_bindhdl(dip, rdip, handle, dmareq,
787 cookiep, ccountp));
788 }
789
790 int
791 iommulib_iommu_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
792 ddi_dma_handle_t handle)
793 {
794 iommulib_nexops_t *nexops;
795
796 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
797 return (nexops->nops_dma_unbindhdl(dip, rdip, handle));
798 }
799
800 void
801 iommulib_iommu_dma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
802 {
803 iommulib_nexops_t *nexops;
804
805 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
806 nexops->nops_dma_reset_cookies(dip, handle);
807 }
808
809 int
810 iommulib_iommu_dma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
811 ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
812 {
813 iommulib_nexops_t *nexops;
814
815 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
816 return (nexops->nops_dma_get_cookies(dip, handle, cookiepp, ccountp));
817 }
818
819 int
820 iommulib_iommu_dma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
821 ddi_dma_cookie_t *cookiep, uint_t ccount)
822 {
823 iommulib_nexops_t *nexops;
824
825 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
826 return (nexops->nops_dma_set_cookies(dip, handle, cookiep, ccount));
827 }
828
829 int
830 iommulib_iommu_dma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
831 {
832 iommulib_nexops_t *nexops;
833
834 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
835 return (nexops->nops_dma_clear_cookies(dip, handle));
836 }
837
838 int
839 iommulib_iommu_dma_get_sleep_flags(dev_info_t *dip, ddi_dma_handle_t handle)
840 {
841 iommulib_nexops_t *nexops;
842
843 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
844 return (nexops->nops_dma_get_sleep_flags(handle));
845 }
846
847 int
848 iommulib_iommu_dma_sync(dev_info_t *dip, dev_info_t *rdip,
849 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags)
850 {
851 iommulib_nexops_t *nexops;
852
853 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
854 return (nexops->nops_dma_sync(dip, rdip, handle, off, len,
855 cache_flags));
856 }
857
858 int
859 iommulib_iommu_dma_win(dev_info_t *dip, dev_info_t *rdip,
860 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
861 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
862 {
863 iommulib_nexops_t *nexops;
864
865 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
866 return (nexops->nops_dma_win(dip, rdip, handle, win, offp, lenp,
867 cookiep, ccountp));
868 }
869
870 int
871 iommulib_iommu_dmahdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
872 ddi_dma_handle_t handle, void *priv)
873 {
874 iommulib_nexops_t *nexops;
875
876 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
877 return (nexops->nops_dmahdl_setprivate(dip, rdip, handle, priv));
878 }
879
880 void *
881 iommulib_iommu_dmahdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
882 ddi_dma_handle_t handle)
883 {
884 iommulib_nexops_t *nexops;
885
886 nexops = &DEVI(dip)->devi_iommulib_nex_handle->nex_ops;
887 return (nexops->nops_dmahdl_getprivate(dip, rdip, handle));
888 }
889
890 int
891 iommulib_iommu_getunitid(iommulib_handle_t handle, uint64_t *unitidp)
892 {
893 iommulib_unit_t *unitp;
894 uint64_t unitid;
895
896 unitp = (iommulib_unit_t *)handle;
897
898 ASSERT(unitp);
899 ASSERT(unitidp);
900
901 mutex_enter(&unitp->ilu_lock);
902 unitid = unitp->ilu_unitid;
903 mutex_exit(&unitp->ilu_lock);
904
905 ASSERT(unitid > 0);
906 *unitidp = (uint64_t)unitid;
907
908 return (DDI_SUCCESS);
909 }
910
911 dev_info_t *
912 iommulib_iommu_getdip(iommulib_handle_t handle)
913 {
914 iommulib_unit_t *unitp;
915 dev_info_t *dip;
916
917 unitp = (iommulib_unit_t *)handle;
918
919 ASSERT(unitp);
920
921 mutex_enter(&unitp->ilu_lock);
922 dip = unitp->ilu_dip;
923 ASSERT(dip);
924 ndi_hold_devi(dip);
925 mutex_exit(&unitp->ilu_lock);
926
927 return (dip);
928 }
929
930 iommulib_ops_t *
931 iommulib_iommu_getops(iommulib_handle_t handle)
932 {
933 iommulib_unit_t *unitp;
934 iommulib_ops_t *ops;
935
936 unitp = (iommulib_unit_t *)handle;
937
938 ASSERT(unitp);
939
940 mutex_enter(&unitp->ilu_lock);
941 ops = unitp->ilu_ops;
942 mutex_exit(&unitp->ilu_lock);
943
944 ASSERT(ops);
945
946 return (ops);
947 }
948
949 void *
950 iommulib_iommu_getdata(iommulib_handle_t handle)
951 {
952 iommulib_unit_t *unitp;
953 void *data;
954
955 unitp = (iommulib_unit_t *)handle;
956
957 ASSERT(unitp);
958
959 mutex_enter(&unitp->ilu_lock);
960 data = unitp->ilu_data;
961 mutex_exit(&unitp->ilu_lock);
962
963 ASSERT(data);
964
965 return (data);
966 }