1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 #include <sys/sunddi.h>
27 #include <sys/sunndi.h>
28 #include <sys/acpi/acpi.h>
29 #include <sys/acpica.h>
30 #include <sys/amd_iommu.h>
31 #include <sys/bootconf.h>
32 #include <sys/sysmacros.h>
33 #include <sys/ddidmareq.h>
34
35 #include "amd_iommu_impl.h"
36 #include "amd_iommu_acpi.h"
37 #include "amd_iommu_page_tables.h"
38
39 ddi_dma_attr_t amd_iommu_pgtable_dma_attr = {
40 DMA_ATTR_V0,
41 0U, /* dma_attr_addr_lo */
42 0xffffffffffffffffULL, /* dma_attr_addr_hi */
43 0xffffffffU, /* dma_attr_count_max */
44 (uint64_t)4096, /* dma_attr_align */
45 1, /* dma_attr_burstsizes */
46 64, /* dma_attr_minxfer */
47 0xffffffffU, /* dma_attr_maxxfer */
48 0xffffffffU, /* dma_attr_seg */
49 1, /* dma_attr_sgllen, variable */
50 64, /* dma_attr_granular */
51 0 /* dma_attr_flags */
52 };
53
54 static amd_iommu_domain_t **amd_iommu_domain_table;
55
56 static struct {
57 int f_count;
58 amd_iommu_page_table_t *f_list;
59 } amd_iommu_pgtable_freelist;
60 int amd_iommu_no_pgtable_freelist;
61
62 /*ARGSUSED*/
63 static int
64 amd_iommu_get_src_bdf(amd_iommu_t *iommu, int32_t bdf, int32_t *src_bdfp)
65 {
66 amd_iommu_acpi_ivhd_t *hinfop;
67
68 hinfop = amd_iommu_lookup_ivhd(bdf);
69 if (hinfop == NULL) {
70 if (bdf == -1) {
71 *src_bdfp = bdf;
72 } else {
73 cmn_err(CE_WARN, "No IVHD entry for 0x%x", bdf);
74 return (DDI_FAILURE);
75 }
76 } else if (hinfop->ach_src_deviceid == -1) {
77 *src_bdfp = bdf;
78 } else {
79 *src_bdfp = hinfop->ach_src_deviceid;
80 }
81
82 return (DDI_SUCCESS);
83 }
84
85 /*ARGSUSED*/
86 static int
87 amd_iommu_get_domain(amd_iommu_t *iommu, dev_info_t *rdip, int alias,
88 uint16_t deviceid, domain_id_t *domainid, const char *path)
89 {
90 const char *f = "amd_iommu_get_domain";
91
92 *domainid = AMD_IOMMU_INVALID_DOMAIN;
93
94 ASSERT(strcmp(ddi_driver_name(rdip), "agpgart") != 0);
95
96 switch (deviceid) {
97 case AMD_IOMMU_INVALID_DOMAIN:
98 case AMD_IOMMU_IDENTITY_DOMAIN:
99 case AMD_IOMMU_PASSTHRU_DOMAIN:
100 case AMD_IOMMU_SYS_DOMAIN:
101 *domainid = AMD_IOMMU_SYS_DOMAIN;
102 break;
103 default:
104 *domainid = deviceid;
105 break;
106 }
107
108 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
109 cmn_err(CE_NOTE, "%s: domainid for %s = %d",
110 f, path, *domainid);
111 }
112
113 return (DDI_SUCCESS);
114 }
115
116 static uint16_t
117 hash_domain(domain_id_t domainid)
118 {
119 return (domainid % AMD_IOMMU_DOMAIN_HASH_SZ);
120 }
121
122 /*ARGSUSED*/
123 void
124 amd_iommu_init_page_tables(amd_iommu_t *iommu)
125 {
126 amd_iommu_domain_table = kmem_zalloc(
127 sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ, KM_SLEEP);
128 }
129
130 /*ARGSUSED*/
131 void
132 amd_iommu_fini_page_tables(amd_iommu_t *iommu)
133 {
134 if (amd_iommu_domain_table) {
135 kmem_free(amd_iommu_domain_table,
136 sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ);
137 amd_iommu_domain_table = NULL;
138 }
139 }
140
141 static amd_iommu_domain_t *
142 amd_iommu_lookup_domain(amd_iommu_t *iommu, domain_id_t domainid,
143 map_type_t type, int km_flags)
144 {
145 uint16_t idx;
146 amd_iommu_domain_t *dp;
147 char name[AMD_IOMMU_VMEM_NAMELEN+1];
148
149 ASSERT(amd_iommu_domain_table);
150
151 idx = hash_domain(domainid);
152
153 for (dp = amd_iommu_domain_table[idx]; dp; dp = dp->d_next) {
154 if (dp->d_domainid == domainid)
155 return (dp);
156 }
157
158 ASSERT(type != AMD_IOMMU_INVALID_MAP);
159
160 dp = kmem_zalloc(sizeof (*dp), km_flags);
161 if (dp == NULL)
162 return (NULL);
163 dp->d_domainid = domainid;
164 dp->d_pgtable_root_4K = 0; /* make this explicit */
165
166 if (type == AMD_IOMMU_VMEM_MAP) {
167 uint64_t base;
168 uint64_t size;
169 (void) snprintf(name, sizeof (name), "dvma_idx%d_domain%d",
170 iommu->aiomt_idx, domainid);
171 base = MMU_PAGESIZE;
172 size = AMD_IOMMU_SIZE_4G - MMU_PAGESIZE;
173 dp->d_vmem = vmem_create(name, (void *)(uintptr_t)base, size,
174 MMU_PAGESIZE, NULL, NULL, NULL, 0,
175 km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
176 if (dp->d_vmem == NULL) {
177 kmem_free(dp, sizeof (*dp));
178 return (NULL);
179 }
180 } else {
181 dp->d_vmem = NULL;
182 }
183
184 dp->d_next = amd_iommu_domain_table[idx];
185 dp->d_prev = NULL;
186 amd_iommu_domain_table[idx] = dp;
187 if (dp->d_next)
188 dp->d_next->d_prev = dp;
189 dp->d_ref = 0;
190
191
192 return (dp);
193 }
194
195 static void
196 amd_iommu_teardown_domain(amd_iommu_t *iommu, amd_iommu_domain_t *dp)
197 {
198 uint16_t idx;
199 int flags;
200 amd_iommu_cmdargs_t cmdargs = {0};
201 domain_id_t domainid = dp->d_domainid;
202 const char *f = "amd_iommu_teardown_domain";
203
204 ASSERT(dp->d_ref == 0);
205
206 idx = hash_domain(dp->d_domainid);
207
208 if (dp->d_prev == NULL)
209 amd_iommu_domain_table[idx] = dp->d_next;
210 else
211 dp->d_prev->d_next = dp->d_next;
212
213 if (dp->d_next)
214 dp->d_next->d_prev = dp->d_prev;
215
216 if (dp->d_vmem != NULL) {
217 vmem_destroy(dp->d_vmem);
218 dp->d_vmem = NULL;
219 }
220
221 kmem_free(dp, sizeof (*dp));
222
223 cmdargs.ca_domainid = (uint16_t)domainid;
224 cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
225 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
226 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
227
228 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
229 &cmdargs, flags, 0) != DDI_SUCCESS) {
230 cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
231 "Failed to invalidate domain in IOMMU HW cache",
232 f, iommu->aiomt_idx, cmdargs.ca_domainid);
233 }
234 }
235
236 static int
237 amd_iommu_get_deviceid(amd_iommu_t *iommu, dev_info_t *rdip, int32_t *deviceid,
238 int *aliasp, const char *path)
239 {
240 int bus = -1;
241 int device = -1;
242 int func = -1;
243 uint16_t bdf;
244 int32_t src_bdf;
245 dev_info_t *idip = iommu->aiomt_dip;
246 const char *driver = ddi_driver_name(idip);
247 int instance = ddi_get_instance(idip);
248 dev_info_t *pci_dip;
249 const char *f = "amd_iommu_get_deviceid";
250
251 /* be conservative. Always assume an alias */
252 *aliasp = 1;
253 *deviceid = 0;
254
255 /* Check for special special devices (rdip == NULL) */
256 if (rdip == NULL) {
257 if (amd_iommu_get_src_bdf(iommu, -1, &src_bdf) != DDI_SUCCESS) {
258 cmn_err(CE_WARN,
259 "%s: %s%d: idx=%d, failed to get SRC BDF "
260 "for special-device",
261 f, driver, instance, iommu->aiomt_idx);
262 return (DDI_DMA_NOMAPPING);
263 }
264 *deviceid = src_bdf;
265 *aliasp = 1;
266 return (DDI_SUCCESS);
267 }
268
269 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
270 cmn_err(CE_NOTE, "%s: attempting to get deviceid for %s",
271 f, path);
272 }
273
274 pci_dip = amd_iommu_pci_dip(rdip, path);
275 if (pci_dip == NULL) {
276 cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip "
277 "for rdip=%p, path = %s",
278 f, driver, instance, iommu->aiomt_idx, (void *)rdip,
279 path);
280 return (DDI_DMA_NOMAPPING);
281 }
282
283 if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) {
284 ndi_rele_devi(pci_dip);
285 cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get BDF for "
286 "PCI dip (%p). rdip path = %s",
287 f, driver, instance, iommu->aiomt_idx,
288 (void *)pci_dip, path);
289 return (DDI_DMA_NOMAPPING);
290 }
291
292 ndi_rele_devi(pci_dip);
293
294 if (bus > UINT8_MAX || bus < 0 ||
295 device > UINT8_MAX || device < 0 ||
296 func > UINT8_MAX || func < 0) {
297 cmn_err(CE_WARN, "%s: %s%d: idx=%d, invalid BDF(%d,%d,%d) "
298 "for PCI dip (%p). rdip path = %s", f, driver, instance,
299 iommu->aiomt_idx,
300 bus, device, func,
301 (void *)pci_dip, path);
302 return (DDI_DMA_NOMAPPING);
303 }
304
305 bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func;
306
307 if (amd_iommu_get_src_bdf(iommu, bdf, &src_bdf) != DDI_SUCCESS) {
308 cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get SRC BDF "
309 "for PCI dip (%p) rdip path = %s.",
310 f, driver, instance, iommu->aiomt_idx, (void *)pci_dip,
311 path);
312 return (DDI_DMA_NOMAPPING);
313 }
314
315 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
316 cmn_err(CE_NOTE, "%s: Deviceid = %u for path = %s",
317 f, src_bdf, path);
318 }
319
320 *deviceid = src_bdf;
321 *aliasp = (src_bdf != bdf);
322
323 return (DDI_SUCCESS);
324 }
325
326 /*ARGSUSED*/
327 static int
328 init_devtbl(amd_iommu_t *iommu, uint64_t *devtbl_entry, domain_id_t domainid,
329 amd_iommu_domain_t *dp)
330 {
331 uint64_t entry[4] = {0};
332 int i;
333
334 /* If already passthru, don't touch */
335 if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 0 &&
336 AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
337 return (0);
338 }
339
340 if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 1 &&
341 AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 1) {
342
343 ASSERT(dp->d_pgtable_root_4K ==
344 AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
345 AMD_IOMMU_DEVTBL_ROOT_PGTBL));
346
347 ASSERT(dp->d_domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
348 AMD_IOMMU_DEVTBL_DOMAINID));
349
350 return (0);
351 }
352
353 /* New devtbl entry for this domain. Bump up the domain ref-count */
354 dp->d_ref++;
355
356 entry[3] = 0;
357 entry[2] = 0;
358 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_EX, 1);
359 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SD, 0);
360 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_CACHE, 0);
361 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOCTL, 1);
362 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SA, 0);
363 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SE, 1);
364 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_DOMAINID,
365 (uint16_t)domainid);
366 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IW, 1);
367 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IR, 1);
368 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL,
369 dp->d_pgtable_root_4K);
370 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_PG_MODE,
371 AMD_IOMMU_PGTABLE_MAXLEVEL);
372 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_TV,
373 domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);
374 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_V,
375 domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);
376
377 for (i = 1; i < 4; i++) {
378 devtbl_entry[i] = entry[i];
379 }
380 devtbl_entry[0] = entry[0];
381
382 /* we did an actual init */
383 return (1);
384 }
385
386 void
387 amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip)
388 {
389 int32_t deviceid;
390 int alias;
391 uint64_t *devtbl_entry;
392 amd_iommu_cmdargs_t cmdargs = {0};
393 char *path;
394 int pathfree;
395 int V;
396 int TV;
397 int instance;
398 const char *driver;
399 const char *f = "amd_iommu_set_passthru";
400
401 if (rdip) {
402 driver = ddi_driver_name(rdip);
403 instance = ddi_get_instance(rdip);
404 } else {
405 driver = "special-device";
406 instance = 0;
407 }
408
409 path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
410 if (path) {
411 if (rdip)
412 (void) ddi_pathname(rdip, path);
413 else
414 (void) strcpy(path, "special-device");
415 pathfree = 1;
416 } else {
417 pathfree = 0;
418 path = "<path-mem-alloc-failed>";
419 }
420
421 if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
422 != DDI_SUCCESS) {
423 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
424 "Failed to get device ID for device %s.", f, driver,
425 instance,
426 iommu->aiomt_idx, (void *)rdip, path);
427 goto out;
428 }
429
430 /* No deviceid */
431 if (deviceid == -1) {
432 goto out;
433 }
434
435 if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
436 iommu->aiomt_devtbl_sz) {
437 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
438 "for rdip (%p) exceeds device table size (%u), path=%s",
439 f, driver,
440 instance, iommu->aiomt_idx, deviceid, (void *)rdip,
441 iommu->aiomt_devtbl_sz, path);
442 goto out;
443 }
444
445 /*LINTED*/
446 devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
447 [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
448
449 V = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V);
450 TV = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV);
451
452 /* Already passthru */
453 if (V == 0 && TV == 0) {
454 goto out;
455 }
456
457 /* Existing translations */
458 if (V == 1 && TV == 1) {
459 goto out;
460 }
461
462 /* Invalid setting */
463 if (V == 0 && TV == 1) {
464 goto out;
465 }
466
467 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 0);
468
469 cmdargs.ca_deviceid = (uint16_t)deviceid;
470 (void) amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
471 &cmdargs, 0, 0);
472
473 out:
474 if (pathfree)
475 kmem_free(path, MAXPATHLEN);
476 }
477
478 static int
479 amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
480 domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
481 const char *path)
482 {
483 uint64_t *devtbl_entry;
484 amd_iommu_cmdargs_t cmdargs = {0};
485 int error, flags;
486 dev_info_t *idip = iommu->aiomt_dip;
487 const char *driver = ddi_driver_name(idip);
488 int instance = ddi_get_instance(idip);
489 const char *f = "amd_iommu_set_devtbl_entry";
490
491 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
492 cmn_err(CE_NOTE, "%s: attempting to set devtbl entry for %s",
493 f, path);
494 }
495
496 if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
497 iommu->aiomt_devtbl_sz) {
498 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
499 "for rdip (%p) exceeds device table size (%u), path=%s",
500 f, driver,
501 instance, iommu->aiomt_idx, deviceid, (void *)rdip,
502 iommu->aiomt_devtbl_sz, path);
503 return (DDI_DMA_NOMAPPING);
504 }
505
506 /*LINTED*/
507 devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
508 [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
509
510 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
511 cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
512 f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
513 }
514
515 /*
516 * Flush internal caches, need to do this if we came up from
517 * fast boot
518 */
519 cmdargs.ca_deviceid = deviceid;
520 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
521 &cmdargs, 0, 0);
522 if (error != DDI_SUCCESS) {
523 cmn_err(CE_WARN, "%s: idx=%d: deviceid=%d"
524 "Failed to invalidate domain in IOMMU HW cache",
525 f, iommu->aiomt_idx, deviceid);
526 return (error);
527 }
528
529 cmdargs.ca_domainid = (uint16_t)domainid;
530 cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
531 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
532 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
533
534 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
535 &cmdargs, flags, 0);
536 if (error != DDI_SUCCESS) {
537 cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
538 "Failed to invalidate translations in IOMMU HW cache",
539 f, iommu->aiomt_idx, cmdargs.ca_domainid);
540 return (error);
541 }
542
543 /* Initialize device table entry */
544 if (init_devtbl(iommu, devtbl_entry, domainid, dp)) {
545 cmdargs.ca_deviceid = deviceid;
546 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
547 &cmdargs, 0, 0);
548 }
549
550 return (error);
551 }
552
553 int
554 amd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
555 domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
556 int *domain_freed, char *path)
557 {
558 uint64_t *devtbl_entry;
559 int error = DDI_SUCCESS;
560 amd_iommu_cmdargs_t cmdargs = {0};
561 const char *driver = ddi_driver_name(iommu->aiomt_dip);
562 int instance = ddi_get_instance(iommu->aiomt_dip);
563 const char *f = "amd_iommu_clear_devtbl_entry";
564
565 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
566 cmn_err(CE_NOTE, "%s: attempting to clear devtbl entry for "
567 "domainid = %d, deviceid = %u, path = %s",
568 f, domainid, deviceid, path);
569 }
570
571 if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
572 iommu->aiomt_devtbl_sz) {
573 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
574 "for rdip (%p) exceeds device table size (%u), path = %s",
575 f, driver, instance,
576 iommu->aiomt_idx, deviceid, (void *)rdip,
577 iommu->aiomt_devtbl_sz, path);
578 return (DDI_FAILURE);
579 }
580
581 /*LINTED*/
582 devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
583 [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
584
585 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
586 cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
587 f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
588 }
589
590 if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
591 /* Nothing to do */
592 return (DDI_SUCCESS);
593 }
594
595 ASSERT(dp->d_pgtable_root_4K == AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
596 AMD_IOMMU_DEVTBL_ROOT_PGTBL));
597
598 ASSERT(domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
599 AMD_IOMMU_DEVTBL_DOMAINID));
600
601 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV, 0);
602 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 0);
603 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 1);
604
605 SYNC_FORDEV(iommu->aiomt_dmahdl);
606
607 dp->d_ref--;
608 ASSERT(dp->d_ref >= 0);
609
610 if (dp->d_ref == 0) {
611 *domain_freed = 1;
612 }
613
614 cmdargs.ca_deviceid = deviceid;
615 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
616 &cmdargs, 0, 0);
617 if (error != DDI_SUCCESS)
618 error = DDI_FAILURE;
619
620 return (error);
621 }
622
623 int
624 amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt)
625 {
626 ampt->ampt_hash = kmem_zalloc(sizeof (amd_iommu_page_table_t *) *
627 AMD_IOMMU_PGTABLE_HASH_SZ, KM_SLEEP);
628 return (DDI_SUCCESS);
629 }
630
631 void
632 amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt)
633 {
634 kmem_free(ampt->ampt_hash,
635 sizeof (amd_iommu_page_table_t *) * AMD_IOMMU_PGTABLE_HASH_SZ);
636 ampt->ampt_hash = NULL;
637 }
638
639 static uint32_t
640 pt_hashfn(uint64_t pa_4K)
641 {
642 return (pa_4K % AMD_IOMMU_PGTABLE_HASH_SZ);
643 }
644
645 static void
646 amd_iommu_insert_pgtable_hash(amd_iommu_page_table_t *pt)
647 {
648 uint64_t pa_4K = ((uint64_t)pt->pt_cookie.dmac_cookie_addr) >> 12;
649 uint32_t idx = pt_hashfn(pa_4K);
650
651 ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
652
653 mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
654
655 pt->pt_next = amd_iommu_page_table_hash.ampt_hash[idx];
656 pt->pt_prev = NULL;
657 amd_iommu_page_table_hash.ampt_hash[idx] = pt;
658 if (pt->pt_next)
659 pt->pt_next->pt_prev = pt;
660
661 mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
662 }
663
664 static void
665 amd_iommu_remove_pgtable_hash(amd_iommu_page_table_t *pt)
666 {
667 uint64_t pa_4K = (pt->pt_cookie.dmac_cookie_addr >> 12);
668 uint32_t idx = pt_hashfn(pa_4K);
669
670 ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
671
672 mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
673
674 if (pt->pt_next)
675 pt->pt_next->pt_prev = pt->pt_prev;
676
677 if (pt->pt_prev)
678 pt->pt_prev->pt_next = pt->pt_next;
679 else
680 amd_iommu_page_table_hash.ampt_hash[idx] = pt->pt_next;
681
682 pt->pt_next = NULL;
683 pt->pt_prev = NULL;
684
685 mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
686 }
687
688 static amd_iommu_page_table_t *
689 amd_iommu_lookup_pgtable_hash(domain_id_t domainid, uint64_t pgtable_pa_4K)
690 {
691 amd_iommu_page_table_t *pt;
692 uint32_t idx = pt_hashfn(pgtable_pa_4K);
693
694 mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
695 pt = amd_iommu_page_table_hash.ampt_hash[idx];
696 for (; pt; pt = pt->pt_next) {
697 if (domainid != pt->pt_domainid)
698 continue;
699 ASSERT((pt->pt_cookie.dmac_cookie_addr &
700 AMD_IOMMU_PGTABLE_ALIGN) == 0);
701 if ((pt->pt_cookie.dmac_cookie_addr >> 12) == pgtable_pa_4K) {
702 break;
703 }
704 }
705 mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
706
707 return (pt);
708 }
709
710 /*ARGSUSED*/
711 static amd_iommu_page_table_t *
712 amd_iommu_lookup_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *ppt,
713 amd_iommu_domain_t *dp, int level, uint16_t index)
714 {
715 uint64_t *pdtep;
716 uint64_t pgtable_pa_4K;
717
718 ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);
719 ASSERT(dp);
720
721 if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
722 ASSERT(ppt == NULL);
723 ASSERT(index == 0);
724 pgtable_pa_4K = dp->d_pgtable_root_4K;
725 } else {
726 ASSERT(ppt);
727 pdtep = &(ppt->pt_pgtblva[index]);
728 if (AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_PR) == 0) {
729 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
730 cmn_err(CE_NOTE, "Skipping PR=0 pdte: 0x%"
731 PRIx64, *pdtep);
732 }
733 return (NULL);
734 }
735 pgtable_pa_4K = AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_ADDR);
736 }
737
738 return (amd_iommu_lookup_pgtable_hash(dp->d_domainid, pgtable_pa_4K));
739 }
740
741 static amd_iommu_page_table_t *
742 amd_iommu_alloc_from_freelist(void)
743 {
744 int i;
745 uint64_t *pte_array;
746 amd_iommu_page_table_t *pt;
747
748 if (amd_iommu_no_pgtable_freelist == 1)
749 return (NULL);
750
751 if (amd_iommu_pgtable_freelist.f_count == 0)
752 return (NULL);
753
754 pt = amd_iommu_pgtable_freelist.f_list;
755 amd_iommu_pgtable_freelist.f_list = pt->pt_next;
756 amd_iommu_pgtable_freelist.f_count--;
757
758 pte_array = pt->pt_pgtblva;
759 for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
760 ASSERT(pt->pt_pte_ref[i] == 0);
761 ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
762 AMD_IOMMU_PTDE_PR) == 0);
763 }
764
765 return (pt);
766 }
767
768 static int
769 amd_iommu_alloc_pgtable(amd_iommu_t *iommu, domain_id_t domainid,
770 const char *path, amd_iommu_page_table_t **ptp, int km_flags)
771 {
772 int err;
773 uint_t ncookies;
774 amd_iommu_page_table_t *pt;
775 dev_info_t *idip = iommu->aiomt_dip;
776 const char *driver = ddi_driver_name(idip);
777 int instance = ddi_get_instance(idip);
778 const char *f = "amd_iommu_alloc_pgtable";
779
780 *ptp = NULL;
781
782 pt = amd_iommu_alloc_from_freelist();
783 if (pt)
784 goto init_pgtable;
785
786 pt = kmem_zalloc(sizeof (amd_iommu_page_table_t), km_flags);
787 if (pt == NULL)
788 return (DDI_DMA_NORESOURCES);
789
790 /*
791 * Each page table is 4K in size
792 */
793 pt->pt_mem_reqsz = AMD_IOMMU_PGTABLE_SZ;
794
795 /*
796 * Alloc a DMA handle. Use the IOMMU dip as we want this DMA
797 * to *not* enter the IOMMU - no recursive entrance.
798 */
799 err = ddi_dma_alloc_handle(idip, &amd_iommu_pgtable_dma_attr,
800 km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
801 NULL, &pt->pt_dma_hdl);
802 if (err != DDI_SUCCESS) {
803 cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path = %s. "
804 "Cannot alloc DMA handle for IO Page Table",
805 f, driver, instance, domainid, path);
806 kmem_free(pt, sizeof (amd_iommu_page_table_t));
807 return (err == DDI_DMA_NORESOURCES ? err : DDI_DMA_NOMAPPING);
808 }
809
810 /*
811 * Alloc memory for IO Page Table.
812 * XXX remove size_t cast kludge
813 */
814 err = ddi_dma_mem_alloc(pt->pt_dma_hdl, pt->pt_mem_reqsz,
815 &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,
816 km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
817 NULL, (caddr_t *)&pt->pt_pgtblva,
818 (size_t *)&pt->pt_mem_realsz, &pt->pt_mem_hdl);
819 if (err != DDI_SUCCESS) {
820 cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
821 "Cannot allocate DMA memory for IO Page table",
822 f, driver, instance, domainid, path);
823 ddi_dma_free_handle(&pt->pt_dma_hdl);
824 kmem_free(pt, sizeof (amd_iommu_page_table_t));
825 return (DDI_DMA_NORESOURCES);
826 }
827
828 /*
829 * The Page table DMA VA must be 4K aligned and
830 * size >= than requested memory.
831 *
832 */
833 ASSERT(((uint64_t)(uintptr_t)pt->pt_pgtblva & AMD_IOMMU_PGTABLE_ALIGN)
834 == 0);
835 ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);
836
837 /*
838 * Now bind the handle
839 */
840 err = ddi_dma_addr_bind_handle(pt->pt_dma_hdl, NULL,
841 (caddr_t)pt->pt_pgtblva, pt->pt_mem_realsz,
842 DDI_DMA_READ | DDI_DMA_CONSISTENT,
843 km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
844 NULL, &pt->pt_cookie, &ncookies);
845 if (err != DDI_DMA_MAPPED) {
846 cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
847 "Cannot bind memory for DMA to IO Page Tables. "
848 "bufrealsz=%p",
849 f, driver, instance, domainid, path,
850 (void *)(uintptr_t)pt->pt_mem_realsz);
851 ddi_dma_mem_free(&pt->pt_mem_hdl);
852 ddi_dma_free_handle(&pt->pt_dma_hdl);
853 kmem_free(pt, sizeof (amd_iommu_page_table_t));
854 return (err == DDI_DMA_PARTIAL_MAP ? DDI_DMA_NOMAPPING :
855 err);
856 }
857
858 /*
859 * We assume the DMA engine on the IOMMU is capable of handling the
860 * whole page table in a single cookie. If not and multiple cookies
861 * are needed we fail.
862 */
863 if (ncookies != 1) {
864 cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path=%s "
865 "Cannot handle multiple "
866 "cookies for DMA to IO page Table, #cookies=%u",
867 f, driver, instance, domainid, path, ncookies);
868 (void) ddi_dma_unbind_handle(pt->pt_dma_hdl);
869 ddi_dma_mem_free(&pt->pt_mem_hdl);
870 ddi_dma_free_handle(&pt->pt_dma_hdl);
871 kmem_free(pt, sizeof (amd_iommu_page_table_t));
872 return (DDI_DMA_NOMAPPING);
873 }
874
875 init_pgtable:
876 /*
877 * The address in the cookie must be 4K aligned and >= table size
878 */
879 ASSERT(pt->pt_cookie.dmac_cookie_addr != NULL);
880 ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
881 ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_realsz);
882 ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_reqsz);
883 ASSERT(pt->pt_mem_reqsz >= AMD_IOMMU_PGTABLE_SIZE);
884 ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);
885 ASSERT(pt->pt_pgtblva);
886
887 pt->pt_domainid = AMD_IOMMU_INVALID_DOMAIN;
888 pt->pt_level = 0x7;
889 pt->pt_index = 0;
890 pt->pt_ref = 0;
891 pt->pt_next = NULL;
892 pt->pt_prev = NULL;
893 pt->pt_parent = NULL;
894
895 bzero(pt->pt_pgtblva, pt->pt_mem_realsz);
896 SYNC_FORDEV(pt->pt_dma_hdl);
897
898 amd_iommu_insert_pgtable_hash(pt);
899
900 *ptp = pt;
901
902 return (DDI_SUCCESS);
903 }
904
905 static int
906 amd_iommu_move_to_freelist(amd_iommu_page_table_t *pt)
907 {
908 if (amd_iommu_no_pgtable_freelist == 1)
909 return (DDI_FAILURE);
910
911 if (amd_iommu_pgtable_freelist.f_count ==
912 AMD_IOMMU_PGTABLE_FREELIST_MAX)
913 return (DDI_FAILURE);
914
915 pt->pt_next = amd_iommu_pgtable_freelist.f_list;
916 amd_iommu_pgtable_freelist.f_list = pt;
917 amd_iommu_pgtable_freelist.f_count++;
918
919 return (DDI_SUCCESS);
920 }
921
922 static void
923 amd_iommu_free_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *pt)
924 {
925 int i;
926 uint64_t *pte_array;
927 dev_info_t *dip = iommu->aiomt_dip;
928 int instance = ddi_get_instance(dip);
929 const char *driver = ddi_driver_name(dip);
930 const char *f = "amd_iommu_free_pgtable";
931
932 ASSERT(pt->pt_ref == 0);
933
934 amd_iommu_remove_pgtable_hash(pt);
935
936 pte_array = pt->pt_pgtblva;
937 for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
938 ASSERT(pt->pt_pte_ref[i] == 0);
939 ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
940 AMD_IOMMU_PTDE_PR) == 0);
941 }
942
943 if (amd_iommu_move_to_freelist(pt) == DDI_SUCCESS)
944 return;
945
946 /* Unbind the handle */
947 if (ddi_dma_unbind_handle(pt->pt_dma_hdl) != DDI_SUCCESS) {
948 cmn_err(CE_WARN, "%s: %s%d: idx=%d, domainid=%d. "
949 "Failed to unbind handle: %p for IOMMU Page Table",
950 f, driver, instance, iommu->aiomt_idx, pt->pt_domainid,
951 (void *)pt->pt_dma_hdl);
952 }
953 /* Free the table memory allocated for DMA */
954 ddi_dma_mem_free(&pt->pt_mem_hdl);
955
956 /* Free the DMA handle */
957 ddi_dma_free_handle(&pt->pt_dma_hdl);
958
959 kmem_free(pt, sizeof (amd_iommu_page_table_t));
960
961 }
962
963 static int
964 init_pde(amd_iommu_page_table_t *ppt, amd_iommu_page_table_t *pt)
965 {
966 uint64_t *pdep = &(ppt->pt_pgtblva[pt->pt_index]);
967 uint64_t next_pgtable_pa_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;
968
969 /* nothing to set. PDE is already set */
970 if (AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1) {
971 ASSERT(PT_REF_VALID(ppt));
972 ASSERT(PT_REF_VALID(pt));
973 ASSERT(ppt->pt_pte_ref[pt->pt_index] == 0);
974 ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_ADDR)
975 == next_pgtable_pa_4K);
976 return (DDI_SUCCESS);
977 }
978
979 ppt->pt_ref++;
980 ASSERT(PT_REF_VALID(ppt));
981
982 /* Page Directories are always RW */
983 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IW, 1);
984 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IR, 1);
985 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_ADDR,
986 next_pgtable_pa_4K);
987 pt->pt_parent = ppt;
988 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_NXT_LVL,
989 pt->pt_level);
990 ppt->pt_pte_ref[pt->pt_index] = 0;
991 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_PR, 1);
992 SYNC_FORDEV(ppt->pt_dma_hdl);
993 ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1);
994
995 return (DDI_SUCCESS);
996 }
997
998 static int
999 init_pte(amd_iommu_page_table_t *pt, uint64_t pa, uint16_t index,
1000 struct ddi_dma_req *dmareq)
1001 {
1002 uint64_t *ptep = &(pt->pt_pgtblva[index]);
1003 uint64_t pa_4K = pa >> 12;
1004 int R;
1005 int W;
1006
1007 /* nothing to set if PTE is already set */
1008 if (AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1) {
1009 /*
1010 * Adjust current permissions
1011 * DDI_DMA_WRITE means direction of DMA is MEM -> I/O
1012 * so that requires Memory READ permissions i.e. sense
1013 * is inverted.
1014 * Note: either or both of DD_DMA_READ/WRITE may be set
1015 */
1016 if (amd_iommu_no_RW_perms == 0) {
1017 R = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IR);
1018 W = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IW);
1019 if (R == 0 && ((dmareq->dmar_flags & DDI_DMA_WRITE) ||
1020 (dmareq->dmar_flags & DDI_DMA_RDWR))) {
1021 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1022 }
1023 if (W == 0 && ((dmareq->dmar_flags & DDI_DMA_READ) ||
1024 (dmareq->dmar_flags & DDI_DMA_RDWR))) {
1025 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1026 }
1027 }
1028 ASSERT(PT_REF_VALID(pt));
1029 pt->pt_pte_ref[index]++;
1030 ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR)
1031 == pa_4K);
1032 return (DDI_SUCCESS);
1033 }
1034
1035 pt->pt_ref++;
1036 ASSERT(PT_REF_VALID(pt));
1037
1038 /* see comment above about inverting sense of RD/WR */
1039 if (amd_iommu_no_RW_perms == 0) {
1040 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 0);
1041 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 0);
1042 if (dmareq->dmar_flags & DDI_DMA_RDWR) {
1043 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1044 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1045 } else {
1046 if (dmareq->dmar_flags & DDI_DMA_WRITE) {
1047 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1048 }
1049 if (dmareq->dmar_flags & DDI_DMA_READ) {
1050 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1051 }
1052 }
1053 } else {
1054 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1055 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1056 }
1057
1058 /* TODO what is correct for FC and U */
1059 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_FC, 0);
1060 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_U, 0);
1061 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_ADDR, pa_4K);
1062 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_NXT_LVL, 0);
1063 ASSERT(pt->pt_pte_ref[index] == 0);
1064 pt->pt_pte_ref[index] = 1;
1065 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_PR, 1);
1066 SYNC_FORDEV(pt->pt_dma_hdl);
1067 ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1);
1068
1069 return (DDI_SUCCESS);
1070 }
1071
1072
1073 static void
1074 init_pt(amd_iommu_page_table_t *pt, amd_iommu_domain_t *dp,
1075 int level, uint16_t index)
1076 {
1077 ASSERT(dp);
1078
1079 if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
1080 dp->d_pgtable_root_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;
1081 } else {
1082 ASSERT(level >= 1 && level < AMD_IOMMU_PGTABLE_MAXLEVEL);
1083 }
1084
1085 pt->pt_domainid = dp->d_domainid;
1086 pt->pt_level = level;
1087 pt->pt_index = index;
1088 }
1089
1090 static int
1091 amd_iommu_setup_1_pgtable(amd_iommu_t *iommu, dev_info_t *rdip,
1092 struct ddi_dma_req *dmareq,
1093 domain_id_t domainid, amd_iommu_domain_t *dp,
1094 amd_iommu_page_table_t *ppt,
1095 uint16_t index, int level, uint64_t va, uint64_t pa,
1096 amd_iommu_page_table_t **ptp, uint16_t *next_idxp, const char *path,
1097 int km_flags)
1098 {
1099 int error;
1100 amd_iommu_page_table_t *pt;
1101 const char *driver = ddi_driver_name(rdip);
1102 int instance = ddi_get_instance(rdip);
1103 const char *f = "amd_iommu_setup_1_pgtable";
1104
1105 *ptp = NULL;
1106 *next_idxp = 0;
1107 error = DDI_SUCCESS;
1108
1109 ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);
1110
1111 ASSERT(dp);
1112 if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
1113 ASSERT(ppt == NULL);
1114 ASSERT(index == 0);
1115 } else {
1116 ASSERT(ppt);
1117 }
1118
1119 /* Check if page table is already allocated */
1120 if (pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index)) {
1121 ASSERT(pt->pt_domainid == domainid);
1122 ASSERT(pt->pt_level == level);
1123 ASSERT(pt->pt_index == index);
1124 goto out;
1125 }
1126
1127 if ((error = amd_iommu_alloc_pgtable(iommu, domainid, path, &pt,
1128 km_flags)) != DDI_SUCCESS) {
1129 cmn_err(CE_WARN, "%s: %s%d: idx = %u, domainid = %d, va = %p "
1130 "path = %s", f, driver, instance, iommu->aiomt_idx,
1131 domainid, (void *)(uintptr_t)va, path);
1132 return (error);
1133 }
1134
1135 ASSERT(dp->d_domainid == domainid);
1136
1137 init_pt(pt, dp, level, index);
1138
1139 out:
1140 if (level != AMD_IOMMU_PGTABLE_MAXLEVEL) {
1141 error = init_pde(ppt, pt);
1142 }
1143
1144 if (level == 1) {
1145 ASSERT(error == DDI_SUCCESS);
1146 error = init_pte(pt, pa, AMD_IOMMU_VA_BITS(va, level), dmareq);
1147 } else {
1148 *next_idxp = AMD_IOMMU_VA_BITS(va, level);
1149 *ptp = pt;
1150 }
1151
1152 return (error);
1153 }
1154
1155 typedef enum {
1156 PDTE_NOT_TORN = 0x1,
1157 PDTE_TORN_DOWN = 0x2,
1158 PGTABLE_TORN_DOWN = 0x4
1159 } pdte_tear_t;
1160
1161 static pdte_tear_t
1162 amd_iommu_teardown_pdte(amd_iommu_t *iommu,
1163 amd_iommu_page_table_t *pt, int index)
1164 {
1165 uint8_t next_level;
1166 pdte_tear_t retval;
1167 uint64_t *ptdep = &(pt->pt_pgtblva[index]);
1168
1169 next_level = AMD_IOMMU_REG_GET64(ptdep,
1170 AMD_IOMMU_PTDE_NXT_LVL);
1171
1172 if (AMD_IOMMU_REG_GET64(ptdep, AMD_IOMMU_PTDE_PR) == 1) {
1173 if (pt->pt_level == 1) {
1174 ASSERT(next_level == 0);
1175 /* PTE */
1176 pt->pt_pte_ref[index]--;
1177 if (pt->pt_pte_ref[index] != 0) {
1178 return (PDTE_NOT_TORN);
1179 }
1180 } else {
1181 ASSERT(next_level != 0 && next_level != 7);
1182 }
1183 ASSERT(pt->pt_pte_ref[index] == 0);
1184 ASSERT(PT_REF_VALID(pt));
1185
1186 AMD_IOMMU_REG_SET64(ptdep, AMD_IOMMU_PTDE_PR, 0);
1187 SYNC_FORDEV(pt->pt_dma_hdl);
1188 ASSERT(AMD_IOMMU_REG_GET64(ptdep,
1189 AMD_IOMMU_PTDE_PR) == 0);
1190 pt->pt_ref--;
1191 ASSERT(PT_REF_VALID(pt));
1192 retval = PDTE_TORN_DOWN;
1193 } else {
1194 ASSERT(0);
1195 ASSERT(pt->pt_pte_ref[index] == 0);
1196 ASSERT(PT_REF_VALID(pt));
1197 retval = PDTE_NOT_TORN;
1198 }
1199
1200 if (pt->pt_ref == 0) {
1201 amd_iommu_free_pgtable(iommu, pt);
1202 return (PGTABLE_TORN_DOWN);
1203 }
1204
1205 return (retval);
1206 }
1207
1208 static int
1209 amd_iommu_create_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
1210 struct ddi_dma_req *dmareq, uint64_t va,
1211 uint64_t pa, uint16_t deviceid, domain_id_t domainid,
1212 amd_iommu_domain_t *dp, const char *path, int km_flags)
1213 {
1214 int level;
1215 uint16_t index;
1216 uint16_t next_idx;
1217 amd_iommu_page_table_t *pt;
1218 amd_iommu_page_table_t *ppt;
1219 int error;
1220 const char *driver = ddi_driver_name(rdip);
1221 int instance = ddi_get_instance(rdip);
1222 const char *f = "amd_iommu_create_pgtables";
1223
1224 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1225 cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
1226 "deviceid = %u, va = %p, pa = %p, path = %s",
1227 f, driver, instance,
1228 iommu->aiomt_idx, domainid, deviceid,
1229 (void *)(uintptr_t)va,
1230 (void *)(uintptr_t)pa, path);
1231 }
1232
1233 if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
1234 /* No need for pagetables. Just set up device table entry */
1235 goto passthru;
1236 }
1237
1238 index = 0;
1239 ppt = NULL;
1240 for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0;
1241 level--, pt = NULL, next_idx = 0) {
1242 if ((error = amd_iommu_setup_1_pgtable(iommu, rdip, dmareq,
1243 domainid, dp, ppt, index, level, va, pa, &pt,
1244 &next_idx, path, km_flags)) != DDI_SUCCESS) {
1245 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
1246 "deviceid=%u, va= %p, pa = %p, Failed to setup "
1247 "page table(s) at level = %d, path = %s.",
1248 f, driver, instance, iommu->aiomt_idx,
1249 domainid, deviceid, (void *)(uintptr_t)va,
1250 (void *)(uintptr_t)pa, level, path);
1251 return (error);
1252 }
1253
1254 if (level > 1) {
1255 ASSERT(pt);
1256 ASSERT(pt->pt_domainid == domainid);
1257 ppt = pt;
1258 index = next_idx;
1259 } else {
1260 ASSERT(level == 1);
1261 ASSERT(pt == NULL);
1262 ASSERT(next_idx == 0);
1263 ppt = NULL;
1264 index = 0;
1265 }
1266 }
1267
1268 passthru:
1269 if ((error = amd_iommu_set_devtbl_entry(iommu, rdip, domainid, deviceid,
1270 dp, path)) != DDI_SUCCESS) {
1271 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, deviceid=%u, "
1272 "domainid=%d."
1273 "Failed to set device table entry for path %s.",
1274 f, driver, instance,
1275 iommu->aiomt_idx, (void *)rdip, deviceid, domainid, path);
1276 return (error);
1277 }
1278
1279 SYNC_FORDEV(iommu->aiomt_dmahdl);
1280
1281 return (DDI_SUCCESS);
1282 }
1283
1284 static int
1285 amd_iommu_destroy_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
1286 uint64_t pageva, uint16_t deviceid, domain_id_t domainid,
1287 amd_iommu_domain_t *dp, map_type_t type, int *domain_freed, char *path)
1288 {
1289 int level;
1290 int flags;
1291 amd_iommu_cmdargs_t cmdargs = {0};
1292 uint16_t index;
1293 uint16_t prev_index;
1294 amd_iommu_page_table_t *pt;
1295 amd_iommu_page_table_t *ppt;
1296 pdte_tear_t retval;
1297 int tear_level;
1298 int invalidate_pte;
1299 int invalidate_pde;
1300 int error = DDI_FAILURE;
1301 const char *driver = ddi_driver_name(iommu->aiomt_dip);
1302 int instance = ddi_get_instance(iommu->aiomt_dip);
1303 const char *f = "amd_iommu_destroy_pgtables";
1304
1305 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1306 cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
1307 "deviceid = %u, va = %p, path = %s",
1308 f, driver, instance,
1309 iommu->aiomt_idx, domainid, deviceid,
1310 (void *)(uintptr_t)pageva, path);
1311 }
1312
1313 if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
1314 /*
1315 * there are no pagetables for the passthru domain.
1316 * Just the device table entry
1317 */
1318 error = DDI_SUCCESS;
1319 goto passthru;
1320 }
1321
1322 ppt = NULL;
1323 index = 0;
1324 for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; level--) {
1325 pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index);
1326 if (pt) {
1327 ppt = pt;
1328 index = AMD_IOMMU_VA_BITS(pageva, level);
1329 continue;
1330 }
1331 break;
1332 }
1333
1334 if (level == 0) {
1335 uint64_t *ptep;
1336 uint64_t pa_4K;
1337
1338 ASSERT(pt);
1339 ASSERT(pt == ppt);
1340 ASSERT(pt->pt_domainid == dp->d_domainid);
1341
1342 ptep = &(pt->pt_pgtblva[index]);
1343
1344 pa_4K = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR);
1345 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
1346 ASSERT(pageva == (pa_4K << MMU_PAGESHIFT));
1347 }
1348 }
1349
1350 tear_level = -1;
1351 invalidate_pde = 0;
1352 invalidate_pte = 0;
1353 for (++level; level <= AMD_IOMMU_PGTABLE_MAXLEVEL; level++) {
1354 prev_index = pt->pt_index;
1355 ppt = pt->pt_parent;
1356 retval = amd_iommu_teardown_pdte(iommu, pt, index);
1357 switch (retval) {
1358 case PDTE_NOT_TORN:
1359 goto invalidate;
1360 case PDTE_TORN_DOWN:
1361 invalidate_pte = 1;
1362 goto invalidate;
1363 case PGTABLE_TORN_DOWN:
1364 invalidate_pte = 1;
1365 invalidate_pde = 1;
1366 tear_level = level;
1367 break;
1368 }
1369 index = prev_index;
1370 pt = ppt;
1371 }
1372
1373 invalidate:
1374 /*
1375 * Now teardown the IOMMU HW caches if applicable
1376 */
1377 if (invalidate_pte) {
1378 cmdargs.ca_domainid = (uint16_t)domainid;
1379 if (amd_iommu_pageva_inval_all) {
1380 cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
1381 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
1382 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
1383 } else if (invalidate_pde) {
1384 cmdargs.ca_addr =
1385 (uintptr_t)AMD_IOMMU_VA_INVAL(pageva, tear_level);
1386 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
1387 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
1388 } else {
1389 cmdargs.ca_addr = (uintptr_t)pageva;
1390 flags = 0;
1391 }
1392 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
1393 &cmdargs, flags, 0) != DDI_SUCCESS) {
1394 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
1395 "rdip=%p. Failed to invalidate IOMMU HW cache "
1396 "for %s", f, driver, instance,
1397 iommu->aiomt_idx, domainid, (void *)rdip, path);
1398 error = DDI_FAILURE;
1399 goto out;
1400 }
1401 }
1402
1403 passthru:
1404 if (tear_level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
1405 error = amd_iommu_clear_devtbl_entry(iommu, rdip, domainid,
1406 deviceid, dp, domain_freed, path);
1407 } else {
1408 error = DDI_SUCCESS;
1409 }
1410
1411 out:
1412 SYNC_FORDEV(iommu->aiomt_dmahdl);
1413
1414 return (error);
1415 }
1416
1417 static int
1418 cvt_bind_error(int error)
1419 {
1420 switch (error) {
1421 case DDI_DMA_MAPPED:
1422 case DDI_DMA_PARTIAL_MAP:
1423 case DDI_DMA_NORESOURCES:
1424 case DDI_DMA_NOMAPPING:
1425 break;
1426 default:
1427 cmn_err(CE_PANIC, "Unsupported error code: %d", error);
1428 /*NOTREACHED*/
1429 }
1430 return (error);
1431 }
1432
1433 int
1434 amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
1435 struct ddi_dma_req *dmareq, uint64_t start_pa, uint64_t pa_sz,
1436 map_type_t type, uint64_t *start_vap, int km_flags)
1437 {
1438 pfn_t pfn_start;
1439 pfn_t pfn_end;
1440 pfn_t pfn;
1441 int alias;
1442 int32_t deviceid;
1443 domain_id_t domainid;
1444 amd_iommu_domain_t *dp;
1445 uint64_t end_pa;
1446 uint64_t start_va;
1447 uint64_t end_va;
1448 uint64_t pg_start;
1449 uint64_t pg_end;
1450 uint64_t pg;
1451 uint64_t va_sz;
1452 char *path;
1453 int error = DDI_DMA_NOMAPPING;
1454 const char *driver = ddi_driver_name(iommu->aiomt_dip);
1455 int instance = ddi_get_instance(iommu->aiomt_dip);
1456 const char *f = "amd_iommu_map_pa2va";
1457
1458 ASSERT(pa_sz != 0);
1459
1460 *start_vap = 0;
1461
1462 ASSERT(rdip);
1463
1464 path = kmem_alloc(MAXPATHLEN, km_flags);
1465 if (path == NULL) {
1466 error = DDI_DMA_NORESOURCES;
1467 goto out;
1468 }
1469 (void) ddi_pathname(rdip, path);
1470
1471 /*
1472 * First get deviceid
1473 */
1474 if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
1475 != DDI_SUCCESS) {
1476 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
1477 "Failed to get device ID for %s.", f, driver, instance,
1478 iommu->aiomt_idx, (void *)rdip, path);
1479 error = DDI_DMA_NOMAPPING;
1480 goto out;
1481 }
1482
1483 /*
1484 * Next get the domain for this rdip
1485 */
1486 if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
1487 != DDI_SUCCESS) {
1488 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
1489 "Failed to get domain.", f, driver, instance,
1490 iommu->aiomt_idx, (void *)rdip, path);
1491 error = DDI_DMA_NOMAPPING;
1492 goto out;
1493 }
1494
1495 dp = amd_iommu_lookup_domain(iommu, domainid, type, km_flags);
1496 if (dp == NULL) {
1497 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
1498 "Failed to get device ID for %s.", f, driver, instance,
1499 iommu->aiomt_idx, domainid, (void *)rdip, path);
1500 error = DDI_DMA_NORESOURCES;
1501 goto out;
1502 }
1503
1504 ASSERT(dp->d_domainid == domainid);
1505
1506 pfn_start = start_pa >> MMU_PAGESHIFT;
1507
1508 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1509 cmn_err(CE_NOTE, "pa = %p, pfn_new = %p, pfn_start = %p, "
1510 "pgshift = %d",
1511 (void *)(uintptr_t)start_pa,
1512 (void *)(uintptr_t)(start_pa >> MMU_PAGESHIFT),
1513 (void *)(uintptr_t)pfn_start, MMU_PAGESHIFT);
1514 }
1515
1516 end_pa = start_pa + pa_sz - 1;
1517 pfn_end = end_pa >> MMU_PAGESHIFT;
1518
1519 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
1520 start_va = start_pa;
1521 end_va = end_pa;
1522 va_sz = pa_sz;
1523 *start_vap = start_va;
1524 } else {
1525 va_sz = mmu_ptob(pfn_end - pfn_start + 1);
1526 start_va = (uintptr_t)vmem_xalloc(dp->d_vmem, va_sz,
1527 MAX(attrp->dma_attr_align, MMU_PAGESIZE),
1528 0,
1529 attrp->dma_attr_seg + 1,
1530 (void *)(uintptr_t)attrp->dma_attr_addr_lo,
1531 (void *)(uintptr_t)MIN((attrp->dma_attr_addr_hi + 1),
1532 AMD_IOMMU_SIZE_4G), /* XXX rollover */
1533 km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
1534 if (start_va == 0) {
1535 cmn_err(CE_WARN, "%s: No VA resources",
1536 amd_iommu_modname);
1537 error = DDI_DMA_NORESOURCES;
1538 goto out;
1539 }
1540 ASSERT((start_va & MMU_PAGEOFFSET) == 0);
1541 end_va = start_va + va_sz - 1;
1542 *start_vap = start_va + (start_pa & MMU_PAGEOFFSET);
1543 }
1544
1545 pg_start = start_va >> MMU_PAGESHIFT;
1546 pg_end = end_va >> MMU_PAGESHIFT;
1547
1548 pg = pg_start;
1549 for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) {
1550
1551 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1552 cmn_err(CE_NOTE, "%s: attempting to create page tables "
1553 "for pfn = %p, va = %p, path = %s",
1554 f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT),
1555 (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path);
1556
1557 }
1558
1559 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
1560 ASSERT(pfn == pg);
1561 }
1562
1563 if ((error = amd_iommu_create_pgtables(iommu, rdip, dmareq,
1564 pg << MMU_PAGESHIFT,
1565 pfn << MMU_PAGESHIFT, deviceid, domainid, dp, path,
1566 km_flags)) != DDI_SUCCESS) {
1567 cmn_err(CE_WARN, "Failed to create_pgtables");
1568 goto out;
1569 }
1570
1571 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1572 cmn_err(CE_NOTE, "%s: successfully created page tables "
1573 "for pfn = %p, vapg = %p, path = %s",
1574 f, (void *)(uintptr_t)pfn,
1575 (void *)(uintptr_t)pg, path);
1576 }
1577
1578 }
1579 ASSERT(pg == pg_end + 1);
1580
1581
1582 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PA2VA) {
1583 cmn_err(CE_NOTE, "pa=%p, va=%p",
1584 (void *)(uintptr_t)start_pa,
1585 (void *)(uintptr_t)(*start_vap));
1586 }
1587 error = DDI_DMA_MAPPED;
1588
1589 out:
1590 kmem_free(path, MAXPATHLEN);
1591 return (cvt_bind_error(error));
1592 }
1593
1594 int
1595 amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, uint64_t start_va,
1596 uint64_t va_sz, map_type_t type)
1597 {
1598 uint64_t end_va;
1599 uint64_t pg_start;
1600 uint64_t pg_end;
1601 uint64_t pg;
1602 uint64_t actual_sz;
1603 char *path;
1604 int pathfree;
1605 int alias;
1606 int32_t deviceid;
1607 domain_id_t domainid;
1608 amd_iommu_domain_t *dp;
1609 int error;
1610 int domain_freed;
1611 const char *driver = ddi_driver_name(iommu->aiomt_dip);
1612 int instance = ddi_get_instance(iommu->aiomt_dip);
1613 const char *f = "amd_iommu_unmap_va";
1614
1615 if (amd_iommu_no_unmap)
1616 return (DDI_SUCCESS);
1617
1618 path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
1619 if (path) {
1620 (void) ddi_pathname(rdip, path);
1621 pathfree = 1;
1622 } else {
1623 pathfree = 0;
1624 path = "<path-mem-alloc-failed>";
1625 }
1626
1627 /*
1628 * First get deviceid
1629 */
1630 if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
1631 != DDI_SUCCESS) {
1632 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
1633 "Failed to get device ID for %s.", f, driver, instance,
1634 iommu->aiomt_idx, (void *)rdip, path);
1635 error = DDI_FAILURE;
1636 goto out;
1637 }
1638
1639 /*
1640 * Next get the domain for this rdip
1641 */
1642 if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
1643 != DDI_SUCCESS) {
1644 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
1645 "Failed to get domain.", f, driver, instance,
1646 iommu->aiomt_idx, (void *)rdip, path);
1647 error = DDI_FAILURE;
1648 goto out;
1649 }
1650
1651 /* should never result in domain allocation/vmem_create */
1652 dp = amd_iommu_lookup_domain(iommu, domainid, AMD_IOMMU_INVALID_MAP,
1653 KM_NOSLEEP);
1654 if (dp == NULL) {
1655 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
1656 "Failed to get device ID for %s.", f, driver, instance,
1657 iommu->aiomt_idx, domainid, (void *)rdip, path);
1658 error = DDI_FAILURE;
1659 goto out;
1660 }
1661
1662 ASSERT(dp->d_domainid == domainid);
1663
1664 pg_start = start_va >> MMU_PAGESHIFT;
1665 end_va = start_va + va_sz - 1;
1666 pg_end = end_va >> MMU_PAGESHIFT;
1667 actual_sz = (pg_end - pg_start + 1) << MMU_PAGESHIFT;
1668
1669 domain_freed = 0;
1670 for (pg = pg_start; pg <= pg_end; pg++) {
1671 domain_freed = 0;
1672 if (amd_iommu_destroy_pgtables(iommu, rdip,
1673 pg << MMU_PAGESHIFT, deviceid, domainid, dp, type,
1674 &domain_freed, path) != DDI_SUCCESS) {
1675 error = DDI_FAILURE;
1676 goto out;
1677 }
1678 if (domain_freed) {
1679 ASSERT(pg == pg_end);
1680 break;
1681 }
1682 }
1683
1684 /*
1685 * vmem_xalloc() must be paired with vmem_xfree
1686 */
1687 if (type == AMD_IOMMU_VMEM_MAP && !amd_iommu_unity_map) {
1688 vmem_xfree(dp->d_vmem,
1689 (void *)(uintptr_t)(pg_start << MMU_PAGESHIFT), actual_sz);
1690 }
1691
1692 if (domain_freed)
1693 amd_iommu_teardown_domain(iommu, dp);
1694
1695 error = DDI_SUCCESS;
1696 out:
1697 if (pathfree)
1698 kmem_free(path, MAXPATHLEN);
1699 return (error);
1700 }