1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 #include <sys/sunddi.h>
  27 #include <sys/sunndi.h>
  28 #include <sys/acpi/acpi.h>
  29 #include <sys/acpica.h>
  30 #include <sys/amd_iommu.h>
  31 #include <sys/bootconf.h>
  32 #include <sys/sysmacros.h>
  33 #include <sys/ddidmareq.h>
  34 
  35 #include "amd_iommu_impl.h"
  36 #include "amd_iommu_acpi.h"
  37 #include "amd_iommu_page_tables.h"
  38 
  39 ddi_dma_attr_t amd_iommu_pgtable_dma_attr = {
  40         DMA_ATTR_V0,
  41         0U,                             /* dma_attr_addr_lo */
  42         0xffffffffffffffffULL,          /* dma_attr_addr_hi */
  43         0xffffffffU,                    /* dma_attr_count_max */
  44         (uint64_t)4096,                 /* dma_attr_align */
  45         1,                              /* dma_attr_burstsizes */
  46         64,                             /* dma_attr_minxfer */
  47         0xffffffffU,                    /* dma_attr_maxxfer */
  48         0xffffffffU,                    /* dma_attr_seg */
  49         1,                              /* dma_attr_sgllen, variable */
  50         64,                             /* dma_attr_granular */
  51         0                               /* dma_attr_flags */
  52 };
  53 
  54 static amd_iommu_domain_t **amd_iommu_domain_table;
  55 
  56 static struct {
  57         int f_count;
  58         amd_iommu_page_table_t *f_list;
  59 } amd_iommu_pgtable_freelist;
  60 int amd_iommu_no_pgtable_freelist;
  61 
  62 /*ARGSUSED*/
  63 static int
  64 amd_iommu_get_src_bdf(amd_iommu_t *iommu, int32_t bdf, int32_t *src_bdfp)
  65 {
  66         amd_iommu_acpi_ivhd_t *hinfop;
  67 
  68         hinfop = amd_iommu_lookup_ivhd(bdf);
  69         if (hinfop == NULL) {
  70                 if (bdf == -1) {
  71                         *src_bdfp = bdf;
  72                 } else {
  73                         cmn_err(CE_WARN, "No IVHD entry for 0x%x", bdf);
  74                         return (DDI_FAILURE);
  75                 }
  76         } else if (hinfop->ach_src_deviceid == -1) {
  77                 *src_bdfp = bdf;
  78         } else {
  79                 *src_bdfp = hinfop->ach_src_deviceid;
  80         }
  81 
  82         return (DDI_SUCCESS);
  83 }
  84 
  85 /*ARGSUSED*/
  86 static int
  87 amd_iommu_get_domain(amd_iommu_t *iommu, dev_info_t *rdip, int alias,
  88     uint16_t deviceid, domain_id_t *domainid, const char *path)
  89 {
  90         const char *f = "amd_iommu_get_domain";
  91 
  92         *domainid = AMD_IOMMU_INVALID_DOMAIN;
  93 
  94         ASSERT(strcmp(ddi_driver_name(rdip), "agpgart") != 0);
  95 
  96         switch (deviceid) {
  97                 case AMD_IOMMU_INVALID_DOMAIN:
  98                 case AMD_IOMMU_IDENTITY_DOMAIN:
  99                 case AMD_IOMMU_PASSTHRU_DOMAIN:
 100                 case AMD_IOMMU_SYS_DOMAIN:
 101                         *domainid = AMD_IOMMU_SYS_DOMAIN;
 102                         break;
 103                 default:
 104                         *domainid = deviceid;
 105                         break;
 106         }
 107 
 108         if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
 109                 cmn_err(CE_NOTE, "%s: domainid for %s = %d",
 110                     f, path, *domainid);
 111         }
 112 
 113         return (DDI_SUCCESS);
 114 }
 115 
 116 static uint16_t
 117 hash_domain(domain_id_t domainid)
 118 {
 119         return (domainid % AMD_IOMMU_DOMAIN_HASH_SZ);
 120 }
 121 
 122 /*ARGSUSED*/
 123 void
 124 amd_iommu_init_page_tables(amd_iommu_t *iommu)
 125 {
 126         amd_iommu_domain_table = kmem_zalloc(
 127             sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ, KM_SLEEP);
 128 }
 129 
 130 /*ARGSUSED*/
 131 void
 132 amd_iommu_fini_page_tables(amd_iommu_t *iommu)
 133 {
 134         if (amd_iommu_domain_table) {
 135                 kmem_free(amd_iommu_domain_table,
 136                     sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ);
 137                 amd_iommu_domain_table = NULL;
 138         }
 139 }
 140 
 141 static amd_iommu_domain_t *
 142 amd_iommu_lookup_domain(amd_iommu_t *iommu, domain_id_t domainid,
 143     map_type_t type, int km_flags)
 144 {
 145         uint16_t idx;
 146         amd_iommu_domain_t *dp;
 147         char name[AMD_IOMMU_VMEM_NAMELEN+1];
 148 
 149         ASSERT(amd_iommu_domain_table);
 150 
 151         idx = hash_domain(domainid);
 152 
 153         for (dp = amd_iommu_domain_table[idx]; dp; dp = dp->d_next) {
 154                 if (dp->d_domainid == domainid)
 155                         return (dp);
 156         }
 157 
 158         ASSERT(type != AMD_IOMMU_INVALID_MAP);
 159 
 160         dp = kmem_zalloc(sizeof (*dp), km_flags);
 161         if (dp == NULL)
 162                 return (NULL);
 163         dp->d_domainid = domainid;
 164         dp->d_pgtable_root_4K = 0;   /* make this explicit */
 165 
 166         if (type == AMD_IOMMU_VMEM_MAP) {
 167                 uint64_t base;
 168                 uint64_t size;
 169                 (void) snprintf(name, sizeof (name), "dvma_idx%d_domain%d",
 170                     iommu->aiomt_idx, domainid);
 171                 base = MMU_PAGESIZE;
 172                 size = AMD_IOMMU_SIZE_4G - MMU_PAGESIZE;
 173                 dp->d_vmem = vmem_create(name, (void *)(uintptr_t)base, size,
 174                     MMU_PAGESIZE, NULL, NULL, NULL, 0,
 175                     km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
 176                 if (dp->d_vmem == NULL) {
 177                         kmem_free(dp, sizeof (*dp));
 178                         return (NULL);
 179                 }
 180         } else {
 181                 dp->d_vmem = NULL;
 182         }
 183 
 184         dp->d_next = amd_iommu_domain_table[idx];
 185         dp->d_prev = NULL;
 186         amd_iommu_domain_table[idx] = dp;
 187         if (dp->d_next)
 188                 dp->d_next->d_prev = dp;
 189         dp->d_ref = 0;
 190 
 191 
 192         return (dp);
 193 }
 194 
 195 static void
 196 amd_iommu_teardown_domain(amd_iommu_t *iommu, amd_iommu_domain_t *dp)
 197 {
 198         uint16_t idx;
 199         int flags;
 200         amd_iommu_cmdargs_t cmdargs = {0};
 201         domain_id_t domainid = dp->d_domainid;
 202         const char *f = "amd_iommu_teardown_domain";
 203 
 204         ASSERT(dp->d_ref == 0);
 205 
 206         idx = hash_domain(dp->d_domainid);
 207 
 208         if (dp->d_prev == NULL)
 209                 amd_iommu_domain_table[idx] = dp->d_next;
 210         else
 211                 dp->d_prev->d_next = dp->d_next;
 212 
 213         if (dp->d_next)
 214                 dp->d_next->d_prev = dp->d_prev;
 215 
 216         if (dp->d_vmem != NULL) {
 217                 vmem_destroy(dp->d_vmem);
 218                 dp->d_vmem = NULL;
 219         }
 220 
 221         kmem_free(dp, sizeof (*dp));
 222 
 223         cmdargs.ca_domainid = (uint16_t)domainid;
 224         cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
 225         flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
 226             AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
 227 
 228         if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
 229             &cmdargs, flags, 0) != DDI_SUCCESS) {
 230                 cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
 231                     "Failed to invalidate domain in IOMMU HW cache",
 232                     f, iommu->aiomt_idx, cmdargs.ca_domainid);
 233         }
 234 }
 235 
 236 static int
 237 amd_iommu_get_deviceid(amd_iommu_t *iommu, dev_info_t *rdip, int32_t *deviceid,
 238     int *aliasp, const char *path)
 239 {
 240         int bus = -1;
 241         int device = -1;
 242         int func = -1;
 243         uint16_t bdf;
 244         int32_t src_bdf;
 245         dev_info_t *idip = iommu->aiomt_dip;
 246         const char *driver = ddi_driver_name(idip);
 247         int instance = ddi_get_instance(idip);
 248         dev_info_t *pci_dip;
 249         const char *f = "amd_iommu_get_deviceid";
 250 
 251         /* be conservative. Always assume an alias */
 252         *aliasp = 1;
 253         *deviceid = 0;
 254 
 255         /* Check for special special devices (rdip == NULL) */
 256         if (rdip == NULL) {
 257                 if (amd_iommu_get_src_bdf(iommu, -1, &src_bdf) != DDI_SUCCESS) {
 258                         cmn_err(CE_WARN,
 259                             "%s: %s%d: idx=%d, failed to get SRC BDF "
 260                             "for special-device",
 261                             f, driver, instance, iommu->aiomt_idx);
 262                         return (DDI_DMA_NOMAPPING);
 263                 }
 264                 *deviceid = src_bdf;
 265                 *aliasp = 1;
 266                 return (DDI_SUCCESS);
 267         }
 268 
 269         if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
 270                 cmn_err(CE_NOTE, "%s: attempting to get deviceid for %s",
 271                     f, path);
 272         }
 273 
 274         pci_dip = amd_iommu_pci_dip(rdip, path);
 275         if (pci_dip == NULL) {
 276                 cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip "
 277                     "for rdip=%p, path = %s",
 278                     f, driver, instance, iommu->aiomt_idx, (void *)rdip,
 279                     path);
 280                 return (DDI_DMA_NOMAPPING);
 281         }
 282 
 283         if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) {
 284                 ndi_rele_devi(pci_dip);
 285                 cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get BDF for "
 286                     "PCI dip (%p). rdip path = %s",
 287                     f, driver, instance, iommu->aiomt_idx,
 288                     (void *)pci_dip, path);
 289                 return (DDI_DMA_NOMAPPING);
 290         }
 291 
 292         ndi_rele_devi(pci_dip);
 293 
 294         if (bus > UINT8_MAX || bus < 0 ||
 295             device > UINT8_MAX || device < 0 ||
 296             func > UINT8_MAX || func < 0) {
 297                 cmn_err(CE_WARN, "%s: %s%d:  idx=%d, invalid BDF(%d,%d,%d) "
 298                     "for PCI dip (%p). rdip path = %s", f, driver, instance,
 299                     iommu->aiomt_idx,
 300                     bus, device, func,
 301                     (void *)pci_dip, path);
 302                 return (DDI_DMA_NOMAPPING);
 303         }
 304 
 305         bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func;
 306 
 307         if (amd_iommu_get_src_bdf(iommu, bdf, &src_bdf) != DDI_SUCCESS) {
 308                 cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get SRC BDF "
 309                     "for PCI dip (%p) rdip path = %s.",
 310                     f, driver, instance, iommu->aiomt_idx, (void *)pci_dip,
 311                     path);
 312                 return (DDI_DMA_NOMAPPING);
 313         }
 314 
 315         if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
 316                 cmn_err(CE_NOTE, "%s: Deviceid = %u for path = %s",
 317                     f, src_bdf, path);
 318         }
 319 
 320         *deviceid = src_bdf;
 321         *aliasp = (src_bdf != bdf);
 322 
 323         return (DDI_SUCCESS);
 324 }
 325 
 326 /*ARGSUSED*/
 327 static int
 328 init_devtbl(amd_iommu_t *iommu, uint64_t *devtbl_entry, domain_id_t domainid,
 329     amd_iommu_domain_t *dp)
 330 {
 331         uint64_t entry[4] = {0};
 332         int i;
 333 
 334         /* If already passthru, don't touch */
 335         if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 0 &&
 336             AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
 337                 return (0);
 338         }
 339 
 340         if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 1 &&
 341             AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 1) {
 342 
 343                 ASSERT(dp->d_pgtable_root_4K ==
 344                     AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
 345                     AMD_IOMMU_DEVTBL_ROOT_PGTBL));
 346 
 347                 ASSERT(dp->d_domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
 348                     AMD_IOMMU_DEVTBL_DOMAINID));
 349 
 350                 return (0);
 351         }
 352 
 353         /* New devtbl entry for this domain. Bump up the domain ref-count */
 354         dp->d_ref++;
 355 
 356         entry[3] = 0;
 357         entry[2] = 0;
 358         AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_EX, 1);
 359         AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SD, 0);
 360         AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_CACHE, 0);
 361         AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOCTL, 1);
 362         AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SA, 0);
 363         AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SE, 1);
 364         AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_DOMAINID,
 365             (uint16_t)domainid);
 366         AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IW, 1);
 367         AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IR, 1);
 368         AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL,
 369             dp->d_pgtable_root_4K);
 370         AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_PG_MODE,
 371             AMD_IOMMU_PGTABLE_MAXLEVEL);
 372         AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_TV,
 373             domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);
 374         AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_V,
 375             domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);
 376 
 377         for (i = 1; i < 4; i++) {
 378                 devtbl_entry[i] = entry[i];
 379         }
 380         devtbl_entry[0] = entry[0];
 381 
 382         /* we did an actual init */
 383         return (1);
 384 }
 385 
 386 void
 387 amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip)
 388 {
 389         int32_t deviceid;
 390         int alias;
 391         uint64_t *devtbl_entry;
 392         amd_iommu_cmdargs_t cmdargs = {0};
 393         char *path;
 394         int pathfree;
 395         int V;
 396         int TV;
 397         int instance;
 398         const char *driver;
 399         const char *f = "amd_iommu_set_passthru";
 400 
 401         if (rdip) {
 402                 driver = ddi_driver_name(rdip);
 403                 instance = ddi_get_instance(rdip);
 404         } else {
 405                 driver = "special-device";
 406                 instance = 0;
 407         }
 408 
 409         path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
 410         if (path) {
 411                 if (rdip)
 412                         (void) ddi_pathname(rdip, path);
 413                 else
 414                         (void) strcpy(path, "special-device");
 415                 pathfree = 1;
 416         } else {
 417                 pathfree = 0;
 418                 path = "<path-mem-alloc-failed>";
 419         }
 420 
 421         if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
 422             != DDI_SUCCESS) {
 423                 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
 424                     "Failed to get device ID for device %s.", f, driver,
 425                     instance,
 426                     iommu->aiomt_idx, (void *)rdip, path);
 427                 goto out;
 428         }
 429 
 430         /* No deviceid */
 431         if (deviceid == -1) {
 432                 goto out;
 433         }
 434 
 435         if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
 436             iommu->aiomt_devtbl_sz) {
 437                 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
 438                     "for rdip (%p) exceeds device table size (%u), path=%s",
 439                     f, driver,
 440                     instance, iommu->aiomt_idx, deviceid, (void *)rdip,
 441                     iommu->aiomt_devtbl_sz, path);
 442                 goto out;
 443         }
 444 
 445         /*LINTED*/
 446         devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
 447             [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
 448 
 449         V = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V);
 450         TV = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV);
 451 
 452         /* Already passthru */
 453         if (V == 0 && TV == 0) {
 454                 goto out;
 455         }
 456 
 457         /* Existing translations */
 458         if (V == 1 && TV == 1) {
 459                 goto out;
 460         }
 461 
 462         /* Invalid setting */
 463         if (V == 0 && TV == 1) {
 464                 goto out;
 465         }
 466 
 467         AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 0);
 468 
 469         cmdargs.ca_deviceid = (uint16_t)deviceid;
 470         (void) amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
 471             &cmdargs, 0, 0);
 472 
 473 out:
 474         if (pathfree)
 475                 kmem_free(path, MAXPATHLEN);
 476 }
 477 
 478 static int
 479 amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
 480     domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
 481     const char *path)
 482 {
 483         uint64_t *devtbl_entry;
 484         amd_iommu_cmdargs_t cmdargs = {0};
 485         int error, flags;
 486         dev_info_t *idip = iommu->aiomt_dip;
 487         const char *driver = ddi_driver_name(idip);
 488         int instance = ddi_get_instance(idip);
 489         const char *f = "amd_iommu_set_devtbl_entry";
 490 
 491         if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
 492                 cmn_err(CE_NOTE, "%s: attempting to set devtbl entry for %s",
 493                     f, path);
 494         }
 495 
 496         if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
 497             iommu->aiomt_devtbl_sz) {
 498                 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
 499                     "for rdip (%p) exceeds device table size (%u), path=%s",
 500                     f, driver,
 501                     instance, iommu->aiomt_idx, deviceid, (void *)rdip,
 502                     iommu->aiomt_devtbl_sz, path);
 503                 return (DDI_DMA_NOMAPPING);
 504         }
 505 
 506         /*LINTED*/
 507         devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
 508             [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
 509 
 510         if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
 511                 cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
 512                     f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
 513         }
 514 
 515         /*
 516          * Flush internal caches, need to do this if we came up from
 517          * fast boot
 518          */
 519         cmdargs.ca_deviceid = deviceid;
 520         error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
 521             &cmdargs, 0, 0);
 522         if (error != DDI_SUCCESS) {
 523                 cmn_err(CE_WARN, "%s: idx=%d: deviceid=%d"
 524                     "Failed to invalidate domain in IOMMU HW cache",
 525                     f, iommu->aiomt_idx, deviceid);
 526                 return (error);
 527         }
 528 
 529         cmdargs.ca_domainid = (uint16_t)domainid;
 530         cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
 531         flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
 532             AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
 533 
 534         error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
 535             &cmdargs, flags, 0);
 536         if (error != DDI_SUCCESS) {
 537                 cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
 538                     "Failed to invalidate translations in IOMMU HW cache",
 539                     f, iommu->aiomt_idx, cmdargs.ca_domainid);
 540                 return (error);
 541         }
 542 
 543         /* Initialize device table entry */
 544         if (init_devtbl(iommu, devtbl_entry, domainid, dp)) {
 545                 cmdargs.ca_deviceid = deviceid;
 546                 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
 547                     &cmdargs, 0, 0);
 548         }
 549 
 550         return (error);
 551 }
 552 
 553 int
 554 amd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
 555     domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
 556     int *domain_freed, char *path)
 557 {
 558         uint64_t *devtbl_entry;
 559         int error = DDI_SUCCESS;
 560         amd_iommu_cmdargs_t cmdargs = {0};
 561         const char *driver = ddi_driver_name(iommu->aiomt_dip);
 562         int instance = ddi_get_instance(iommu->aiomt_dip);
 563         const char *f = "amd_iommu_clear_devtbl_entry";
 564 
 565         if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
 566                 cmn_err(CE_NOTE, "%s: attempting to clear devtbl entry for "
 567                     "domainid = %d, deviceid = %u, path = %s",
 568                     f, domainid, deviceid, path);
 569         }
 570 
 571         if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
 572             iommu->aiomt_devtbl_sz) {
 573                 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
 574                     "for rdip (%p) exceeds device table size (%u), path = %s",
 575                     f, driver, instance,
 576                     iommu->aiomt_idx, deviceid, (void *)rdip,
 577                     iommu->aiomt_devtbl_sz, path);
 578                 return (DDI_FAILURE);
 579         }
 580 
 581         /*LINTED*/
 582         devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
 583             [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
 584 
 585         if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
 586                 cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
 587                     f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
 588         }
 589 
 590         if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
 591                 /* Nothing to do */
 592                 return (DDI_SUCCESS);
 593         }
 594 
 595         ASSERT(dp->d_pgtable_root_4K == AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
 596             AMD_IOMMU_DEVTBL_ROOT_PGTBL));
 597 
 598         ASSERT(domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
 599             AMD_IOMMU_DEVTBL_DOMAINID));
 600 
 601         AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV, 0);
 602         AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 0);
 603         AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 1);
 604 
 605         SYNC_FORDEV(iommu->aiomt_dmahdl);
 606 
 607         dp->d_ref--;
 608         ASSERT(dp->d_ref >= 0);
 609 
 610         if (dp->d_ref == 0) {
 611                 *domain_freed = 1;
 612         }
 613 
 614         cmdargs.ca_deviceid = deviceid;
 615         error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
 616             &cmdargs, 0, 0);
 617         if (error != DDI_SUCCESS)
 618                 error = DDI_FAILURE;
 619 
 620         return (error);
 621 }
 622 
 623 int
 624 amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt)
 625 {
 626         ampt->ampt_hash = kmem_zalloc(sizeof (amd_iommu_page_table_t *) *
 627             AMD_IOMMU_PGTABLE_HASH_SZ, KM_SLEEP);
 628         return (DDI_SUCCESS);
 629 }
 630 
 631 void
 632 amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt)
 633 {
 634         kmem_free(ampt->ampt_hash,
 635             sizeof (amd_iommu_page_table_t *) * AMD_IOMMU_PGTABLE_HASH_SZ);
 636         ampt->ampt_hash = NULL;
 637 }
 638 
 639 static uint32_t
 640 pt_hashfn(uint64_t pa_4K)
 641 {
 642         return (pa_4K % AMD_IOMMU_PGTABLE_HASH_SZ);
 643 }
 644 
 645 static void
 646 amd_iommu_insert_pgtable_hash(amd_iommu_page_table_t *pt)
 647 {
 648         uint64_t pa_4K = ((uint64_t)pt->pt_cookie.dmac_cookie_addr) >> 12;
 649         uint32_t idx = pt_hashfn(pa_4K);
 650 
 651         ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
 652 
 653         mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
 654 
 655         pt->pt_next = amd_iommu_page_table_hash.ampt_hash[idx];
 656         pt->pt_prev = NULL;
 657         amd_iommu_page_table_hash.ampt_hash[idx] = pt;
 658         if (pt->pt_next)
 659                 pt->pt_next->pt_prev = pt;
 660 
 661         mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
 662 }
 663 
 664 static void
 665 amd_iommu_remove_pgtable_hash(amd_iommu_page_table_t *pt)
 666 {
 667         uint64_t pa_4K = (pt->pt_cookie.dmac_cookie_addr >> 12);
 668         uint32_t idx = pt_hashfn(pa_4K);
 669 
 670         ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
 671 
 672         mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
 673 
 674         if (pt->pt_next)
 675                 pt->pt_next->pt_prev = pt->pt_prev;
 676 
 677         if (pt->pt_prev)
 678                 pt->pt_prev->pt_next = pt->pt_next;
 679         else
 680                 amd_iommu_page_table_hash.ampt_hash[idx] = pt->pt_next;
 681 
 682         pt->pt_next = NULL;
 683         pt->pt_prev = NULL;
 684 
 685         mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
 686 }
 687 
 688 static amd_iommu_page_table_t *
 689 amd_iommu_lookup_pgtable_hash(domain_id_t domainid, uint64_t pgtable_pa_4K)
 690 {
 691         amd_iommu_page_table_t *pt;
 692         uint32_t idx = pt_hashfn(pgtable_pa_4K);
 693 
 694         mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
 695         pt = amd_iommu_page_table_hash.ampt_hash[idx];
 696         for (; pt; pt = pt->pt_next) {
 697                 if (domainid != pt->pt_domainid)
 698                         continue;
 699                 ASSERT((pt->pt_cookie.dmac_cookie_addr &
 700                     AMD_IOMMU_PGTABLE_ALIGN) == 0);
 701                 if ((pt->pt_cookie.dmac_cookie_addr >> 12) == pgtable_pa_4K) {
 702                         break;
 703                 }
 704         }
 705         mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
 706 
 707         return (pt);
 708 }
 709 
 710 /*ARGSUSED*/
 711 static amd_iommu_page_table_t *
 712 amd_iommu_lookup_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *ppt,
 713     amd_iommu_domain_t *dp, int level, uint16_t index)
 714 {
 715         uint64_t *pdtep;
 716         uint64_t pgtable_pa_4K;
 717 
 718         ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);
 719         ASSERT(dp);
 720 
 721         if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
 722                 ASSERT(ppt == NULL);
 723                 ASSERT(index == 0);
 724                 pgtable_pa_4K = dp->d_pgtable_root_4K;
 725         } else {
 726                 ASSERT(ppt);
 727                 pdtep = &(ppt->pt_pgtblva[index]);
 728                 if (AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_PR) == 0) {
 729                         if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
 730                                 cmn_err(CE_NOTE, "Skipping PR=0 pdte: 0x%"
 731                                     PRIx64, *pdtep);
 732                         }
 733                         return (NULL);
 734                 }
 735                 pgtable_pa_4K = AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_ADDR);
 736         }
 737 
 738         return (amd_iommu_lookup_pgtable_hash(dp->d_domainid, pgtable_pa_4K));
 739 }
 740 
 741 static amd_iommu_page_table_t *
 742 amd_iommu_alloc_from_freelist(void)
 743 {
 744         int i;
 745         uint64_t *pte_array;
 746         amd_iommu_page_table_t *pt;
 747 
 748         if (amd_iommu_no_pgtable_freelist == 1)
 749                 return (NULL);
 750 
 751         if (amd_iommu_pgtable_freelist.f_count == 0)
 752                 return (NULL);
 753 
 754         pt = amd_iommu_pgtable_freelist.f_list;
 755         amd_iommu_pgtable_freelist.f_list = pt->pt_next;
 756         amd_iommu_pgtable_freelist.f_count--;
 757 
 758         pte_array = pt->pt_pgtblva;
 759         for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
 760                 ASSERT(pt->pt_pte_ref[i] == 0);
 761                 ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
 762                     AMD_IOMMU_PTDE_PR)  == 0);
 763         }
 764 
 765         return (pt);
 766 }
 767 
 768 static int
 769 amd_iommu_alloc_pgtable(amd_iommu_t *iommu, domain_id_t domainid,
 770     const char *path, amd_iommu_page_table_t **ptp, int km_flags)
 771 {
 772         int err;
 773         uint_t ncookies;
 774         amd_iommu_page_table_t *pt;
 775         dev_info_t *idip = iommu->aiomt_dip;
 776         const char *driver = ddi_driver_name(idip);
 777         int instance = ddi_get_instance(idip);
 778         const char *f = "amd_iommu_alloc_pgtable";
 779 
 780         *ptp = NULL;
 781 
 782         pt = amd_iommu_alloc_from_freelist();
 783         if (pt)
 784                 goto init_pgtable;
 785 
 786         pt = kmem_zalloc(sizeof (amd_iommu_page_table_t), km_flags);
 787         if (pt == NULL)
 788                 return (DDI_DMA_NORESOURCES);
 789 
 790         /*
 791          * Each page table is 4K in size
 792          */
 793         pt->pt_mem_reqsz = AMD_IOMMU_PGTABLE_SZ;
 794 
 795         /*
 796          * Alloc a DMA handle. Use the IOMMU dip as we want this DMA
 797          * to *not* enter the IOMMU - no recursive entrance.
 798          */
 799         err = ddi_dma_alloc_handle(idip, &amd_iommu_pgtable_dma_attr,
 800             km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
 801             NULL, &pt->pt_dma_hdl);
 802         if (err != DDI_SUCCESS) {
 803                 cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path = %s. "
 804                     "Cannot alloc DMA handle for IO Page Table",
 805                     f, driver, instance, domainid, path);
 806                 kmem_free(pt, sizeof (amd_iommu_page_table_t));
 807                 return (err == DDI_DMA_NORESOURCES ? err : DDI_DMA_NOMAPPING);
 808         }
 809 
 810         /*
 811          * Alloc memory for IO Page Table.
 812          * XXX remove size_t cast kludge
 813          */
 814         err = ddi_dma_mem_alloc(pt->pt_dma_hdl, pt->pt_mem_reqsz,
 815             &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,
 816             km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
 817             NULL, (caddr_t *)&pt->pt_pgtblva,
 818             (size_t *)&pt->pt_mem_realsz, &pt->pt_mem_hdl);
 819         if (err != DDI_SUCCESS) {
 820                 cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
 821                     "Cannot allocate DMA memory for IO Page table",
 822                     f, driver, instance, domainid, path);
 823                 ddi_dma_free_handle(&pt->pt_dma_hdl);
 824                 kmem_free(pt, sizeof (amd_iommu_page_table_t));
 825                 return (DDI_DMA_NORESOURCES);
 826         }
 827 
 828         /*
 829          * The Page table DMA VA must be 4K aligned and
 830          * size >= than requested memory.
 831          *
 832          */
 833         ASSERT(((uint64_t)(uintptr_t)pt->pt_pgtblva & AMD_IOMMU_PGTABLE_ALIGN)
 834             == 0);
 835         ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);
 836 
 837         /*
 838          * Now bind the handle
 839          */
 840         err = ddi_dma_addr_bind_handle(pt->pt_dma_hdl, NULL,
 841             (caddr_t)pt->pt_pgtblva, pt->pt_mem_realsz,
 842             DDI_DMA_READ | DDI_DMA_CONSISTENT,
 843             km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
 844             NULL, &pt->pt_cookie, &ncookies);
 845         if (err != DDI_DMA_MAPPED) {
 846                 cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
 847                     "Cannot bind memory for DMA to IO Page Tables. "
 848                     "bufrealsz=%p",
 849                     f, driver, instance, domainid, path,
 850                     (void *)(uintptr_t)pt->pt_mem_realsz);
 851                 ddi_dma_mem_free(&pt->pt_mem_hdl);
 852                 ddi_dma_free_handle(&pt->pt_dma_hdl);
 853                 kmem_free(pt, sizeof (amd_iommu_page_table_t));
 854                 return (err == DDI_DMA_PARTIAL_MAP ? DDI_DMA_NOMAPPING :
 855                     err);
 856         }
 857 
 858         /*
 859          * We assume the DMA engine on the IOMMU is capable of handling the
 860          * whole page table in a single cookie. If not and multiple cookies
 861          * are needed we fail.
 862          */
 863         if (ncookies != 1) {
 864                 cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path=%s "
 865                     "Cannot handle multiple "
 866                     "cookies for DMA to IO page Table, #cookies=%u",
 867                     f, driver, instance, domainid, path, ncookies);
 868                 (void) ddi_dma_unbind_handle(pt->pt_dma_hdl);
 869                 ddi_dma_mem_free(&pt->pt_mem_hdl);
 870                 ddi_dma_free_handle(&pt->pt_dma_hdl);
 871                 kmem_free(pt, sizeof (amd_iommu_page_table_t));
 872                 return (DDI_DMA_NOMAPPING);
 873         }
 874 
 875 init_pgtable:
 876         /*
 877          * The address in the cookie must be 4K aligned and >= table size
 878          */
 879         ASSERT(pt->pt_cookie.dmac_cookie_addr != NULL);
 880         ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
 881         ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_realsz);
 882         ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_reqsz);
 883         ASSERT(pt->pt_mem_reqsz >= AMD_IOMMU_PGTABLE_SIZE);
 884         ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);
 885         ASSERT(pt->pt_pgtblva);
 886 
 887         pt->pt_domainid = AMD_IOMMU_INVALID_DOMAIN;
 888         pt->pt_level = 0x7;
 889         pt->pt_index = 0;
 890         pt->pt_ref = 0;
 891         pt->pt_next = NULL;
 892         pt->pt_prev = NULL;
 893         pt->pt_parent = NULL;
 894 
 895         bzero(pt->pt_pgtblva, pt->pt_mem_realsz);
 896         SYNC_FORDEV(pt->pt_dma_hdl);
 897 
 898         amd_iommu_insert_pgtable_hash(pt);
 899 
 900         *ptp = pt;
 901 
 902         return (DDI_SUCCESS);
 903 }
 904 
 905 static int
 906 amd_iommu_move_to_freelist(amd_iommu_page_table_t *pt)
 907 {
 908         if (amd_iommu_no_pgtable_freelist == 1)
 909                 return (DDI_FAILURE);
 910 
 911         if (amd_iommu_pgtable_freelist.f_count ==
 912             AMD_IOMMU_PGTABLE_FREELIST_MAX)
 913                 return (DDI_FAILURE);
 914 
 915         pt->pt_next = amd_iommu_pgtable_freelist.f_list;
 916         amd_iommu_pgtable_freelist.f_list = pt;
 917         amd_iommu_pgtable_freelist.f_count++;
 918 
 919         return (DDI_SUCCESS);
 920 }
 921 
 922 static void
 923 amd_iommu_free_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *pt)
 924 {
 925         int i;
 926         uint64_t *pte_array;
 927         dev_info_t *dip = iommu->aiomt_dip;
 928         int instance = ddi_get_instance(dip);
 929         const char *driver = ddi_driver_name(dip);
 930         const char *f = "amd_iommu_free_pgtable";
 931 
 932         ASSERT(pt->pt_ref == 0);
 933 
 934         amd_iommu_remove_pgtable_hash(pt);
 935 
 936         pte_array = pt->pt_pgtblva;
 937         for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
 938                 ASSERT(pt->pt_pte_ref[i] == 0);
 939                 ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
 940                     AMD_IOMMU_PTDE_PR)  == 0);
 941         }
 942 
 943         if (amd_iommu_move_to_freelist(pt) == DDI_SUCCESS)
 944                 return;
 945 
 946         /* Unbind the handle */
 947         if (ddi_dma_unbind_handle(pt->pt_dma_hdl) != DDI_SUCCESS) {
 948                 cmn_err(CE_WARN, "%s: %s%d: idx=%d, domainid=%d. "
 949                     "Failed to unbind handle: %p for IOMMU Page Table",
 950                     f, driver, instance, iommu->aiomt_idx, pt->pt_domainid,
 951                     (void *)pt->pt_dma_hdl);
 952         }
 953         /* Free the table memory allocated for DMA */
 954         ddi_dma_mem_free(&pt->pt_mem_hdl);
 955 
 956         /* Free the DMA handle */
 957         ddi_dma_free_handle(&pt->pt_dma_hdl);
 958 
 959         kmem_free(pt, sizeof (amd_iommu_page_table_t));
 960 
 961 }
 962 
 963 static int
 964 init_pde(amd_iommu_page_table_t *ppt, amd_iommu_page_table_t *pt)
 965 {
 966         uint64_t *pdep = &(ppt->pt_pgtblva[pt->pt_index]);
 967         uint64_t next_pgtable_pa_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;
 968 
 969         /* nothing to set. PDE is already set */
 970         if (AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1) {
 971                 ASSERT(PT_REF_VALID(ppt));
 972                 ASSERT(PT_REF_VALID(pt));
 973                 ASSERT(ppt->pt_pte_ref[pt->pt_index] == 0);
 974                 ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_ADDR)
 975                     == next_pgtable_pa_4K);
 976                 return (DDI_SUCCESS);
 977         }
 978 
 979         ppt->pt_ref++;
 980         ASSERT(PT_REF_VALID(ppt));
 981 
 982         /* Page Directories are always RW */
 983         AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IW, 1);
 984         AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IR, 1);
 985         AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_ADDR,
 986             next_pgtable_pa_4K);
 987         pt->pt_parent = ppt;
 988         AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_NXT_LVL,
 989             pt->pt_level);
 990         ppt->pt_pte_ref[pt->pt_index] = 0;
 991         AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_PR, 1);
 992         SYNC_FORDEV(ppt->pt_dma_hdl);
 993         ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1);
 994 
 995         return (DDI_SUCCESS);
 996 }
 997 
 998 static int
 999 init_pte(amd_iommu_page_table_t *pt, uint64_t pa, uint16_t index,
1000     struct ddi_dma_req *dmareq)
1001 {
1002         uint64_t *ptep = &(pt->pt_pgtblva[index]);
1003         uint64_t pa_4K = pa >> 12;
1004         int R;
1005         int W;
1006 
1007         /* nothing to set if PTE is already set */
1008         if (AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1) {
1009                 /*
1010                  * Adjust current permissions
1011                  * DDI_DMA_WRITE means direction of DMA is MEM -> I/O
1012                  * so that requires Memory READ permissions i.e. sense
1013                  * is inverted.
1014                  * Note: either or both of DD_DMA_READ/WRITE may be set
1015                  */
1016                 if (amd_iommu_no_RW_perms == 0) {
1017                         R = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IR);
1018                         W = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IW);
1019                         if (R == 0 && ((dmareq->dmar_flags & DDI_DMA_WRITE) ||
1020                             (dmareq->dmar_flags & DDI_DMA_RDWR))) {
1021                                 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1022                         }
1023                         if (W  == 0 && ((dmareq->dmar_flags & DDI_DMA_READ) ||
1024                             (dmareq->dmar_flags & DDI_DMA_RDWR))) {
1025                                 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1026                         }
1027                 }
1028                 ASSERT(PT_REF_VALID(pt));
1029                 pt->pt_pte_ref[index]++;
1030                 ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR)
1031                     == pa_4K);
1032                 return (DDI_SUCCESS);
1033         }
1034 
1035         pt->pt_ref++;
1036         ASSERT(PT_REF_VALID(pt));
1037 
1038         /* see comment above about inverting sense of RD/WR */
1039         if (amd_iommu_no_RW_perms == 0) {
1040                 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 0);
1041                 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 0);
1042                 if (dmareq->dmar_flags & DDI_DMA_RDWR) {
1043                         AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1044                         AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1045                 } else {
1046                         if (dmareq->dmar_flags & DDI_DMA_WRITE) {
1047                                 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1048                         }
1049                         if (dmareq->dmar_flags & DDI_DMA_READ) {
1050                                 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1051                         }
1052                 }
1053         } else {
1054                 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1055                 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1056         }
1057 
1058         /* TODO what is correct for FC and U */
1059         AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_FC, 0);
1060         AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_U, 0);
1061         AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_ADDR, pa_4K);
1062         AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_NXT_LVL, 0);
1063         ASSERT(pt->pt_pte_ref[index] == 0);
1064         pt->pt_pte_ref[index] = 1;
1065         AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_PR, 1);
1066         SYNC_FORDEV(pt->pt_dma_hdl);
1067         ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1);
1068 
1069         return (DDI_SUCCESS);
1070 }
1071 
1072 
1073 static void
1074 init_pt(amd_iommu_page_table_t *pt, amd_iommu_domain_t *dp,
1075     int level, uint16_t index)
1076 {
1077         ASSERT(dp);
1078 
1079         if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
1080                 dp->d_pgtable_root_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;
1081         } else {
1082                 ASSERT(level >= 1 && level < AMD_IOMMU_PGTABLE_MAXLEVEL);
1083         }
1084 
1085         pt->pt_domainid = dp->d_domainid;
1086         pt->pt_level = level;
1087         pt->pt_index = index;
1088 }
1089 
1090 static int
1091 amd_iommu_setup_1_pgtable(amd_iommu_t *iommu, dev_info_t *rdip,
1092     struct ddi_dma_req *dmareq,
1093     domain_id_t domainid, amd_iommu_domain_t *dp,
1094     amd_iommu_page_table_t *ppt,
1095     uint16_t index, int level, uint64_t va, uint64_t pa,
1096     amd_iommu_page_table_t **ptp,  uint16_t *next_idxp, const char *path,
1097     int km_flags)
1098 {
1099         int error;
1100         amd_iommu_page_table_t *pt;
1101         const char *driver = ddi_driver_name(rdip);
1102         int instance = ddi_get_instance(rdip);
1103         const char *f = "amd_iommu_setup_1_pgtable";
1104 
1105         *ptp = NULL;
1106         *next_idxp = 0;
1107         error = DDI_SUCCESS;
1108 
1109         ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);
1110 
1111         ASSERT(dp);
1112         if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
1113                 ASSERT(ppt == NULL);
1114                 ASSERT(index == 0);
1115         } else {
1116                 ASSERT(ppt);
1117         }
1118 
1119         /* Check if page table is already allocated */
1120         if (pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index)) {
1121                 ASSERT(pt->pt_domainid == domainid);
1122                 ASSERT(pt->pt_level == level);
1123                 ASSERT(pt->pt_index == index);
1124                 goto out;
1125         }
1126 
1127         if ((error = amd_iommu_alloc_pgtable(iommu, domainid, path, &pt,
1128             km_flags)) != DDI_SUCCESS) {
1129                 cmn_err(CE_WARN, "%s: %s%d: idx = %u, domainid = %d, va = %p "
1130                     "path = %s", f, driver, instance, iommu->aiomt_idx,
1131                     domainid, (void *)(uintptr_t)va, path);
1132                 return (error);
1133         }
1134 
1135         ASSERT(dp->d_domainid == domainid);
1136 
1137         init_pt(pt, dp, level, index);
1138 
1139 out:
1140         if (level != AMD_IOMMU_PGTABLE_MAXLEVEL) {
1141                 error = init_pde(ppt, pt);
1142         }
1143 
1144         if (level == 1) {
1145                 ASSERT(error == DDI_SUCCESS);
1146                 error = init_pte(pt, pa, AMD_IOMMU_VA_BITS(va, level), dmareq);
1147         } else {
1148                 *next_idxp = AMD_IOMMU_VA_BITS(va, level);
1149                 *ptp = pt;
1150         }
1151 
1152         return (error);
1153 }
1154 
1155 typedef enum {
1156         PDTE_NOT_TORN = 0x1,
1157         PDTE_TORN_DOWN = 0x2,
1158         PGTABLE_TORN_DOWN = 0x4
1159 } pdte_tear_t;
1160 
1161 static pdte_tear_t
1162 amd_iommu_teardown_pdte(amd_iommu_t *iommu,
1163     amd_iommu_page_table_t *pt, int index)
1164 {
1165         uint8_t next_level;
1166         pdte_tear_t retval;
1167         uint64_t *ptdep = &(pt->pt_pgtblva[index]);
1168 
1169         next_level = AMD_IOMMU_REG_GET64(ptdep,
1170             AMD_IOMMU_PTDE_NXT_LVL);
1171 
1172         if (AMD_IOMMU_REG_GET64(ptdep, AMD_IOMMU_PTDE_PR) == 1) {
1173                 if (pt->pt_level == 1) {
1174                         ASSERT(next_level == 0);
1175                         /* PTE */
1176                         pt->pt_pte_ref[index]--;
1177                         if (pt->pt_pte_ref[index] != 0) {
1178                                 return (PDTE_NOT_TORN);
1179                         }
1180                 } else {
1181                         ASSERT(next_level != 0 && next_level != 7);
1182                 }
1183                 ASSERT(pt->pt_pte_ref[index] == 0);
1184                 ASSERT(PT_REF_VALID(pt));
1185 
1186                 AMD_IOMMU_REG_SET64(ptdep, AMD_IOMMU_PTDE_PR, 0);
1187                 SYNC_FORDEV(pt->pt_dma_hdl);
1188                 ASSERT(AMD_IOMMU_REG_GET64(ptdep,
1189                     AMD_IOMMU_PTDE_PR) == 0);
1190                 pt->pt_ref--;
1191                 ASSERT(PT_REF_VALID(pt));
1192                 retval = PDTE_TORN_DOWN;
1193         } else {
1194                 ASSERT(0);
1195                 ASSERT(pt->pt_pte_ref[index] == 0);
1196                 ASSERT(PT_REF_VALID(pt));
1197                 retval = PDTE_NOT_TORN;
1198         }
1199 
1200         if (pt->pt_ref == 0) {
1201                 amd_iommu_free_pgtable(iommu, pt);
1202                 return (PGTABLE_TORN_DOWN);
1203         }
1204 
1205         return (retval);
1206 }
1207 
1208 static int
1209 amd_iommu_create_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
1210     struct ddi_dma_req *dmareq, uint64_t va,
1211     uint64_t pa, uint16_t deviceid, domain_id_t domainid,
1212     amd_iommu_domain_t *dp, const char *path, int km_flags)
1213 {
1214         int level;
1215         uint16_t index;
1216         uint16_t next_idx;
1217         amd_iommu_page_table_t *pt;
1218         amd_iommu_page_table_t *ppt;
1219         int error;
1220         const char *driver = ddi_driver_name(rdip);
1221         int instance = ddi_get_instance(rdip);
1222         const char *f = "amd_iommu_create_pgtables";
1223 
1224         if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1225                 cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
1226                     "deviceid = %u, va = %p, pa = %p, path = %s",
1227                     f, driver, instance,
1228                     iommu->aiomt_idx, domainid, deviceid,
1229                     (void *)(uintptr_t)va,
1230                     (void *)(uintptr_t)pa, path);
1231         }
1232 
1233         if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
1234                 /* No need for pagetables. Just set up device table entry */
1235                 goto passthru;
1236         }
1237 
1238         index = 0;
1239         ppt = NULL;
1240         for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0;
1241             level--, pt = NULL, next_idx = 0) {
1242                 if ((error = amd_iommu_setup_1_pgtable(iommu, rdip, dmareq,
1243                     domainid, dp, ppt, index, level, va, pa, &pt,
1244                     &next_idx, path, km_flags)) != DDI_SUCCESS) {
1245                         cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
1246                             "deviceid=%u, va= %p, pa = %p, Failed to setup "
1247                             "page table(s) at level = %d, path = %s.",
1248                             f, driver, instance, iommu->aiomt_idx,
1249                             domainid, deviceid, (void *)(uintptr_t)va,
1250                             (void *)(uintptr_t)pa, level, path);
1251                         return (error);
1252                 }
1253 
1254                 if (level > 1) {
1255                         ASSERT(pt);
1256                         ASSERT(pt->pt_domainid == domainid);
1257                         ppt = pt;
1258                         index = next_idx;
1259                 } else {
1260                         ASSERT(level == 1);
1261                         ASSERT(pt == NULL);
1262                         ASSERT(next_idx == 0);
1263                         ppt = NULL;
1264                         index = 0;
1265                 }
1266         }
1267 
1268 passthru:
1269         if ((error = amd_iommu_set_devtbl_entry(iommu, rdip, domainid, deviceid,
1270             dp, path)) != DDI_SUCCESS) {
1271                 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, deviceid=%u, "
1272                     "domainid=%d."
1273                     "Failed to set device table entry for path %s.",
1274                     f, driver, instance,
1275                     iommu->aiomt_idx, (void *)rdip, deviceid, domainid, path);
1276                 return (error);
1277         }
1278 
1279         SYNC_FORDEV(iommu->aiomt_dmahdl);
1280 
1281         return (DDI_SUCCESS);
1282 }
1283 
1284 static int
1285 amd_iommu_destroy_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
1286     uint64_t pageva, uint16_t deviceid, domain_id_t domainid,
1287     amd_iommu_domain_t *dp, map_type_t type, int *domain_freed, char *path)
1288 {
1289         int level;
1290         int flags;
1291         amd_iommu_cmdargs_t cmdargs = {0};
1292         uint16_t index;
1293         uint16_t prev_index;
1294         amd_iommu_page_table_t *pt;
1295         amd_iommu_page_table_t *ppt;
1296         pdte_tear_t retval;
1297         int tear_level;
1298         int invalidate_pte;
1299         int invalidate_pde;
1300         int error = DDI_FAILURE;
1301         const char *driver = ddi_driver_name(iommu->aiomt_dip);
1302         int instance = ddi_get_instance(iommu->aiomt_dip);
1303         const char *f = "amd_iommu_destroy_pgtables";
1304 
1305         if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1306                 cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
1307                     "deviceid = %u, va = %p, path = %s",
1308                     f, driver, instance,
1309                     iommu->aiomt_idx, domainid, deviceid,
1310                     (void *)(uintptr_t)pageva, path);
1311         }
1312 
1313         if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
1314                 /*
1315                  * there are no pagetables for the passthru domain.
1316                  * Just the device table entry
1317                  */
1318                 error = DDI_SUCCESS;
1319                 goto passthru;
1320         }
1321 
1322         ppt = NULL;
1323         index = 0;
1324         for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; level--) {
1325                 pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index);
1326                 if (pt) {
1327                         ppt = pt;
1328                         index = AMD_IOMMU_VA_BITS(pageva, level);
1329                         continue;
1330                 }
1331                 break;
1332         }
1333 
1334         if (level == 0) {
1335                 uint64_t *ptep;
1336                 uint64_t pa_4K;
1337 
1338                 ASSERT(pt);
1339                 ASSERT(pt == ppt);
1340                 ASSERT(pt->pt_domainid == dp->d_domainid);
1341 
1342                 ptep = &(pt->pt_pgtblva[index]);
1343 
1344                 pa_4K = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR);
1345                 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
1346                         ASSERT(pageva == (pa_4K << MMU_PAGESHIFT));
1347                 }
1348         }
1349 
1350         tear_level = -1;
1351         invalidate_pde = 0;
1352         invalidate_pte = 0;
1353         for (++level; level <= AMD_IOMMU_PGTABLE_MAXLEVEL; level++) {
1354                 prev_index = pt->pt_index;
1355                 ppt = pt->pt_parent;
1356                 retval = amd_iommu_teardown_pdte(iommu, pt, index);
1357                 switch (retval) {
1358                         case PDTE_NOT_TORN:
1359                                 goto invalidate;
1360                         case PDTE_TORN_DOWN:
1361                                 invalidate_pte = 1;
1362                                 goto invalidate;
1363                         case PGTABLE_TORN_DOWN:
1364                                 invalidate_pte = 1;
1365                                 invalidate_pde = 1;
1366                                 tear_level = level;
1367                                 break;
1368                 }
1369                 index = prev_index;
1370                 pt = ppt;
1371         }
1372 
1373 invalidate:
1374         /*
1375          * Now teardown the IOMMU HW caches if applicable
1376          */
1377         if (invalidate_pte) {
1378                 cmdargs.ca_domainid = (uint16_t)domainid;
1379                 if (amd_iommu_pageva_inval_all) {
1380                         cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
1381                         flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
1382                             AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
1383                 } else if (invalidate_pde) {
1384                         cmdargs.ca_addr =
1385                             (uintptr_t)AMD_IOMMU_VA_INVAL(pageva, tear_level);
1386                         flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
1387                             AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
1388                 } else {
1389                         cmdargs.ca_addr = (uintptr_t)pageva;
1390                         flags = 0;
1391                 }
1392                 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
1393                     &cmdargs, flags, 0) != DDI_SUCCESS) {
1394                         cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
1395                             "rdip=%p. Failed to invalidate IOMMU HW cache "
1396                             "for %s", f, driver, instance,
1397                             iommu->aiomt_idx, domainid, (void *)rdip, path);
1398                         error = DDI_FAILURE;
1399                         goto out;
1400                 }
1401         }
1402 
1403 passthru:
1404         if (tear_level ==  AMD_IOMMU_PGTABLE_MAXLEVEL) {
1405                 error = amd_iommu_clear_devtbl_entry(iommu, rdip, domainid,
1406                     deviceid, dp, domain_freed, path);
1407         } else {
1408                 error = DDI_SUCCESS;
1409         }
1410 
1411 out:
1412         SYNC_FORDEV(iommu->aiomt_dmahdl);
1413 
1414         return (error);
1415 }
1416 
1417 static int
1418 cvt_bind_error(int error)
1419 {
1420         switch (error) {
1421         case DDI_DMA_MAPPED:
1422         case DDI_DMA_PARTIAL_MAP:
1423         case DDI_DMA_NORESOURCES:
1424         case DDI_DMA_NOMAPPING:
1425                 break;
1426         default:
1427                 cmn_err(CE_PANIC, "Unsupported error code: %d", error);
1428                 /*NOTREACHED*/
1429         }
1430         return (error);
1431 }
1432 
1433 int
1434 amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
1435     struct ddi_dma_req *dmareq, uint64_t start_pa, uint64_t pa_sz,
1436     map_type_t type, uint64_t *start_vap, int km_flags)
1437 {
1438         pfn_t pfn_start;
1439         pfn_t pfn_end;
1440         pfn_t pfn;
1441         int alias;
1442         int32_t deviceid;
1443         domain_id_t domainid;
1444         amd_iommu_domain_t *dp;
1445         uint64_t end_pa;
1446         uint64_t start_va;
1447         uint64_t end_va;
1448         uint64_t pg_start;
1449         uint64_t pg_end;
1450         uint64_t pg;
1451         uint64_t va_sz;
1452         char *path;
1453         int error = DDI_DMA_NOMAPPING;
1454         const char *driver = ddi_driver_name(iommu->aiomt_dip);
1455         int instance = ddi_get_instance(iommu->aiomt_dip);
1456         const char *f = "amd_iommu_map_pa2va";
1457 
1458         ASSERT(pa_sz != 0);
1459 
1460         *start_vap = 0;
1461 
1462         ASSERT(rdip);
1463 
1464         path = kmem_alloc(MAXPATHLEN, km_flags);
1465         if (path == NULL) {
1466                 error = DDI_DMA_NORESOURCES;
1467                 goto out;
1468         }
1469         (void) ddi_pathname(rdip, path);
1470 
1471         /*
1472          * First get deviceid
1473          */
1474         if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
1475             != DDI_SUCCESS) {
1476                 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
1477                     "Failed to get device ID for %s.", f, driver, instance,
1478                     iommu->aiomt_idx, (void *)rdip, path);
1479                 error = DDI_DMA_NOMAPPING;
1480                 goto out;
1481         }
1482 
1483         /*
1484          * Next get the domain for this rdip
1485          */
1486         if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
1487             != DDI_SUCCESS) {
1488                 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
1489                     "Failed to get domain.", f, driver, instance,
1490                     iommu->aiomt_idx, (void *)rdip, path);
1491                 error = DDI_DMA_NOMAPPING;
1492                 goto out;
1493         }
1494 
1495         dp = amd_iommu_lookup_domain(iommu, domainid, type, km_flags);
1496         if (dp == NULL) {
1497                 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
1498                     "Failed to get device ID for %s.", f, driver, instance,
1499                     iommu->aiomt_idx, domainid, (void *)rdip, path);
1500                 error = DDI_DMA_NORESOURCES;
1501                 goto out;
1502         }
1503 
1504         ASSERT(dp->d_domainid == domainid);
1505 
1506         pfn_start = start_pa >> MMU_PAGESHIFT;
1507 
1508         if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1509                 cmn_err(CE_NOTE, "pa = %p, pfn_new = %p, pfn_start = %p, "
1510                     "pgshift = %d",
1511                     (void *)(uintptr_t)start_pa,
1512                     (void *)(uintptr_t)(start_pa >> MMU_PAGESHIFT),
1513                     (void *)(uintptr_t)pfn_start, MMU_PAGESHIFT);
1514         }
1515 
1516         end_pa = start_pa + pa_sz - 1;
1517         pfn_end = end_pa >> MMU_PAGESHIFT;
1518 
1519         if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
1520                 start_va = start_pa;
1521                 end_va = end_pa;
1522                 va_sz = pa_sz;
1523                 *start_vap = start_va;
1524         } else {
1525                 va_sz = mmu_ptob(pfn_end - pfn_start + 1);
1526                 start_va = (uintptr_t)vmem_xalloc(dp->d_vmem, va_sz,
1527                     MAX(attrp->dma_attr_align, MMU_PAGESIZE),
1528                     0,
1529                     attrp->dma_attr_seg + 1,
1530                     (void *)(uintptr_t)attrp->dma_attr_addr_lo,
1531                     (void *)(uintptr_t)MIN((attrp->dma_attr_addr_hi + 1),
1532                     AMD_IOMMU_SIZE_4G), /* XXX rollover */
1533                     km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
1534                 if (start_va == 0) {
1535                         cmn_err(CE_WARN, "%s: No VA resources",
1536                             amd_iommu_modname);
1537                         error = DDI_DMA_NORESOURCES;
1538                         goto out;
1539                 }
1540                 ASSERT((start_va & MMU_PAGEOFFSET) == 0);
1541                 end_va = start_va + va_sz - 1;
1542                 *start_vap = start_va + (start_pa & MMU_PAGEOFFSET);
1543         }
1544 
1545         pg_start = start_va >> MMU_PAGESHIFT;
1546         pg_end = end_va >> MMU_PAGESHIFT;
1547 
1548         pg = pg_start;
1549         for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) {
1550 
1551                 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1552                         cmn_err(CE_NOTE, "%s: attempting to create page tables "
1553                             "for pfn = %p, va = %p, path = %s",
1554                             f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT),
1555                             (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path);
1556 
1557                 }
1558 
1559                 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
1560                         ASSERT(pfn == pg);
1561                 }
1562 
1563                 if ((error = amd_iommu_create_pgtables(iommu, rdip, dmareq,
1564                     pg << MMU_PAGESHIFT,
1565                     pfn << MMU_PAGESHIFT, deviceid, domainid, dp, path,
1566                     km_flags)) != DDI_SUCCESS) {
1567                         cmn_err(CE_WARN, "Failed to create_pgtables");
1568                         goto out;
1569                 }
1570 
1571                 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1572                         cmn_err(CE_NOTE, "%s: successfully created page tables "
1573                             "for pfn = %p, vapg = %p, path = %s",
1574                             f, (void *)(uintptr_t)pfn,
1575                             (void *)(uintptr_t)pg, path);
1576                 }
1577 
1578         }
1579         ASSERT(pg == pg_end + 1);
1580 
1581 
1582         if (amd_iommu_debug & AMD_IOMMU_DEBUG_PA2VA) {
1583                 cmn_err(CE_NOTE, "pa=%p, va=%p",
1584                     (void *)(uintptr_t)start_pa,
1585                     (void *)(uintptr_t)(*start_vap));
1586         }
1587         error = DDI_DMA_MAPPED;
1588 
1589 out:
1590         kmem_free(path, MAXPATHLEN);
1591         return (cvt_bind_error(error));
1592 }
1593 
1594 int
1595 amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, uint64_t start_va,
1596     uint64_t va_sz, map_type_t type)
1597 {
1598         uint64_t end_va;
1599         uint64_t pg_start;
1600         uint64_t pg_end;
1601         uint64_t pg;
1602         uint64_t actual_sz;
1603         char *path;
1604         int pathfree;
1605         int alias;
1606         int32_t deviceid;
1607         domain_id_t domainid;
1608         amd_iommu_domain_t *dp;
1609         int error;
1610         int domain_freed;
1611         const char *driver = ddi_driver_name(iommu->aiomt_dip);
1612         int instance = ddi_get_instance(iommu->aiomt_dip);
1613         const char *f = "amd_iommu_unmap_va";
1614 
1615         if (amd_iommu_no_unmap)
1616                 return (DDI_SUCCESS);
1617 
1618         path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
1619         if (path) {
1620                 (void) ddi_pathname(rdip, path);
1621                 pathfree = 1;
1622         } else {
1623                 pathfree = 0;
1624                 path = "<path-mem-alloc-failed>";
1625         }
1626 
1627         /*
1628          * First get deviceid
1629          */
1630         if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
1631             != DDI_SUCCESS) {
1632                 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
1633                     "Failed to get device ID for %s.", f, driver, instance,
1634                     iommu->aiomt_idx, (void *)rdip, path);
1635                 error = DDI_FAILURE;
1636                 goto out;
1637         }
1638 
1639         /*
1640          * Next get the domain for this rdip
1641          */
1642         if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
1643             != DDI_SUCCESS) {
1644                 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
1645                     "Failed to get domain.", f, driver, instance,
1646                     iommu->aiomt_idx, (void *)rdip, path);
1647                 error = DDI_FAILURE;
1648                 goto out;
1649         }
1650 
1651         /* should never result in domain allocation/vmem_create */
1652         dp = amd_iommu_lookup_domain(iommu, domainid, AMD_IOMMU_INVALID_MAP,
1653             KM_NOSLEEP);
1654         if (dp == NULL) {
1655                 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
1656                     "Failed to get device ID for %s.", f, driver, instance,
1657                     iommu->aiomt_idx, domainid, (void *)rdip, path);
1658                 error = DDI_FAILURE;
1659                 goto out;
1660         }
1661 
1662         ASSERT(dp->d_domainid == domainid);
1663 
1664         pg_start = start_va >> MMU_PAGESHIFT;
1665         end_va = start_va + va_sz - 1;
1666         pg_end = end_va >> MMU_PAGESHIFT;
1667         actual_sz = (pg_end - pg_start + 1) << MMU_PAGESHIFT;
1668 
1669         domain_freed = 0;
1670         for (pg = pg_start; pg <= pg_end; pg++) {
1671                 domain_freed = 0;
1672                 if (amd_iommu_destroy_pgtables(iommu, rdip,
1673                     pg << MMU_PAGESHIFT, deviceid, domainid, dp, type,
1674                     &domain_freed, path) != DDI_SUCCESS) {
1675                         error = DDI_FAILURE;
1676                         goto out;
1677                 }
1678                 if (domain_freed) {
1679                         ASSERT(pg == pg_end);
1680                         break;
1681                 }
1682         }
1683 
1684         /*
1685          * vmem_xalloc() must be paired with vmem_xfree
1686          */
1687         if (type == AMD_IOMMU_VMEM_MAP && !amd_iommu_unity_map) {
1688                 vmem_xfree(dp->d_vmem,
1689                     (void *)(uintptr_t)(pg_start << MMU_PAGESHIFT), actual_sz);
1690         }
1691 
1692         if (domain_freed)
1693                 amd_iommu_teardown_domain(iommu, dp);
1694 
1695         error = DDI_SUCCESS;
1696 out:
1697         if (pathfree)
1698                 kmem_free(path, MAXPATHLEN);
1699         return (error);
1700 }