1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
  25  */
  26 
  27 #include <sys/sunddi.h>
  28 #include <sys/sunndi.h>
  29 #include <sys/iommulib.h>
  30 #include <sys/amd_iommu.h>
  31 #include <sys/pci_cap.h>
  32 #include <sys/bootconf.h>
  33 #include <sys/ddidmareq.h>
  34 
  35 #include "amd_iommu_impl.h"
  36 #include "amd_iommu_acpi.h"
  37 #include "amd_iommu_page_tables.h"
  38 
  39 static int amd_iommu_fini(amd_iommu_t *iommu, int type);
  40 static void amd_iommu_teardown_interrupts(amd_iommu_t *iommu);
  41 static void amd_iommu_stop(amd_iommu_t *iommu);
  42 
  43 static int amd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip);
  44 static int amd_iommu_allochdl(iommulib_handle_t handle,
  45     dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
  46     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep);
  47 static int amd_iommu_freehdl(iommulib_handle_t handle,
  48     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
  49 static int amd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
  50     dev_info_t *rdip, ddi_dma_handle_t dma_handle,
  51     struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep,
  52     uint_t *ccountp);
  53 static int amd_iommu_unbindhdl(iommulib_handle_t handle,
  54     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
  55 static int amd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip,
  56     dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
  57     size_t len, uint_t cache_flags);
  58 static int amd_iommu_win(iommulib_handle_t handle, dev_info_t *dip,
  59     dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
  60     off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
  61     uint_t *ccountp);
  62 static int amd_iommu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
  63     dev_info_t *rdip, ddi_dma_handle_t dma_handle,
  64     struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao);
  65 static int amd_iommu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
  66     dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao);
  67 
  68 static int unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
  69     ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked);
  70 
  71 extern void *device_arena_alloc(size_t size, int vm_flag);
  72 extern void device_arena_free(void * vaddr, size_t size);
  73 
  74 ddi_dma_attr_t amd_iommu_dma_attr = {
  75         DMA_ATTR_V0,
  76         0U,                             /* dma_attr_addr_lo */
  77         0xffffffffffffffffULL,          /* dma_attr_addr_hi */
  78         0xffffffffU,                    /* dma_attr_count_max */
  79         (uint64_t)4096,                 /* dma_attr_align */
  80         1,                              /* dma_attr_burstsizes */
  81         64,                             /* dma_attr_minxfer */
  82         0xffffffffU,                    /* dma_attr_maxxfer */
  83         0xffffffffU,                    /* dma_attr_seg */
  84         1,                              /* dma_attr_sgllen, variable */
  85         64,                             /* dma_attr_granular */
  86         0                               /* dma_attr_flags */
  87 };
  88 
  89 ddi_device_acc_attr_t amd_iommu_devacc = {
  90         DDI_DEVICE_ATTR_V0,
  91         DDI_NEVERSWAP_ACC,
  92         DDI_STRICTORDER_ACC
  93 };
  94 
  95 struct iommulib_ops amd_iommulib_ops = {
  96         IOMMU_OPS_VERSION,
  97         AMD_IOMMU,
  98         "AMD IOMMU Vers. 1",
  99         NULL,
 100         amd_iommu_probe,
 101         amd_iommu_allochdl,
 102         amd_iommu_freehdl,
 103         amd_iommu_bindhdl,
 104         amd_iommu_unbindhdl,
 105         amd_iommu_sync,
 106         amd_iommu_win,
 107         amd_iommu_mapobject,
 108         amd_iommu_unmapobject,
 109 };
 110 
 111 static kmutex_t amd_iommu_pgtable_lock;
 112 
 113 static int
 114 amd_iommu_register(amd_iommu_t *iommu)
 115 {
 116         dev_info_t *dip = iommu->aiomt_dip;
 117         const char *driver = ddi_driver_name(dip);
 118         int instance = ddi_get_instance(dip);
 119         iommulib_ops_t *iommulib_ops;
 120         iommulib_handle_t handle;
 121         const char *f = "amd_iommu_register";
 122 
 123         iommulib_ops = kmem_zalloc(sizeof (iommulib_ops_t), KM_SLEEP);
 124 
 125         *iommulib_ops = amd_iommulib_ops;
 126 
 127         iommulib_ops->ilops_data = (void *)iommu;
 128         iommu->aiomt_iommulib_ops = iommulib_ops;
 129 
 130         if (iommulib_iommu_register(dip, iommulib_ops, &handle)
 131             != DDI_SUCCESS) {
 132                 cmn_err(CE_WARN, "%s: %s%d: Register with iommulib "
 133                     "failed idx=%d", f, driver, instance, iommu->aiomt_idx);
 134                 kmem_free(iommulib_ops, sizeof (iommulib_ops_t));
 135                 return (DDI_FAILURE);
 136         }
 137 
 138         iommu->aiomt_iommulib_handle = handle;
 139 
 140         return (DDI_SUCCESS);
 141 }
 142 
 143 static int
 144 amd_iommu_unregister(amd_iommu_t *iommu)
 145 {
 146         if (iommu->aiomt_iommulib_handle == NULL) {
 147                 /* we never registered */
 148                 return (DDI_SUCCESS);
 149         }
 150 
 151         if (iommulib_iommu_unregister(iommu->aiomt_iommulib_handle)
 152             != DDI_SUCCESS) {
 153                 return (DDI_FAILURE);
 154         }
 155 
 156         kmem_free(iommu->aiomt_iommulib_ops, sizeof (iommulib_ops_t));
 157         iommu->aiomt_iommulib_ops = NULL;
 158         iommu->aiomt_iommulib_handle = NULL;
 159 
 160         return (DDI_SUCCESS);
 161 }
 162 
 163 static int
 164 amd_iommu_setup_passthru(amd_iommu_t *iommu)
 165 {
 166         gfx_entry_t *gfxp;
 167         dev_info_t *dip;
 168 
 169         /*
 170          * Setup passthru mapping for "special" devices
 171          */
 172         amd_iommu_set_passthru(iommu, NULL);
 173 
 174         for (gfxp = gfx_devinfo_list; gfxp; gfxp = gfxp->g_next) {
 175                 gfxp->g_ref++;
 176                 dip = gfxp->g_dip;
 177                 if (dip) {
 178                         amd_iommu_set_passthru(iommu, dip);
 179                 }
 180                 gfxp->g_ref--;
 181         }
 182 
 183         return (DDI_SUCCESS);
 184 }
 185 
 186 static int
 187 amd_iommu_start(amd_iommu_t *iommu)
 188 {
 189         dev_info_t *dip = iommu->aiomt_dip;
 190         int instance = ddi_get_instance(dip);
 191         const char *driver = ddi_driver_name(dip);
 192         amd_iommu_acpi_ivhd_t *hinfop;
 193         const char *f = "amd_iommu_start";
 194 
 195         hinfop = amd_iommu_lookup_all_ivhd();
 196 
 197         /*
 198          * Disable HT tunnel translation.
 199          * XXX use ACPI
 200          */
 201         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 202             AMD_IOMMU_HT_TUN_ENABLE, 0);
 203 
 204         if (hinfop) {
 205                 if (amd_iommu_debug) {
 206                         cmn_err(CE_NOTE,
 207                             "amd_iommu: using ACPI for CTRL registers");
 208                 }
 209                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 210                     AMD_IOMMU_ISOC, hinfop->ach_Isoc);
 211                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 212                     AMD_IOMMU_RESPASSPW, hinfop->ach_ResPassPW);
 213                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 214                     AMD_IOMMU_PASSPW, hinfop->ach_PassPW);
 215         }
 216 
 217         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 218             AMD_IOMMU_INVTO, 5);
 219 
 220 
 221         /*
 222          * The Device table entry bit 0 (V) controls whether the device
 223          * table entry is valid for address translation and Device table
 224          * entry bit 128 (IV) controls whether interrupt remapping is valid.
 225          * By setting both to zero we are essentially doing pass-thru. Since
 226          * this table is zeroed on allocation, essentially we will have
 227          * pass-thru when IOMMU is enabled.
 228          */
 229 
 230         /* Finally enable the IOMMU ... */
 231         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 232             AMD_IOMMU_ENABLE, 1);
 233 
 234         if (amd_iommu_debug) {
 235                 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
 236                     "Successfully started AMD IOMMU", f, driver, instance,
 237                     iommu->aiomt_idx);
 238         }
 239         cmn_err(CE_NOTE, "AMD IOMMU (%d,%d) enabled",
 240             instance, iommu->aiomt_idx);
 241 
 242         return (DDI_SUCCESS);
 243 }
 244 
 245 static void
 246 amd_iommu_stop(amd_iommu_t *iommu)
 247 {
 248         dev_info_t *dip = iommu->aiomt_dip;
 249         int instance = ddi_get_instance(dip);
 250         const char *driver = ddi_driver_name(dip);
 251         const char *f = "amd_iommu_stop";
 252 
 253         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 254             AMD_IOMMU_ENABLE, 0);
 255         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 256             AMD_IOMMU_EVENTINT_ENABLE, 0);
 257         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 258             AMD_IOMMU_COMWAITINT_ENABLE, 0);
 259         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 260             AMD_IOMMU_EVENTLOG_ENABLE, 0);
 261 
 262         /*
 263          * Disable translation on HT tunnel traffic
 264          */
 265         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 266             AMD_IOMMU_HT_TUN_ENABLE, 0);
 267         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 268             AMD_IOMMU_CMDBUF_ENABLE, 0);
 269 
 270         cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMYU idx=%d. "
 271             "Successfully stopped AMD IOMMU", f, driver, instance,
 272             iommu->aiomt_idx);
 273 }
 274 
 275 static int
 276 amd_iommu_setup_tables_and_buffers(amd_iommu_t *iommu)
 277 {
 278         dev_info_t *dip = iommu->aiomt_dip;
 279         int instance = ddi_get_instance(dip);
 280         const char *driver = ddi_driver_name(dip);
 281         uint32_t dma_bufsz;
 282         caddr_t addr;
 283         uint32_t sz;
 284         uint32_t p2sz;
 285         int i;
 286         uint64_t *dentry;
 287         int err;
 288         const char *f = "amd_iommu_setup_tables_and_buffers";
 289 
 290         /*
 291          * We will put the Device Table, Command Buffer and
 292          * Event Log in contiguous memory. Allocate the maximum
 293          * size allowed for such structures
 294          * Device Table:  256b * 64K = 32B * 64K
 295          * Command Buffer: 128b * 32K = 16B * 32K
 296          * Event Log:  128b * 32K = 16B * 32K
 297          */
 298         iommu->aiomt_devtbl_sz = (1<<AMD_IOMMU_DEVTBL_SZ) * AMD_IOMMU_DEVENT_SZ;
 299         iommu->aiomt_cmdbuf_sz = (1<<AMD_IOMMU_CMDBUF_SZ) * AMD_IOMMU_CMD_SZ;
 300         iommu->aiomt_eventlog_sz =
 301             (1<<AMD_IOMMU_EVENTLOG_SZ) * AMD_IOMMU_EVENT_SZ;
 302 
 303         dma_bufsz = iommu->aiomt_devtbl_sz + iommu->aiomt_cmdbuf_sz
 304             + iommu->aiomt_eventlog_sz;
 305 
 306         /*
 307          * Alloc a DMA handle.
 308          */
 309         err = ddi_dma_alloc_handle(dip, &amd_iommu_dma_attr,
 310             DDI_DMA_SLEEP, NULL, &iommu->aiomt_dmahdl);
 311         if (err != DDI_SUCCESS) {
 312                 cmn_err(CE_WARN, "%s: %s%d: Cannot alloc DMA handle for "
 313                     "AMD IOMMU tables and buffers", f, driver, instance);
 314                 return (DDI_FAILURE);
 315         }
 316 
 317         /*
 318          * Alloc memory for tables and buffers
 319          * XXX remove cast to size_t
 320          */
 321         err = ddi_dma_mem_alloc(iommu->aiomt_dmahdl, dma_bufsz,
 322             &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,
 323             DDI_DMA_SLEEP,  NULL, (caddr_t *)&iommu->aiomt_dma_bufva,
 324             (size_t *)&iommu->aiomt_dma_mem_realsz, &iommu->aiomt_dma_mem_hdl);
 325         if (err != DDI_SUCCESS) {
 326                 cmn_err(CE_WARN, "%s: %s%d: Cannot alloc memory for DMA "
 327                     "to AMD IOMMU tables and buffers", f, driver, instance);
 328                 iommu->aiomt_dma_bufva = NULL;
 329                 iommu->aiomt_dma_mem_realsz = 0;
 330                 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
 331                 iommu->aiomt_dmahdl = NULL;
 332                 return (DDI_FAILURE);
 333         }
 334 
 335         /*
 336          * The VA must be 4K aligned and >= table size
 337          */
 338         ASSERT(((uintptr_t)iommu->aiomt_dma_bufva &
 339             AMD_IOMMU_TABLE_ALIGN) == 0);
 340         ASSERT(iommu->aiomt_dma_mem_realsz >= dma_bufsz);
 341 
 342         /*
 343          * Now bind the handle
 344          */
 345         err = ddi_dma_addr_bind_handle(iommu->aiomt_dmahdl, NULL,
 346             iommu->aiomt_dma_bufva, iommu->aiomt_dma_mem_realsz,
 347             DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP,
 348             NULL, &iommu->aiomt_buf_dma_cookie, &iommu->aiomt_buf_dma_ncookie);
 349         if (err != DDI_DMA_MAPPED) {
 350                 cmn_err(CE_WARN, "%s: %s%d: Cannot bind memory for DMA "
 351                     "to AMD IOMMU tables and buffers. bufrealsz=%p",
 352                     f, driver, instance,
 353                     (void *)(uintptr_t)iommu->aiomt_dma_mem_realsz);
 354                 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
 355                 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
 356                 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
 357                 iommu->aiomt_buf_dma_ncookie = 0;
 358                 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
 359                 iommu->aiomt_dma_mem_hdl = NULL;
 360                 iommu->aiomt_dma_bufva = NULL;
 361                 iommu->aiomt_dma_mem_realsz = 0;
 362                 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
 363                 iommu->aiomt_dmahdl = NULL;
 364                 return (DDI_FAILURE);
 365         }
 366 
 367         /*
 368          * We assume the DMA engine on the IOMMU is capable of handling the
 369          * whole table buffer in a single cookie. If not and multiple cookies
 370          * are needed we fail.
 371          */
 372         if (iommu->aiomt_buf_dma_ncookie != 1) {
 373                 cmn_err(CE_WARN, "%s: %s%d: Cannot handle multiple "
 374                     "cookies for DMA to AMD IOMMU tables and buffers. "
 375                     "#cookies=%u", f, driver, instance,
 376                     iommu->aiomt_buf_dma_ncookie);
 377                 (void) ddi_dma_unbind_handle(iommu->aiomt_dmahdl);
 378                 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
 379                 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
 380                 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
 381                 iommu->aiomt_buf_dma_ncookie = 0;
 382                 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
 383                 iommu->aiomt_dma_mem_hdl = NULL;
 384                 iommu->aiomt_dma_bufva = NULL;
 385                 iommu->aiomt_dma_mem_realsz = 0;
 386                 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
 387                 iommu->aiomt_dmahdl = NULL;
 388                 return (DDI_FAILURE);
 389         }
 390 
 391         /*
 392          * The address in the cookie must be 4K aligned and >= table size
 393          */
 394         ASSERT((iommu->aiomt_buf_dma_cookie.dmac_cookie_addr
 395             & AMD_IOMMU_TABLE_ALIGN) == 0);
 396         ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size
 397             <= iommu->aiomt_dma_mem_realsz);
 398         ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size >= dma_bufsz);
 399 
 400         /*
 401          * Setup the device table pointers in the iommu struct as
 402          * well as the IOMMU device table register
 403          */
 404         iommu->aiomt_devtbl = iommu->aiomt_dma_bufva;
 405         bzero(iommu->aiomt_devtbl, iommu->aiomt_devtbl_sz);
 406 
 407         /*
 408          * Set V=1 and TV = 0, so any inadvertant pass-thrus cause
 409          * page faults. Also set SE bit so we aren't swamped with
 410          * page fault messages
 411          */
 412         for (i = 0; i <= AMD_IOMMU_MAX_DEVICEID; i++) {
 413                 /*LINTED*/
 414                 dentry = (uint64_t *)&iommu->aiomt_devtbl
 415                     [i * AMD_IOMMU_DEVTBL_ENTRY_SZ];
 416                 AMD_IOMMU_REG_SET64(dentry, AMD_IOMMU_DEVTBL_V, 1);
 417                 AMD_IOMMU_REG_SET64(&(dentry[1]), AMD_IOMMU_DEVTBL_SE, 1);
 418         }
 419 
 420         addr = (caddr_t)(uintptr_t)iommu->aiomt_buf_dma_cookie.dmac_cookie_addr;
 421         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
 422             AMD_IOMMU_DEVTABBASE, ((uint64_t)(uintptr_t)addr) >> 12);
 423         sz = (iommu->aiomt_devtbl_sz >> 12) - 1;
 424         ASSERT(sz <= ((1 << 9) - 1));
 425         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
 426             AMD_IOMMU_DEVTABSIZE, sz);
 427 
 428         /*
 429          * Setup the command buffer pointers
 430          */
 431         iommu->aiomt_cmdbuf = iommu->aiomt_devtbl +
 432             iommu->aiomt_devtbl_sz;
 433         bzero(iommu->aiomt_cmdbuf, iommu->aiomt_cmdbuf_sz);
 434         addr += iommu->aiomt_devtbl_sz;
 435         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
 436             AMD_IOMMU_COMBASE, ((uint64_t)(uintptr_t)addr) >> 12);
 437 
 438         p2sz = AMD_IOMMU_CMDBUF_SZ;
 439         ASSERT(p2sz >= AMD_IOMMU_CMDBUF_MINSZ &&
 440             p2sz <= AMD_IOMMU_CMDBUF_MAXSZ);
 441         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
 442             AMD_IOMMU_COMLEN, p2sz);
 443         /*LINTED*/
 444         iommu->aiomt_cmd_tail = (uint32_t *)iommu->aiomt_cmdbuf;
 445         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
 446             AMD_IOMMU_CMDHEADPTR, 0);
 447         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_tail_va),
 448             AMD_IOMMU_CMDTAILPTR, 0);
 449 
 450         /*
 451          * Setup the event log pointers
 452          */
 453         iommu->aiomt_eventlog = iommu->aiomt_cmdbuf +
 454             iommu->aiomt_eventlog_sz;
 455         bzero(iommu->aiomt_eventlog, iommu->aiomt_eventlog_sz);
 456         addr += iommu->aiomt_cmdbuf_sz;
 457         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
 458             AMD_IOMMU_EVENTBASE, ((uint64_t)(uintptr_t)addr) >> 12);
 459         p2sz = AMD_IOMMU_EVENTLOG_SZ;
 460         ASSERT(p2sz >= AMD_IOMMU_EVENTLOG_MINSZ &&
 461             p2sz <= AMD_IOMMU_EVENTLOG_MAXSZ);
 462         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
 463             AMD_IOMMU_EVENTLEN, sz);
 464         /*LINTED*/
 465         iommu->aiomt_event_head = (uint32_t *)iommu->aiomt_eventlog;
 466         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
 467             AMD_IOMMU_EVENTHEADPTR, 0);
 468         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_tail_va),
 469             AMD_IOMMU_EVENTTAILPTR, 0);
 470 
 471         /* dma sync so device sees this init */
 472         SYNC_FORDEV(iommu->aiomt_dmahdl);
 473 
 474         if (amd_iommu_debug & AMD_IOMMU_DEBUG_TABLES) {
 475                 cmn_err(CE_NOTE, "%s: %s%d: successfully setup AMD IOMMU "
 476                     "tables, idx=%d", f, driver, instance, iommu->aiomt_idx);
 477         }
 478 
 479         return (DDI_SUCCESS);
 480 }
 481 
 482 static void
 483 amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu, int type)
 484 {
 485         dev_info_t *dip = iommu->aiomt_dip;
 486         int instance = ddi_get_instance(dip);
 487         const char *driver = ddi_driver_name(dip);
 488         const char *f = "amd_iommu_teardown_tables_and_buffers";
 489 
 490         iommu->aiomt_eventlog = NULL;
 491         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
 492             AMD_IOMMU_EVENTBASE, 0);
 493         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
 494             AMD_IOMMU_EVENTLEN, 0);
 495         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
 496             AMD_IOMMU_EVENTHEADPTR, 0);
 497         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
 498             AMD_IOMMU_EVENTTAILPTR, 0);
 499 
 500 
 501         iommu->aiomt_cmdbuf = NULL;
 502         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
 503             AMD_IOMMU_COMBASE, 0);
 504         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
 505             AMD_IOMMU_COMLEN, 0);
 506         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
 507             AMD_IOMMU_CMDHEADPTR, 0);
 508         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
 509             AMD_IOMMU_CMDTAILPTR, 0);
 510 
 511 
 512         iommu->aiomt_devtbl = NULL;
 513         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
 514             AMD_IOMMU_DEVTABBASE, 0);
 515         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
 516             AMD_IOMMU_DEVTABSIZE, 0);
 517 
 518         if (iommu->aiomt_dmahdl == NULL || type == AMD_IOMMU_QUIESCE)
 519                 return;
 520 
 521         /* Unbind the handle */
 522         if (ddi_dma_unbind_handle(iommu->aiomt_dmahdl) != DDI_SUCCESS) {
 523                 cmn_err(CE_WARN, "%s: %s%d: failed to unbind handle: "
 524                     "%p for IOMMU idx=%d", f, driver, instance,
 525                     (void *)iommu->aiomt_dmahdl, iommu->aiomt_idx);
 526         }
 527         iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
 528         iommu->aiomt_buf_dma_cookie.dmac_size = 0;
 529         iommu->aiomt_buf_dma_cookie.dmac_type = 0;
 530         iommu->aiomt_buf_dma_ncookie = 0;
 531 
 532         /* Free the table memory allocated for DMA */
 533         ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
 534         iommu->aiomt_dma_mem_hdl = NULL;
 535         iommu->aiomt_dma_bufva = NULL;
 536         iommu->aiomt_dma_mem_realsz = 0;
 537 
 538         /* Free the DMA handle */
 539         ddi_dma_free_handle(&iommu->aiomt_dmahdl);
 540         iommu->aiomt_dmahdl = NULL;
 541 }
 542 
 543 static void
 544 amd_iommu_enable_interrupts(amd_iommu_t *iommu)
 545 {
 546         ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
 547             AMD_IOMMU_CMDBUF_RUN) == 0);
 548         ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
 549             AMD_IOMMU_EVENT_LOG_RUN) == 0);
 550 
 551         /* Must be set prior to enabling command buffer */
 552         /* Must be set prior to enabling event logging */
 553         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 554             AMD_IOMMU_CMDBUF_ENABLE, 1);
 555         /* No interrupts for completion wait  - too heavy weight. use polling */
 556         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 557             AMD_IOMMU_COMWAITINT_ENABLE, 0);
 558         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 559             AMD_IOMMU_EVENTLOG_ENABLE, 1);
 560         AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
 561             AMD_IOMMU_EVENTINT_ENABLE, 1);
 562 }
 563 
 564 static int
 565 amd_iommu_setup_exclusion(amd_iommu_t *iommu)
 566 {
 567         amd_iommu_acpi_ivmd_t *minfop;
 568 
 569         minfop = amd_iommu_lookup_all_ivmd();
 570 
 571         if (minfop && minfop->acm_ExclRange == 1) {
 572                 cmn_err(CE_NOTE, "Programming exclusion range");
 573                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
 574                     AMD_IOMMU_EXCL_BASE_ADDR,
 575                     minfop->acm_ivmd_phys_start >> 12);
 576                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
 577                     AMD_IOMMU_EXCL_BASE_ALLOW, 1);
 578                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
 579                     AMD_IOMMU_EXCL_BASE_EXEN, 1);
 580                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
 581                     AMD_IOMMU_EXCL_LIM, (minfop->acm_ivmd_phys_start +
 582                     minfop->acm_ivmd_phys_len) >> 12);
 583         } else {
 584                 if (amd_iommu_debug) {
 585                         cmn_err(CE_NOTE, "Skipping exclusion range");
 586                 }
 587                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
 588                     AMD_IOMMU_EXCL_BASE_ADDR, 0);
 589                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
 590                     AMD_IOMMU_EXCL_BASE_ALLOW, 1);
 591                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
 592                     AMD_IOMMU_EXCL_BASE_EXEN, 0);
 593                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
 594                     AMD_IOMMU_EXCL_LIM, 0);
 595         }
 596 
 597         return (DDI_SUCCESS);
 598 }
 599 
 600 static void
 601 amd_iommu_teardown_exclusion(amd_iommu_t *iommu)
 602 {
 603         (void) amd_iommu_setup_exclusion(iommu);
 604 }
 605 
 606 static uint_t
 607 amd_iommu_intr_handler(caddr_t arg1, caddr_t arg2)
 608 {
 609         /*LINTED*/
 610         amd_iommu_t *iommu = (amd_iommu_t *)arg1;
 611         dev_info_t *dip = iommu->aiomt_dip;
 612         int instance = ddi_get_instance(dip);
 613         const char *driver = ddi_driver_name(dip);
 614         const char *f = "amd_iommu_intr_handler";
 615 
 616         ASSERT(arg1);
 617         ASSERT(arg2 == NULL);
 618 
 619         if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 620                 cmn_err(CE_NOTE, "%s: %s%d: IOMMU unit idx=%d. In INTR handler",
 621                     f, driver, instance, iommu->aiomt_idx);
 622         }
 623 
 624         if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
 625             AMD_IOMMU_EVENT_LOG_INT) == 1) {
 626                 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 627                         cmn_err(CE_NOTE, "%s: %s%d: IOMMU unit idx=%d "
 628                             "Event Log Interrupt", f, driver, instance,
 629                             iommu->aiomt_idx);
 630                 }
 631                 (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISPLAY);
 632                 WAIT_SEC(1);
 633                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
 634                     AMD_IOMMU_EVENT_LOG_INT, 1);
 635                 return (DDI_INTR_CLAIMED);
 636         }
 637 
 638         if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
 639             AMD_IOMMU_EVENT_OVERFLOW_INT) == 1) {
 640                 cmn_err(CE_NOTE, "!%s: %s%d: IOMMU unit idx=%d "
 641                     "Event Overflow Interrupt", f, driver, instance,
 642                     iommu->aiomt_idx);
 643                 (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISCARD);
 644                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
 645                     AMD_IOMMU_EVENT_LOG_INT, 1);
 646                 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
 647                     AMD_IOMMU_EVENT_OVERFLOW_INT, 1);
 648                 return (DDI_INTR_CLAIMED);
 649         }
 650 
 651         return (DDI_INTR_UNCLAIMED);
 652 }
 653 
 654 
 655 static int
 656 amd_iommu_setup_interrupts(amd_iommu_t *iommu)
 657 {
 658         dev_info_t *dip = iommu->aiomt_dip;
 659         int instance = ddi_get_instance(dip);
 660         const char *driver = ddi_driver_name(dip);
 661         int intrcap0;
 662         int intrcapN;
 663         int type;
 664         int err;
 665         int req;
 666         int avail;
 667         int p2req;
 668         int actual;
 669         int i;
 670         int j;
 671         const char *f = "amd_iommu_setup_interrupts";
 672 
 673         if (ddi_intr_get_supported_types(dip, &type) != DDI_SUCCESS) {
 674                 cmn_err(CE_WARN, "%s: %s%d: ddi_intr_get_supported_types "
 675                     "failed: idx=%d", f, driver, instance, iommu->aiomt_idx);
 676                 return (DDI_FAILURE);
 677         }
 678 
 679         if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 680                 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
 681                     "Interrupt types supported = 0x%x", f, driver, instance,
 682                     iommu->aiomt_idx, type);
 683         }
 684 
 685         /*
 686          * for now we only support MSI
 687          */
 688         if ((type & DDI_INTR_TYPE_MSI) == 0) {
 689                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d. "
 690                     "MSI interrupts not supported. Failing init.",
 691                     f, driver, instance, iommu->aiomt_idx);
 692                 return (DDI_FAILURE);
 693         }
 694 
 695         if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 696                 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. MSI supported",
 697                     f, driver, instance, iommu->aiomt_idx);
 698         }
 699 
 700         err = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSI, &req);
 701         if (err != DDI_SUCCESS) {
 702                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d. "
 703                     "ddi_intr_get_nintrs failed err = %d",
 704                     f, driver, instance, iommu->aiomt_idx, err);
 705                 return (DDI_FAILURE);
 706         }
 707 
 708         if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 709                 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
 710                     "MSI number of interrupts requested: %d",
 711                     f, driver, instance, iommu->aiomt_idx, req);
 712         }
 713 
 714         if (req == 0) {
 715                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: 0 MSI "
 716                     "interrupts requested. Failing init", f,
 717                     driver, instance, iommu->aiomt_idx);
 718                 return (DDI_FAILURE);
 719         }
 720 
 721         err = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSI, &avail);
 722         if (err != DDI_SUCCESS) {
 723                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d "
 724                     "ddi_intr_get_navail failed err = %d", f,
 725                     driver, instance, iommu->aiomt_idx, err);
 726                 return (DDI_FAILURE);
 727         }
 728 
 729         if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 730                 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
 731                     "MSI number of interrupts available: %d",
 732                     f, driver, instance, iommu->aiomt_idx, avail);
 733         }
 734 
 735         if (avail == 0) {
 736                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: 0 MSI "
 737                     "interrupts available. Failing init", f,
 738                     driver, instance, iommu->aiomt_idx);
 739                 return (DDI_FAILURE);
 740         }
 741 
 742         if (avail < req) {
 743                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: MSI "
 744                     "interrupts: requested (%d) > available (%d). "
 745                     "Failing init", f, driver, instance, iommu->aiomt_idx,
 746                     req, avail);
 747                 return (DDI_FAILURE);
 748         }
 749 
 750         /* Allocate memory for DDI interrupt handles */
 751         iommu->aiomt_intr_htable_sz = req * sizeof (ddi_intr_handle_t);
 752         iommu->aiomt_intr_htable = kmem_zalloc(iommu->aiomt_intr_htable_sz,
 753             KM_SLEEP);
 754 
 755         iommu->aiomt_intr_state = AMD_IOMMU_INTR_TABLE;
 756 
 757         /* Convert req to a power of two as required by ddi_intr_alloc */
 758         p2req = 0;
 759         while (1<<p2req <= req)
 760                 p2req++;
 761         p2req--;
 762         req = 1<<p2req;
 763 
 764         if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 765                 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
 766                     "MSI power of 2 number of interrupts: %d,%d",
 767                     f, driver, instance, iommu->aiomt_idx, p2req, req);
 768         }
 769 
 770         err = ddi_intr_alloc(iommu->aiomt_dip, iommu->aiomt_intr_htable,
 771             DDI_INTR_TYPE_MSI, 0, req, &actual, DDI_INTR_ALLOC_STRICT);
 772         if (err != DDI_SUCCESS) {
 773                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
 774                     "ddi_intr_alloc failed: err = %d",
 775                     f, driver, instance, iommu->aiomt_idx, err);
 776                 amd_iommu_teardown_interrupts(iommu);
 777                 return (DDI_FAILURE);
 778         }
 779 
 780         iommu->aiomt_actual_intrs = actual;
 781         iommu->aiomt_intr_state = AMD_IOMMU_INTR_ALLOCED;
 782 
 783         if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 784                 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
 785                     "number of interrupts actually allocated %d",
 786                     f, driver, instance, iommu->aiomt_idx, actual);
 787         }
 788 
 789         if (iommu->aiomt_actual_intrs < req) {
 790                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
 791                     "ddi_intr_alloc failed: actual (%d) < req (%d)",
 792                     f, driver, instance, iommu->aiomt_idx,
 793                     iommu->aiomt_actual_intrs, req);
 794                 amd_iommu_teardown_interrupts(iommu);
 795                 return (DDI_FAILURE);
 796         }
 797 
 798         for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
 799                 if (ddi_intr_add_handler(iommu->aiomt_intr_htable[i],
 800                     amd_iommu_intr_handler, (void *)iommu, NULL)
 801                     != DDI_SUCCESS) {
 802                         cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
 803                             "ddi_intr_add_handler failed: intr = %d, err = %d",
 804                             f, driver, instance, iommu->aiomt_idx, i, err);
 805                         for (j = 0; j < i; j++) {
 806                                 (void) ddi_intr_remove_handler(
 807                                     iommu->aiomt_intr_htable[j]);
 808                         }
 809                         amd_iommu_teardown_interrupts(iommu);
 810                         return (DDI_FAILURE);
 811                 }
 812         }
 813         iommu->aiomt_intr_state = AMD_IOMMU_INTR_HANDLER;
 814 
 815         intrcap0 = intrcapN = -1;
 816         if (ddi_intr_get_cap(iommu->aiomt_intr_htable[0], &intrcap0)
 817             != DDI_SUCCESS ||
 818             ddi_intr_get_cap(
 819             iommu->aiomt_intr_htable[iommu->aiomt_actual_intrs - 1], &intrcapN)
 820             != DDI_SUCCESS || intrcap0 != intrcapN) {
 821                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
 822                     "ddi_intr_get_cap failed or inconsistent cap among "
 823                     "interrupts: intrcap0 (%d) < intrcapN (%d)",
 824                     f, driver, instance, iommu->aiomt_idx, intrcap0, intrcapN);
 825                 amd_iommu_teardown_interrupts(iommu);
 826                 return (DDI_FAILURE);
 827         }
 828         iommu->aiomt_intr_cap = intrcap0;
 829 
 830         if (intrcap0 & DDI_INTR_FLAG_BLOCK) {
 831                 /* Need to call block enable */
 832                 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 833                         cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: "
 834                             "Need to call block enable",
 835                             f, driver, instance, iommu->aiomt_idx);
 836                 }
 837                 if (ddi_intr_block_enable(iommu->aiomt_intr_htable,
 838                     iommu->aiomt_actual_intrs) != DDI_SUCCESS) {
 839                         cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
 840                             "ddi_intr_block enable failed ", f, driver,
 841                             instance, iommu->aiomt_idx);
 842                         (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
 843                             iommu->aiomt_actual_intrs);
 844                         amd_iommu_teardown_interrupts(iommu);
 845                         return (DDI_FAILURE);
 846                 }
 847         } else {
 848                 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 849                         cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: "
 850                             "Need to call individual enable",
 851                             f, driver, instance, iommu->aiomt_idx);
 852                 }
 853                 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
 854                         if (ddi_intr_enable(iommu->aiomt_intr_htable[i])
 855                             != DDI_SUCCESS) {
 856                                 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
 857                                     "ddi_intr_enable failed: intr = %d", f,
 858                                     driver, instance, iommu->aiomt_idx, i);
 859                                 for (j = 0; j < i; j++) {
 860                                         (void) ddi_intr_disable(
 861                                             iommu->aiomt_intr_htable[j]);
 862                                 }
 863                                 amd_iommu_teardown_interrupts(iommu);
 864                                 return (DDI_FAILURE);
 865                         }
 866                 }
 867         }
 868         iommu->aiomt_intr_state = AMD_IOMMU_INTR_ENABLED;
 869 
 870         if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
 871                 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: "
 872                     "Interrupts successfully %s enabled. # of interrupts = %d",
 873                     f, driver, instance, iommu->aiomt_idx,
 874                     (intrcap0 & DDI_INTR_FLAG_BLOCK) ? "(block)" :
 875                     "(individually)", iommu->aiomt_actual_intrs);
 876         }
 877 
 878         return (DDI_SUCCESS);
 879 }
 880 
 881 static void
 882 amd_iommu_teardown_interrupts(amd_iommu_t *iommu)
 883 {
 884         int i;
 885 
 886         if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ENABLED) {
 887                 if (iommu->aiomt_intr_cap & DDI_INTR_FLAG_BLOCK) {
 888                         (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
 889                             iommu->aiomt_actual_intrs);
 890                 } else {
 891                         for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
 892                                 (void) ddi_intr_disable(
 893                                     iommu->aiomt_intr_htable[i]);
 894                         }
 895                 }
 896         }
 897 
 898         if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_HANDLER) {
 899                 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
 900                         (void) ddi_intr_remove_handler(
 901                             iommu->aiomt_intr_htable[i]);
 902                 }
 903         }
 904 
 905         if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ALLOCED) {
 906                 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
 907                         (void) ddi_intr_free(iommu->aiomt_intr_htable[i]);
 908                 }
 909         }
 910         if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_TABLE) {
 911                 kmem_free(iommu->aiomt_intr_htable,
 912                     iommu->aiomt_intr_htable_sz);
 913         }
 914         iommu->aiomt_intr_htable = NULL;
 915         iommu->aiomt_intr_htable_sz = 0;
 916         iommu->aiomt_intr_state = AMD_IOMMU_INTR_INVALID;
 917 }
 918 
 919 static amd_iommu_t *
 920 amd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx,
 921     uint16_t cap_base)
 922 {
 923         amd_iommu_t *iommu;
 924         int instance = ddi_get_instance(dip);
 925         const char *driver = ddi_driver_name(dip);
 926         uint32_t caphdr;
 927         uint32_t low_addr32;
 928         uint32_t hi_addr32;
 929         uint32_t range;
 930         uint32_t misc;
 931         uint64_t pgoffset;
 932         amd_iommu_acpi_global_t *global;
 933         amd_iommu_acpi_ivhd_t *hinfop;
 934         int bus, device, func;
 935         const char *f = "amd_iommu_init";
 936 
 937         low_addr32 = PCI_CAP_GET32(handle, 0, cap_base,
 938             AMD_IOMMU_CAP_ADDR_LOW_OFF);
 939         if (!(low_addr32 & AMD_IOMMU_REG_ADDR_LOCKED)) {
 940                 cmn_err(CE_WARN, "%s: %s%d: capability registers not locked. "
 941                     "Unable to use IOMMU unit idx=%d - skipping ...", f, driver,
 942                     instance, idx);
 943                 return (NULL);
 944         }
 945 
 946         iommu = kmem_zalloc(sizeof (amd_iommu_t), KM_SLEEP);
 947         mutex_init(&iommu->aiomt_mutex, NULL, MUTEX_DRIVER, NULL);
 948         mutex_enter(&iommu->aiomt_mutex);
 949 
 950         mutex_init(&iommu->aiomt_cmdlock, NULL, MUTEX_DRIVER, NULL);
 951         mutex_init(&iommu->aiomt_eventlock, NULL, MUTEX_DRIVER, NULL);
 952 
 953         iommu->aiomt_dip = dip;
 954         iommu->aiomt_idx = idx;
 955 
 956         if (acpica_get_bdf(iommu->aiomt_dip, &bus, &device, &func)
 957             != DDI_SUCCESS) {
 958                 cmn_err(CE_WARN, "%s: %s%d: Failed to get BDF"
 959                     "Unable to use IOMMU unit idx=%d - skipping ...",
 960                     f, driver, instance, idx);
 961                 return (NULL);
 962         }
 963 
 964         iommu->aiomt_bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) |
 965             (uint8_t)func;
 966 
 967         /*
 968          * Since everything in the capability block is locked and RO at this
 969          * point, copy everything into the IOMMU struct
 970          */
 971 
 972         /* Get cap header */
 973         caphdr = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_HDR_OFF);
 974         iommu->aiomt_cap_hdr = caphdr;
 975         iommu->aiomt_npcache = AMD_IOMMU_REG_GET32(&caphdr,
 976             AMD_IOMMU_CAP_NPCACHE);
 977         iommu->aiomt_httun = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_HTTUN);
 978 
 979         global = amd_iommu_lookup_acpi_global();
 980         hinfop = amd_iommu_lookup_any_ivhd(iommu);
 981 
 982         if (hinfop)
 983                 iommu->aiomt_iotlb = hinfop->ach_IotlbSup;
 984         else
 985                 iommu->aiomt_iotlb =
 986                     AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_IOTLB);
 987 
 988         iommu->aiomt_captype = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
 989         iommu->aiomt_capid = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
 990 
 991         /*
 992          * Get address of IOMMU control registers
 993          */
 994         hi_addr32 = PCI_CAP_GET32(handle, 0, cap_base,
 995             AMD_IOMMU_CAP_ADDR_HI_OFF);
 996         iommu->aiomt_low_addr32 = low_addr32;
 997         iommu->aiomt_hi_addr32 = hi_addr32;
 998         low_addr32 &= ~AMD_IOMMU_REG_ADDR_LOCKED;
 999 
1000         if (hinfop) {
1001                 iommu->aiomt_reg_pa =  hinfop->ach_IOMMU_reg_base;
1002                 ASSERT(hinfop->ach_IOMMU_pci_seg == 0);
1003         } else {
1004                 iommu->aiomt_reg_pa =  ((uint64_t)hi_addr32 << 32 | low_addr32);
1005         }
1006 
1007         /*
1008          * Get cap range reg
1009          */
1010         range = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_RANGE_OFF);
1011         iommu->aiomt_range = range;
1012         iommu->aiomt_rng_valid = AMD_IOMMU_REG_GET32(&range,
1013             AMD_IOMMU_RNG_VALID);
1014         if (iommu->aiomt_rng_valid) {
1015                 iommu->aiomt_rng_bus = AMD_IOMMU_REG_GET32(&range,
1016                     AMD_IOMMU_RNG_BUS);
1017                 iommu->aiomt_first_devfn = AMD_IOMMU_REG_GET32(&range,
1018                     AMD_IOMMU_FIRST_DEVFN);
1019                 iommu->aiomt_last_devfn = AMD_IOMMU_REG_GET32(&range,
1020                     AMD_IOMMU_LAST_DEVFN);
1021         } else {
1022                 iommu->aiomt_rng_bus = 0;
1023                 iommu->aiomt_first_devfn = 0;
1024                 iommu->aiomt_last_devfn = 0;
1025         }
1026 
1027         if (hinfop)
1028                 iommu->aiomt_ht_unitid = hinfop->ach_IOMMU_UnitID;
1029         else
1030                 iommu->aiomt_ht_unitid = AMD_IOMMU_REG_GET32(&range,
1031                     AMD_IOMMU_HT_UNITID);
1032 
1033         /*
1034          * Get cap misc reg
1035          */
1036         misc = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_MISC_OFF);
1037         iommu->aiomt_misc = misc;
1038 
1039         if (global) {
1040                 iommu->aiomt_htatsresv = global->acg_HtAtsResv;
1041                 iommu->aiomt_vasize = global->acg_VAsize;
1042                 iommu->aiomt_pasize = global->acg_PAsize;
1043         } else {
1044                 iommu->aiomt_htatsresv = AMD_IOMMU_REG_GET32(&misc,
1045                     AMD_IOMMU_HT_ATSRSV);
1046                 iommu->aiomt_vasize = AMD_IOMMU_REG_GET32(&misc,
1047                     AMD_IOMMU_VA_SIZE);
1048                 iommu->aiomt_pasize = AMD_IOMMU_REG_GET32(&misc,
1049                     AMD_IOMMU_PA_SIZE);
1050         }
1051 
1052         if (hinfop) {
1053                 iommu->aiomt_msinum = hinfop->ach_IOMMU_MSInum;
1054         } else {
1055                 iommu->aiomt_msinum =
1056                     AMD_IOMMU_REG_GET32(&misc, AMD_IOMMU_MSINUM);
1057         }
1058 
1059         /*
1060          * Set up mapping between control registers PA and VA
1061          */
1062         pgoffset = iommu->aiomt_reg_pa & MMU_PAGEOFFSET;
1063         ASSERT(pgoffset == 0);
1064         iommu->aiomt_reg_pages = mmu_btopr(AMD_IOMMU_REG_SIZE + pgoffset);
1065         iommu->aiomt_reg_size = mmu_ptob(iommu->aiomt_reg_pages);
1066 
1067         iommu->aiomt_va = (uintptr_t)device_arena_alloc(
1068             ptob(iommu->aiomt_reg_pages), VM_SLEEP);
1069         if (iommu->aiomt_va == 0) {
1070                 cmn_err(CE_WARN, "%s: %s%d: Failed to alloc VA for IOMMU "
1071                     "control regs. Skipping IOMMU idx=%d", f, driver,
1072                     instance, idx);
1073                 mutex_exit(&iommu->aiomt_mutex);
1074                 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1075                 return (NULL);
1076         }
1077 
1078         hat_devload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
1079             iommu->aiomt_reg_size,
1080             mmu_btop(iommu->aiomt_reg_pa), PROT_READ | PROT_WRITE
1081             | HAT_STRICTORDER, HAT_LOAD_LOCK);
1082 
1083         iommu->aiomt_reg_va = iommu->aiomt_va + pgoffset;
1084 
1085         /*
1086          * Setup the various control register's VA
1087          */
1088         iommu->aiomt_reg_devtbl_va = iommu->aiomt_reg_va +
1089             AMD_IOMMU_DEVTBL_REG_OFF;
1090         iommu->aiomt_reg_cmdbuf_va = iommu->aiomt_reg_va +
1091             AMD_IOMMU_CMDBUF_REG_OFF;
1092         iommu->aiomt_reg_eventlog_va = iommu->aiomt_reg_va +
1093             AMD_IOMMU_EVENTLOG_REG_OFF;
1094         iommu->aiomt_reg_ctrl_va = iommu->aiomt_reg_va +
1095             AMD_IOMMU_CTRL_REG_OFF;
1096         iommu->aiomt_reg_excl_base_va = iommu->aiomt_reg_va +
1097             AMD_IOMMU_EXCL_BASE_REG_OFF;
1098         iommu->aiomt_reg_excl_lim_va = iommu->aiomt_reg_va +
1099             AMD_IOMMU_EXCL_LIM_REG_OFF;
1100         iommu->aiomt_reg_cmdbuf_head_va = iommu->aiomt_reg_va +
1101             AMD_IOMMU_CMDBUF_HEAD_REG_OFF;
1102         iommu->aiomt_reg_cmdbuf_tail_va = iommu->aiomt_reg_va +
1103             AMD_IOMMU_CMDBUF_TAIL_REG_OFF;
1104         iommu->aiomt_reg_eventlog_head_va = iommu->aiomt_reg_va +
1105             AMD_IOMMU_EVENTLOG_HEAD_REG_OFF;
1106         iommu->aiomt_reg_eventlog_tail_va = iommu->aiomt_reg_va +
1107             AMD_IOMMU_EVENTLOG_TAIL_REG_OFF;
1108         iommu->aiomt_reg_status_va = iommu->aiomt_reg_va +
1109             AMD_IOMMU_STATUS_REG_OFF;
1110 
1111 
1112         /*
1113          * Setup the DEVICE table, CMD buffer, and LOG buffer in
1114          * memory and setup DMA access to this memory location
1115          */
1116         if (amd_iommu_setup_tables_and_buffers(iommu) != DDI_SUCCESS) {
1117                 mutex_exit(&iommu->aiomt_mutex);
1118                 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1119                 return (NULL);
1120         }
1121 
1122         if (amd_iommu_setup_exclusion(iommu) != DDI_SUCCESS) {
1123                 mutex_exit(&iommu->aiomt_mutex);
1124                 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1125                 return (NULL);
1126         }
1127 
1128         amd_iommu_enable_interrupts(iommu);
1129 
1130         if (amd_iommu_setup_interrupts(iommu) != DDI_SUCCESS) {
1131                 mutex_exit(&iommu->aiomt_mutex);
1132                 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1133                 return (NULL);
1134         }
1135 
1136         /*
1137          * need to setup domain table before gfx bypass
1138          */
1139         amd_iommu_init_page_tables(iommu);
1140 
1141         /*
1142          * Set pass-thru for special devices like IOAPIC and HPET
1143          *
1144          * Also, gfx devices don't use DDI for DMA. No need to register
1145          * before setting up gfx passthru
1146          */
1147         if (amd_iommu_setup_passthru(iommu) != DDI_SUCCESS) {
1148                 mutex_exit(&iommu->aiomt_mutex);
1149                 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1150                 return (NULL);
1151         }
1152 
1153         /* Initialize device table entries based on ACPI settings */
1154         if (amd_iommu_acpi_init_devtbl(iommu) !=  DDI_SUCCESS) {
1155                 cmn_err(CE_WARN, "%s: %s%d: Can't initialize device table",
1156                     f, driver, instance);
1157                 mutex_exit(&iommu->aiomt_mutex);
1158                 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1159                 return (NULL);
1160         }
1161 
1162         if (amd_iommu_start(iommu) != DDI_SUCCESS) {
1163                 mutex_exit(&iommu->aiomt_mutex);
1164                 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1165                 return (NULL);
1166         }
1167 
1168         /* xxx register/start race  */
1169         if (amd_iommu_register(iommu) != DDI_SUCCESS) {
1170                 mutex_exit(&iommu->aiomt_mutex);
1171                 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1172                 return (NULL);
1173         }
1174 
1175         if (amd_iommu_debug) {
1176                 cmn_err(CE_NOTE, "%s: %s%d: IOMMU idx=%d inited.", f, driver,
1177                     instance, idx);
1178         }
1179 
1180         return (iommu);
1181 }
1182 
1183 static int
1184 amd_iommu_fini(amd_iommu_t *iommu, int type)
1185 {
1186         int idx = iommu->aiomt_idx;
1187         dev_info_t *dip = iommu->aiomt_dip;
1188         int instance = ddi_get_instance(dip);
1189         const char *driver = ddi_driver_name(dip);
1190         const char *f = "amd_iommu_fini";
1191 
1192         if (type == AMD_IOMMU_TEARDOWN) {
1193                 mutex_enter(&iommu->aiomt_mutex);
1194                 if (amd_iommu_unregister(iommu) != DDI_SUCCESS) {
1195                         cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit failed. "
1196                             "idx = %d", f, driver, instance, idx);
1197                         return (DDI_FAILURE);
1198                 }
1199         }
1200 
1201         amd_iommu_stop(iommu);
1202 
1203         if (type == AMD_IOMMU_TEARDOWN) {
1204                 amd_iommu_fini_page_tables(iommu);
1205                 amd_iommu_teardown_interrupts(iommu);
1206                 amd_iommu_teardown_exclusion(iommu);
1207         }
1208 
1209         amd_iommu_teardown_tables_and_buffers(iommu, type);
1210 
1211         if (type == AMD_IOMMU_QUIESCE)
1212                 return (DDI_SUCCESS);
1213 
1214         if (iommu->aiomt_va != 0) {
1215                 hat_unload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
1216                     iommu->aiomt_reg_size, HAT_UNLOAD_UNLOCK);
1217                 device_arena_free((void *)(uintptr_t)iommu->aiomt_va,
1218                     ptob(iommu->aiomt_reg_pages));
1219                 iommu->aiomt_va = 0;
1220                 iommu->aiomt_reg_va = 0;
1221         }
1222         mutex_destroy(&iommu->aiomt_eventlock);
1223         mutex_destroy(&iommu->aiomt_cmdlock);
1224         mutex_exit(&iommu->aiomt_mutex);
1225         mutex_destroy(&iommu->aiomt_mutex);
1226         kmem_free(iommu, sizeof (amd_iommu_t));
1227 
1228         cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit complete. idx = %d",
1229             f, driver, instance, idx);
1230 
1231         return (DDI_SUCCESS);
1232 }
1233 
1234 int
1235 amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep)
1236 {
1237         int instance = ddi_get_instance(dip);
1238         const char *driver = ddi_driver_name(dip);
1239         ddi_acc_handle_t handle;
1240         uint8_t base_class;
1241         uint8_t sub_class;
1242         uint8_t prog_class;
1243         int idx;
1244         uint32_t id;
1245         uint16_t cap_base;
1246         uint32_t caphdr;
1247         uint8_t cap_type;
1248         uint8_t cap_id;
1249         amd_iommu_t *iommu;
1250         const char *f = "amd_iommu_setup";
1251 
1252         ASSERT(instance >= 0);
1253         ASSERT(driver);
1254 
1255         /* First setup PCI access to config space */
1256 
1257         if (pci_config_setup(dip, &handle) != DDI_SUCCESS) {
1258                 cmn_err(CE_WARN, "%s: PCI config setup failed: %s%d",
1259                     f, driver, instance);
1260                 return (DDI_FAILURE);
1261         }
1262 
1263         /*
1264          * The AMD IOMMU is part of an independent PCI function. There may be
1265          * more than one IOMMU in that PCI function
1266          */
1267         base_class = pci_config_get8(handle, PCI_CONF_BASCLASS);
1268         sub_class = pci_config_get8(handle, PCI_CONF_SUBCLASS);
1269         prog_class = pci_config_get8(handle, PCI_CONF_PROGCLASS);
1270 
1271         if (base_class != PCI_CLASS_PERIPH || sub_class != PCI_PERIPH_IOMMU ||
1272             prog_class != AMD_IOMMU_PCI_PROG_IF) {
1273                 cmn_err(CE_WARN, "%s: %s%d: invalid PCI class(0x%x)/"
1274                     "subclass(0x%x)/programming interface(0x%x)", f, driver,
1275                     instance, base_class, sub_class, prog_class);
1276                 pci_config_teardown(&handle);
1277                 return (DDI_FAILURE);
1278         }
1279 
1280         /*
1281          * Find and initialize all IOMMU units in this function
1282          */
1283         for (idx = 0; ; idx++) {
1284                 if (pci_cap_probe(handle, idx, &id, &cap_base) != DDI_SUCCESS)
1285                         break;
1286 
1287                 /* check if cap ID is secure device cap id */
1288                 if (id != PCI_CAP_ID_SECURE_DEV) {
1289                         if (amd_iommu_debug) {
1290                                 cmn_err(CE_NOTE,
1291                                     "%s: %s%d: skipping IOMMU: idx(0x%x) "
1292                                     "cap ID (0x%x) != secure dev capid (0x%x)",
1293                                     f, driver, instance, idx, id,
1294                                     PCI_CAP_ID_SECURE_DEV);
1295                         }
1296                         continue;
1297                 }
1298 
1299                 /* check if cap type is IOMMU cap type */
1300                 caphdr = PCI_CAP_GET32(handle, 0, cap_base,
1301                     AMD_IOMMU_CAP_HDR_OFF);
1302                 cap_type = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
1303                 cap_id = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
1304 
1305                 if (cap_type != AMD_IOMMU_CAP) {
1306                         cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) "
1307                             "cap type (0x%x) != AMD IOMMU CAP (0x%x)", f,
1308                             driver, instance, idx, cap_type, AMD_IOMMU_CAP);
1309                         continue;
1310                 }
1311                 ASSERT(cap_id == PCI_CAP_ID_SECURE_DEV);
1312                 ASSERT(cap_id == id);
1313 
1314                 iommu = amd_iommu_init(dip, handle, idx, cap_base);
1315                 if (iommu == NULL) {
1316                         cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) "
1317                             "failed to init IOMMU", f,
1318                             driver, instance, idx);
1319                         continue;
1320                 }
1321 
1322                 if (statep->aioms_iommu_start == NULL) {
1323                         statep->aioms_iommu_start = iommu;
1324                 } else {
1325                         statep->aioms_iommu_end->aiomt_next = iommu;
1326                 }
1327                 statep->aioms_iommu_end = iommu;
1328 
1329                 statep->aioms_nunits++;
1330         }
1331 
1332         pci_config_teardown(&handle);
1333 
1334         if (amd_iommu_debug) {
1335                 cmn_err(CE_NOTE, "%s: %s%d: state=%p: setup %d IOMMU units",
1336                     f, driver, instance, (void *)statep, statep->aioms_nunits);
1337         }
1338 
1339         return (DDI_SUCCESS);
1340 }
1341 
1342 int
1343 amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep, int type)
1344 {
1345         int instance = ddi_get_instance(dip);
1346         const char *driver = ddi_driver_name(dip);
1347         amd_iommu_t *iommu, *next_iommu;
1348         int teardown;
1349         int error = DDI_SUCCESS;
1350         const char *f = "amd_iommu_teardown";
1351 
1352         teardown = 0;
1353         for (iommu = statep->aioms_iommu_start; iommu;
1354             iommu = next_iommu) {
1355                 ASSERT(statep->aioms_nunits > 0);
1356                 next_iommu = iommu->aiomt_next;
1357                 if (amd_iommu_fini(iommu, type) != DDI_SUCCESS) {
1358                         error = DDI_FAILURE;
1359                         continue;
1360                 }
1361                 statep->aioms_nunits--;
1362                 teardown++;
1363         }
1364 
1365         cmn_err(CE_NOTE, "%s: %s%d: state=%p: toredown %d units. "
1366             "%d units left", f, driver, instance, (void *)statep,
1367             teardown, statep->aioms_nunits);
1368 
1369         return (error);
1370 }
1371 
1372 dev_info_t *
1373 amd_iommu_pci_dip(dev_info_t *rdip, const char *path)
1374 {
1375         dev_info_t *pdip;
1376         const char *driver = ddi_driver_name(rdip);
1377         int instance = ddi_get_instance(rdip);
1378         const char *f = "amd_iommu_pci_dip";
1379 
1380         /* Hold rdip so it and its parents don't go away */
1381         ndi_hold_devi(rdip);
1382 
1383         if (ddi_is_pci_dip(rdip))
1384                 return (rdip);
1385 
1386         pdip = rdip;
1387         while (pdip = ddi_get_parent(pdip)) {
1388                 if (ddi_is_pci_dip(pdip)) {
1389                         ndi_hold_devi(pdip);
1390                         ndi_rele_devi(rdip);
1391                         return (pdip);
1392                 }
1393         }
1394 
1395         cmn_err(
1396 #ifdef  DEBUG
1397             CE_PANIC,
1398 #else
1399             CE_WARN,
1400 #endif  /* DEBUG */
1401             "%s: %s%d dip = %p has no PCI parent, path = %s",
1402             f, driver, instance, (void *)rdip, path);
1403 
1404         ndi_rele_devi(rdip);
1405 
1406         return (NULL);
1407 }
1408 
1409 /* Interface with IOMMULIB */
1410 /*ARGSUSED*/
1411 static int
1412 amd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip)
1413 {
1414         const char *driver = ddi_driver_name(rdip);
1415         char *s;
1416         int bus, device, func, bdf;
1417         amd_iommu_acpi_ivhd_t *hinfop;
1418         dev_info_t *pci_dip;
1419         amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1420         const char *f = "amd_iommu_probe";
1421         int instance = ddi_get_instance(iommu->aiomt_dip);
1422         const char *idriver = ddi_driver_name(iommu->aiomt_dip);
1423         char *path, *pathp;
1424 
1425         if (amd_iommu_disable_list) {
1426                 s = strstr(amd_iommu_disable_list, driver);
1427                 if (s == NULL)
1428                         return (DDI_SUCCESS);
1429                 if (s == amd_iommu_disable_list || *(s - 1) == ':') {
1430                         s += strlen(driver);
1431                         if (*s == '\0' || *s == ':') {
1432                                 amd_iommu_set_passthru(iommu, rdip);
1433                                 return (DDI_FAILURE);
1434                         }
1435                 }
1436         }
1437 
1438         path = kmem_alloc(MAXPATHLEN, KM_SLEEP);
1439         if ((pathp = ddi_pathname(rdip, path)) == NULL)
1440                 pathp = "<unknown>";
1441 
1442         pci_dip = amd_iommu_pci_dip(rdip, path);
1443         if (pci_dip == NULL) {
1444                 cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip "
1445                     "for rdip=%p, path = %s",
1446                     f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
1447                     pathp);
1448                 kmem_free(path, MAXPATHLEN);
1449                 return (DDI_FAILURE);
1450         }
1451 
1452         if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) {
1453                 cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get BDF "
1454                     "for rdip=%p, path = %s",
1455                     f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
1456                     pathp);
1457                 kmem_free(path, MAXPATHLEN);
1458                 return (DDI_FAILURE);
1459         }
1460         kmem_free(path, MAXPATHLEN);
1461 
1462         /*
1463          * See whether device is described by IVRS as being managed
1464          * by this IOMMU
1465          */
1466         bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func;
1467         hinfop = amd_iommu_lookup_ivhd(bdf);
1468         if (hinfop && hinfop->ach_IOMMU_deviceid == iommu->aiomt_bdf)
1469                 return (DDI_SUCCESS);
1470 
1471         return (DDI_FAILURE);
1472 }
1473 
1474 /*ARGSUSED*/
1475 static int
1476 amd_iommu_allochdl(iommulib_handle_t handle,
1477     dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1478     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep)
1479 {
1480         return (iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp,
1481             arg, dma_handlep));
1482 }
1483 
1484 /*ARGSUSED*/
1485 static int
1486 amd_iommu_freehdl(iommulib_handle_t handle,
1487     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
1488 {
1489         return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle));
1490 }
1491 
1492 /*ARGSUSED*/
1493 static int
1494 map_current_window(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
1495     struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookie_array, uint_t ccount,
1496     int km_flags)
1497 {
1498         const char *driver = ddi_driver_name(iommu->aiomt_dip);
1499         int instance = ddi_get_instance(iommu->aiomt_dip);
1500         int idx = iommu->aiomt_idx;
1501         int i;
1502         uint64_t start_va;
1503         char *path;
1504         int error = DDI_FAILURE;
1505         const char *f = "map_current_window";
1506 
1507         path = kmem_alloc(MAXPATHLEN, km_flags);
1508         if (path == NULL) {
1509                 return (DDI_DMA_NORESOURCES);
1510         }
1511 
1512         (void) ddi_pathname(rdip, path);
1513         mutex_enter(&amd_iommu_pgtable_lock);
1514 
1515         if (amd_iommu_debug == AMD_IOMMU_DEBUG_PAGE_TABLES) {
1516                 cmn_err(CE_NOTE, "%s: %s%d: idx=%d Attempting to get cookies "
1517                     "from handle for device %s",
1518                     f, driver, instance, idx, path);
1519         }
1520 
1521         start_va = 0;
1522         for (i = 0; i < ccount; i++) {
1523                 if ((error = amd_iommu_map_pa2va(iommu, rdip, attrp, dmareq,
1524                     cookie_array[i].dmac_cookie_addr,
1525                     cookie_array[i].dmac_size,
1526                     AMD_IOMMU_VMEM_MAP, &start_va, km_flags)) != DDI_SUCCESS) {
1527                         break;
1528                 }
1529                 cookie_array[i].dmac_cookie_addr = (uintptr_t)start_va;
1530                 cookie_array[i].dmac_type = 0;
1531         }
1532 
1533         if (i != ccount) {
1534                 cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot map cookie# %d "
1535                     "for device %s", f, driver, instance, idx, i, path);
1536                 (void) unmap_current_window(iommu, rdip, cookie_array,
1537                     ccount, i, 1);
1538                 goto out;
1539         }
1540 
1541         if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1542                 cmn_err(CE_NOTE, "%s: return SUCCESS", f);
1543         }
1544 
1545         error = DDI_DMA_MAPPED;
1546 out:
1547         mutex_exit(&amd_iommu_pgtable_lock);
1548         kmem_free(path, MAXPATHLEN);
1549         return (error);
1550 }
1551 
1552 /*ARGSUSED*/
1553 static int
1554 unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
1555     ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked)
1556 {
1557         const char *driver = ddi_driver_name(iommu->aiomt_dip);
1558         int instance = ddi_get_instance(iommu->aiomt_dip);
1559         int idx = iommu->aiomt_idx;
1560         int i;
1561         int error = DDI_FAILURE;
1562         char *path;
1563         int pathfree;
1564         const char *f = "unmap_current_window";
1565 
1566         if (!locked)
1567                 mutex_enter(&amd_iommu_pgtable_lock);
1568 
1569         path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
1570         if (path) {
1571                 (void) ddi_pathname(rdip, path);
1572                 pathfree = 1;
1573         } else {
1574                 path = "<path-mem-alloc-failed>";
1575                 pathfree = 0;
1576         }
1577 
1578         if (ncookies == -1)
1579                 ncookies = ccount;
1580 
1581         for (i = 0; i < ncookies; i++) {
1582                 if (amd_iommu_unmap_va(iommu, rdip,
1583                     cookie_array[i].dmac_cookie_addr,
1584                     cookie_array[i].dmac_size,
1585                     AMD_IOMMU_VMEM_MAP) != DDI_SUCCESS) {
1586                         break;
1587                 }
1588         }
1589 
1590         if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_COMPL_WAIT, NULL, 0, 0)
1591             != DDI_SUCCESS) {
1592                 cmn_err(CE_WARN, "%s: AMD IOMMU completion wait failed for: %s",
1593                     f, path);
1594         }
1595 
1596         if (i != ncookies) {
1597                 cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot unmap cookie# %d "
1598                     "for device %s", f, driver, instance, idx, i, path);
1599                 error = DDI_FAILURE;
1600                 goto out;
1601         }
1602 
1603         error = DDI_SUCCESS;
1604 
1605 out:
1606         if (pathfree)
1607                 kmem_free(path, MAXPATHLEN);
1608         if (!locked)
1609                 mutex_exit(&amd_iommu_pgtable_lock);
1610         return (error);
1611 }
1612 
1613 /*ARGSUSED*/
1614 static int
1615 amd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
1616     dev_info_t *rdip, ddi_dma_handle_t dma_handle,
1617     struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep,
1618     uint_t *ccountp)
1619 {
1620         int dma_error = DDI_DMA_NOMAPPING;
1621         int error;
1622         char *path;
1623         ddi_dma_cookie_t *cookie_array = NULL;
1624         uint_t ccount = 0;
1625         ddi_dma_impl_t *hp;
1626         ddi_dma_attr_t *attrp;
1627         int km_flags;
1628         amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1629         int instance = ddi_get_instance(rdip);
1630         const char *driver = ddi_driver_name(rdip);
1631         const char *f = "amd_iommu_bindhdl";
1632 
1633         dma_error = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle,
1634             dmareq, cookiep, ccountp);
1635 
1636         if (dma_error != DDI_DMA_MAPPED && dma_error != DDI_DMA_PARTIAL_MAP)
1637                 return (dma_error);
1638 
1639         km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle);
1640 
1641         path = kmem_alloc(MAXPATHLEN, km_flags);
1642         if (path) {
1643                 (void) ddi_pathname(rdip, path);
1644         } else {
1645                 dma_error = DDI_DMA_NORESOURCES;
1646                 goto unbind;
1647         }
1648 
1649         if (amd_iommu_debug & AMD_IOMMU_DEBUG_BIND) {
1650                 cmn_err(CE_NOTE, "%s: %s got cookie (%p), #cookies: %d",
1651                     f, path,
1652                     (void *)cookiep->dmac_cookie_addr,
1653                     *ccountp);
1654         }
1655 
1656         cookie_array = NULL;
1657         ccount = 0;
1658         if ((error = iommulib_iommu_dma_get_cookies(dip, dma_handle,
1659             &cookie_array, &ccount)) != DDI_SUCCESS) {
1660                 cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
1661                     "for device %s", f, driver, instance, path);
1662                 dma_error = error;
1663                 goto unbind;
1664         }
1665 
1666         hp = (ddi_dma_impl_t *)dma_handle;
1667         attrp = &hp->dmai_attr;
1668 
1669         error = map_current_window(iommu, rdip, attrp, dmareq,
1670             cookie_array, ccount, km_flags);
1671         if (error != DDI_SUCCESS) {
1672                 dma_error = error;
1673                 goto unbind;
1674         }
1675 
1676         if ((error = iommulib_iommu_dma_set_cookies(dip, dma_handle,
1677             cookie_array, ccount)) != DDI_SUCCESS) {
1678                 cmn_err(CE_WARN, "%s: %s%d: Cannot set cookies "
1679                     "for device %s", f, driver, instance, path);
1680                 dma_error = error;
1681                 goto unbind;
1682         }
1683 
1684         *cookiep = cookie_array[0];
1685 
1686         if (amd_iommu_debug & AMD_IOMMU_DEBUG_BIND) {
1687                 cmn_err(CE_NOTE, "%s: %s remapped cookie (%p), #cookies: %d",
1688                     f, path,
1689                     (void *)(uintptr_t)cookiep->dmac_cookie_addr,
1690                     *ccountp);
1691         }
1692 
1693         kmem_free(path, MAXPATHLEN);
1694         ASSERT(dma_error == DDI_DMA_MAPPED || dma_error == DDI_DMA_PARTIAL_MAP);
1695         return (dma_error);
1696 unbind:
1697         kmem_free(path, MAXPATHLEN);
1698         (void) iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle);
1699         return (dma_error);
1700 }
1701 
1702 /*ARGSUSED*/
1703 static int
1704 amd_iommu_unbindhdl(iommulib_handle_t handle,
1705     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
1706 {
1707         amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1708         ddi_dma_cookie_t *cookie_array = NULL;
1709         uint_t ccount = 0;
1710         int error = DDI_FAILURE;
1711         int instance = ddi_get_instance(rdip);
1712         const char *driver = ddi_driver_name(rdip);
1713         const char *f = "amd_iommu_unbindhdl";
1714 
1715         cookie_array = NULL;
1716         ccount = 0;
1717         if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
1718             &ccount) != DDI_SUCCESS) {
1719                 cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
1720                     "for device %p", f, driver, instance, (void *)rdip);
1721                 error = DDI_FAILURE;
1722                 goto out;
1723         }
1724 
1725         if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
1726                 cmn_err(CE_WARN, "%s: %s%d: Cannot clear cookies "
1727                     "for device %p", f, driver, instance, (void *)rdip);
1728                 error = DDI_FAILURE;
1729                 goto out;
1730         }
1731 
1732         if (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle)
1733             != DDI_SUCCESS) {
1734                 cmn_err(CE_WARN, "%s: %s%d: failed to unbindhdl for dip=%p",
1735                     f, driver, instance, (void *)rdip);
1736                 error = DDI_FAILURE;
1737                 goto out;
1738         }
1739 
1740         if (unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0)
1741             != DDI_SUCCESS) {
1742                 cmn_err(CE_WARN, "%s: %s%d: failed to unmap current window "
1743                     "for dip=%p", f, driver, instance, (void *)rdip);
1744                 error = DDI_FAILURE;
1745         } else {
1746                 error = DDI_SUCCESS;
1747         }
1748 out:
1749         if (cookie_array)
1750                 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
1751         return (error);
1752 }
1753 
1754 /*ARGSUSED*/
1755 static int
1756 amd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip,
1757     dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
1758     size_t len, uint_t cache_flags)
1759 {
1760         ddi_dma_cookie_t *cookie_array = NULL;
1761         uint_t ccount = 0;
1762         int error;
1763         const char *f = "amd_iommu_sync";
1764 
1765         cookie_array = NULL;
1766         ccount = 0;
1767         if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
1768             &ccount) != DDI_SUCCESS) {
1769                 ASSERT(cookie_array == NULL);
1770                 cmn_err(CE_WARN, "%s: Cannot get cookies "
1771                     "for device %p", f, (void *)rdip);
1772                 error = DDI_FAILURE;
1773                 goto out;
1774         }
1775 
1776         if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
1777                 cmn_err(CE_WARN, "%s: Cannot clear cookies "
1778                     "for device %p", f, (void *)rdip);
1779                 error = DDI_FAILURE;
1780                 goto out;
1781         }
1782 
1783         error = iommulib_iommu_dma_sync(dip, rdip, dma_handle, off,
1784             len, cache_flags);
1785 
1786         if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array,
1787             ccount) != DDI_SUCCESS) {
1788                 cmn_err(CE_WARN, "%s: Cannot set cookies "
1789                     "for device %p", f, (void *)rdip);
1790                 error = DDI_FAILURE;
1791         } else {
1792                 cookie_array = NULL;
1793                 ccount = 0;
1794         }
1795 
1796 out:
1797         if (cookie_array)
1798                 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
1799         return (error);
1800 }
1801 
1802 /*ARGSUSED*/
1803 static int
1804 amd_iommu_win(iommulib_handle_t handle, dev_info_t *dip,
1805     dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
1806     off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
1807     uint_t *ccountp)
1808 {
1809         int error = DDI_FAILURE;
1810         amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1811         ddi_dma_cookie_t *cookie_array = NULL;
1812         uint_t ccount = 0;
1813         int km_flags;
1814         ddi_dma_impl_t *hp;
1815         ddi_dma_attr_t *attrp;
1816         struct ddi_dma_req sdmareq = {0};
1817         int instance = ddi_get_instance(rdip);
1818         const char *driver = ddi_driver_name(rdip);
1819         const char *f = "amd_iommu_win";
1820 
1821         km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle);
1822 
1823         cookie_array = NULL;
1824         ccount = 0;
1825         if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
1826             &ccount) != DDI_SUCCESS) {
1827                 cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
1828                     "for device %p", f, driver, instance, (void *)rdip);
1829                 error = DDI_FAILURE;
1830                 goto out;
1831         }
1832 
1833         if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
1834                 cmn_err(CE_WARN, "%s: %s%d: Cannot clear cookies "
1835                     "for device %p", f, driver, instance, (void *)rdip);
1836                 error = DDI_FAILURE;
1837                 goto out;
1838         }
1839 
1840         if (iommulib_iommu_dma_win(dip, rdip, dma_handle, win,
1841             offp, lenp, cookiep, ccountp) != DDI_SUCCESS) {
1842                 cmn_err(CE_WARN, "%s: %s%d: failed switch windows for dip=%p",
1843                     f, driver, instance, (void *)rdip);
1844                 error = DDI_FAILURE;
1845                 goto out;
1846         }
1847 
1848         (void) unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0);
1849 
1850         if (cookie_array) {
1851                 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
1852                 cookie_array = NULL;
1853                 ccount = 0;
1854         }
1855 
1856         cookie_array = NULL;
1857         ccount = 0;
1858         if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
1859             &ccount) != DDI_SUCCESS) {
1860                 cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
1861                     "for device %p", f, driver, instance, (void *)rdip);
1862                 error = DDI_FAILURE;
1863                 goto out;
1864         }
1865 
1866         hp = (ddi_dma_impl_t *)dma_handle;
1867         attrp = &hp->dmai_attr;
1868 
1869         sdmareq.dmar_flags = DDI_DMA_RDWR;
1870         error = map_current_window(iommu, rdip, attrp, &sdmareq,
1871             cookie_array, ccount, km_flags);
1872 
1873         if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array,
1874             ccount) != DDI_SUCCESS) {
1875                 cmn_err(CE_WARN, "%s: %s%d: Cannot set cookies "
1876                     "for device %p", f, driver, instance, (void *)rdip);
1877                 error = DDI_FAILURE;
1878                 goto out;
1879         }
1880 
1881         *cookiep = cookie_array[0];
1882 
1883         return (error == DDI_SUCCESS ? DDI_SUCCESS : DDI_FAILURE);
1884 out:
1885         if (cookie_array)
1886                 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
1887 
1888         return (error);
1889 }
1890 
1891 /*ARGSUSED*/
1892 static int
1893 amd_iommu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
1894     dev_info_t *rdip, ddi_dma_handle_t dma_handle,
1895     struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao)
1896 {
1897         return (DDI_ENOTSUP);
1898 }
1899 
1900 /*ARGSUSED*/
1901 static int
1902 amd_iommu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
1903     dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao)
1904 {
1905         return (DDI_ENOTSUP);
1906 }
1907 
1908 uint64_t
1909 amd_iommu_reg_get64_workaround(uint64_t *regp, uint32_t bits)
1910 {
1911         split_t s;
1912         uint32_t *ptr32 = (uint32_t *)regp;
1913         uint64_t *s64p = &(s.u64);
1914 
1915         s.u32[0] = ptr32[0];
1916         s.u32[1] = ptr32[1];
1917 
1918         return (AMD_IOMMU_REG_GET64_IMPL(s64p, bits));
1919 }
1920 
1921 uint64_t
1922 amd_iommu_reg_set64_workaround(uint64_t *regp, uint32_t bits, uint64_t value)
1923 {
1924         split_t s;
1925         uint32_t *ptr32 = (uint32_t *)regp;
1926         uint64_t *s64p = &(s.u64);
1927 
1928         s.u32[0] = ptr32[0];
1929         s.u32[1] = ptr32[1];
1930 
1931         AMD_IOMMU_REG_SET64_IMPL(s64p, bits, value);
1932 
1933         *regp = s.u64;
1934 
1935         return (s.u64);
1936 }
1937 
1938 void
1939 amd_iommu_read_boot_props(void)
1940 {
1941         char *propval;
1942 
1943         /*
1944          * if "amd-iommu = no/false" boot property is set,
1945          * ignore AMD iommu
1946          */
1947         if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
1948             DDI_PROP_DONTPASS, "amd-iommu", &propval) == DDI_SUCCESS) {
1949                 if (strcmp(propval, "no") == 0 ||
1950                     strcmp(propval, "false") == 0) {
1951                         amd_iommu_disable = 1;
1952                 }
1953                 ddi_prop_free(propval);
1954         }
1955 
1956         /*
1957          * Copy the list of drivers for which IOMMU is disabled by user.
1958          */
1959         if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
1960             DDI_PROP_DONTPASS, "amd-iommu-disable-list", &propval)
1961             == DDI_SUCCESS) {
1962                 amd_iommu_disable_list = kmem_alloc(strlen(propval) + 1,
1963                     KM_SLEEP);
1964                 (void) strcpy(amd_iommu_disable_list, propval);
1965                 ddi_prop_free(propval);
1966         }
1967 
1968 }
1969 
1970 void
1971 amd_iommu_lookup_conf_props(dev_info_t *dip)
1972 {
1973         char *disable;
1974 
1975         if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip,
1976             DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu", &disable)
1977             == DDI_PROP_SUCCESS) {
1978                 if (strcmp(disable, "no") == 0) {
1979                         amd_iommu_disable = 1;
1980                 }
1981                 ddi_prop_free(disable);
1982         }
1983 
1984         if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip,
1985             DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu-disable-list",
1986             &disable) == DDI_PROP_SUCCESS) {
1987                 amd_iommu_disable_list = kmem_alloc(strlen(disable) + 1,
1988                     KM_SLEEP);
1989                 (void) strcpy(amd_iommu_disable_list, disable);
1990                 ddi_prop_free(disable);
1991         }
1992 }