1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
25 */
26
27 #include <sys/sunddi.h>
28 #include <sys/sunndi.h>
29 #include <sys/iommulib.h>
30 #include <sys/amd_iommu.h>
31 #include <sys/pci_cap.h>
32 #include <sys/bootconf.h>
33 #include <sys/ddidmareq.h>
34
35 #include "amd_iommu_impl.h"
36 #include "amd_iommu_acpi.h"
37 #include "amd_iommu_page_tables.h"
38
39 static int amd_iommu_fini(amd_iommu_t *iommu, int type);
40 static void amd_iommu_teardown_interrupts(amd_iommu_t *iommu);
41 static void amd_iommu_stop(amd_iommu_t *iommu);
42
43 static int amd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip);
44 static int amd_iommu_allochdl(iommulib_handle_t handle,
45 dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
46 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep);
47 static int amd_iommu_freehdl(iommulib_handle_t handle,
48 dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
49 static int amd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
50 dev_info_t *rdip, ddi_dma_handle_t dma_handle,
51 struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep,
52 uint_t *ccountp);
53 static int amd_iommu_unbindhdl(iommulib_handle_t handle,
54 dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
55 static int amd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip,
56 dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
57 size_t len, uint_t cache_flags);
58 static int amd_iommu_win(iommulib_handle_t handle, dev_info_t *dip,
59 dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
60 off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
61 uint_t *ccountp);
62 static int amd_iommu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
63 dev_info_t *rdip, ddi_dma_handle_t dma_handle,
64 struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao);
65 static int amd_iommu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
66 dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao);
67
68 static int unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
69 ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked);
70
71 extern void *device_arena_alloc(size_t size, int vm_flag);
72 extern void device_arena_free(void * vaddr, size_t size);
73
74 ddi_dma_attr_t amd_iommu_dma_attr = {
75 DMA_ATTR_V0,
76 0U, /* dma_attr_addr_lo */
77 0xffffffffffffffffULL, /* dma_attr_addr_hi */
78 0xffffffffU, /* dma_attr_count_max */
79 (uint64_t)4096, /* dma_attr_align */
80 1, /* dma_attr_burstsizes */
81 64, /* dma_attr_minxfer */
82 0xffffffffU, /* dma_attr_maxxfer */
83 0xffffffffU, /* dma_attr_seg */
84 1, /* dma_attr_sgllen, variable */
85 64, /* dma_attr_granular */
86 0 /* dma_attr_flags */
87 };
88
89 ddi_device_acc_attr_t amd_iommu_devacc = {
90 DDI_DEVICE_ATTR_V0,
91 DDI_NEVERSWAP_ACC,
92 DDI_STRICTORDER_ACC
93 };
94
95 struct iommulib_ops amd_iommulib_ops = {
96 IOMMU_OPS_VERSION,
97 AMD_IOMMU,
98 "AMD IOMMU Vers. 1",
99 NULL,
100 amd_iommu_probe,
101 amd_iommu_allochdl,
102 amd_iommu_freehdl,
103 amd_iommu_bindhdl,
104 amd_iommu_unbindhdl,
105 amd_iommu_sync,
106 amd_iommu_win,
107 amd_iommu_mapobject,
108 amd_iommu_unmapobject,
109 };
110
111 static kmutex_t amd_iommu_pgtable_lock;
112
113 static int
114 amd_iommu_register(amd_iommu_t *iommu)
115 {
116 dev_info_t *dip = iommu->aiomt_dip;
117 const char *driver = ddi_driver_name(dip);
118 int instance = ddi_get_instance(dip);
119 iommulib_ops_t *iommulib_ops;
120 iommulib_handle_t handle;
121 const char *f = "amd_iommu_register";
122
123 iommulib_ops = kmem_zalloc(sizeof (iommulib_ops_t), KM_SLEEP);
124
125 *iommulib_ops = amd_iommulib_ops;
126
127 iommulib_ops->ilops_data = (void *)iommu;
128 iommu->aiomt_iommulib_ops = iommulib_ops;
129
130 if (iommulib_iommu_register(dip, iommulib_ops, &handle)
131 != DDI_SUCCESS) {
132 cmn_err(CE_WARN, "%s: %s%d: Register with iommulib "
133 "failed idx=%d", f, driver, instance, iommu->aiomt_idx);
134 kmem_free(iommulib_ops, sizeof (iommulib_ops_t));
135 return (DDI_FAILURE);
136 }
137
138 iommu->aiomt_iommulib_handle = handle;
139
140 return (DDI_SUCCESS);
141 }
142
143 static int
144 amd_iommu_unregister(amd_iommu_t *iommu)
145 {
146 if (iommu->aiomt_iommulib_handle == NULL) {
147 /* we never registered */
148 return (DDI_SUCCESS);
149 }
150
151 if (iommulib_iommu_unregister(iommu->aiomt_iommulib_handle)
152 != DDI_SUCCESS) {
153 return (DDI_FAILURE);
154 }
155
156 kmem_free(iommu->aiomt_iommulib_ops, sizeof (iommulib_ops_t));
157 iommu->aiomt_iommulib_ops = NULL;
158 iommu->aiomt_iommulib_handle = NULL;
159
160 return (DDI_SUCCESS);
161 }
162
163 static int
164 amd_iommu_setup_passthru(amd_iommu_t *iommu)
165 {
166 gfx_entry_t *gfxp;
167 dev_info_t *dip;
168
169 /*
170 * Setup passthru mapping for "special" devices
171 */
172 amd_iommu_set_passthru(iommu, NULL);
173
174 for (gfxp = gfx_devinfo_list; gfxp; gfxp = gfxp->g_next) {
175 gfxp->g_ref++;
176 dip = gfxp->g_dip;
177 if (dip) {
178 amd_iommu_set_passthru(iommu, dip);
179 }
180 gfxp->g_ref--;
181 }
182
183 return (DDI_SUCCESS);
184 }
185
186 static int
187 amd_iommu_start(amd_iommu_t *iommu)
188 {
189 dev_info_t *dip = iommu->aiomt_dip;
190 int instance = ddi_get_instance(dip);
191 const char *driver = ddi_driver_name(dip);
192 amd_iommu_acpi_ivhd_t *hinfop;
193 const char *f = "amd_iommu_start";
194
195 hinfop = amd_iommu_lookup_all_ivhd();
196
197 /*
198 * Disable HT tunnel translation.
199 * XXX use ACPI
200 */
201 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
202 AMD_IOMMU_HT_TUN_ENABLE, 0);
203
204 if (hinfop) {
205 if (amd_iommu_debug) {
206 cmn_err(CE_NOTE,
207 "amd_iommu: using ACPI for CTRL registers");
208 }
209 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
210 AMD_IOMMU_ISOC, hinfop->ach_Isoc);
211 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
212 AMD_IOMMU_RESPASSPW, hinfop->ach_ResPassPW);
213 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
214 AMD_IOMMU_PASSPW, hinfop->ach_PassPW);
215 }
216
217 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
218 AMD_IOMMU_INVTO, 5);
219
220
221 /*
222 * The Device table entry bit 0 (V) controls whether the device
223 * table entry is valid for address translation and Device table
224 * entry bit 128 (IV) controls whether interrupt remapping is valid.
225 * By setting both to zero we are essentially doing pass-thru. Since
226 * this table is zeroed on allocation, essentially we will have
227 * pass-thru when IOMMU is enabled.
228 */
229
230 /* Finally enable the IOMMU ... */
231 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
232 AMD_IOMMU_ENABLE, 1);
233
234 if (amd_iommu_debug) {
235 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
236 "Successfully started AMD IOMMU", f, driver, instance,
237 iommu->aiomt_idx);
238 }
239 cmn_err(CE_NOTE, "AMD IOMMU (%d,%d) enabled",
240 instance, iommu->aiomt_idx);
241
242 return (DDI_SUCCESS);
243 }
244
245 static void
246 amd_iommu_stop(amd_iommu_t *iommu)
247 {
248 dev_info_t *dip = iommu->aiomt_dip;
249 int instance = ddi_get_instance(dip);
250 const char *driver = ddi_driver_name(dip);
251 const char *f = "amd_iommu_stop";
252
253 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
254 AMD_IOMMU_ENABLE, 0);
255 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
256 AMD_IOMMU_EVENTINT_ENABLE, 0);
257 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
258 AMD_IOMMU_COMWAITINT_ENABLE, 0);
259 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
260 AMD_IOMMU_EVENTLOG_ENABLE, 0);
261
262 /*
263 * Disable translation on HT tunnel traffic
264 */
265 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
266 AMD_IOMMU_HT_TUN_ENABLE, 0);
267 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
268 AMD_IOMMU_CMDBUF_ENABLE, 0);
269
270 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMYU idx=%d. "
271 "Successfully stopped AMD IOMMU", f, driver, instance,
272 iommu->aiomt_idx);
273 }
274
275 static int
276 amd_iommu_setup_tables_and_buffers(amd_iommu_t *iommu)
277 {
278 dev_info_t *dip = iommu->aiomt_dip;
279 int instance = ddi_get_instance(dip);
280 const char *driver = ddi_driver_name(dip);
281 uint32_t dma_bufsz;
282 caddr_t addr;
283 uint32_t sz;
284 uint32_t p2sz;
285 int i;
286 uint64_t *dentry;
287 int err;
288 const char *f = "amd_iommu_setup_tables_and_buffers";
289
290 /*
291 * We will put the Device Table, Command Buffer and
292 * Event Log in contiguous memory. Allocate the maximum
293 * size allowed for such structures
294 * Device Table: 256b * 64K = 32B * 64K
295 * Command Buffer: 128b * 32K = 16B * 32K
296 * Event Log: 128b * 32K = 16B * 32K
297 */
298 iommu->aiomt_devtbl_sz = (1<<AMD_IOMMU_DEVTBL_SZ) * AMD_IOMMU_DEVENT_SZ;
299 iommu->aiomt_cmdbuf_sz = (1<<AMD_IOMMU_CMDBUF_SZ) * AMD_IOMMU_CMD_SZ;
300 iommu->aiomt_eventlog_sz =
301 (1<<AMD_IOMMU_EVENTLOG_SZ) * AMD_IOMMU_EVENT_SZ;
302
303 dma_bufsz = iommu->aiomt_devtbl_sz + iommu->aiomt_cmdbuf_sz
304 + iommu->aiomt_eventlog_sz;
305
306 /*
307 * Alloc a DMA handle.
308 */
309 err = ddi_dma_alloc_handle(dip, &amd_iommu_dma_attr,
310 DDI_DMA_SLEEP, NULL, &iommu->aiomt_dmahdl);
311 if (err != DDI_SUCCESS) {
312 cmn_err(CE_WARN, "%s: %s%d: Cannot alloc DMA handle for "
313 "AMD IOMMU tables and buffers", f, driver, instance);
314 return (DDI_FAILURE);
315 }
316
317 /*
318 * Alloc memory for tables and buffers
319 * XXX remove cast to size_t
320 */
321 err = ddi_dma_mem_alloc(iommu->aiomt_dmahdl, dma_bufsz,
322 &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,
323 DDI_DMA_SLEEP, NULL, (caddr_t *)&iommu->aiomt_dma_bufva,
324 (size_t *)&iommu->aiomt_dma_mem_realsz, &iommu->aiomt_dma_mem_hdl);
325 if (err != DDI_SUCCESS) {
326 cmn_err(CE_WARN, "%s: %s%d: Cannot alloc memory for DMA "
327 "to AMD IOMMU tables and buffers", f, driver, instance);
328 iommu->aiomt_dma_bufva = NULL;
329 iommu->aiomt_dma_mem_realsz = 0;
330 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
331 iommu->aiomt_dmahdl = NULL;
332 return (DDI_FAILURE);
333 }
334
335 /*
336 * The VA must be 4K aligned and >= table size
337 */
338 ASSERT(((uintptr_t)iommu->aiomt_dma_bufva &
339 AMD_IOMMU_TABLE_ALIGN) == 0);
340 ASSERT(iommu->aiomt_dma_mem_realsz >= dma_bufsz);
341
342 /*
343 * Now bind the handle
344 */
345 err = ddi_dma_addr_bind_handle(iommu->aiomt_dmahdl, NULL,
346 iommu->aiomt_dma_bufva, iommu->aiomt_dma_mem_realsz,
347 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP,
348 NULL, &iommu->aiomt_buf_dma_cookie, &iommu->aiomt_buf_dma_ncookie);
349 if (err != DDI_DMA_MAPPED) {
350 cmn_err(CE_WARN, "%s: %s%d: Cannot bind memory for DMA "
351 "to AMD IOMMU tables and buffers. bufrealsz=%p",
352 f, driver, instance,
353 (void *)(uintptr_t)iommu->aiomt_dma_mem_realsz);
354 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
355 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
356 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
357 iommu->aiomt_buf_dma_ncookie = 0;
358 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
359 iommu->aiomt_dma_mem_hdl = NULL;
360 iommu->aiomt_dma_bufva = NULL;
361 iommu->aiomt_dma_mem_realsz = 0;
362 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
363 iommu->aiomt_dmahdl = NULL;
364 return (DDI_FAILURE);
365 }
366
367 /*
368 * We assume the DMA engine on the IOMMU is capable of handling the
369 * whole table buffer in a single cookie. If not and multiple cookies
370 * are needed we fail.
371 */
372 if (iommu->aiomt_buf_dma_ncookie != 1) {
373 cmn_err(CE_WARN, "%s: %s%d: Cannot handle multiple "
374 "cookies for DMA to AMD IOMMU tables and buffers. "
375 "#cookies=%u", f, driver, instance,
376 iommu->aiomt_buf_dma_ncookie);
377 (void) ddi_dma_unbind_handle(iommu->aiomt_dmahdl);
378 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
379 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
380 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
381 iommu->aiomt_buf_dma_ncookie = 0;
382 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
383 iommu->aiomt_dma_mem_hdl = NULL;
384 iommu->aiomt_dma_bufva = NULL;
385 iommu->aiomt_dma_mem_realsz = 0;
386 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
387 iommu->aiomt_dmahdl = NULL;
388 return (DDI_FAILURE);
389 }
390
391 /*
392 * The address in the cookie must be 4K aligned and >= table size
393 */
394 ASSERT((iommu->aiomt_buf_dma_cookie.dmac_cookie_addr
395 & AMD_IOMMU_TABLE_ALIGN) == 0);
396 ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size
397 <= iommu->aiomt_dma_mem_realsz);
398 ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size >= dma_bufsz);
399
400 /*
401 * Setup the device table pointers in the iommu struct as
402 * well as the IOMMU device table register
403 */
404 iommu->aiomt_devtbl = iommu->aiomt_dma_bufva;
405 bzero(iommu->aiomt_devtbl, iommu->aiomt_devtbl_sz);
406
407 /*
408 * Set V=1 and TV = 0, so any inadvertant pass-thrus cause
409 * page faults. Also set SE bit so we aren't swamped with
410 * page fault messages
411 */
412 for (i = 0; i <= AMD_IOMMU_MAX_DEVICEID; i++) {
413 /*LINTED*/
414 dentry = (uint64_t *)&iommu->aiomt_devtbl
415 [i * AMD_IOMMU_DEVTBL_ENTRY_SZ];
416 AMD_IOMMU_REG_SET64(dentry, AMD_IOMMU_DEVTBL_V, 1);
417 AMD_IOMMU_REG_SET64(&(dentry[1]), AMD_IOMMU_DEVTBL_SE, 1);
418 }
419
420 addr = (caddr_t)(uintptr_t)iommu->aiomt_buf_dma_cookie.dmac_cookie_addr;
421 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
422 AMD_IOMMU_DEVTABBASE, ((uint64_t)(uintptr_t)addr) >> 12);
423 sz = (iommu->aiomt_devtbl_sz >> 12) - 1;
424 ASSERT(sz <= ((1 << 9) - 1));
425 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
426 AMD_IOMMU_DEVTABSIZE, sz);
427
428 /*
429 * Setup the command buffer pointers
430 */
431 iommu->aiomt_cmdbuf = iommu->aiomt_devtbl +
432 iommu->aiomt_devtbl_sz;
433 bzero(iommu->aiomt_cmdbuf, iommu->aiomt_cmdbuf_sz);
434 addr += iommu->aiomt_devtbl_sz;
435 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
436 AMD_IOMMU_COMBASE, ((uint64_t)(uintptr_t)addr) >> 12);
437
438 p2sz = AMD_IOMMU_CMDBUF_SZ;
439 ASSERT(p2sz >= AMD_IOMMU_CMDBUF_MINSZ &&
440 p2sz <= AMD_IOMMU_CMDBUF_MAXSZ);
441 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
442 AMD_IOMMU_COMLEN, p2sz);
443 /*LINTED*/
444 iommu->aiomt_cmd_tail = (uint32_t *)iommu->aiomt_cmdbuf;
445 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
446 AMD_IOMMU_CMDHEADPTR, 0);
447 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_tail_va),
448 AMD_IOMMU_CMDTAILPTR, 0);
449
450 /*
451 * Setup the event log pointers
452 */
453 iommu->aiomt_eventlog = iommu->aiomt_cmdbuf +
454 iommu->aiomt_eventlog_sz;
455 bzero(iommu->aiomt_eventlog, iommu->aiomt_eventlog_sz);
456 addr += iommu->aiomt_cmdbuf_sz;
457 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
458 AMD_IOMMU_EVENTBASE, ((uint64_t)(uintptr_t)addr) >> 12);
459 p2sz = AMD_IOMMU_EVENTLOG_SZ;
460 ASSERT(p2sz >= AMD_IOMMU_EVENTLOG_MINSZ &&
461 p2sz <= AMD_IOMMU_EVENTLOG_MAXSZ);
462 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
463 AMD_IOMMU_EVENTLEN, sz);
464 /*LINTED*/
465 iommu->aiomt_event_head = (uint32_t *)iommu->aiomt_eventlog;
466 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
467 AMD_IOMMU_EVENTHEADPTR, 0);
468 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_tail_va),
469 AMD_IOMMU_EVENTTAILPTR, 0);
470
471 /* dma sync so device sees this init */
472 SYNC_FORDEV(iommu->aiomt_dmahdl);
473
474 if (amd_iommu_debug & AMD_IOMMU_DEBUG_TABLES) {
475 cmn_err(CE_NOTE, "%s: %s%d: successfully setup AMD IOMMU "
476 "tables, idx=%d", f, driver, instance, iommu->aiomt_idx);
477 }
478
479 return (DDI_SUCCESS);
480 }
481
482 static void
483 amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu, int type)
484 {
485 dev_info_t *dip = iommu->aiomt_dip;
486 int instance = ddi_get_instance(dip);
487 const char *driver = ddi_driver_name(dip);
488 const char *f = "amd_iommu_teardown_tables_and_buffers";
489
490 iommu->aiomt_eventlog = NULL;
491 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
492 AMD_IOMMU_EVENTBASE, 0);
493 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
494 AMD_IOMMU_EVENTLEN, 0);
495 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
496 AMD_IOMMU_EVENTHEADPTR, 0);
497 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
498 AMD_IOMMU_EVENTTAILPTR, 0);
499
500
501 iommu->aiomt_cmdbuf = NULL;
502 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
503 AMD_IOMMU_COMBASE, 0);
504 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
505 AMD_IOMMU_COMLEN, 0);
506 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
507 AMD_IOMMU_CMDHEADPTR, 0);
508 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
509 AMD_IOMMU_CMDTAILPTR, 0);
510
511
512 iommu->aiomt_devtbl = NULL;
513 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
514 AMD_IOMMU_DEVTABBASE, 0);
515 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
516 AMD_IOMMU_DEVTABSIZE, 0);
517
518 if (iommu->aiomt_dmahdl == NULL || type == AMD_IOMMU_QUIESCE)
519 return;
520
521 /* Unbind the handle */
522 if (ddi_dma_unbind_handle(iommu->aiomt_dmahdl) != DDI_SUCCESS) {
523 cmn_err(CE_WARN, "%s: %s%d: failed to unbind handle: "
524 "%p for IOMMU idx=%d", f, driver, instance,
525 (void *)iommu->aiomt_dmahdl, iommu->aiomt_idx);
526 }
527 iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
528 iommu->aiomt_buf_dma_cookie.dmac_size = 0;
529 iommu->aiomt_buf_dma_cookie.dmac_type = 0;
530 iommu->aiomt_buf_dma_ncookie = 0;
531
532 /* Free the table memory allocated for DMA */
533 ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
534 iommu->aiomt_dma_mem_hdl = NULL;
535 iommu->aiomt_dma_bufva = NULL;
536 iommu->aiomt_dma_mem_realsz = 0;
537
538 /* Free the DMA handle */
539 ddi_dma_free_handle(&iommu->aiomt_dmahdl);
540 iommu->aiomt_dmahdl = NULL;
541 }
542
543 static void
544 amd_iommu_enable_interrupts(amd_iommu_t *iommu)
545 {
546 ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
547 AMD_IOMMU_CMDBUF_RUN) == 0);
548 ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
549 AMD_IOMMU_EVENT_LOG_RUN) == 0);
550
551 /* Must be set prior to enabling command buffer */
552 /* Must be set prior to enabling event logging */
553 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
554 AMD_IOMMU_CMDBUF_ENABLE, 1);
555 /* No interrupts for completion wait - too heavy weight. use polling */
556 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
557 AMD_IOMMU_COMWAITINT_ENABLE, 0);
558 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
559 AMD_IOMMU_EVENTLOG_ENABLE, 1);
560 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
561 AMD_IOMMU_EVENTINT_ENABLE, 1);
562 }
563
564 static int
565 amd_iommu_setup_exclusion(amd_iommu_t *iommu)
566 {
567 amd_iommu_acpi_ivmd_t *minfop;
568
569 minfop = amd_iommu_lookup_all_ivmd();
570
571 if (minfop && minfop->acm_ExclRange == 1) {
572 cmn_err(CE_NOTE, "Programming exclusion range");
573 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
574 AMD_IOMMU_EXCL_BASE_ADDR,
575 minfop->acm_ivmd_phys_start >> 12);
576 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
577 AMD_IOMMU_EXCL_BASE_ALLOW, 1);
578 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
579 AMD_IOMMU_EXCL_BASE_EXEN, 1);
580 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
581 AMD_IOMMU_EXCL_LIM, (minfop->acm_ivmd_phys_start +
582 minfop->acm_ivmd_phys_len) >> 12);
583 } else {
584 if (amd_iommu_debug) {
585 cmn_err(CE_NOTE, "Skipping exclusion range");
586 }
587 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
588 AMD_IOMMU_EXCL_BASE_ADDR, 0);
589 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
590 AMD_IOMMU_EXCL_BASE_ALLOW, 1);
591 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
592 AMD_IOMMU_EXCL_BASE_EXEN, 0);
593 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
594 AMD_IOMMU_EXCL_LIM, 0);
595 }
596
597 return (DDI_SUCCESS);
598 }
599
600 static void
601 amd_iommu_teardown_exclusion(amd_iommu_t *iommu)
602 {
603 (void) amd_iommu_setup_exclusion(iommu);
604 }
605
606 static uint_t
607 amd_iommu_intr_handler(caddr_t arg1, caddr_t arg2)
608 {
609 /*LINTED*/
610 amd_iommu_t *iommu = (amd_iommu_t *)arg1;
611 dev_info_t *dip = iommu->aiomt_dip;
612 int instance = ddi_get_instance(dip);
613 const char *driver = ddi_driver_name(dip);
614 const char *f = "amd_iommu_intr_handler";
615
616 ASSERT(arg1);
617 ASSERT(arg2 == NULL);
618
619 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
620 cmn_err(CE_NOTE, "%s: %s%d: IOMMU unit idx=%d. In INTR handler",
621 f, driver, instance, iommu->aiomt_idx);
622 }
623
624 if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
625 AMD_IOMMU_EVENT_LOG_INT) == 1) {
626 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
627 cmn_err(CE_NOTE, "%s: %s%d: IOMMU unit idx=%d "
628 "Event Log Interrupt", f, driver, instance,
629 iommu->aiomt_idx);
630 }
631 (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISPLAY);
632 WAIT_SEC(1);
633 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
634 AMD_IOMMU_EVENT_LOG_INT, 1);
635 return (DDI_INTR_CLAIMED);
636 }
637
638 if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
639 AMD_IOMMU_EVENT_OVERFLOW_INT) == 1) {
640 cmn_err(CE_NOTE, "!%s: %s%d: IOMMU unit idx=%d "
641 "Event Overflow Interrupt", f, driver, instance,
642 iommu->aiomt_idx);
643 (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISCARD);
644 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
645 AMD_IOMMU_EVENT_LOG_INT, 1);
646 AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
647 AMD_IOMMU_EVENT_OVERFLOW_INT, 1);
648 return (DDI_INTR_CLAIMED);
649 }
650
651 return (DDI_INTR_UNCLAIMED);
652 }
653
654
655 static int
656 amd_iommu_setup_interrupts(amd_iommu_t *iommu)
657 {
658 dev_info_t *dip = iommu->aiomt_dip;
659 int instance = ddi_get_instance(dip);
660 const char *driver = ddi_driver_name(dip);
661 int intrcap0;
662 int intrcapN;
663 int type;
664 int err;
665 int req;
666 int avail;
667 int p2req;
668 int actual;
669 int i;
670 int j;
671 const char *f = "amd_iommu_setup_interrupts";
672
673 if (ddi_intr_get_supported_types(dip, &type) != DDI_SUCCESS) {
674 cmn_err(CE_WARN, "%s: %s%d: ddi_intr_get_supported_types "
675 "failed: idx=%d", f, driver, instance, iommu->aiomt_idx);
676 return (DDI_FAILURE);
677 }
678
679 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
680 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
681 "Interrupt types supported = 0x%x", f, driver, instance,
682 iommu->aiomt_idx, type);
683 }
684
685 /*
686 * for now we only support MSI
687 */
688 if ((type & DDI_INTR_TYPE_MSI) == 0) {
689 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d. "
690 "MSI interrupts not supported. Failing init.",
691 f, driver, instance, iommu->aiomt_idx);
692 return (DDI_FAILURE);
693 }
694
695 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
696 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. MSI supported",
697 f, driver, instance, iommu->aiomt_idx);
698 }
699
700 err = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSI, &req);
701 if (err != DDI_SUCCESS) {
702 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d. "
703 "ddi_intr_get_nintrs failed err = %d",
704 f, driver, instance, iommu->aiomt_idx, err);
705 return (DDI_FAILURE);
706 }
707
708 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
709 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
710 "MSI number of interrupts requested: %d",
711 f, driver, instance, iommu->aiomt_idx, req);
712 }
713
714 if (req == 0) {
715 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: 0 MSI "
716 "interrupts requested. Failing init", f,
717 driver, instance, iommu->aiomt_idx);
718 return (DDI_FAILURE);
719 }
720
721 err = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSI, &avail);
722 if (err != DDI_SUCCESS) {
723 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d "
724 "ddi_intr_get_navail failed err = %d", f,
725 driver, instance, iommu->aiomt_idx, err);
726 return (DDI_FAILURE);
727 }
728
729 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
730 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
731 "MSI number of interrupts available: %d",
732 f, driver, instance, iommu->aiomt_idx, avail);
733 }
734
735 if (avail == 0) {
736 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: 0 MSI "
737 "interrupts available. Failing init", f,
738 driver, instance, iommu->aiomt_idx);
739 return (DDI_FAILURE);
740 }
741
742 if (avail < req) {
743 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: MSI "
744 "interrupts: requested (%d) > available (%d). "
745 "Failing init", f, driver, instance, iommu->aiomt_idx,
746 req, avail);
747 return (DDI_FAILURE);
748 }
749
750 /* Allocate memory for DDI interrupt handles */
751 iommu->aiomt_intr_htable_sz = req * sizeof (ddi_intr_handle_t);
752 iommu->aiomt_intr_htable = kmem_zalloc(iommu->aiomt_intr_htable_sz,
753 KM_SLEEP);
754
755 iommu->aiomt_intr_state = AMD_IOMMU_INTR_TABLE;
756
757 /* Convert req to a power of two as required by ddi_intr_alloc */
758 p2req = 0;
759 while (1<<p2req <= req)
760 p2req++;
761 p2req--;
762 req = 1<<p2req;
763
764 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
765 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
766 "MSI power of 2 number of interrupts: %d,%d",
767 f, driver, instance, iommu->aiomt_idx, p2req, req);
768 }
769
770 err = ddi_intr_alloc(iommu->aiomt_dip, iommu->aiomt_intr_htable,
771 DDI_INTR_TYPE_MSI, 0, req, &actual, DDI_INTR_ALLOC_STRICT);
772 if (err != DDI_SUCCESS) {
773 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
774 "ddi_intr_alloc failed: err = %d",
775 f, driver, instance, iommu->aiomt_idx, err);
776 amd_iommu_teardown_interrupts(iommu);
777 return (DDI_FAILURE);
778 }
779
780 iommu->aiomt_actual_intrs = actual;
781 iommu->aiomt_intr_state = AMD_IOMMU_INTR_ALLOCED;
782
783 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
784 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
785 "number of interrupts actually allocated %d",
786 f, driver, instance, iommu->aiomt_idx, actual);
787 }
788
789 if (iommu->aiomt_actual_intrs < req) {
790 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
791 "ddi_intr_alloc failed: actual (%d) < req (%d)",
792 f, driver, instance, iommu->aiomt_idx,
793 iommu->aiomt_actual_intrs, req);
794 amd_iommu_teardown_interrupts(iommu);
795 return (DDI_FAILURE);
796 }
797
798 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
799 if (ddi_intr_add_handler(iommu->aiomt_intr_htable[i],
800 amd_iommu_intr_handler, (void *)iommu, NULL)
801 != DDI_SUCCESS) {
802 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
803 "ddi_intr_add_handler failed: intr = %d, err = %d",
804 f, driver, instance, iommu->aiomt_idx, i, err);
805 for (j = 0; j < i; j++) {
806 (void) ddi_intr_remove_handler(
807 iommu->aiomt_intr_htable[j]);
808 }
809 amd_iommu_teardown_interrupts(iommu);
810 return (DDI_FAILURE);
811 }
812 }
813 iommu->aiomt_intr_state = AMD_IOMMU_INTR_HANDLER;
814
815 intrcap0 = intrcapN = -1;
816 if (ddi_intr_get_cap(iommu->aiomt_intr_htable[0], &intrcap0)
817 != DDI_SUCCESS ||
818 ddi_intr_get_cap(
819 iommu->aiomt_intr_htable[iommu->aiomt_actual_intrs - 1], &intrcapN)
820 != DDI_SUCCESS || intrcap0 != intrcapN) {
821 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
822 "ddi_intr_get_cap failed or inconsistent cap among "
823 "interrupts: intrcap0 (%d) < intrcapN (%d)",
824 f, driver, instance, iommu->aiomt_idx, intrcap0, intrcapN);
825 amd_iommu_teardown_interrupts(iommu);
826 return (DDI_FAILURE);
827 }
828 iommu->aiomt_intr_cap = intrcap0;
829
830 if (intrcap0 & DDI_INTR_FLAG_BLOCK) {
831 /* Need to call block enable */
832 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
833 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: "
834 "Need to call block enable",
835 f, driver, instance, iommu->aiomt_idx);
836 }
837 if (ddi_intr_block_enable(iommu->aiomt_intr_htable,
838 iommu->aiomt_actual_intrs) != DDI_SUCCESS) {
839 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
840 "ddi_intr_block enable failed ", f, driver,
841 instance, iommu->aiomt_idx);
842 (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
843 iommu->aiomt_actual_intrs);
844 amd_iommu_teardown_interrupts(iommu);
845 return (DDI_FAILURE);
846 }
847 } else {
848 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
849 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: "
850 "Need to call individual enable",
851 f, driver, instance, iommu->aiomt_idx);
852 }
853 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
854 if (ddi_intr_enable(iommu->aiomt_intr_htable[i])
855 != DDI_SUCCESS) {
856 cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
857 "ddi_intr_enable failed: intr = %d", f,
858 driver, instance, iommu->aiomt_idx, i);
859 for (j = 0; j < i; j++) {
860 (void) ddi_intr_disable(
861 iommu->aiomt_intr_htable[j]);
862 }
863 amd_iommu_teardown_interrupts(iommu);
864 return (DDI_FAILURE);
865 }
866 }
867 }
868 iommu->aiomt_intr_state = AMD_IOMMU_INTR_ENABLED;
869
870 if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
871 cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: "
872 "Interrupts successfully %s enabled. # of interrupts = %d",
873 f, driver, instance, iommu->aiomt_idx,
874 (intrcap0 & DDI_INTR_FLAG_BLOCK) ? "(block)" :
875 "(individually)", iommu->aiomt_actual_intrs);
876 }
877
878 return (DDI_SUCCESS);
879 }
880
881 static void
882 amd_iommu_teardown_interrupts(amd_iommu_t *iommu)
883 {
884 int i;
885
886 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ENABLED) {
887 if (iommu->aiomt_intr_cap & DDI_INTR_FLAG_BLOCK) {
888 (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
889 iommu->aiomt_actual_intrs);
890 } else {
891 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
892 (void) ddi_intr_disable(
893 iommu->aiomt_intr_htable[i]);
894 }
895 }
896 }
897
898 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_HANDLER) {
899 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
900 (void) ddi_intr_remove_handler(
901 iommu->aiomt_intr_htable[i]);
902 }
903 }
904
905 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ALLOCED) {
906 for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
907 (void) ddi_intr_free(iommu->aiomt_intr_htable[i]);
908 }
909 }
910 if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_TABLE) {
911 kmem_free(iommu->aiomt_intr_htable,
912 iommu->aiomt_intr_htable_sz);
913 }
914 iommu->aiomt_intr_htable = NULL;
915 iommu->aiomt_intr_htable_sz = 0;
916 iommu->aiomt_intr_state = AMD_IOMMU_INTR_INVALID;
917 }
918
919 static amd_iommu_t *
920 amd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx,
921 uint16_t cap_base)
922 {
923 amd_iommu_t *iommu;
924 int instance = ddi_get_instance(dip);
925 const char *driver = ddi_driver_name(dip);
926 uint32_t caphdr;
927 uint32_t low_addr32;
928 uint32_t hi_addr32;
929 uint32_t range;
930 uint32_t misc;
931 uint64_t pgoffset;
932 amd_iommu_acpi_global_t *global;
933 amd_iommu_acpi_ivhd_t *hinfop;
934 int bus, device, func;
935 const char *f = "amd_iommu_init";
936
937 low_addr32 = PCI_CAP_GET32(handle, 0, cap_base,
938 AMD_IOMMU_CAP_ADDR_LOW_OFF);
939 if (!(low_addr32 & AMD_IOMMU_REG_ADDR_LOCKED)) {
940 cmn_err(CE_WARN, "%s: %s%d: capability registers not locked. "
941 "Unable to use IOMMU unit idx=%d - skipping ...", f, driver,
942 instance, idx);
943 return (NULL);
944 }
945
946 iommu = kmem_zalloc(sizeof (amd_iommu_t), KM_SLEEP);
947 mutex_init(&iommu->aiomt_mutex, NULL, MUTEX_DRIVER, NULL);
948 mutex_enter(&iommu->aiomt_mutex);
949
950 mutex_init(&iommu->aiomt_cmdlock, NULL, MUTEX_DRIVER, NULL);
951 mutex_init(&iommu->aiomt_eventlock, NULL, MUTEX_DRIVER, NULL);
952
953 iommu->aiomt_dip = dip;
954 iommu->aiomt_idx = idx;
955
956 if (acpica_get_bdf(iommu->aiomt_dip, &bus, &device, &func)
957 != DDI_SUCCESS) {
958 cmn_err(CE_WARN, "%s: %s%d: Failed to get BDF"
959 "Unable to use IOMMU unit idx=%d - skipping ...",
960 f, driver, instance, idx);
961 return (NULL);
962 }
963
964 iommu->aiomt_bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) |
965 (uint8_t)func;
966
967 /*
968 * Since everything in the capability block is locked and RO at this
969 * point, copy everything into the IOMMU struct
970 */
971
972 /* Get cap header */
973 caphdr = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_HDR_OFF);
974 iommu->aiomt_cap_hdr = caphdr;
975 iommu->aiomt_npcache = AMD_IOMMU_REG_GET32(&caphdr,
976 AMD_IOMMU_CAP_NPCACHE);
977 iommu->aiomt_httun = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_HTTUN);
978
979 global = amd_iommu_lookup_acpi_global();
980 hinfop = amd_iommu_lookup_any_ivhd(iommu);
981
982 if (hinfop)
983 iommu->aiomt_iotlb = hinfop->ach_IotlbSup;
984 else
985 iommu->aiomt_iotlb =
986 AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_IOTLB);
987
988 iommu->aiomt_captype = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
989 iommu->aiomt_capid = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
990
991 /*
992 * Get address of IOMMU control registers
993 */
994 hi_addr32 = PCI_CAP_GET32(handle, 0, cap_base,
995 AMD_IOMMU_CAP_ADDR_HI_OFF);
996 iommu->aiomt_low_addr32 = low_addr32;
997 iommu->aiomt_hi_addr32 = hi_addr32;
998 low_addr32 &= ~AMD_IOMMU_REG_ADDR_LOCKED;
999
1000 if (hinfop) {
1001 iommu->aiomt_reg_pa = hinfop->ach_IOMMU_reg_base;
1002 ASSERT(hinfop->ach_IOMMU_pci_seg == 0);
1003 } else {
1004 iommu->aiomt_reg_pa = ((uint64_t)hi_addr32 << 32 | low_addr32);
1005 }
1006
1007 /*
1008 * Get cap range reg
1009 */
1010 range = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_RANGE_OFF);
1011 iommu->aiomt_range = range;
1012 iommu->aiomt_rng_valid = AMD_IOMMU_REG_GET32(&range,
1013 AMD_IOMMU_RNG_VALID);
1014 if (iommu->aiomt_rng_valid) {
1015 iommu->aiomt_rng_bus = AMD_IOMMU_REG_GET32(&range,
1016 AMD_IOMMU_RNG_BUS);
1017 iommu->aiomt_first_devfn = AMD_IOMMU_REG_GET32(&range,
1018 AMD_IOMMU_FIRST_DEVFN);
1019 iommu->aiomt_last_devfn = AMD_IOMMU_REG_GET32(&range,
1020 AMD_IOMMU_LAST_DEVFN);
1021 } else {
1022 iommu->aiomt_rng_bus = 0;
1023 iommu->aiomt_first_devfn = 0;
1024 iommu->aiomt_last_devfn = 0;
1025 }
1026
1027 if (hinfop)
1028 iommu->aiomt_ht_unitid = hinfop->ach_IOMMU_UnitID;
1029 else
1030 iommu->aiomt_ht_unitid = AMD_IOMMU_REG_GET32(&range,
1031 AMD_IOMMU_HT_UNITID);
1032
1033 /*
1034 * Get cap misc reg
1035 */
1036 misc = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_MISC_OFF);
1037 iommu->aiomt_misc = misc;
1038
1039 if (global) {
1040 iommu->aiomt_htatsresv = global->acg_HtAtsResv;
1041 iommu->aiomt_vasize = global->acg_VAsize;
1042 iommu->aiomt_pasize = global->acg_PAsize;
1043 } else {
1044 iommu->aiomt_htatsresv = AMD_IOMMU_REG_GET32(&misc,
1045 AMD_IOMMU_HT_ATSRSV);
1046 iommu->aiomt_vasize = AMD_IOMMU_REG_GET32(&misc,
1047 AMD_IOMMU_VA_SIZE);
1048 iommu->aiomt_pasize = AMD_IOMMU_REG_GET32(&misc,
1049 AMD_IOMMU_PA_SIZE);
1050 }
1051
1052 if (hinfop) {
1053 iommu->aiomt_msinum = hinfop->ach_IOMMU_MSInum;
1054 } else {
1055 iommu->aiomt_msinum =
1056 AMD_IOMMU_REG_GET32(&misc, AMD_IOMMU_MSINUM);
1057 }
1058
1059 /*
1060 * Set up mapping between control registers PA and VA
1061 */
1062 pgoffset = iommu->aiomt_reg_pa & MMU_PAGEOFFSET;
1063 ASSERT(pgoffset == 0);
1064 iommu->aiomt_reg_pages = mmu_btopr(AMD_IOMMU_REG_SIZE + pgoffset);
1065 iommu->aiomt_reg_size = mmu_ptob(iommu->aiomt_reg_pages);
1066
1067 iommu->aiomt_va = (uintptr_t)device_arena_alloc(
1068 ptob(iommu->aiomt_reg_pages), VM_SLEEP);
1069 if (iommu->aiomt_va == 0) {
1070 cmn_err(CE_WARN, "%s: %s%d: Failed to alloc VA for IOMMU "
1071 "control regs. Skipping IOMMU idx=%d", f, driver,
1072 instance, idx);
1073 mutex_exit(&iommu->aiomt_mutex);
1074 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1075 return (NULL);
1076 }
1077
1078 hat_devload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
1079 iommu->aiomt_reg_size,
1080 mmu_btop(iommu->aiomt_reg_pa), PROT_READ | PROT_WRITE
1081 | HAT_STRICTORDER, HAT_LOAD_LOCK);
1082
1083 iommu->aiomt_reg_va = iommu->aiomt_va + pgoffset;
1084
1085 /*
1086 * Setup the various control register's VA
1087 */
1088 iommu->aiomt_reg_devtbl_va = iommu->aiomt_reg_va +
1089 AMD_IOMMU_DEVTBL_REG_OFF;
1090 iommu->aiomt_reg_cmdbuf_va = iommu->aiomt_reg_va +
1091 AMD_IOMMU_CMDBUF_REG_OFF;
1092 iommu->aiomt_reg_eventlog_va = iommu->aiomt_reg_va +
1093 AMD_IOMMU_EVENTLOG_REG_OFF;
1094 iommu->aiomt_reg_ctrl_va = iommu->aiomt_reg_va +
1095 AMD_IOMMU_CTRL_REG_OFF;
1096 iommu->aiomt_reg_excl_base_va = iommu->aiomt_reg_va +
1097 AMD_IOMMU_EXCL_BASE_REG_OFF;
1098 iommu->aiomt_reg_excl_lim_va = iommu->aiomt_reg_va +
1099 AMD_IOMMU_EXCL_LIM_REG_OFF;
1100 iommu->aiomt_reg_cmdbuf_head_va = iommu->aiomt_reg_va +
1101 AMD_IOMMU_CMDBUF_HEAD_REG_OFF;
1102 iommu->aiomt_reg_cmdbuf_tail_va = iommu->aiomt_reg_va +
1103 AMD_IOMMU_CMDBUF_TAIL_REG_OFF;
1104 iommu->aiomt_reg_eventlog_head_va = iommu->aiomt_reg_va +
1105 AMD_IOMMU_EVENTLOG_HEAD_REG_OFF;
1106 iommu->aiomt_reg_eventlog_tail_va = iommu->aiomt_reg_va +
1107 AMD_IOMMU_EVENTLOG_TAIL_REG_OFF;
1108 iommu->aiomt_reg_status_va = iommu->aiomt_reg_va +
1109 AMD_IOMMU_STATUS_REG_OFF;
1110
1111
1112 /*
1113 * Setup the DEVICE table, CMD buffer, and LOG buffer in
1114 * memory and setup DMA access to this memory location
1115 */
1116 if (amd_iommu_setup_tables_and_buffers(iommu) != DDI_SUCCESS) {
1117 mutex_exit(&iommu->aiomt_mutex);
1118 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1119 return (NULL);
1120 }
1121
1122 if (amd_iommu_setup_exclusion(iommu) != DDI_SUCCESS) {
1123 mutex_exit(&iommu->aiomt_mutex);
1124 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1125 return (NULL);
1126 }
1127
1128 amd_iommu_enable_interrupts(iommu);
1129
1130 if (amd_iommu_setup_interrupts(iommu) != DDI_SUCCESS) {
1131 mutex_exit(&iommu->aiomt_mutex);
1132 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1133 return (NULL);
1134 }
1135
1136 /*
1137 * need to setup domain table before gfx bypass
1138 */
1139 amd_iommu_init_page_tables(iommu);
1140
1141 /*
1142 * Set pass-thru for special devices like IOAPIC and HPET
1143 *
1144 * Also, gfx devices don't use DDI for DMA. No need to register
1145 * before setting up gfx passthru
1146 */
1147 if (amd_iommu_setup_passthru(iommu) != DDI_SUCCESS) {
1148 mutex_exit(&iommu->aiomt_mutex);
1149 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1150 return (NULL);
1151 }
1152
1153 /* Initialize device table entries based on ACPI settings */
1154 if (amd_iommu_acpi_init_devtbl(iommu) != DDI_SUCCESS) {
1155 cmn_err(CE_WARN, "%s: %s%d: Can't initialize device table",
1156 f, driver, instance);
1157 mutex_exit(&iommu->aiomt_mutex);
1158 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1159 return (NULL);
1160 }
1161
1162 if (amd_iommu_start(iommu) != DDI_SUCCESS) {
1163 mutex_exit(&iommu->aiomt_mutex);
1164 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1165 return (NULL);
1166 }
1167
1168 /* xxx register/start race */
1169 if (amd_iommu_register(iommu) != DDI_SUCCESS) {
1170 mutex_exit(&iommu->aiomt_mutex);
1171 (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN);
1172 return (NULL);
1173 }
1174
1175 if (amd_iommu_debug) {
1176 cmn_err(CE_NOTE, "%s: %s%d: IOMMU idx=%d inited.", f, driver,
1177 instance, idx);
1178 }
1179
1180 return (iommu);
1181 }
1182
1183 static int
1184 amd_iommu_fini(amd_iommu_t *iommu, int type)
1185 {
1186 int idx = iommu->aiomt_idx;
1187 dev_info_t *dip = iommu->aiomt_dip;
1188 int instance = ddi_get_instance(dip);
1189 const char *driver = ddi_driver_name(dip);
1190 const char *f = "amd_iommu_fini";
1191
1192 if (type == AMD_IOMMU_TEARDOWN) {
1193 mutex_enter(&iommu->aiomt_mutex);
1194 if (amd_iommu_unregister(iommu) != DDI_SUCCESS) {
1195 cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit failed. "
1196 "idx = %d", f, driver, instance, idx);
1197 return (DDI_FAILURE);
1198 }
1199 }
1200
1201 amd_iommu_stop(iommu);
1202
1203 if (type == AMD_IOMMU_TEARDOWN) {
1204 amd_iommu_fini_page_tables(iommu);
1205 amd_iommu_teardown_interrupts(iommu);
1206 amd_iommu_teardown_exclusion(iommu);
1207 }
1208
1209 amd_iommu_teardown_tables_and_buffers(iommu, type);
1210
1211 if (type == AMD_IOMMU_QUIESCE)
1212 return (DDI_SUCCESS);
1213
1214 if (iommu->aiomt_va != 0) {
1215 hat_unload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
1216 iommu->aiomt_reg_size, HAT_UNLOAD_UNLOCK);
1217 device_arena_free((void *)(uintptr_t)iommu->aiomt_va,
1218 ptob(iommu->aiomt_reg_pages));
1219 iommu->aiomt_va = 0;
1220 iommu->aiomt_reg_va = 0;
1221 }
1222 mutex_destroy(&iommu->aiomt_eventlock);
1223 mutex_destroy(&iommu->aiomt_cmdlock);
1224 mutex_exit(&iommu->aiomt_mutex);
1225 mutex_destroy(&iommu->aiomt_mutex);
1226 kmem_free(iommu, sizeof (amd_iommu_t));
1227
1228 cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit complete. idx = %d",
1229 f, driver, instance, idx);
1230
1231 return (DDI_SUCCESS);
1232 }
1233
1234 int
1235 amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep)
1236 {
1237 int instance = ddi_get_instance(dip);
1238 const char *driver = ddi_driver_name(dip);
1239 ddi_acc_handle_t handle;
1240 uint8_t base_class;
1241 uint8_t sub_class;
1242 uint8_t prog_class;
1243 int idx;
1244 uint32_t id;
1245 uint16_t cap_base;
1246 uint32_t caphdr;
1247 uint8_t cap_type;
1248 uint8_t cap_id;
1249 amd_iommu_t *iommu;
1250 const char *f = "amd_iommu_setup";
1251
1252 ASSERT(instance >= 0);
1253 ASSERT(driver);
1254
1255 /* First setup PCI access to config space */
1256
1257 if (pci_config_setup(dip, &handle) != DDI_SUCCESS) {
1258 cmn_err(CE_WARN, "%s: PCI config setup failed: %s%d",
1259 f, driver, instance);
1260 return (DDI_FAILURE);
1261 }
1262
1263 /*
1264 * The AMD IOMMU is part of an independent PCI function. There may be
1265 * more than one IOMMU in that PCI function
1266 */
1267 base_class = pci_config_get8(handle, PCI_CONF_BASCLASS);
1268 sub_class = pci_config_get8(handle, PCI_CONF_SUBCLASS);
1269 prog_class = pci_config_get8(handle, PCI_CONF_PROGCLASS);
1270
1271 if (base_class != PCI_CLASS_PERIPH || sub_class != PCI_PERIPH_IOMMU ||
1272 prog_class != AMD_IOMMU_PCI_PROG_IF) {
1273 cmn_err(CE_WARN, "%s: %s%d: invalid PCI class(0x%x)/"
1274 "subclass(0x%x)/programming interface(0x%x)", f, driver,
1275 instance, base_class, sub_class, prog_class);
1276 pci_config_teardown(&handle);
1277 return (DDI_FAILURE);
1278 }
1279
1280 /*
1281 * Find and initialize all IOMMU units in this function
1282 */
1283 for (idx = 0; ; idx++) {
1284 if (pci_cap_probe(handle, idx, &id, &cap_base) != DDI_SUCCESS)
1285 break;
1286
1287 /* check if cap ID is secure device cap id */
1288 if (id != PCI_CAP_ID_SECURE_DEV) {
1289 if (amd_iommu_debug) {
1290 cmn_err(CE_NOTE,
1291 "%s: %s%d: skipping IOMMU: idx(0x%x) "
1292 "cap ID (0x%x) != secure dev capid (0x%x)",
1293 f, driver, instance, idx, id,
1294 PCI_CAP_ID_SECURE_DEV);
1295 }
1296 continue;
1297 }
1298
1299 /* check if cap type is IOMMU cap type */
1300 caphdr = PCI_CAP_GET32(handle, 0, cap_base,
1301 AMD_IOMMU_CAP_HDR_OFF);
1302 cap_type = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
1303 cap_id = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
1304
1305 if (cap_type != AMD_IOMMU_CAP) {
1306 cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) "
1307 "cap type (0x%x) != AMD IOMMU CAP (0x%x)", f,
1308 driver, instance, idx, cap_type, AMD_IOMMU_CAP);
1309 continue;
1310 }
1311 ASSERT(cap_id == PCI_CAP_ID_SECURE_DEV);
1312 ASSERT(cap_id == id);
1313
1314 iommu = amd_iommu_init(dip, handle, idx, cap_base);
1315 if (iommu == NULL) {
1316 cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) "
1317 "failed to init IOMMU", f,
1318 driver, instance, idx);
1319 continue;
1320 }
1321
1322 if (statep->aioms_iommu_start == NULL) {
1323 statep->aioms_iommu_start = iommu;
1324 } else {
1325 statep->aioms_iommu_end->aiomt_next = iommu;
1326 }
1327 statep->aioms_iommu_end = iommu;
1328
1329 statep->aioms_nunits++;
1330 }
1331
1332 pci_config_teardown(&handle);
1333
1334 if (amd_iommu_debug) {
1335 cmn_err(CE_NOTE, "%s: %s%d: state=%p: setup %d IOMMU units",
1336 f, driver, instance, (void *)statep, statep->aioms_nunits);
1337 }
1338
1339 return (DDI_SUCCESS);
1340 }
1341
1342 int
1343 amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep, int type)
1344 {
1345 int instance = ddi_get_instance(dip);
1346 const char *driver = ddi_driver_name(dip);
1347 amd_iommu_t *iommu, *next_iommu;
1348 int teardown;
1349 int error = DDI_SUCCESS;
1350 const char *f = "amd_iommu_teardown";
1351
1352 teardown = 0;
1353 for (iommu = statep->aioms_iommu_start; iommu;
1354 iommu = next_iommu) {
1355 ASSERT(statep->aioms_nunits > 0);
1356 next_iommu = iommu->aiomt_next;
1357 if (amd_iommu_fini(iommu, type) != DDI_SUCCESS) {
1358 error = DDI_FAILURE;
1359 continue;
1360 }
1361 statep->aioms_nunits--;
1362 teardown++;
1363 }
1364
1365 cmn_err(CE_NOTE, "%s: %s%d: state=%p: toredown %d units. "
1366 "%d units left", f, driver, instance, (void *)statep,
1367 teardown, statep->aioms_nunits);
1368
1369 return (error);
1370 }
1371
1372 dev_info_t *
1373 amd_iommu_pci_dip(dev_info_t *rdip, const char *path)
1374 {
1375 dev_info_t *pdip;
1376 const char *driver = ddi_driver_name(rdip);
1377 int instance = ddi_get_instance(rdip);
1378 const char *f = "amd_iommu_pci_dip";
1379
1380 /* Hold rdip so it and its parents don't go away */
1381 ndi_hold_devi(rdip);
1382
1383 if (ddi_is_pci_dip(rdip))
1384 return (rdip);
1385
1386 pdip = rdip;
1387 while (pdip = ddi_get_parent(pdip)) {
1388 if (ddi_is_pci_dip(pdip)) {
1389 ndi_hold_devi(pdip);
1390 ndi_rele_devi(rdip);
1391 return (pdip);
1392 }
1393 }
1394
1395 cmn_err(
1396 #ifdef DEBUG
1397 CE_PANIC,
1398 #else
1399 CE_WARN,
1400 #endif /* DEBUG */
1401 "%s: %s%d dip = %p has no PCI parent, path = %s",
1402 f, driver, instance, (void *)rdip, path);
1403
1404 ndi_rele_devi(rdip);
1405
1406 return (NULL);
1407 }
1408
1409 /* Interface with IOMMULIB */
1410 /*ARGSUSED*/
1411 static int
1412 amd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip)
1413 {
1414 const char *driver = ddi_driver_name(rdip);
1415 char *s;
1416 int bus, device, func, bdf;
1417 amd_iommu_acpi_ivhd_t *hinfop;
1418 dev_info_t *pci_dip;
1419 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1420 const char *f = "amd_iommu_probe";
1421 int instance = ddi_get_instance(iommu->aiomt_dip);
1422 const char *idriver = ddi_driver_name(iommu->aiomt_dip);
1423 char *path, *pathp;
1424
1425 if (amd_iommu_disable_list) {
1426 s = strstr(amd_iommu_disable_list, driver);
1427 if (s == NULL)
1428 return (DDI_SUCCESS);
1429 if (s == amd_iommu_disable_list || *(s - 1) == ':') {
1430 s += strlen(driver);
1431 if (*s == '\0' || *s == ':') {
1432 amd_iommu_set_passthru(iommu, rdip);
1433 return (DDI_FAILURE);
1434 }
1435 }
1436 }
1437
1438 path = kmem_alloc(MAXPATHLEN, KM_SLEEP);
1439 if ((pathp = ddi_pathname(rdip, path)) == NULL)
1440 pathp = "<unknown>";
1441
1442 pci_dip = amd_iommu_pci_dip(rdip, path);
1443 if (pci_dip == NULL) {
1444 cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip "
1445 "for rdip=%p, path = %s",
1446 f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
1447 pathp);
1448 kmem_free(path, MAXPATHLEN);
1449 return (DDI_FAILURE);
1450 }
1451
1452 if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) {
1453 cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get BDF "
1454 "for rdip=%p, path = %s",
1455 f, idriver, instance, iommu->aiomt_idx, (void *)rdip,
1456 pathp);
1457 kmem_free(path, MAXPATHLEN);
1458 return (DDI_FAILURE);
1459 }
1460 kmem_free(path, MAXPATHLEN);
1461
1462 /*
1463 * See whether device is described by IVRS as being managed
1464 * by this IOMMU
1465 */
1466 bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func;
1467 hinfop = amd_iommu_lookup_ivhd(bdf);
1468 if (hinfop && hinfop->ach_IOMMU_deviceid == iommu->aiomt_bdf)
1469 return (DDI_SUCCESS);
1470
1471 return (DDI_FAILURE);
1472 }
1473
1474 /*ARGSUSED*/
1475 static int
1476 amd_iommu_allochdl(iommulib_handle_t handle,
1477 dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1478 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep)
1479 {
1480 return (iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp,
1481 arg, dma_handlep));
1482 }
1483
1484 /*ARGSUSED*/
1485 static int
1486 amd_iommu_freehdl(iommulib_handle_t handle,
1487 dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
1488 {
1489 return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle));
1490 }
1491
1492 /*ARGSUSED*/
1493 static int
1494 map_current_window(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
1495 struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookie_array, uint_t ccount,
1496 int km_flags)
1497 {
1498 const char *driver = ddi_driver_name(iommu->aiomt_dip);
1499 int instance = ddi_get_instance(iommu->aiomt_dip);
1500 int idx = iommu->aiomt_idx;
1501 int i;
1502 uint64_t start_va;
1503 char *path;
1504 int error = DDI_FAILURE;
1505 const char *f = "map_current_window";
1506
1507 path = kmem_alloc(MAXPATHLEN, km_flags);
1508 if (path == NULL) {
1509 return (DDI_DMA_NORESOURCES);
1510 }
1511
1512 (void) ddi_pathname(rdip, path);
1513 mutex_enter(&amd_iommu_pgtable_lock);
1514
1515 if (amd_iommu_debug == AMD_IOMMU_DEBUG_PAGE_TABLES) {
1516 cmn_err(CE_NOTE, "%s: %s%d: idx=%d Attempting to get cookies "
1517 "from handle for device %s",
1518 f, driver, instance, idx, path);
1519 }
1520
1521 start_va = 0;
1522 for (i = 0; i < ccount; i++) {
1523 if ((error = amd_iommu_map_pa2va(iommu, rdip, attrp, dmareq,
1524 cookie_array[i].dmac_cookie_addr,
1525 cookie_array[i].dmac_size,
1526 AMD_IOMMU_VMEM_MAP, &start_va, km_flags)) != DDI_SUCCESS) {
1527 break;
1528 }
1529 cookie_array[i].dmac_cookie_addr = (uintptr_t)start_va;
1530 cookie_array[i].dmac_type = 0;
1531 }
1532
1533 if (i != ccount) {
1534 cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot map cookie# %d "
1535 "for device %s", f, driver, instance, idx, i, path);
1536 (void) unmap_current_window(iommu, rdip, cookie_array,
1537 ccount, i, 1);
1538 goto out;
1539 }
1540
1541 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1542 cmn_err(CE_NOTE, "%s: return SUCCESS", f);
1543 }
1544
1545 error = DDI_DMA_MAPPED;
1546 out:
1547 mutex_exit(&amd_iommu_pgtable_lock);
1548 kmem_free(path, MAXPATHLEN);
1549 return (error);
1550 }
1551
1552 /*ARGSUSED*/
1553 static int
1554 unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
1555 ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked)
1556 {
1557 const char *driver = ddi_driver_name(iommu->aiomt_dip);
1558 int instance = ddi_get_instance(iommu->aiomt_dip);
1559 int idx = iommu->aiomt_idx;
1560 int i;
1561 int error = DDI_FAILURE;
1562 char *path;
1563 int pathfree;
1564 const char *f = "unmap_current_window";
1565
1566 if (!locked)
1567 mutex_enter(&amd_iommu_pgtable_lock);
1568
1569 path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
1570 if (path) {
1571 (void) ddi_pathname(rdip, path);
1572 pathfree = 1;
1573 } else {
1574 path = "<path-mem-alloc-failed>";
1575 pathfree = 0;
1576 }
1577
1578 if (ncookies == -1)
1579 ncookies = ccount;
1580
1581 for (i = 0; i < ncookies; i++) {
1582 if (amd_iommu_unmap_va(iommu, rdip,
1583 cookie_array[i].dmac_cookie_addr,
1584 cookie_array[i].dmac_size,
1585 AMD_IOMMU_VMEM_MAP) != DDI_SUCCESS) {
1586 break;
1587 }
1588 }
1589
1590 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_COMPL_WAIT, NULL, 0, 0)
1591 != DDI_SUCCESS) {
1592 cmn_err(CE_WARN, "%s: AMD IOMMU completion wait failed for: %s",
1593 f, path);
1594 }
1595
1596 if (i != ncookies) {
1597 cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot unmap cookie# %d "
1598 "for device %s", f, driver, instance, idx, i, path);
1599 error = DDI_FAILURE;
1600 goto out;
1601 }
1602
1603 error = DDI_SUCCESS;
1604
1605 out:
1606 if (pathfree)
1607 kmem_free(path, MAXPATHLEN);
1608 if (!locked)
1609 mutex_exit(&amd_iommu_pgtable_lock);
1610 return (error);
1611 }
1612
1613 /*ARGSUSED*/
1614 static int
1615 amd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
1616 dev_info_t *rdip, ddi_dma_handle_t dma_handle,
1617 struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep,
1618 uint_t *ccountp)
1619 {
1620 int dma_error = DDI_DMA_NOMAPPING;
1621 int error;
1622 char *path;
1623 ddi_dma_cookie_t *cookie_array = NULL;
1624 uint_t ccount = 0;
1625 ddi_dma_impl_t *hp;
1626 ddi_dma_attr_t *attrp;
1627 int km_flags;
1628 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1629 int instance = ddi_get_instance(rdip);
1630 const char *driver = ddi_driver_name(rdip);
1631 const char *f = "amd_iommu_bindhdl";
1632
1633 dma_error = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle,
1634 dmareq, cookiep, ccountp);
1635
1636 if (dma_error != DDI_DMA_MAPPED && dma_error != DDI_DMA_PARTIAL_MAP)
1637 return (dma_error);
1638
1639 km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle);
1640
1641 path = kmem_alloc(MAXPATHLEN, km_flags);
1642 if (path) {
1643 (void) ddi_pathname(rdip, path);
1644 } else {
1645 dma_error = DDI_DMA_NORESOURCES;
1646 goto unbind;
1647 }
1648
1649 if (amd_iommu_debug & AMD_IOMMU_DEBUG_BIND) {
1650 cmn_err(CE_NOTE, "%s: %s got cookie (%p), #cookies: %d",
1651 f, path,
1652 (void *)cookiep->dmac_cookie_addr,
1653 *ccountp);
1654 }
1655
1656 cookie_array = NULL;
1657 ccount = 0;
1658 if ((error = iommulib_iommu_dma_get_cookies(dip, dma_handle,
1659 &cookie_array, &ccount)) != DDI_SUCCESS) {
1660 cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
1661 "for device %s", f, driver, instance, path);
1662 dma_error = error;
1663 goto unbind;
1664 }
1665
1666 hp = (ddi_dma_impl_t *)dma_handle;
1667 attrp = &hp->dmai_attr;
1668
1669 error = map_current_window(iommu, rdip, attrp, dmareq,
1670 cookie_array, ccount, km_flags);
1671 if (error != DDI_SUCCESS) {
1672 dma_error = error;
1673 goto unbind;
1674 }
1675
1676 if ((error = iommulib_iommu_dma_set_cookies(dip, dma_handle,
1677 cookie_array, ccount)) != DDI_SUCCESS) {
1678 cmn_err(CE_WARN, "%s: %s%d: Cannot set cookies "
1679 "for device %s", f, driver, instance, path);
1680 dma_error = error;
1681 goto unbind;
1682 }
1683
1684 *cookiep = cookie_array[0];
1685
1686 if (amd_iommu_debug & AMD_IOMMU_DEBUG_BIND) {
1687 cmn_err(CE_NOTE, "%s: %s remapped cookie (%p), #cookies: %d",
1688 f, path,
1689 (void *)(uintptr_t)cookiep->dmac_cookie_addr,
1690 *ccountp);
1691 }
1692
1693 kmem_free(path, MAXPATHLEN);
1694 ASSERT(dma_error == DDI_DMA_MAPPED || dma_error == DDI_DMA_PARTIAL_MAP);
1695 return (dma_error);
1696 unbind:
1697 kmem_free(path, MAXPATHLEN);
1698 (void) iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle);
1699 return (dma_error);
1700 }
1701
1702 /*ARGSUSED*/
1703 static int
1704 amd_iommu_unbindhdl(iommulib_handle_t handle,
1705 dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
1706 {
1707 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1708 ddi_dma_cookie_t *cookie_array = NULL;
1709 uint_t ccount = 0;
1710 int error = DDI_FAILURE;
1711 int instance = ddi_get_instance(rdip);
1712 const char *driver = ddi_driver_name(rdip);
1713 const char *f = "amd_iommu_unbindhdl";
1714
1715 cookie_array = NULL;
1716 ccount = 0;
1717 if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
1718 &ccount) != DDI_SUCCESS) {
1719 cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
1720 "for device %p", f, driver, instance, (void *)rdip);
1721 error = DDI_FAILURE;
1722 goto out;
1723 }
1724
1725 if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
1726 cmn_err(CE_WARN, "%s: %s%d: Cannot clear cookies "
1727 "for device %p", f, driver, instance, (void *)rdip);
1728 error = DDI_FAILURE;
1729 goto out;
1730 }
1731
1732 if (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle)
1733 != DDI_SUCCESS) {
1734 cmn_err(CE_WARN, "%s: %s%d: failed to unbindhdl for dip=%p",
1735 f, driver, instance, (void *)rdip);
1736 error = DDI_FAILURE;
1737 goto out;
1738 }
1739
1740 if (unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0)
1741 != DDI_SUCCESS) {
1742 cmn_err(CE_WARN, "%s: %s%d: failed to unmap current window "
1743 "for dip=%p", f, driver, instance, (void *)rdip);
1744 error = DDI_FAILURE;
1745 } else {
1746 error = DDI_SUCCESS;
1747 }
1748 out:
1749 if (cookie_array)
1750 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
1751 return (error);
1752 }
1753
1754 /*ARGSUSED*/
1755 static int
1756 amd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip,
1757 dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
1758 size_t len, uint_t cache_flags)
1759 {
1760 ddi_dma_cookie_t *cookie_array = NULL;
1761 uint_t ccount = 0;
1762 int error;
1763 const char *f = "amd_iommu_sync";
1764
1765 cookie_array = NULL;
1766 ccount = 0;
1767 if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
1768 &ccount) != DDI_SUCCESS) {
1769 ASSERT(cookie_array == NULL);
1770 cmn_err(CE_WARN, "%s: Cannot get cookies "
1771 "for device %p", f, (void *)rdip);
1772 error = DDI_FAILURE;
1773 goto out;
1774 }
1775
1776 if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
1777 cmn_err(CE_WARN, "%s: Cannot clear cookies "
1778 "for device %p", f, (void *)rdip);
1779 error = DDI_FAILURE;
1780 goto out;
1781 }
1782
1783 error = iommulib_iommu_dma_sync(dip, rdip, dma_handle, off,
1784 len, cache_flags);
1785
1786 if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array,
1787 ccount) != DDI_SUCCESS) {
1788 cmn_err(CE_WARN, "%s: Cannot set cookies "
1789 "for device %p", f, (void *)rdip);
1790 error = DDI_FAILURE;
1791 } else {
1792 cookie_array = NULL;
1793 ccount = 0;
1794 }
1795
1796 out:
1797 if (cookie_array)
1798 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
1799 return (error);
1800 }
1801
1802 /*ARGSUSED*/
1803 static int
1804 amd_iommu_win(iommulib_handle_t handle, dev_info_t *dip,
1805 dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
1806 off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
1807 uint_t *ccountp)
1808 {
1809 int error = DDI_FAILURE;
1810 amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
1811 ddi_dma_cookie_t *cookie_array = NULL;
1812 uint_t ccount = 0;
1813 int km_flags;
1814 ddi_dma_impl_t *hp;
1815 ddi_dma_attr_t *attrp;
1816 struct ddi_dma_req sdmareq = {0};
1817 int instance = ddi_get_instance(rdip);
1818 const char *driver = ddi_driver_name(rdip);
1819 const char *f = "amd_iommu_win";
1820
1821 km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle);
1822
1823 cookie_array = NULL;
1824 ccount = 0;
1825 if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
1826 &ccount) != DDI_SUCCESS) {
1827 cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
1828 "for device %p", f, driver, instance, (void *)rdip);
1829 error = DDI_FAILURE;
1830 goto out;
1831 }
1832
1833 if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
1834 cmn_err(CE_WARN, "%s: %s%d: Cannot clear cookies "
1835 "for device %p", f, driver, instance, (void *)rdip);
1836 error = DDI_FAILURE;
1837 goto out;
1838 }
1839
1840 if (iommulib_iommu_dma_win(dip, rdip, dma_handle, win,
1841 offp, lenp, cookiep, ccountp) != DDI_SUCCESS) {
1842 cmn_err(CE_WARN, "%s: %s%d: failed switch windows for dip=%p",
1843 f, driver, instance, (void *)rdip);
1844 error = DDI_FAILURE;
1845 goto out;
1846 }
1847
1848 (void) unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0);
1849
1850 if (cookie_array) {
1851 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
1852 cookie_array = NULL;
1853 ccount = 0;
1854 }
1855
1856 cookie_array = NULL;
1857 ccount = 0;
1858 if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
1859 &ccount) != DDI_SUCCESS) {
1860 cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
1861 "for device %p", f, driver, instance, (void *)rdip);
1862 error = DDI_FAILURE;
1863 goto out;
1864 }
1865
1866 hp = (ddi_dma_impl_t *)dma_handle;
1867 attrp = &hp->dmai_attr;
1868
1869 sdmareq.dmar_flags = DDI_DMA_RDWR;
1870 error = map_current_window(iommu, rdip, attrp, &sdmareq,
1871 cookie_array, ccount, km_flags);
1872
1873 if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array,
1874 ccount) != DDI_SUCCESS) {
1875 cmn_err(CE_WARN, "%s: %s%d: Cannot set cookies "
1876 "for device %p", f, driver, instance, (void *)rdip);
1877 error = DDI_FAILURE;
1878 goto out;
1879 }
1880
1881 *cookiep = cookie_array[0];
1882
1883 return (error == DDI_SUCCESS ? DDI_SUCCESS : DDI_FAILURE);
1884 out:
1885 if (cookie_array)
1886 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
1887
1888 return (error);
1889 }
1890
1891 /*ARGSUSED*/
1892 static int
1893 amd_iommu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
1894 dev_info_t *rdip, ddi_dma_handle_t dma_handle,
1895 struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao)
1896 {
1897 return (DDI_ENOTSUP);
1898 }
1899
1900 /*ARGSUSED*/
1901 static int
1902 amd_iommu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
1903 dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao)
1904 {
1905 return (DDI_ENOTSUP);
1906 }
1907
1908 uint64_t
1909 amd_iommu_reg_get64_workaround(uint64_t *regp, uint32_t bits)
1910 {
1911 split_t s;
1912 uint32_t *ptr32 = (uint32_t *)regp;
1913 uint64_t *s64p = &(s.u64);
1914
1915 s.u32[0] = ptr32[0];
1916 s.u32[1] = ptr32[1];
1917
1918 return (AMD_IOMMU_REG_GET64_IMPL(s64p, bits));
1919 }
1920
1921 uint64_t
1922 amd_iommu_reg_set64_workaround(uint64_t *regp, uint32_t bits, uint64_t value)
1923 {
1924 split_t s;
1925 uint32_t *ptr32 = (uint32_t *)regp;
1926 uint64_t *s64p = &(s.u64);
1927
1928 s.u32[0] = ptr32[0];
1929 s.u32[1] = ptr32[1];
1930
1931 AMD_IOMMU_REG_SET64_IMPL(s64p, bits, value);
1932
1933 *regp = s.u64;
1934
1935 return (s.u64);
1936 }
1937
1938 void
1939 amd_iommu_read_boot_props(void)
1940 {
1941 char *propval;
1942
1943 /*
1944 * if "amd-iommu = no/false" boot property is set,
1945 * ignore AMD iommu
1946 */
1947 if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
1948 DDI_PROP_DONTPASS, "amd-iommu", &propval) == DDI_SUCCESS) {
1949 if (strcmp(propval, "no") == 0 ||
1950 strcmp(propval, "false") == 0) {
1951 amd_iommu_disable = 1;
1952 }
1953 ddi_prop_free(propval);
1954 }
1955
1956 /*
1957 * Copy the list of drivers for which IOMMU is disabled by user.
1958 */
1959 if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
1960 DDI_PROP_DONTPASS, "amd-iommu-disable-list", &propval)
1961 == DDI_SUCCESS) {
1962 amd_iommu_disable_list = kmem_alloc(strlen(propval) + 1,
1963 KM_SLEEP);
1964 (void) strcpy(amd_iommu_disable_list, propval);
1965 ddi_prop_free(propval);
1966 }
1967
1968 }
1969
1970 void
1971 amd_iommu_lookup_conf_props(dev_info_t *dip)
1972 {
1973 char *disable;
1974
1975 if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip,
1976 DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu", &disable)
1977 == DDI_PROP_SUCCESS) {
1978 if (strcmp(disable, "no") == 0) {
1979 amd_iommu_disable = 1;
1980 }
1981 ddi_prop_free(disable);
1982 }
1983
1984 if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip,
1985 DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu-disable-list",
1986 &disable) == DDI_PROP_SUCCESS) {
1987 amd_iommu_disable_list = kmem_alloc(strlen(disable) + 1,
1988 KM_SLEEP);
1989 (void) strcpy(amd_iommu_disable_list, disable);
1990 ddi_prop_free(disable);
1991 }
1992 }