1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
29 */
30
31 /*
32 * Host to PCI-Express local bus driver
33 */
34
35 #include <sys/conf.h>
36 #include <sys/modctl.h>
37 #include <sys/file.h>
38 #include <sys/pci_impl.h>
39 #include <sys/pcie_impl.h>
40 #include <sys/sysmacros.h>
41 #include <sys/ddi_intr.h>
42 #include <sys/sunndi.h>
43 #include <sys/sunddi.h>
44 #include <sys/ddifm.h>
45 #include <sys/ndifm.h>
46 #include <sys/fm/util.h>
47 #include <sys/hotplug/pci/pcie_hp.h>
48 #include <io/pci/pci_tools_ext.h>
49 #include <io/pci/pci_common.h>
50 #include <io/pciex/pcie_nvidia.h>
51
52 /*
53 * Helper Macros
54 */
55 #define NPE_IS_HANDLE_FOR_STDCFG_ACC(hp) \
56 ((hp) != NULL && \
57 ((ddi_acc_hdl_t *)(hp))->ah_platform_private != NULL && \
58 (((ddi_acc_impl_t *)((ddi_acc_hdl_t *)(hp))-> \
59 ah_platform_private)-> \
60 ahi_acc_attr &(DDI_ACCATTR_CPU_VADDR|DDI_ACCATTR_CONFIG_SPACE)) \
61 == DDI_ACCATTR_CONFIG_SPACE)
62
63 /*
64 * Bus Operation functions
65 */
66 static int npe_bus_map(dev_info_t *, dev_info_t *, ddi_map_req_t *,
67 off_t, off_t, caddr_t *);
68 static int npe_ctlops(dev_info_t *, dev_info_t *, ddi_ctl_enum_t,
69 void *, void *);
70 static int npe_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
71 ddi_intr_handle_impl_t *, void *);
72 static int npe_fm_init(dev_info_t *, dev_info_t *, int,
73 ddi_iblock_cookie_t *);
74
75 static int npe_fm_callback(dev_info_t *, ddi_fm_error_t *, const void *);
76
77 /*
78 * Disable URs and Received MA for all PCIe devices. Until x86 SW is changed so
79 * that random drivers do not do PIO accesses on devices that it does not own,
80 * these error bits must be disabled. SERR must also be disabled if URs have
81 * been masked.
82 */
83 uint32_t npe_aer_uce_mask = PCIE_AER_UCE_UR;
84 uint32_t npe_aer_ce_mask = 0;
85 uint32_t npe_aer_suce_mask = PCIE_AER_SUCE_RCVD_MA;
86
87 struct bus_ops npe_bus_ops = {
88 BUSO_REV,
89 npe_bus_map,
90 NULL,
91 NULL,
92 NULL,
93 i_ddi_map_fault,
94 NULL,
95 ddi_dma_allochdl,
96 ddi_dma_freehdl,
97 ddi_dma_bindhdl,
98 ddi_dma_unbindhdl,
99 ddi_dma_flush,
100 ddi_dma_win,
101 ddi_dma_mctl,
102 npe_ctlops,
103 ddi_bus_prop_op,
104 0, /* (*bus_get_eventcookie)(); */
105 0, /* (*bus_add_eventcall)(); */
106 0, /* (*bus_remove_eventcall)(); */
107 0, /* (*bus_post_event)(); */
108 0, /* (*bus_intr_ctl)(); */
109 0, /* (*bus_config)(); */
110 0, /* (*bus_unconfig)(); */
111 npe_fm_init, /* (*bus_fm_init)(); */
112 NULL, /* (*bus_fm_fini)(); */
113 NULL, /* (*bus_fm_access_enter)(); */
114 NULL, /* (*bus_fm_access_exit)(); */
115 NULL, /* (*bus_power)(); */
116 npe_intr_ops, /* (*bus_intr_op)(); */
117 pcie_hp_common_ops /* (*bus_hp_op)(); */
118 };
119
120 static int npe_open(dev_t *, int, int, cred_t *);
121 static int npe_close(dev_t, int, int, cred_t *);
122 static int npe_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
123
124 struct cb_ops npe_cb_ops = {
125 npe_open, /* open */
126 npe_close, /* close */
127 nodev, /* strategy */
128 nodev, /* print */
129 nodev, /* dump */
130 nodev, /* read */
131 nodev, /* write */
132 npe_ioctl, /* ioctl */
133 nodev, /* devmap */
134 nodev, /* mmap */
135 nodev, /* segmap */
136 nochpoll, /* poll */
137 pcie_prop_op, /* cb_prop_op */
138 NULL, /* streamtab */
139 D_NEW | D_MP | D_HOTPLUG, /* Driver compatibility flag */
140 CB_REV, /* rev */
141 nodev, /* int (*cb_aread)() */
142 nodev /* int (*cb_awrite)() */
143 };
144
145
146 /*
147 * Device Node Operation functions
148 */
149 static int npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
150 static int npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
151 static int npe_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
152
153 struct dev_ops npe_ops = {
154 DEVO_REV, /* devo_rev */
155 0, /* refcnt */
156 npe_info, /* info */
157 nulldev, /* identify */
158 nulldev, /* probe */
159 npe_attach, /* attach */
160 npe_detach, /* detach */
161 nulldev, /* reset */
162 &npe_cb_ops, /* driver operations */
163 &npe_bus_ops, /* bus operations */
164 NULL, /* power */
165 ddi_quiesce_not_needed, /* quiesce */
166 };
167
168 /*
169 * Internal routines in support of particular npe_ctlops.
170 */
171 static int npe_removechild(dev_info_t *child);
172 static int npe_initchild(dev_info_t *child);
173
174 /*
175 * External support routine
176 */
177 extern void npe_query_acpi_mcfg(dev_info_t *dip);
178 extern void npe_ck804_fix_aer_ptr(ddi_acc_handle_t cfg_hdl);
179 extern int npe_disable_empty_bridges_workaround(dev_info_t *child);
180 extern void npe_nvidia_error_workaround(ddi_acc_handle_t cfg_hdl);
181 extern void npe_intel_error_workaround(ddi_acc_handle_t cfg_hdl);
182 extern boolean_t npe_is_mmcfg_supported(dev_info_t *dip);
183 extern void npe_enable_htmsi_children(dev_info_t *dip);
184 extern int npe_save_htconfig_children(dev_info_t *dip);
185 extern int npe_restore_htconfig_children(dev_info_t *dip);
186
187 /*
188 * Module linkage information for the kernel.
189 */
190 static struct modldrv modldrv = {
191 &mod_driverops, /* Type of module */
192 "Host to PCIe nexus driver", /* Name of module */
193 &npe_ops, /* driver ops */
194 };
195
196 static struct modlinkage modlinkage = {
197 MODREV_1,
198 (void *)&modldrv,
199 NULL
200 };
201
202 /* Save minimal state. */
203 void *npe_statep;
204
205 int
206 _init(void)
207 {
208 int e;
209
210 /*
211 * Initialize per-pci bus soft state pointer.
212 */
213 e = ddi_soft_state_init(&npe_statep, sizeof (pci_state_t), 1);
214 if (e != 0)
215 return (e);
216
217 if ((e = mod_install(&modlinkage)) != 0)
218 ddi_soft_state_fini(&npe_statep);
219
220 return (e);
221 }
222
223
224 int
225 _fini(void)
226 {
227 int rc;
228
229 rc = mod_remove(&modlinkage);
230 if (rc != 0)
231 return (rc);
232
233 ddi_soft_state_fini(&npe_statep);
234 return (rc);
235 }
236
237
238 int
239 _info(struct modinfo *modinfop)
240 {
241 return (mod_info(&modlinkage, modinfop));
242 }
243
244 /*ARGSUSED*/
245 static int
246 npe_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
247 {
248 minor_t minor = getminor((dev_t)arg);
249 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
250 pci_state_t *pcip = ddi_get_soft_state(npe_statep, instance);
251 int ret = DDI_SUCCESS;
252
253 switch (cmd) {
254 case DDI_INFO_DEVT2INSTANCE:
255 *result = (void *)(intptr_t)instance;
256 break;
257 case DDI_INFO_DEVT2DEVINFO:
258 if (pcip == NULL) {
259 ret = DDI_FAILURE;
260 break;
261 }
262
263 *result = (void *)pcip->pci_dip;
264 break;
265 default:
266 ret = DDI_FAILURE;
267 break;
268 }
269
270 return (ret);
271 }
272
273 /*ARGSUSED*/
274 static int
275 npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
276 {
277 int instance = ddi_get_instance(devi);
278 pci_state_t *pcip = NULL;
279
280 if (cmd == DDI_RESUME) {
281 /*
282 * the system might still be able to resume even if this fails
283 */
284 (void) npe_restore_htconfig_children(devi);
285 return (DDI_SUCCESS);
286 }
287
288 /*
289 * We must do this here in order to ensure that all top level devices
290 * get their HyperTransport MSI mapping regs programmed first.
291 * "Memory controller" and "hostbridge" class devices are leaf devices
292 * that may affect MSI translation functionality for devices
293 * connected to the same link/bus.
294 *
295 * This will also program HT MSI mapping registers on root buses
296 * devices (basically sitting on an HT bus) that are not dependent
297 * on the aforementioned HT devices for MSI translation.
298 */
299 npe_enable_htmsi_children(devi);
300
301 if (ddi_prop_update_string(DDI_DEV_T_NONE, devi, "device_type",
302 "pciex") != DDI_PROP_SUCCESS) {
303 cmn_err(CE_WARN, "npe: 'device_type' prop create failed");
304 }
305
306 if (ddi_soft_state_zalloc(npe_statep, instance) == DDI_SUCCESS)
307 pcip = ddi_get_soft_state(npe_statep, instance);
308
309 if (pcip == NULL)
310 return (DDI_FAILURE);
311
312 pcip->pci_dip = devi;
313 pcip->pci_soft_state = PCI_SOFT_STATE_CLOSED;
314
315 if (pcie_init(devi, NULL) != DDI_SUCCESS)
316 goto fail1;
317
318 /* Second arg: initialize for pci_express root nexus */
319 if (pcitool_init(devi, B_TRUE) != DDI_SUCCESS)
320 goto fail2;
321
322 pcip->pci_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
323 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
324 ddi_fm_init(devi, &pcip->pci_fmcap, &pcip->pci_fm_ibc);
325
326 if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE) {
327 ddi_fm_handler_register(devi, npe_fm_callback, NULL);
328 }
329
330 PCIE_DIP2PFD(devi) = kmem_zalloc(sizeof (pf_data_t), KM_SLEEP);
331 pcie_rc_init_pfd(devi, PCIE_DIP2PFD(devi));
332
333 npe_query_acpi_mcfg(devi);
334 ddi_report_dev(devi);
335 pcie_fab_init_bus(devi, PCIE_BUS_FINAL);
336
337 return (DDI_SUCCESS);
338
339 fail2:
340 (void) pcie_uninit(devi);
341 fail1:
342 pcie_rc_fini_bus(devi);
343 ddi_soft_state_free(npe_statep, instance);
344
345 return (DDI_FAILURE);
346 }
347
348 /*ARGSUSED*/
349 static int
350 npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
351 {
352 int instance = ddi_get_instance(devi);
353 pci_state_t *pcip;
354
355 pcip = ddi_get_soft_state(npe_statep, ddi_get_instance(devi));
356
357 switch (cmd) {
358 case DDI_DETACH:
359 pcie_fab_fini_bus(devi, PCIE_BUS_INITIAL);
360
361 /* Uninitialize pcitool support. */
362 pcitool_uninit(devi);
363
364 if (pcie_uninit(devi) != DDI_SUCCESS)
365 return (DDI_FAILURE);
366
367 if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE)
368 ddi_fm_handler_unregister(devi);
369
370 pcie_rc_fini_pfd(PCIE_DIP2PFD(devi));
371 kmem_free(PCIE_DIP2PFD(devi), sizeof (pf_data_t));
372
373 ddi_fm_fini(devi);
374 ddi_soft_state_free(npe_statep, instance);
375 return (DDI_SUCCESS);
376
377 case DDI_SUSPEND:
378 /*
379 * the system might still be able to suspend/resume even if
380 * this fails
381 */
382 (void) npe_save_htconfig_children(devi);
383 return (DDI_SUCCESS);
384 default:
385 return (DDI_FAILURE);
386 }
387 }
388
389 /*
390 * Configure the access handle for standard configuration space
391 * access (see pci_fm_acc_setup for code that initializes the
392 * access-function pointers).
393 */
394 static int
395 npe_setup_std_pcicfg_acc(dev_info_t *rdip, ddi_map_req_t *mp,
396 ddi_acc_hdl_t *hp, off_t offset, off_t len)
397 {
398 int ret;
399
400 if ((ret = pci_fm_acc_setup(hp, offset, len)) ==
401 DDI_SUCCESS) {
402 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
403 mp->map_handlep->ah_acc.devacc_attr_access
404 != DDI_DEFAULT_ACC) {
405 ndi_fmc_insert(rdip, ACC_HANDLE,
406 (void *)mp->map_handlep, NULL);
407 }
408 }
409 return (ret);
410 }
411
412 static int
413 npe_bus_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
414 off_t offset, off_t len, caddr_t *vaddrp)
415 {
416 int rnumber;
417 int length;
418 int space;
419 ddi_acc_impl_t *ap;
420 ddi_acc_hdl_t *hp;
421 ddi_map_req_t mr;
422 pci_regspec_t pci_reg;
423 pci_regspec_t *pci_rp;
424 struct regspec reg;
425 pci_acc_cfblk_t *cfp;
426 int retval;
427 int64_t *ecfginfo;
428 uint_t nelem;
429
430 mr = *mp; /* Get private copy of request */
431 mp = &mr;
432
433 /*
434 * check for register number
435 */
436 switch (mp->map_type) {
437 case DDI_MT_REGSPEC:
438 pci_reg = *(pci_regspec_t *)(mp->map_obj.rp);
439 pci_rp = &pci_reg;
440 if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
441 return (DDI_FAILURE);
442 break;
443 case DDI_MT_RNUMBER:
444 rnumber = mp->map_obj.rnumber;
445 /*
446 * get ALL "reg" properties for dip, select the one of
447 * of interest. In x86, "assigned-addresses" property
448 * is identical to the "reg" property, so there is no
449 * need to cross check the two to determine the physical
450 * address of the registers.
451 * This routine still performs some validity checks to
452 * make sure that everything is okay.
453 */
454 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
455 DDI_PROP_DONTPASS, "reg", (int **)&pci_rp,
456 (uint_t *)&length) != DDI_PROP_SUCCESS)
457 return (DDI_FAILURE);
458
459 /*
460 * validate the register number.
461 */
462 length /= (sizeof (pci_regspec_t) / sizeof (int));
463 if (rnumber >= length) {
464 ddi_prop_free(pci_rp);
465 return (DDI_FAILURE);
466 }
467
468 /*
469 * copy the required entry.
470 */
471 pci_reg = pci_rp[rnumber];
472
473 /*
474 * free the memory allocated by ddi_prop_lookup_int_array
475 */
476 ddi_prop_free(pci_rp);
477
478 pci_rp = &pci_reg;
479 if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
480 return (DDI_FAILURE);
481 mp->map_type = DDI_MT_REGSPEC;
482 break;
483 default:
484 return (DDI_ME_INVAL);
485 }
486
487 space = pci_rp->pci_phys_hi & PCI_REG_ADDR_M;
488
489 /*
490 * check for unmap and unlock of address space
491 */
492 if ((mp->map_op == DDI_MO_UNMAP) || (mp->map_op == DDI_MO_UNLOCK)) {
493 switch (space) {
494 case PCI_ADDR_IO:
495 reg.regspec_bustype = 1;
496 break;
497
498 case PCI_ADDR_CONFIG:
499 /*
500 * If this is an unmap/unlock of a standard config
501 * space mapping (memory-mapped config space mappings
502 * would have the DDI_ACCATTR_CPU_VADDR bit set in the
503 * acc_attr), undo that setup here.
504 */
505 if (NPE_IS_HANDLE_FOR_STDCFG_ACC(mp->map_handlep)) {
506
507 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
508 mp->map_handlep->ah_acc.devacc_attr_access
509 != DDI_DEFAULT_ACC) {
510 ndi_fmc_remove(rdip, ACC_HANDLE,
511 (void *)mp->map_handlep);
512 }
513 return (DDI_SUCCESS);
514 }
515
516 pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE;
517
518 /* FALLTHROUGH */
519 case PCI_ADDR_MEM64:
520 /*
521 * MEM64 requires special treatment on map, to check
522 * that the device is below 4G. On unmap, however,
523 * we can assume that everything is OK... the map
524 * must have succeeded.
525 */
526 /* FALLTHROUGH */
527 case PCI_ADDR_MEM32:
528 reg.regspec_bustype = 0;
529 break;
530
531 default:
532 return (DDI_FAILURE);
533 }
534
535 /*
536 * Adjust offset and length
537 * A non-zero length means override the one in the regspec.
538 */
539 pci_rp->pci_phys_low += (uint_t)offset;
540 if (len != 0)
541 pci_rp->pci_size_low = len;
542
543 reg.regspec_addr = pci_rp->pci_phys_low;
544 reg.regspec_size = pci_rp->pci_size_low;
545
546 mp->map_obj.rp = ®
547 retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
548 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
549 mp->map_handlep->ah_acc.devacc_attr_access !=
550 DDI_DEFAULT_ACC) {
551 ndi_fmc_remove(rdip, ACC_HANDLE,
552 (void *)mp->map_handlep);
553 }
554 return (retval);
555
556 }
557
558 /* check for user mapping request - not legal for Config */
559 if (mp->map_op == DDI_MO_MAP_HANDLE && space == PCI_ADDR_CONFIG) {
560 cmn_err(CE_NOTE, "npe: Config mapping request from user\n");
561 return (DDI_FAILURE);
562 }
563
564
565 /*
566 * Note that pci_fm_acc_setup() is called to serve two purposes
567 * i) enable legacy PCI I/O style config space access
568 * ii) register with FMA
569 */
570 if (space == PCI_ADDR_CONFIG) {
571
572 /* Can't map config space without a handle */
573 hp = (ddi_acc_hdl_t *)mp->map_handlep;
574 if (hp == NULL)
575 return (DDI_FAILURE);
576
577 /* record the device address for future reference */
578 cfp = (pci_acc_cfblk_t *)&hp->ah_bus_private;
579 cfp->c_busnum = PCI_REG_BUS_G(pci_rp->pci_phys_hi);
580 cfp->c_devnum = PCI_REG_DEV_G(pci_rp->pci_phys_hi);
581 cfp->c_funcnum = PCI_REG_FUNC_G(pci_rp->pci_phys_hi);
582
583 *vaddrp = (caddr_t)offset;
584
585 /* Check if MMCFG is supported */
586 if (!npe_is_mmcfg_supported(rdip)) {
587 return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
588 offset, len));
589 }
590
591
592 if (ddi_prop_lookup_int64_array(DDI_DEV_T_ANY, rdip, 0,
593 "ecfg", &ecfginfo, &nelem) == DDI_PROP_SUCCESS) {
594
595 if (nelem != 4 ||
596 cfp->c_busnum < ecfginfo[2] ||
597 cfp->c_busnum > ecfginfo[3]) {
598 /*
599 * Invalid property or Doesn't contain the
600 * requested bus; fall back to standard
601 * (I/O-based) config access.
602 */
603 ddi_prop_free(ecfginfo);
604 return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
605 offset, len));
606 } else {
607 pci_rp->pci_phys_low = ecfginfo[0];
608
609 ddi_prop_free(ecfginfo);
610
611 pci_rp->pci_phys_low += ((cfp->c_busnum << 20) |
612 (cfp->c_devnum) << 15 |
613 (cfp->c_funcnum << 12));
614
615 pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE;
616 }
617 } else {
618 /*
619 * Couldn't find the MMCFG property -- fall back to
620 * standard config access
621 */
622 return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
623 offset, len));
624 }
625 }
626
627 length = pci_rp->pci_size_low;
628
629 /*
630 * range check
631 */
632 if ((offset >= length) || (len > length) || (offset + len > length))
633 return (DDI_FAILURE);
634
635 /*
636 * Adjust offset and length
637 * A non-zero length means override the one in the regspec.
638 */
639 pci_rp->pci_phys_low += (uint_t)offset;
640 if (len != 0)
641 pci_rp->pci_size_low = len;
642
643 /*
644 * convert the pci regsec into the generic regspec used by the
645 * parent root nexus driver.
646 */
647 switch (space) {
648 case PCI_ADDR_IO:
649 reg.regspec_bustype = 1;
650 break;
651 case PCI_ADDR_CONFIG:
652 case PCI_ADDR_MEM64:
653 /*
654 * We can't handle 64-bit devices that are mapped above
655 * 4G or that are larger than 4G.
656 */
657 if (pci_rp->pci_phys_mid != 0 || pci_rp->pci_size_hi != 0)
658 return (DDI_FAILURE);
659 /*
660 * Other than that, we can treat them as 32-bit mappings
661 */
662 /* FALLTHROUGH */
663 case PCI_ADDR_MEM32:
664 reg.regspec_bustype = 0;
665 break;
666 default:
667 return (DDI_FAILURE);
668 }
669
670 reg.regspec_addr = pci_rp->pci_phys_low;
671 reg.regspec_size = pci_rp->pci_size_low;
672
673 mp->map_obj.rp = ®
674 retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
675 if (retval == DDI_SUCCESS) {
676 /*
677 * For config space gets force use of cautious access routines.
678 * These will handle default and protected mode accesses too.
679 */
680 if (space == PCI_ADDR_CONFIG) {
681 ap = (ddi_acc_impl_t *)mp->map_handlep;
682 ap->ahi_acc_attr &= ~DDI_ACCATTR_DIRECT;
683 ap->ahi_acc_attr |= DDI_ACCATTR_CONFIG_SPACE;
684 ap->ahi_get8 = i_ddi_caut_get8;
685 ap->ahi_get16 = i_ddi_caut_get16;
686 ap->ahi_get32 = i_ddi_caut_get32;
687 ap->ahi_get64 = i_ddi_caut_get64;
688 ap->ahi_rep_get8 = i_ddi_caut_rep_get8;
689 ap->ahi_rep_get16 = i_ddi_caut_rep_get16;
690 ap->ahi_rep_get32 = i_ddi_caut_rep_get32;
691 ap->ahi_rep_get64 = i_ddi_caut_rep_get64;
692 }
693 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
694 mp->map_handlep->ah_acc.devacc_attr_access !=
695 DDI_DEFAULT_ACC) {
696 ndi_fmc_insert(rdip, ACC_HANDLE,
697 (void *)mp->map_handlep, NULL);
698 }
699 }
700 return (retval);
701 }
702
703
704
705 /*ARGSUSED*/
706 static int
707 npe_ctlops(dev_info_t *dip, dev_info_t *rdip,
708 ddi_ctl_enum_t ctlop, void *arg, void *result)
709 {
710 int rn;
711 int totreg;
712 uint_t reglen;
713 pci_regspec_t *drv_regp;
714 struct attachspec *asp;
715 struct detachspec *dsp;
716 pci_state_t *pci_p = ddi_get_soft_state(npe_statep,
717 ddi_get_instance(dip));
718
719 switch (ctlop) {
720 case DDI_CTLOPS_REPORTDEV:
721 if (rdip == (dev_info_t *)0)
722 return (DDI_FAILURE);
723 cmn_err(CE_CONT, "?PCI Express-device: %s@%s, %s%d\n",
724 ddi_node_name(rdip), ddi_get_name_addr(rdip),
725 ddi_driver_name(rdip), ddi_get_instance(rdip));
726 return (DDI_SUCCESS);
727
728 case DDI_CTLOPS_INITCHILD:
729 return (npe_initchild((dev_info_t *)arg));
730
731 case DDI_CTLOPS_UNINITCHILD:
732 return (npe_removechild((dev_info_t *)arg));
733
734 case DDI_CTLOPS_SIDDEV:
735 return (DDI_SUCCESS);
736
737 case DDI_CTLOPS_REGSIZE:
738 case DDI_CTLOPS_NREGS:
739 if (rdip == (dev_info_t *)0)
740 return (DDI_FAILURE);
741
742 *(int *)result = 0;
743 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
744 DDI_PROP_DONTPASS, "reg", (int **)&drv_regp,
745 ®len) != DDI_PROP_SUCCESS) {
746 return (DDI_FAILURE);
747 }
748
749 totreg = (reglen * sizeof (int)) / sizeof (pci_regspec_t);
750 if (ctlop == DDI_CTLOPS_NREGS)
751 *(int *)result = totreg;
752 else if (ctlop == DDI_CTLOPS_REGSIZE) {
753 rn = *(int *)arg;
754 if (rn >= totreg) {
755 ddi_prop_free(drv_regp);
756 return (DDI_FAILURE);
757 }
758 *(off_t *)result = drv_regp[rn].pci_size_low;
759 }
760 ddi_prop_free(drv_regp);
761
762 return (DDI_SUCCESS);
763
764 case DDI_CTLOPS_POWER:
765 {
766 power_req_t *reqp = (power_req_t *)arg;
767 /*
768 * We currently understand reporting of PCI_PM_IDLESPEED
769 * capability. Everything else is passed up.
770 */
771 if ((reqp->request_type == PMR_REPORT_PMCAP) &&
772 (reqp->req.report_pmcap_req.cap == PCI_PM_IDLESPEED))
773 return (DDI_SUCCESS);
774
775 break;
776 }
777
778 case DDI_CTLOPS_PEEK:
779 case DDI_CTLOPS_POKE:
780 return (pci_common_peekpoke(dip, rdip, ctlop, arg, result));
781
782 /* X86 systems support PME wakeup from suspended state */
783 case DDI_CTLOPS_ATTACH:
784 if (!pcie_is_child(dip, rdip))
785 return (DDI_SUCCESS);
786
787 asp = (struct attachspec *)arg;
788 if ((asp->when == DDI_POST) && (asp->result == DDI_SUCCESS)) {
789 pf_init(rdip, (void *)pci_p->pci_fm_ibc, asp->cmd);
790 (void) pcie_postattach_child(rdip);
791 }
792
793 /* only do this for immediate children */
794 if (asp->cmd == DDI_RESUME && asp->when == DDI_PRE &&
795 ddi_get_parent(rdip) == dip)
796 if (pci_pre_resume(rdip) != DDI_SUCCESS) {
797 /* Not good, better stop now. */
798 cmn_err(CE_PANIC,
799 "Couldn't pre-resume device %p",
800 (void *) dip);
801 /* NOTREACHED */
802 }
803
804 return (DDI_SUCCESS);
805
806 case DDI_CTLOPS_DETACH:
807 if (!pcie_is_child(dip, rdip))
808 return (DDI_SUCCESS);
809
810 dsp = (struct detachspec *)arg;
811
812 if (dsp->when == DDI_PRE)
813 pf_fini(rdip, dsp->cmd);
814
815 /* only do this for immediate children */
816 if (dsp->cmd == DDI_SUSPEND && dsp->when == DDI_POST &&
817 ddi_get_parent(rdip) == dip)
818 if (pci_post_suspend(rdip) != DDI_SUCCESS)
819 return (DDI_FAILURE);
820
821 return (DDI_SUCCESS);
822
823 default:
824 break;
825 }
826
827 return (ddi_ctlops(dip, rdip, ctlop, arg, result));
828
829 }
830
831
832 /*
833 * npe_intr_ops
834 */
835 static int
836 npe_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
837 ddi_intr_handle_impl_t *hdlp, void *result)
838 {
839 return (pci_common_intr_ops(pdip, rdip, intr_op, hdlp, result));
840 }
841
842
843 static int
844 npe_initchild(dev_info_t *child)
845 {
846 char name[80];
847 pcie_bus_t *bus_p;
848 uint32_t regs;
849 ddi_acc_handle_t cfg_hdl;
850
851 /*
852 * Do not bind drivers to empty bridges.
853 * Fail above, if the bridge is found to be hotplug capable
854 */
855 if (npe_disable_empty_bridges_workaround(child) == 1)
856 return (DDI_FAILURE);
857
858 if (pci_common_name_child(child, name, 80) != DDI_SUCCESS)
859 return (DDI_FAILURE);
860
861 ddi_set_name_addr(child, name);
862
863 /*
864 * Pseudo nodes indicate a prototype node with per-instance
865 * properties to be merged into the real h/w device node.
866 * The interpretation of the unit-address is DD[,F]
867 * where DD is the device id and F is the function.
868 */
869 if (ndi_dev_is_persistent_node(child) == 0) {
870 extern int pci_allow_pseudo_children;
871
872 ddi_set_parent_data(child, NULL);
873
874 /*
875 * Try to merge the properties from this prototype
876 * node into real h/w nodes.
877 */
878 if (ndi_merge_node(child, pci_common_name_child) ==
879 DDI_SUCCESS) {
880 /*
881 * Merged ok - return failure to remove the node.
882 */
883 ddi_set_name_addr(child, NULL);
884 return (DDI_FAILURE);
885 }
886
887 /* workaround for DDIVS to run under PCI Express */
888 if (pci_allow_pseudo_children) {
889 /*
890 * If the "interrupts" property doesn't exist,
891 * this must be the ddivs no-intr case, and it returns
892 * DDI_SUCCESS instead of DDI_FAILURE.
893 */
894 if (ddi_prop_get_int(DDI_DEV_T_ANY, child,
895 DDI_PROP_DONTPASS, "interrupts", -1) == -1)
896 return (DDI_SUCCESS);
897 /*
898 * Create the ddi_parent_private_data for a pseudo
899 * child.
900 */
901 pci_common_set_parent_private_data(child);
902 return (DDI_SUCCESS);
903 }
904
905 /*
906 * The child was not merged into a h/w node,
907 * but there's not much we can do with it other
908 * than return failure to cause the node to be removed.
909 */
910 cmn_err(CE_WARN, "!%s@%s: %s.conf properties not merged",
911 ddi_get_name(child), ddi_get_name_addr(child),
912 ddi_get_name(child));
913 ddi_set_name_addr(child, NULL);
914 return (DDI_NOT_WELL_FORMED);
915 }
916
917 if (ddi_prop_get_int(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
918 "interrupts", -1) != -1)
919 pci_common_set_parent_private_data(child);
920 else
921 ddi_set_parent_data(child, NULL);
922
923 /* Disable certain errors on PCIe drivers for x86 platforms */
924 regs = pcie_get_aer_uce_mask() | npe_aer_uce_mask;
925 pcie_set_aer_uce_mask(regs);
926 regs = pcie_get_aer_ce_mask() | npe_aer_ce_mask;
927 pcie_set_aer_ce_mask(regs);
928 regs = pcie_get_aer_suce_mask() | npe_aer_suce_mask;
929 pcie_set_aer_suce_mask(regs);
930
931 /*
932 * If URs are disabled, mask SERRs as well, otherwise the system will
933 * still be notified of URs
934 */
935 if (npe_aer_uce_mask & PCIE_AER_UCE_UR)
936 pcie_set_serr_mask(1);
937
938 if (pci_config_setup(child, &cfg_hdl) == DDI_SUCCESS) {
939 npe_ck804_fix_aer_ptr(cfg_hdl);
940 npe_nvidia_error_workaround(cfg_hdl);
941 npe_intel_error_workaround(cfg_hdl);
942 pci_config_teardown(&cfg_hdl);
943 }
944
945 bus_p = PCIE_DIP2BUS(child);
946 if (bus_p) {
947 uint16_t device_id = (uint16_t)(bus_p->bus_dev_ven_id >> 16);
948 uint16_t vendor_id = (uint16_t)(bus_p->bus_dev_ven_id & 0xFFFF);
949 uint16_t rev_id = bus_p->bus_rev_id;
950
951 /* Disable AER for certain NVIDIA Chipsets */
952 if ((vendor_id == NVIDIA_VENDOR_ID) &&
953 (device_id == NVIDIA_CK804_DEVICE_ID) &&
954 (rev_id < NVIDIA_CK804_AER_VALID_REVID))
955 bus_p->bus_aer_off = 0;
956
957 pcie_init_dom(child);
958 (void) pcie_initchild(child);
959 }
960
961 return (DDI_SUCCESS);
962 }
963
964
965 static int
966 npe_removechild(dev_info_t *dip)
967 {
968 pcie_uninitchild(dip);
969
970 ddi_set_name_addr(dip, NULL);
971
972 /*
973 * Strip the node to properly convert it back to prototype form
974 */
975 ddi_remove_minor_node(dip, NULL);
976
977 ddi_prop_remove_all(dip);
978
979 return (DDI_SUCCESS);
980 }
981
982 static int
983 npe_open(dev_t *devp, int flags, int otyp, cred_t *credp)
984 {
985 minor_t minor = getminor(*devp);
986 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
987 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, instance);
988 int rv;
989
990 /*
991 * Make sure the open is for the right file type.
992 */
993 if (otyp != OTYP_CHR)
994 return (EINVAL);
995
996 if (pci_p == NULL)
997 return (ENXIO);
998
999 mutex_enter(&pci_p->pci_mutex);
1000 switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
1001 case PCI_TOOL_REG_MINOR_NUM:
1002 case PCI_TOOL_INTR_MINOR_NUM:
1003 break;
1004 default:
1005 /* Handle devctl ioctls */
1006 rv = pcie_open(pci_p->pci_dip, devp, flags, otyp, credp);
1007 mutex_exit(&pci_p->pci_mutex);
1008 return (rv);
1009 }
1010
1011 /* Handle pcitool ioctls */
1012 if (flags & FEXCL) {
1013 if (pci_p->pci_soft_state != PCI_SOFT_STATE_CLOSED) {
1014 mutex_exit(&pci_p->pci_mutex);
1015 cmn_err(CE_NOTE, "npe_open: busy");
1016 return (EBUSY);
1017 }
1018 pci_p->pci_soft_state = PCI_SOFT_STATE_OPEN_EXCL;
1019 } else {
1020 if (pci_p->pci_soft_state == PCI_SOFT_STATE_OPEN_EXCL) {
1021 mutex_exit(&pci_p->pci_mutex);
1022 cmn_err(CE_NOTE, "npe_open: busy");
1023 return (EBUSY);
1024 }
1025 pci_p->pci_soft_state = PCI_SOFT_STATE_OPEN;
1026 }
1027 mutex_exit(&pci_p->pci_mutex);
1028
1029 return (0);
1030 }
1031
1032 static int
1033 npe_close(dev_t dev, int flags, int otyp, cred_t *credp)
1034 {
1035 minor_t minor = getminor(dev);
1036 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
1037 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, instance);
1038 int rv;
1039
1040 if (pci_p == NULL)
1041 return (ENXIO);
1042
1043 mutex_enter(&pci_p->pci_mutex);
1044
1045 switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
1046 case PCI_TOOL_REG_MINOR_NUM:
1047 case PCI_TOOL_INTR_MINOR_NUM:
1048 break;
1049 default:
1050 /* Handle devctl ioctls */
1051 rv = pcie_close(pci_p->pci_dip, dev, flags, otyp, credp);
1052 mutex_exit(&pci_p->pci_mutex);
1053 return (rv);
1054 }
1055
1056 /* Handle pcitool ioctls */
1057 pci_p->pci_soft_state = PCI_SOFT_STATE_CLOSED;
1058 mutex_exit(&pci_p->pci_mutex);
1059 return (0);
1060 }
1061
1062 static int
1063 npe_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
1064 {
1065 minor_t minor = getminor(dev);
1066 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
1067 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, instance);
1068 int ret = ENOTTY;
1069
1070 if (pci_p == NULL)
1071 return (ENXIO);
1072
1073 switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
1074 case PCI_TOOL_REG_MINOR_NUM:
1075 case PCI_TOOL_INTR_MINOR_NUM:
1076 /* To handle pcitool related ioctls */
1077 ret = pci_common_ioctl(pci_p->pci_dip, dev, cmd, arg, mode,
1078 credp, rvalp);
1079 break;
1080 default:
1081 /* To handle devctl and hotplug related ioctls */
1082 ret = pcie_ioctl(pci_p->pci_dip, dev, cmd, arg, mode, credp,
1083 rvalp);
1084 break;
1085 }
1086
1087 return (ret);
1088 }
1089
1090 /*ARGSUSED*/
1091 static int
1092 npe_fm_init(dev_info_t *dip, dev_info_t *tdip, int cap,
1093 ddi_iblock_cookie_t *ibc)
1094 {
1095 pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1096 ddi_get_instance(dip));
1097
1098 ASSERT(ibc != NULL);
1099 *ibc = pcip->pci_fm_ibc;
1100
1101 return (pcip->pci_fmcap);
1102 }
1103
1104 /*ARGSUSED*/
1105 static int
1106 npe_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr, const void *no_used)
1107 {
1108 /*
1109 * On current x86 systems, npe's callback does not get called for failed
1110 * loads. If in the future this feature is used, the fault PA should be
1111 * logged in the derr->fme_bus_specific field. The appropriate PCIe
1112 * error handling code should be called and needs to be coordinated with
1113 * safe access handling.
1114 */
1115
1116 return (DDI_FM_OK);
1117 }