1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 /*
  26  * Copyright 2019, Joyent, Inc.
  27  */
  28 
  29 #ifndef _SYS_PCIE_IMPL_H
  30 #define _SYS_PCIE_IMPL_H
  31 
  32 #ifdef  __cplusplus
  33 extern "C" {
  34 #endif
  35 
  36 #include <sys/pcie.h>
  37 #include <sys/pciev.h>
  38 
  39 #define PCI_GET_BDF(dip)        \
  40         PCIE_DIP2BUS(dip)->bus_bdf
  41 #define PCI_GET_SEC_BUS(dip)    \
  42         PCIE_DIP2BUS(dip)->bus_bdg_secbus
  43 #define PCI_GET_PCIE2PCI_SECBUS(dip) \
  44         PCIE_DIP2BUS(dip)->bus_pcie2pci_secbus
  45 
  46 #define DEVI_PORT_TYPE_PCI \
  47         ((PCI_CLASS_BRIDGE << 16) | (PCI_BRIDGE_PCI << 8) | \
  48         PCI_BRIDGE_PCI_IF_PCI2PCI)
  49 
  50 #define PCIE_DIP2BUS(dip) \
  51         (ndi_port_type(dip, B_TRUE, DEVI_PORT_TYPE_PCI) ? \
  52         PCIE_DIP2UPBUS(dip) : \
  53         ndi_port_type(dip, B_FALSE, DEVI_PORT_TYPE_PCI) ? \
  54         PCIE_DIP2DOWNBUS(dip) : NULL)
  55 
  56 #define PCIE_DIP2UPBUS(dip) \
  57         ((pcie_bus_t *)ndi_get_bus_private(dip, B_TRUE))
  58 #define PCIE_DIP2DOWNBUS(dip) \
  59         ((pcie_bus_t *)ndi_get_bus_private(dip, B_FALSE))
  60 #define PCIE_DIP2PFD(dip) (PCIE_DIP2BUS(dip))->bus_pfd
  61 #define PCIE_PFD2BUS(pfd_p) pfd_p->pe_bus_p
  62 #define PCIE_PFD2DIP(pfd_p) PCIE_PFD2BUS(pfd_p)->bus_dip
  63 #define PCIE_BUS2DIP(bus_p) bus_p->bus_dip
  64 #define PCIE_BUS2PFD(bus_p) PCIE_DIP2PFD(PCIE_BUS2DIP(bus_p))
  65 #define PCIE_BUS2DOM(bus_p) bus_p->bus_dom
  66 #define PCIE_DIP2DOM(dip) PCIE_BUS2DOM(PCIE_DIP2BUS(dip))
  67 
  68 /*
  69  * These macros depend on initialization of type related data in bus_p.
  70  */
  71 #define PCIE_IS_PCIE(bus_p) (bus_p->bus_pcie_off)
  72 #define PCIE_IS_PCIX(bus_p) (bus_p->bus_pcix_off)
  73 #define PCIE_IS_PCI(bus_p) (!PCIE_IS_PCIE(bus_p))
  74 #define PCIE_HAS_AER(bus_p) (bus_p->bus_aer_off)
  75 /* IS_ROOT = is RC or RP */
  76 #define PCIE_IS_ROOT(bus_p) (PCIE_IS_RC(bus_p) || PCIE_IS_RP(bus_p))
  77 
  78 #define PCIE_IS_HOTPLUG_CAPABLE(dip) \
  79         (PCIE_DIP2BUS(dip)->bus_hp_sup_modes)
  80 
  81 #define PCIE_IS_HOTPLUG_ENABLED(dip) \
  82         ((PCIE_DIP2BUS(dip)->bus_hp_curr_mode == PCIE_PCI_HP_MODE) || \
  83         (PCIE_DIP2BUS(dip)->bus_hp_curr_mode == PCIE_NATIVE_HP_MODE))
  84 
  85 /*
  86  * This is a pseudo pcie "device type", but it's needed to explain describe
  87  * nodes such as PX and NPE, which aren't really PCI devices but do control or
  88  * interaction with PCI error handling.
  89  */
  90 #define PCIE_IS_RC(bus_p) \
  91         (bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO)
  92 #define PCIE_IS_RP(bus_p) \
  93         ((bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_ROOT) && \
  94             PCIE_IS_PCIE(bus_p))
  95 #define PCIE_IS_SWU(bus_p) \
  96         (bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_UP)
  97 #define PCIE_IS_SWD(bus_p) \
  98         (bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_DOWN)
  99 #define PCIE_IS_SW(bus_p) \
 100         (PCIE_IS_SWU(bus_p) || PCIE_IS_SWD(bus_p))
 101 #define PCIE_IS_BDG(bus_p)  (bus_p->bus_hdr_type == PCI_HEADER_ONE)
 102 #define PCIE_IS_PCI_BDG(bus_p) (PCIE_IS_PCI(bus_p) && PCIE_IS_BDG(bus_p))
 103 #define PCIE_IS_PCIE_BDG(bus_p) \
 104         (bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_PCIE2PCI)
 105 #define PCIE_IS_PCI2PCIE(bus_p) \
 106         (bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_PCI2PCIE)
 107 #define PCIE_IS_PCIE_SEC(bus_p) \
 108         (PCIE_IS_PCIE(bus_p) && PCIE_IS_BDG(bus_p) && !PCIE_IS_PCIE_BDG(bus_p))
 109 #define PCIX_ECC_VERSION_CHECK(bus_p) \
 110         ((bus_p->bus_ecc_ver == PCI_PCIX_VER_1) || \
 111             (bus_p->bus_ecc_ver == PCI_PCIX_VER_2))
 112 
 113 #define PCIE_VENID(bus_p)       (bus_p->bus_dev_ven_id & 0xffff)
 114 #define PCIE_DEVID(bus_p)       ((bus_p->bus_dev_ven_id >> 16) & 0xffff)
 115 
 116 /* PCIE Cap/AER shortcuts */
 117 #define PCIE_GET(sz, bus_p, off) \
 118         pci_config_get ## sz(bus_p->bus_cfg_hdl, off)
 119 #define PCIE_PUT(sz, bus_p, off, val) \
 120         pci_config_put ## sz(bus_p->bus_cfg_hdl, off, val)
 121 #define PCIE_CAP_GET(sz, bus_p, off) \
 122         PCI_CAP_GET ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_pcie_off, off)
 123 #define PCIE_CAP_PUT(sz, bus_p, off, val) \
 124         PCI_CAP_PUT ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_pcie_off, off, \
 125             val)
 126 #define PCIE_AER_GET(sz, bus_p, off) \
 127         PCI_XCAP_GET ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_aer_off, off)
 128 #define PCIE_AER_PUT(sz, bus_p, off, val) \
 129         PCI_XCAP_PUT ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_aer_off, off, \
 130             val)
 131 #define PCIX_CAP_GET(sz, bus_p, off) \
 132         PCI_CAP_GET ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_pcix_off, off)
 133 #define PCIX_CAP_PUT(sz, bus_p, off, val) \
 134         PCI_CAP_PUT ## sz(bus_p->bus_cfg_hdl, 0, bus_p->bus_pcix_off, off, \
 135             val)
 136 
 137 /* Translate PF error return values to DDI_FM values */
 138 #define PF_ERR2DDIFM_ERR(sts) \
 139         (sts & PF_ERR_FATAL_FLAGS ? DDI_FM_FATAL :  \
 140         (sts == PF_ERR_NO_ERROR ? DDI_FM_OK : DDI_FM_NONFATAL))
 141 
 142 /*
 143  * The following flag is used for Broadcom 5714/5715 bridge prefetch issue.
 144  * This flag will be used both by px and pcieb nexus drivers.
 145  */
 146 #define PX_DMAI_FLAGS_MAP_BUFZONE       0x40000
 147 
 148 /*
 149  * PCI(e/-X) structures used to to gather and report errors detected by
 150  * PCI(e/-X) compliant devices.  These registers only contain "dynamic" data.
 151  * Static data such as Capability Offsets and Version #s is saved in the parent
 152  * private data.
 153  */
 154 #define PCI_ERR_REG(pfd_p)         pfd_p->pe_pci_regs
 155 #define PCI_BDG_ERR_REG(pfd_p)     PCI_ERR_REG(pfd_p)->pci_bdg_regs
 156 #define PCIX_ERR_REG(pfd_p)        pfd_p->pe_ext.pe_pcix_regs
 157 #define PCIX_ECC_REG(pfd_p)        PCIX_ERR_REG(pfd_p)->pcix_ecc_regs
 158 #define PCIX_BDG_ERR_REG(pfd_p)    pfd_p->pe_pcix_bdg_regs
 159 #define PCIX_BDG_ECC_REG(pfd_p, n) PCIX_BDG_ERR_REG(pfd_p)->pcix_bdg_ecc_regs[n]
 160 #define PCIE_ERR_REG(pfd_p)        pfd_p->pe_ext.pe_pcie_regs
 161 #define PCIE_RP_REG(pfd_p)         PCIE_ERR_REG(pfd_p)->pcie_rp_regs
 162 #define PCIE_ROOT_FAULT(pfd_p)     pfd_p->pe_root_fault
 163 #define PCIE_ROOT_EH_SRC(pfd_p)    pfd_p->pe_root_eh_src
 164 #define PCIE_ADV_REG(pfd_p)        PCIE_ERR_REG(pfd_p)->pcie_adv_regs
 165 #define PCIE_ADV_HDR(pfd_p, n)     PCIE_ADV_REG(pfd_p)->pcie_ue_hdr[n]
 166 #define PCIE_ADV_BDG_REG(pfd_p) \
 167         PCIE_ADV_REG(pfd_p)->pcie_ext.pcie_adv_bdg_regs
 168 #define PCIE_ADV_BDG_HDR(pfd_p, n) PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_hdr[n]
 169 #define PCIE_ADV_RP_REG(pfd_p) \
 170         PCIE_ADV_REG(pfd_p)->pcie_ext.pcie_adv_rp_regs
 171 #define PFD_AFFECTED_DEV(pfd_p)    pfd_p->pe_affected_dev
 172 #define PFD_SET_AFFECTED_FLAG(pfd_p, aff_flag) \
 173         PFD_AFFECTED_DEV(pfd_p)->pe_affected_flags = aff_flag
 174 #define PFD_SET_AFFECTED_BDF(pfd_p, bdf) \
 175         PFD_AFFECTED_DEV(pfd_p)->pe_affected_bdf = bdf
 176 
 177 #define PFD_IS_ROOT(pfd_p)         PCIE_IS_ROOT(PCIE_PFD2BUS(pfd_p))
 178 #define PFD_IS_RC(pfd_p)           PCIE_IS_RC(PCIE_PFD2BUS(pfd_p))
 179 #define PFD_IS_RP(pfd_p)           PCIE_IS_RP(PCIE_PFD2BUS(pfd_p))
 180 
 181 /* bus_hp_mode field */
 182 typedef enum {
 183         PCIE_NONE_HP_MODE       = 0x0,
 184         PCIE_ACPI_HP_MODE       = 0x1,
 185         PCIE_PCI_HP_MODE        = 0x2,
 186         PCIE_NATIVE_HP_MODE     = 0x4
 187 } pcie_hp_mode_t;
 188 
 189 typedef struct pf_pci_bdg_err_regs {
 190         uint16_t pci_bdg_sec_stat;      /* PCI secondary status reg */
 191         uint16_t pci_bdg_ctrl;          /* PCI bridge control reg */
 192 } pf_pci_bdg_err_regs_t;
 193 
 194 typedef struct pf_pci_err_regs {
 195         uint16_t pci_err_status;        /* pci status register */
 196         uint16_t pci_cfg_comm;          /* pci command register */
 197         pf_pci_bdg_err_regs_t *pci_bdg_regs;
 198 } pf_pci_err_regs_t;
 199 
 200 typedef struct pf_pcix_ecc_regs {
 201         uint32_t pcix_ecc_ctlstat;      /* pcix ecc control status reg */
 202         uint32_t pcix_ecc_fstaddr;      /* pcix ecc first address reg */
 203         uint32_t pcix_ecc_secaddr;      /* pcix ecc second address reg */
 204         uint32_t pcix_ecc_attr;         /* pcix ecc attributes reg */
 205 } pf_pcix_ecc_regs_t;
 206 
 207 typedef struct pf_pcix_err_regs {
 208         uint16_t pcix_command;          /* pcix command register */
 209         uint32_t pcix_status;           /* pcix status register */
 210         pf_pcix_ecc_regs_t *pcix_ecc_regs;      /* pcix ecc registers */
 211 } pf_pcix_err_regs_t;
 212 
 213 typedef struct pf_pcix_bdg_err_regs {
 214         uint16_t pcix_bdg_sec_stat;     /* pcix bridge secondary status reg */
 215         uint32_t pcix_bdg_stat;         /* pcix bridge status reg */
 216         pf_pcix_ecc_regs_t *pcix_bdg_ecc_regs[2];       /* pcix ecc registers */
 217 } pf_pcix_bdg_err_regs_t;
 218 
 219 typedef struct pf_pcie_adv_bdg_err_regs {
 220         uint32_t pcie_sue_ctl;          /* pcie bridge secondary ue control */
 221         uint32_t pcie_sue_status;       /* pcie bridge secondary ue status */
 222         uint32_t pcie_sue_mask;         /* pcie bridge secondary ue mask */
 223         uint32_t pcie_sue_sev;          /* pcie bridge secondary ue severity */
 224         uint32_t pcie_sue_hdr[4];       /* pcie bridge secondary ue hdr log */
 225         uint32_t pcie_sue_tgt_trans;    /* Fault trans type from SAER Logs */
 226         uint64_t pcie_sue_tgt_addr;     /* Fault addr from SAER Logs */
 227         pcie_req_id_t pcie_sue_tgt_bdf; /* Fault bdf from SAER Logs */
 228 } pf_pcie_adv_bdg_err_regs_t;
 229 
 230 typedef struct pf_pcie_adv_rp_err_regs {
 231         uint32_t pcie_rp_err_status;    /* pcie root complex error status reg */
 232         uint32_t pcie_rp_err_cmd;       /* pcie root complex error cmd reg */
 233         uint16_t pcie_rp_ce_src_id;     /* pcie root complex ce sourpe id */
 234         uint16_t pcie_rp_ue_src_id;     /* pcie root complex ue sourpe id */
 235 } pf_pcie_adv_rp_err_regs_t;
 236 
 237 typedef struct pf_pcie_adv_err_regs {
 238         uint32_t pcie_adv_ctl;          /* pcie advanced control reg */
 239         uint32_t pcie_ue_status;        /* pcie ue error status reg */
 240         uint32_t pcie_ue_mask;          /* pcie ue error mask reg */
 241         uint32_t pcie_ue_sev;           /* pcie ue error severity reg */
 242         uint32_t pcie_ue_hdr[4];        /* pcie ue header log */
 243         uint32_t pcie_ce_status;        /* pcie ce error status reg */
 244         uint32_t pcie_ce_mask;          /* pcie ce error mask reg */
 245         union {
 246                 pf_pcie_adv_bdg_err_regs_t *pcie_adv_bdg_regs; /* bdg regs */
 247                 pf_pcie_adv_rp_err_regs_t *pcie_adv_rp_regs;     /* rp regs */
 248         } pcie_ext;
 249         uint32_t pcie_ue_tgt_trans;     /* Fault trans type from AER Logs */
 250         uint64_t pcie_ue_tgt_addr;      /* Fault addr from AER Logs */
 251         pcie_req_id_t pcie_ue_tgt_bdf;  /* Fault bdf from AER Logs */
 252 } pf_pcie_adv_err_regs_t;
 253 
 254 typedef struct pf_pcie_rp_err_regs {
 255         uint32_t pcie_rp_status;        /* root complex status register */
 256         uint16_t pcie_rp_ctl;           /* root complex control register */
 257 } pf_pcie_rp_err_regs_t;
 258 
 259 typedef struct pf_pcie_err_regs {
 260         uint16_t pcie_err_status;       /* pcie device status register */
 261         uint16_t pcie_err_ctl;          /* pcie error control register */
 262         uint32_t pcie_dev_cap;          /* pcie device capabilities register */
 263         pf_pcie_rp_err_regs_t *pcie_rp_regs;     /* pcie root complex regs */
 264         pf_pcie_adv_err_regs_t *pcie_adv_regs; /* pcie aer regs */
 265 } pf_pcie_err_regs_t;
 266 
 267 typedef enum {
 268         PF_INTR_TYPE_NONE = 0,
 269         PF_INTR_TYPE_FABRIC = 1,        /* Fabric Message */
 270         PF_INTR_TYPE_DATA,              /* Data Access Failure, failed loads */
 271         PF_INTR_TYPE_AER,               /* Root Port AER MSI */
 272         PF_INTR_TYPE_INTERNAL           /* Chip specific internal errors */
 273 } pf_intr_type_t;
 274 
 275 typedef struct pf_root_eh_src {
 276         pf_intr_type_t  intr_type;
 277         void            *intr_data;     /* Interrupt Data */
 278 } pf_root_eh_src_t;
 279 
 280 typedef struct pf_root_fault {
 281         pcie_req_id_t   scan_bdf;       /* BDF from error logs */
 282         uint64_t        scan_addr;      /* Addr from error logs */
 283         boolean_t       full_scan;      /* Option to do a full scan */
 284 } pf_root_fault_t;
 285 
 286 typedef struct pf_data pf_data_t;
 287 
 288 /*
 289  * For hot plugged device, these data are init'ed during during probe
 290  * For non-hotplugged device, these data are init'ed in pci_autoconfig (on x86),
 291  * or in px_attach()(on sparc).
 292  *
 293  * For root complex the fields are initialized in pcie_rc_init_bus();
 294  * for others part of the fields are initialized in pcie_init_bus(),
 295  * and part of fields initialized in pcie_post_init_bus(). See comments
 296  * on top of respective functions for details.
 297  */
 298 typedef struct pcie_bus {
 299         /* Needed for PCI/PCIe fabric error handling */
 300         dev_info_t      *bus_dip;
 301         dev_info_t      *bus_rp_dip;
 302         ddi_acc_handle_t bus_cfg_hdl;           /* error handling acc hdle */
 303         uint_t          bus_fm_flags;
 304         uint_t          bus_soft_state;
 305 
 306         /* Static PCI/PCIe information */
 307         pcie_req_id_t   bus_bdf;
 308         pcie_req_id_t   bus_rp_bdf;             /* BDF of device's Root Port */
 309         uint32_t        bus_dev_ven_id;         /* device/vendor ID */
 310         uint8_t         bus_rev_id;             /* revision ID */
 311         uint8_t         bus_hdr_type;           /* pci header type, see pci.h */
 312         uint16_t        bus_dev_type;           /* PCI-E dev type, see pcie.h */
 313         uint8_t         bus_bdg_secbus;         /* Bridge secondary bus num */
 314         uint16_t        bus_pcie_off;           /* PCIe Capability Offset */
 315         uint16_t        bus_aer_off;            /* PCIe Advanced Error Offset */
 316         uint16_t        bus_pcix_off;           /* PCIx Capability Offset */
 317         uint16_t        bus_pci_hp_off;         /* PCI HP (SHPC) Cap Offset */
 318         uint16_t        bus_ecc_ver;            /* PCIX ecc version */
 319         pci_bus_range_t bus_bus_range;          /* pci bus-range property */
 320         ppb_ranges_t    *bus_addr_ranges;       /* pci range property */
 321         int             bus_addr_entries;       /* number of range prop */
 322         pci_regspec_t   *bus_assigned_addr;     /* "assigned-address" prop */
 323         int             bus_assigned_entries;   /* number of prop entries */
 324 
 325         /* Cache of last fault data */
 326         pf_data_t       *bus_pfd;
 327         pcie_domain_t   *bus_dom;
 328 
 329         int             bus_mps;                /* Maximum Payload Size */
 330 
 331         void            *bus_plat_private;      /* Platform specific */
 332         /* Hotplug specific fields */
 333         pcie_hp_mode_t  bus_hp_sup_modes;       /* HP modes supported */
 334         pcie_hp_mode_t  bus_hp_curr_mode;       /* HP mode used */
 335         void            *bus_hp_ctrl;           /* HP bus ctrl data */
 336         int             bus_ari;                /* ARI device */
 337 
 338         uint64_t        bus_cfgacc_base;        /* config space base address */
 339 
 340         /* workaround for PCI/PCI-X devs behind PCIe2PCI Bridge */
 341         pcie_req_id_t   bus_pcie2pci_secbus;
 342 } pcie_bus_t;
 343 
 344 /*
 345  * Data structure to log what devices are affected in relationship to the
 346  * severity after all the errors bits have been analyzed.
 347  */
 348 #define PF_AFFECTED_ROOT        (1 << 0) /* RP/RC is affected */
 349 #define PF_AFFECTED_SELF        (1 << 1) /* Reporting Device is affected */
 350 #define PF_AFFECTED_PARENT      (1 << 2) /* Parent device is affected */
 351 #define PF_AFFECTED_CHILDREN    (1 << 3) /* All children below are affected */
 352 #define PF_AFFECTED_BDF         (1 << 4) /* See affected_bdf */
 353 #define PF_AFFECTED_AER         (1 << 5) /* See AER Registers */
 354 #define PF_AFFECTED_SAER        (1 << 6) /* See SAER Registers */
 355 #define PF_AFFECTED_ADDR        (1 << 7) /* Device targeted by addr */
 356 
 357 #define PF_MAX_AFFECTED_FLAG    PF_AFFECTED_ADDR
 358 
 359 typedef struct pf_affected_dev {
 360         uint16_t                pe_affected_flags;
 361         pcie_req_id_t           pe_affected_bdf;
 362 } pf_affected_dev_t;
 363 
 364 struct pf_data {
 365         boolean_t               pe_lock;
 366         boolean_t               pe_valid;
 367         uint32_t                pe_severity_flags;      /* Severity of error */
 368         uint32_t                pe_orig_severity_flags; /* Original severity */
 369         pf_affected_dev_t       *pe_affected_dev;
 370         pcie_bus_t              *pe_bus_p;
 371         pf_root_fault_t         *pe_root_fault; /* Only valid for RC and RP */
 372         pf_root_eh_src_t        *pe_root_eh_src; /* Only valid for RC and RP */
 373         pf_pci_err_regs_t       *pe_pci_regs;   /* PCI error reg */
 374         union {
 375                 pf_pcix_err_regs_t      *pe_pcix_regs;  /* PCI-X error reg */
 376                 pf_pcie_err_regs_t      *pe_pcie_regs;  /* PCIe error reg */
 377         } pe_ext;
 378         pf_pcix_bdg_err_regs_t *pe_pcix_bdg_regs; /* PCI-X bridge regs */
 379         pf_data_t               *pe_prev;       /* Next error in queue */
 380         pf_data_t               *pe_next;       /* Next error in queue */
 381         boolean_t               pe_rber_fatal;
 382 };
 383 
 384 /* Information used while handling errors in the fabric. */
 385 typedef struct pf_impl {
 386         ddi_fm_error_t  *pf_derr;
 387         pf_root_fault_t *pf_fault;      /* captured fault bdf/addr to scan */
 388         pf_data_t       *pf_dq_head_p;  /* ptr to fault data queue */
 389         pf_data_t       *pf_dq_tail_p;  /* ptr pt last fault data q */
 390         uint32_t        pf_total;       /* total non RC pf_datas */
 391 } pf_impl_t;
 392 
 393 /* bus_fm_flags field */
 394 #define PF_FM_READY             (1 << 0)  /* bus_fm_lock initialized */
 395 #define PF_FM_IS_NH             (1 << 1)  /* known as non-hardened */
 396 
 397 /*
 398  * PCIe fabric handle lookup address flags.  Used to define what type of
 399  * transaction the address is for.  These same value are defined again in
 400  * fabric-xlate FM module.  Do not modify these variables, without modifying
 401  * those.
 402  */
 403 #define PF_ADDR_DMA             (1 << 0)
 404 #define PF_ADDR_PIO             (1 << 1)
 405 #define PF_ADDR_CFG             (1 << 2)
 406 
 407 /* PCIe fabric error scanning status flags */
 408 #define PF_SCAN_SUCCESS         (1 << 0)
 409 #define PF_SCAN_CB_FAILURE      (1 << 1) /* hardened device callback failure */
 410 #define PF_SCAN_NO_ERR_IN_CHILD (1 << 2) /* no errors in bridge sec stat reg */
 411 #define PF_SCAN_IN_DQ           (1 << 3) /* already present in the faultq */
 412 #define PF_SCAN_DEADLOCK        (1 << 4) /* deadlock detected */
 413 #define PF_SCAN_BAD_RESPONSE    (1 << 5) /* Incorrect device response */
 414 
 415 /* PCIe fabric error handling severity return flags */
 416 #define PF_ERR_NO_ERROR         (1 << 0) /* No error seen */
 417 #define PF_ERR_CE               (1 << 1) /* Correctable Error */
 418 #define PF_ERR_NO_PANIC         (1 << 2) /* Error should not panic sys */
 419 #define PF_ERR_MATCHED_DEVICE   (1 << 3) /* Error Handled By Device */
 420 #define PF_ERR_MATCHED_RC       (1 << 4) /* Error Handled By RC */
 421 #define PF_ERR_MATCHED_PARENT   (1 << 5) /* Error Handled By Parent */
 422 #define PF_ERR_PANIC            (1 << 6) /* Error should panic system */
 423 #define PF_ERR_PANIC_DEADLOCK   (1 << 7) /* deadlock detected */
 424 #define PF_ERR_PANIC_BAD_RESPONSE (1 << 8) /* Device no response */
 425 #define PF_ERR_MATCH_DOM        (1 << 9) /* Error Handled By IO domain */
 426 
 427 #define PF_ERR_FATAL_FLAGS              \
 428         (PF_ERR_PANIC | PF_ERR_PANIC_DEADLOCK | PF_ERR_PANIC_BAD_RESPONSE)
 429 
 430 #define PF_HDL_FOUND            1
 431 #define PF_HDL_NOTFOUND         2
 432 
 433 /*
 434  * PCIe Capability Device Type Pseudo Definitions.
 435  *
 436  * PCI_PSEUDO is used on real PCI devices.  The Legacy PCI definition in the
 437  * PCIe spec really refers to PCIe devices that *require* IO Space access.  IO
 438  * Space access is usually frowned upon now in PCIe, but there for legacy
 439  * purposes.
 440  */
 441 #define PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO         0x100
 442 #define PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO        0x101
 443 
 444 #define PCIE_INVALID_BDF        0xFFFF
 445 #define PCIE_CHECK_VALID_BDF(x) (x != PCIE_INVALID_BDF)
 446 
 447 typedef struct {
 448         dev_info_t      *dip;
 449         int             highest_common_mps;
 450 } pcie_max_supported_t;
 451 
 452 /*
 453  * Default interrupt priority for all PCI and PCIe nexus drivers including
 454  * hotplug interrupts.
 455  */
 456 #define PCIE_INTR_PRI           (LOCK_LEVEL - 1)
 457 
 458 /*
 459  * XXX - PCIE_IS_PCIE check is required in order not to invoke these macros
 460  * for non-standard PCI or PCI Express Hotplug Controllers.
 461  */
 462 #define PCIE_ENABLE_ERRORS(dip) \
 463         if (PCIE_IS_PCIE(PCIE_DIP2BUS(dip))) {  \
 464                 pcie_enable_errors(dip);        \
 465                 (void) pcie_enable_ce(dip);     \
 466         }
 467 
 468 #define PCIE_DISABLE_ERRORS(dip)                \
 469         if (PCIE_IS_PCIE(PCIE_DIP2BUS(dip))) {  \
 470                 pcie_disable_errors(dip);       \
 471         }
 472 
 473 /*
 474  * pcie_init_buspcie_fini_bus specific flags
 475  */
 476 #define PCIE_BUS_INITIAL        0x0001
 477 #define PCIE_BUS_FINAL          0x0002
 478 #define PCIE_BUS_ALL            (PCIE_BUS_INITIAL | PCIE_BUS_FINAL)
 479 
 480 #ifdef  DEBUG
 481 #define PCIE_DBG pcie_dbg
 482 /* Common Debugging shortcuts */
 483 #define PCIE_DBG_CFG(dip, bus_p, name, sz, off, org) \
 484         PCIE_DBG("%s:%d:(0x%x) %s(0x%x) 0x%x -> 0x%x\n", ddi_node_name(dip), \
 485             ddi_get_instance(dip), bus_p->bus_bdf, name, off, org, \
 486             PCIE_GET(sz, bus_p, off))
 487 #define PCIE_DBG_CAP(dip, bus_p, name, sz, off, org) \
 488         PCIE_DBG("%s:%d:(0x%x) %s(0x%x) 0x%x -> 0x%x\n", ddi_node_name(dip), \
 489             ddi_get_instance(dip), bus_p->bus_bdf, name, off, org, \
 490             PCIE_CAP_GET(sz, bus_p, off))
 491 #define PCIE_DBG_AER(dip, bus_p, name, sz, off, org) \
 492         PCIE_DBG("%s:%d:(0x%x) %s(0x%x) 0x%x -> 0x%x\n", ddi_node_name(dip), \
 493             ddi_get_instance(dip), bus_p->bus_bdf, name, off, org, \
 494             PCIE_AER_GET(sz, bus_p, off))
 495 
 496 #else   /* DEBUG */
 497 
 498 #define PCIE_DBG_CFG(...)
 499 #define PCIE_DBG(...)
 500 #define PCIE_ARI_DBG(...)
 501 #define PCIE_DBG_CAP(...)
 502 #define PCIE_DBG_AER(...)
 503 
 504 #endif  /* DEBUG */
 505 
 506 /* PCIe Friendly Functions */
 507 extern int pcie_init(dev_info_t *dip, caddr_t arg);
 508 extern int pcie_uninit(dev_info_t *dip);
 509 extern int pcie_hpintr_enable(dev_info_t *dip);
 510 extern int pcie_hpintr_disable(dev_info_t *dip);
 511 extern int pcie_intr(dev_info_t *dip);
 512 extern int pcie_open(dev_info_t *dip, dev_t *devp, int flags, int otyp,
 513     cred_t *credp);
 514 extern int pcie_close(dev_info_t *dip, dev_t dev, int flags, int otyp,
 515     cred_t *credp);
 516 extern int pcie_ioctl(dev_info_t *dip, dev_t dev, int cmd, intptr_t arg,
 517     int mode, cred_t *credp, int *rvalp);
 518 extern int pcie_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op,
 519     int flags, char *name, caddr_t valuep, int *lengthp);
 520 
 521 extern void pcie_init_root_port_mps(dev_info_t *dip);
 522 extern int pcie_initchild(dev_info_t *dip);
 523 extern void pcie_uninitchild(dev_info_t *dip);
 524 extern int pcie_init_cfghdl(dev_info_t *dip);
 525 extern void pcie_fini_cfghdl(dev_info_t *dip);
 526 extern void pcie_clear_errors(dev_info_t *dip);
 527 extern int pcie_postattach_child(dev_info_t *dip);
 528 extern void pcie_enable_errors(dev_info_t *dip);
 529 extern void pcie_disable_errors(dev_info_t *dip);
 530 extern int pcie_enable_ce(dev_info_t *dip);
 531 extern boolean_t pcie_bridge_is_link_disabled(dev_info_t *);
 532 
 533 extern pcie_bus_t *pcie_init_bus(dev_info_t *dip, pcie_req_id_t bdf,
 534     uint8_t flags);
 535 extern void pcie_fini_bus(dev_info_t *dip, uint8_t flags);
 536 extern void pcie_fab_init_bus(dev_info_t *dip, uint8_t flags);
 537 extern void pcie_fab_fini_bus(dev_info_t *dip, uint8_t flags);
 538 extern void pcie_rc_init_bus(dev_info_t *dip);
 539 extern void pcie_rc_fini_bus(dev_info_t *dip);
 540 extern void pcie_rc_init_pfd(dev_info_t *dip, pf_data_t *pfd);
 541 extern void pcie_rc_fini_pfd(pf_data_t *pfd);
 542 extern boolean_t pcie_is_child(dev_info_t *dip, dev_info_t *rdip);
 543 extern int pcie_get_bdf_from_dip(dev_info_t *dip, pcie_req_id_t *bdf);
 544 extern dev_info_t *pcie_get_my_childs_dip(dev_info_t *dip, dev_info_t *rdip);
 545 extern uint32_t pcie_get_bdf_for_dma_xfer(dev_info_t *dip, dev_info_t *rdip);
 546 extern int pcie_dev(dev_info_t *dip);
 547 extern void pcie_get_fabric_mps(dev_info_t *rc_dip, dev_info_t *dip,
 548         int *max_supported);
 549 extern int pcie_root_port(dev_info_t *dip);
 550 extern int pcie_initchild_mps(dev_info_t *dip);
 551 extern void pcie_set_rber_fatal(dev_info_t *dip, boolean_t val);
 552 extern boolean_t pcie_get_rber_fatal(dev_info_t *dip);
 553 
 554 extern uint32_t pcie_get_aer_uce_mask();
 555 extern uint32_t pcie_get_aer_ce_mask();
 556 extern uint32_t pcie_get_aer_suce_mask();
 557 extern uint32_t pcie_get_serr_mask();
 558 extern void pcie_set_aer_uce_mask(uint32_t mask);
 559 extern void pcie_set_aer_ce_mask(uint32_t mask);
 560 extern void pcie_set_aer_suce_mask(uint32_t mask);
 561 extern void pcie_set_serr_mask(uint32_t mask);
 562 extern void pcie_init_plat(dev_info_t *dip);
 563 extern void pcie_fini_plat(dev_info_t *dip);
 564 extern int pcie_read_only_probe(dev_info_t *, char *, dev_info_t **);
 565 extern dev_info_t *pcie_func_to_dip(dev_info_t *dip, pcie_req_id_t function);
 566 extern int pcie_ari_disable(dev_info_t *dip);
 567 extern int pcie_ari_enable(dev_info_t *dip);
 568 
 569 #define PCIE_ARI_FORW_NOT_SUPPORTED     0
 570 #define PCIE_ARI_FORW_SUPPORTED         1
 571 
 572 extern int pcie_ari_supported(dev_info_t *dip);
 573 
 574 #define PCIE_ARI_FORW_DISABLED  0
 575 #define PCIE_ARI_FORW_ENABLED   1
 576 
 577 extern int pcie_ari_is_enabled(dev_info_t *dip);
 578 
 579 #define PCIE_NOT_ARI_DEVICE             0
 580 #define PCIE_ARI_DEVICE                 1
 581 
 582 extern int pcie_ari_device(dev_info_t *dip);
 583 extern int pcie_ari_get_next_function(dev_info_t *dip, int *func);
 584 
 585 /* PCIe error handling functions */
 586 extern void pf_eh_enter(pcie_bus_t *bus_p);
 587 extern void pf_eh_exit(pcie_bus_t *bus_p);
 588 extern int pf_scan_fabric(dev_info_t *rpdip, ddi_fm_error_t *derr,
 589     pf_data_t *root_pfd_p);
 590 extern void pf_init(dev_info_t *, ddi_iblock_cookie_t, ddi_attach_cmd_t);
 591 extern void pf_fini(dev_info_t *, ddi_detach_cmd_t);
 592 extern int pf_hdl_lookup(dev_info_t *, uint64_t, uint32_t, uint64_t,
 593     pcie_req_id_t);
 594 extern int pf_tlp_decode(pcie_bus_t *, pf_pcie_adv_err_regs_t *);
 595 extern void pcie_force_fullscan();
 596 
 597 #ifdef  DEBUG
 598 extern uint_t pcie_debug_flags;
 599 extern void pcie_dbg(char *fmt, ...);
 600 #endif  /* DEBUG */
 601 
 602 /* PCIe IOV functions */
 603 extern dev_info_t *pcie_find_dip_by_bdf(dev_info_t *rootp, pcie_req_id_t bdf);
 604 
 605 extern boolean_t pf_in_bus_range(pcie_bus_t *, pcie_req_id_t);
 606 extern boolean_t pf_in_assigned_addr(pcie_bus_t *, uint64_t);
 607 extern int pf_pci_decode(pf_data_t *, uint16_t *);
 608 extern pcie_bus_t *pf_find_busp_by_bdf(pf_impl_t *, pcie_req_id_t);
 609 extern pcie_bus_t *pf_find_busp_by_addr(pf_impl_t *, uint64_t);
 610 extern pcie_bus_t *pf_find_busp_by_aer(pf_impl_t *, pf_data_t *);
 611 extern pcie_bus_t *pf_find_busp_by_saer(pf_impl_t *, pf_data_t *);
 612 
 613 extern int pciev_eh(pf_data_t *, pf_impl_t *);
 614 extern pcie_bus_t *pciev_get_affected_dev(pf_impl_t *, pf_data_t *,
 615     uint16_t, uint16_t);
 616 extern void pciev_eh_exit(pf_data_t *, uint_t);
 617 extern boolean_t pcie_in_domain(pcie_bus_t *, uint_t);
 618 
 619 #define PCIE_ZALLOC(data) kmem_zalloc(sizeof (data), KM_SLEEP)
 620 
 621 
 622 #ifdef  __cplusplus
 623 }
 624 #endif
 625 
 626 #endif  /* _SYS_PCIE_IMPL_H */