1 /*
   2  * This file and its contents are supplied under the terms of the
   3  * Common Development and Distribution License ("CDDL"), version 1.0.
   4  * You may only use this file in accordance with the terms of version
   5  * 1.0 of the CDDL.
   6  *
   7  * A full copy of the text of the CDDL should have accompanied this
   8  * source. A copy of the CDDL is also available via the Internet at
   9  * http://www.illumos.org/license/CDDL.
  10  */
  11 
  12 /*
  13  * This file is part of the Chelsio T4 support code.
  14  *
  15  * Copyright (C) 2011-2013 Chelsio Communications.  All rights reserved.
  16  *
  17  * This program is distributed in the hope that it will be useful, but WITHOUT
  18  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  19  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
  20  * release for licensing terms and conditions.
  21  */
  22 
  23 #ifndef __CXGBE_ADAPTER_H
  24 #define __CXGBE_ADAPTER_H
  25 
  26 #include <sys/ddi.h>
  27 #include <sys/mac_provider.h>
  28 #include <sys/ethernet.h>
  29 #include <sys/queue.h>
  30 #include <sys/containerof.h>
  31 
  32 #include "offload.h"
  33 #include "firmware/t4fw_interface.h"
  34 #include "shared.h"
  35 
  36 struct adapter;
  37 typedef struct adapter adapter_t;
  38 
  39 enum {
  40         FW_IQ_QSIZE = 256,
  41         FW_IQ_ESIZE = 64,       /* At least 64 mandated by the firmware spec */
  42 
  43         RX_IQ_QSIZE = 1024,
  44         RX_IQ_ESIZE = 64,       /* At least 64 so CPL_RX_PKT will fit */
  45 
  46         EQ_ESIZE = 64,          /* All egres queues use this entry size */
  47 
  48         RX_FL_ESIZE = 64,       /* 8 64bit addresses */
  49 
  50         FL_BUF_SIZES = 4,
  51 
  52         CTRL_EQ_QSIZE = 128,
  53 
  54         TX_EQ_QSIZE = 1024,
  55         TX_SGL_SEGS = 36,
  56         TX_WR_FLITS = SGE_MAX_WR_LEN / 8
  57 };
  58 
  59 enum {
  60         /* adapter flags */
  61         FULL_INIT_DONE  = (1 << 0),
  62         FW_OK           = (1 << 1),
  63         INTR_FWD        = (1 << 2),
  64         INTR_ALLOCATED  = (1 << 3),
  65         MASTER_PF       = (1 << 4),
  66 
  67         CXGBE_BUSY      = (1 << 9),
  68 
  69         /* port flags */
  70         DOOMED          = (1 << 0),
  71         PORT_INIT_DONE  = (1 << 1),
  72 };
  73 
  74 enum {
  75         /* Features */
  76         CXGBE_HW_LSO    = (1 << 0),
  77         CXGBE_HW_CSUM   = (1 << 1),
  78 };
  79 
  80 enum {
  81         UDBS_SEG_SHIFT  = 7,    /* log2(UDBS_SEG_SIZE) */
  82         UDBS_DB_OFFSET  = 8,    /* offset of the 4B doorbell in a segment */
  83         UDBS_WR_OFFSET  = 64,   /* offset of the work request in a segment */
  84 };
  85 
  86 #define IS_DOOMED(pi)   (pi->flags & DOOMED)
  87 #define SET_DOOMED(pi)  do { pi->flags |= DOOMED; } while (0)
  88 #define IS_BUSY(sc)     (sc->flags & CXGBE_BUSY)
  89 #define SET_BUSY(sc)    do { sc->flags |= CXGBE_BUSY; } while (0)
  90 #define CLR_BUSY(sc)    do { sc->flags &= ~CXGBE_BUSY; } while (0)
  91 
  92 struct port_info {
  93         PORT_INFO_HDR;
  94 
  95         kmutex_t lock;
  96         struct adapter *adapter;
  97 
  98 #ifdef TCP_OFFLOAD_ENABLE
  99         void *tdev;
 100 #endif
 101 
 102         unsigned int flags;
 103 
 104         uint16_t viid;
 105         int16_t  xact_addr_filt; /* index of exact MAC address filter */
 106         uint16_t rss_size;      /* size of VI's RSS table slice */
 107         uint16_t ntxq;          /* # of tx queues */
 108         uint16_t first_txq;     /* index of first tx queue */
 109         uint16_t nrxq;          /* # of rx queues */
 110         uint16_t first_rxq;     /* index of first rx queue */
 111 #ifdef TCP_OFFLOAD_ENABLE
 112         uint16_t nofldtxq;              /* # of offload tx queues */
 113         uint16_t first_ofld_txq;        /* index of first offload tx queue */
 114         uint16_t nofldrxq;              /* # of offload rx queues */
 115         uint16_t first_ofld_rxq;        /* index of first offload rx queue */
 116 #endif
 117         uint8_t  lport;         /* associated offload logical port */
 118         int8_t   mdio_addr;
 119         uint8_t  port_type;
 120         uint8_t  mod_type;
 121         uint8_t  port_id;
 122         uint8_t  tx_chan;
 123         uint8_t  rx_chan;
 124         uint8_t instance; /* Associated adapter instance */
 125         uint8_t child_inst; /* Associated child instance */
 126         uint8_t tmr_idx;
 127         int8_t  pktc_idx;
 128         struct link_config link_cfg;
 129         struct port_stats stats;
 130         uint32_t features;
 131         uint8_t macaddr_cnt;
 132         u8 rss_mode;
 133         u16 viid_mirror;
 134         kstat_t *ksp_config;
 135         kstat_t *ksp_info;
 136 };
 137 
 138 struct fl_sdesc {
 139         struct rxbuf *rxb;
 140 };
 141 
 142 struct tx_desc {
 143         __be64 flit[8];
 144 };
 145 
 146 /* DMA maps used for tx */
 147 struct tx_maps {
 148         ddi_dma_handle_t *map;
 149         uint32_t map_total;     /* # of DMA maps */
 150         uint32_t map_pidx;      /* next map to be used */
 151         uint32_t map_cidx;      /* reclaimed up to this index */
 152         uint32_t map_avail;     /* # of available maps */
 153 };
 154 
 155 struct tx_sdesc {
 156         mblk_t *m;
 157         uint32_t txb_used;      /* # of bytes of tx copy buffer used */
 158         uint16_t hdls_used;     /* # of dma handles used */
 159         uint16_t desc_used;     /* # of hardware descriptors used */
 160 };
 161 
 162 enum {
 163         /* iq flags */
 164         IQ_ALLOCATED    = (1 << 0),       /* firmware resources allocated */
 165         IQ_INTR         = (1 << 1),       /* iq takes direct interrupt */
 166         IQ_HAS_FL       = (1 << 2),       /* iq has fl */
 167 
 168         /* iq state */
 169         IQS_DISABLED    = 0,
 170         IQS_BUSY        = 1,
 171         IQS_IDLE        = 2,
 172 };
 173 
 174 /*
 175  * Ingress Queue: T4 is producer, driver is consumer.
 176  */
 177 struct sge_iq {
 178         unsigned int flags;
 179         ddi_dma_handle_t dhdl;
 180         ddi_acc_handle_t ahdl;
 181 
 182         volatile uint_t state;
 183         __be64 *desc;           /* KVA of descriptor ring */
 184         uint64_t ba;            /* bus address of descriptor ring */
 185         const __be64 *cdesc;    /* current descriptor */
 186         struct adapter *adapter; /* associated  adapter */
 187         uint8_t  gen;           /* generation bit */
 188         uint8_t  intr_params;   /* interrupt holdoff parameters */
 189         int8_t   intr_pktc_idx; /* packet count threshold index */
 190         uint8_t  intr_next;     /* holdoff for next interrupt */
 191         uint8_t  esize;         /* size (bytes) of each entry in the queue */
 192         uint16_t qsize;         /* size (# of entries) of the queue */
 193         uint16_t cidx;          /* consumer index */
 194         uint16_t pending;       /* # of descs processed since last doorbell */
 195         uint16_t cntxt_id;      /* SGE context id  for the iq */
 196         uint16_t abs_id;        /* absolute SGE id for the iq */
 197         kmutex_t lock;          /* Rx access lock */
 198         uint8_t polling;
 199 
 200         STAILQ_ENTRY(sge_iq) link;
 201 };
 202 
 203 enum {
 204         EQ_CTRL         = 1,
 205         EQ_ETH          = 2,
 206 #ifdef TCP_OFFLOAD_ENABLE
 207         EQ_OFLD         = 3,
 208 #endif
 209 
 210         /* eq flags */
 211         EQ_TYPEMASK     = 7,            /* 3 lsbits hold the type */
 212         EQ_ALLOCATED    = (1 << 3),       /* firmware resources allocated */
 213         EQ_DOOMED       = (1 << 4),       /* about to be destroyed */
 214         EQ_CRFLUSHED    = (1 << 5),       /* expecting an update from SGE */
 215         EQ_STALLED      = (1 << 6),       /* out of hw descriptors or dmamaps */
 216         EQ_MTX          = (1 << 7),       /* mutex has been initialized */
 217         EQ_STARTED      = (1 << 8),       /* started */
 218 };
 219 
 220 /* Listed in order of preference.  Update t4_sysctls too if you change these */
 221 enum {DOORBELL_UDB=0x1 , DOORBELL_WCWR=0x2, DOORBELL_UDBWC=0x4, DOORBELL_KDB=0x8};
 222 
 223 /*
 224  * Egress Queue: driver is producer, T4 is consumer.
 225  *
 226  * Note: A free list is an egress queue (driver produces the buffers and T4
 227  * consumes them) but it's special enough to have its own struct (see sge_fl).
 228  */
 229 struct sge_eq {
 230         ddi_dma_handle_t desc_dhdl;
 231         ddi_acc_handle_t desc_ahdl;
 232         unsigned int flags;
 233         kmutex_t lock;
 234 
 235         struct tx_desc *desc;   /* KVA of descriptor ring */
 236         uint64_t ba;            /* bus address of descriptor ring */
 237         struct sge_qstat *spg;  /* status page, for convenience */
 238         int doorbells;
 239         volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */
 240         u_int udb_qid;          /* relative qid within the doorbell page */
 241         uint16_t cap;           /* max # of desc, for convenience */
 242         uint16_t avail;         /* available descriptors, for convenience */
 243         uint16_t qsize;         /* size (# of entries) of the queue */
 244         uint16_t cidx;          /* consumer idx (desc idx) */
 245         uint16_t pidx;          /* producer idx (desc idx) */
 246         uint16_t pending;       /* # of descriptors used since last doorbell */
 247         uint16_t iqid;          /* iq that gets egr_update for the eq */
 248         uint8_t tx_chan;        /* tx channel used by the eq */
 249         uint32_t cntxt_id;      /* SGE context id for the eq */
 250 };
 251 
 252 enum {
 253         /* fl flags */
 254         FL_MTX          = (1 << 0),       /* mutex has been initialized */
 255         FL_STARVING     = (1 << 1),       /* on the list of starving fl's */
 256         FL_DOOMED       = (1 << 2),       /* about to be destroyed */
 257 };
 258 
 259 #define FL_RUNNING_LOW(fl)      (fl->cap - fl->needed <= fl->lowat)
 260 #define FL_NOT_RUNNING_LOW(fl)  (fl->cap - fl->needed >= 2 * fl->lowat)
 261 
 262 struct sge_fl {
 263         unsigned int flags;
 264         kmutex_t lock;
 265         ddi_dma_handle_t dhdl;
 266         ddi_acc_handle_t ahdl;
 267 
 268         __be64 *desc;           /* KVA of descriptor ring, ptr to addresses */
 269         uint64_t ba;            /* bus address of descriptor ring */
 270         struct fl_sdesc *sdesc; /* KVA of software descriptor ring */
 271         uint32_t cap;           /* max # of buffers, for convenience */
 272         uint16_t qsize;         /* size (# of entries) of the queue */
 273         uint16_t cntxt_id;      /* SGE context id for the freelist */
 274         uint32_t cidx;          /* consumer idx (buffer idx, NOT hw desc idx) */
 275         uint32_t pidx;          /* producer idx (buffer idx, NOT hw desc idx) */
 276         uint32_t needed;        /* # of buffers needed to fill up fl. */
 277         uint32_t lowat;         /* # of buffers <= this means fl needs help */
 278         uint32_t pending;       /* # of bufs allocated since last doorbell */
 279         uint32_t offset;        /* current packet within the larger buffer */
 280         uint16_t copy_threshold; /* anything this size or less is copied up */
 281 
 282         uint64_t copied_up;     /* # of frames copied into mblk and handed up */
 283         uint64_t passed_up;     /* # of frames wrapped in mblk and handed up */
 284         uint64_t allocb_fail;   /* # of mblk allocation failures */
 285 
 286         TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
 287 };
 288 
 289 /* txq: SGE egress queue + miscellaneous items */
 290 struct sge_txq {
 291         struct sge_eq eq;       /* MUST be first */
 292 
 293         struct port_info *port; /* the port this txq belongs to */
 294         struct tx_sdesc *sdesc; /* KVA of software descriptor ring */
 295         mac_ring_handle_t ring_handle;
 296 
 297         /* DMA handles used for tx */
 298         ddi_dma_handle_t *tx_dhdl;
 299         uint32_t tx_dhdl_total; /* Total # of handles */
 300         uint32_t tx_dhdl_pidx;  /* next handle to be used */
 301         uint32_t tx_dhdl_cidx;  /* reclaimed up to this index */
 302         uint32_t tx_dhdl_avail; /* # of available handles */
 303 
 304         /* Copy buffers for tx */
 305         ddi_dma_handle_t txb_dhdl;
 306         ddi_acc_handle_t txb_ahdl;
 307         caddr_t txb_va;         /* KVA of copy buffers area */
 308         uint64_t txb_ba;        /* bus address of copy buffers area */
 309         uint32_t txb_size;      /* total size */
 310         uint32_t txb_next;      /* offset of next useable area in the buffer */
 311         uint32_t txb_avail;     /* # of bytes available */
 312         uint16_t copy_threshold; /* anything this size or less is copied up */
 313 
 314         uint64_t txpkts;        /* # of ethernet packets */
 315         uint64_t txbytes;       /* # of ethernet bytes */
 316         kstat_t *ksp;
 317 
 318         /* stats for common events first */
 319 
 320         uint64_t txcsum;        /* # of times hardware assisted with checksum */
 321         uint64_t tso_wrs;       /* # of IPv4 TSO work requests */
 322         uint64_t imm_wrs;       /* # of work requests with immediate data */
 323         uint64_t sgl_wrs;       /* # of work requests with direct SGL */
 324         uint64_t txpkt_wrs;     /* # of txpkt work requests (not coalesced) */
 325         uint64_t txpkts_wrs;    /* # of coalesced tx work requests */
 326         uint64_t txpkts_pkts;   /* # of frames in coalesced tx work requests */
 327         uint64_t txb_used;      /* # of tx copy buffers used (64 byte each) */
 328         uint64_t hdl_used;      /* # of DMA handles used */
 329 
 330         /* stats for not-that-common events */
 331 
 332         uint32_t txb_full;      /* txb ran out of space */
 333         uint32_t dma_hdl_failed; /* couldn't obtain DMA handle */
 334         uint32_t dma_map_failed; /* couldn't obtain DMA mapping */
 335         uint32_t qfull;         /* out of hardware descriptors */
 336         uint32_t qflush;        /* # of SGE_EGR_UPDATE notifications for txq */
 337         uint32_t pullup_early;  /* # of pullups before starting frame's SGL */
 338         uint32_t pullup_late;   /* # of pullups while building frame's SGL */
 339         uint32_t pullup_failed; /* # of failed pullups */
 340 };
 341 
 342 /* rxq: SGE ingress queue + SGE free list + miscellaneous items */
 343 struct sge_rxq {
 344         struct sge_iq iq;       /* MUST be first */
 345         struct sge_fl fl;
 346 
 347         struct port_info *port; /* the port this rxq belongs to */
 348         kstat_t *ksp;
 349 
 350         mac_ring_handle_t ring_handle;
 351         uint64_t ring_gen_num;
 352 
 353         /* stats for common events first */
 354 
 355         uint64_t rxcsum;        /* # of times hardware assisted with checksum */
 356         uint64_t rxpkts;        /* # of ethernet packets */
 357         uint64_t rxbytes;       /* # of ethernet bytes */
 358 
 359         /* stats for not-that-common events */
 360 
 361         uint32_t nomem;         /* mblk allocation during rx failed */
 362 };
 363 
 364 #ifdef TCP_OFFLOAD_ENABLE
 365 /* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */
 366 struct sge_ofld_rxq {
 367         struct sge_iq iq;       /* MUST be first */
 368         struct sge_fl fl;
 369 };
 370 
 371 /*
 372  * wrq: SGE egress queue that is given prebuilt work requests.  Both the control
 373  * and offload tx queues are of this type.
 374  */
 375 struct sge_wrq {
 376         struct sge_eq eq;       /* MUST be first */
 377 
 378         struct adapter *adapter;
 379 
 380         /* List of WRs held up due to lack of tx descriptors */
 381         struct mblk_pair wr_list;
 382 
 383         /* stats for common events first */
 384 
 385         uint64_t tx_wrs;        /* # of tx work requests */
 386 
 387         /* stats for not-that-common events */
 388 
 389         uint32_t no_desc;       /* out of hardware descriptors */
 390 };
 391 #endif
 392 
 393 struct sge {
 394         int fl_starve_threshold;
 395         int s_qpp;
 396 
 397         int nrxq;       /* total rx queues (all ports and the rest) */
 398         int ntxq;       /* total tx queues (all ports and the rest) */
 399 #ifdef TCP_OFFLOAD_ENABLE
 400         int nofldrxq;   /* total # of TOE rx queues */
 401         int nofldtxq;   /* total # of TOE tx queues */
 402 #endif
 403         int niq;        /* total ingress queues */
 404         int neq;        /* total egress queues */
 405         int stat_len;   /* length of status page at ring end */
 406         int pktshift;   /* padding between CPL & packet data */
 407         int fl_align;   /* response queue message alignment */
 408 
 409         struct sge_iq fwq;      /* Firmware event queue */
 410 #ifdef TCP_OFFLOAD_ENABLE
 411         struct sge_wrq mgmtq;   /* Management queue (Control queue) */
 412 #endif
 413         struct sge_txq *txq;    /* NIC tx queues */
 414         struct sge_rxq *rxq;    /* NIC rx queues */
 415 #ifdef TCP_OFFLOAD_ENABLE
 416         struct sge_wrq *ctrlq;  /* Control queues */
 417         struct sge_wrq *ofld_txq;       /* TOE tx queues */
 418         struct sge_ofld_rxq *ofld_rxq;  /* TOE rx queues */
 419 #endif
 420 
 421         uint16_t iq_start;
 422         int eq_start;
 423         struct sge_iq **iqmap;  /* iq->cntxt_id to iq mapping */
 424         struct sge_eq **eqmap;  /* eq->cntxt_id to eq mapping */
 425 
 426         /* Device access and DMA attributes for all the descriptor rings */
 427         ddi_device_acc_attr_t acc_attr_desc;
 428         ddi_dma_attr_t  dma_attr_desc;
 429 
 430         /* Device access and DMA attributes for tx buffers */
 431         ddi_device_acc_attr_t acc_attr_tx;
 432         ddi_dma_attr_t  dma_attr_tx;
 433 
 434         /* Device access and DMA attributes for rx buffers are in rxb_params */
 435         kmem_cache_t *rxbuf_cache;
 436         struct rxbuf_cache_params rxb_params;
 437 };
 438 
 439 struct driver_properties {
 440         /* There is a driver.conf variable for each of these */
 441         int max_ntxq_10g;
 442         int max_nrxq_10g;
 443         int max_ntxq_1g;
 444         int max_nrxq_1g;
 445 #ifdef TCP_OFFLOAD_ENABLE
 446         int max_nofldtxq_10g;
 447         int max_nofldrxq_10g;
 448         int max_nofldtxq_1g;
 449         int max_nofldrxq_1g;
 450 #endif
 451         int intr_types;
 452         int tmr_idx_10g;
 453         int pktc_idx_10g;
 454         int tmr_idx_1g;
 455         int pktc_idx_1g;
 456         int qsize_txq;
 457         int qsize_rxq;
 458 
 459         int timer_val[SGE_NTIMERS];
 460         int counter_val[SGE_NCOUNTERS];
 461 
 462         int wc;
 463 
 464         int multi_rings;
 465         int t4_fw_install;
 466 };
 467 
 468 struct rss_header;
 469 typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
 470     mblk_t *);
 471 typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *);
 472 
 473 struct adapter {
 474         SLIST_ENTRY(adapter) link;
 475         dev_info_t *dip;
 476         dev_t dev;
 477 
 478         unsigned int pf;
 479         unsigned int mbox;
 480 
 481         unsigned int vpd_busy;
 482         unsigned int vpd_flag;
 483 
 484         u32 t4_bar0;
 485 
 486         uint_t open;    /* character device is open */
 487 
 488         /* PCI config space access handle */
 489         ddi_acc_handle_t pci_regh;
 490 
 491         /* MMIO register access handle */
 492         ddi_acc_handle_t regh;
 493         caddr_t regp;
 494         /* BAR1 register access handle */
 495         ddi_acc_handle_t reg1h;
 496         caddr_t reg1p;
 497 
 498         /* Interrupt information */
 499         int intr_type;
 500         int intr_count;
 501         int intr_cap;
 502         uint_t intr_pri;
 503         ddi_intr_handle_t *intr_handle;
 504 
 505         struct driver_properties props;
 506         kstat_t *ksp;
 507         kstat_t *ksp_stat;
 508 
 509         struct sge sge;
 510 
 511         struct port_info *port[MAX_NPORTS];
 512         ddi_taskq_t *tq[NCHAN];
 513         uint8_t chan_map[NCHAN];
 514         uint32_t filter_mode;
 515 
 516         struct l2t_data *l2t;   /* L2 table */
 517         struct tid_info tids;
 518 
 519         int doorbells;
 520         int registered_device_map;
 521         int open_device_map;
 522         int flags;
 523 
 524         unsigned int cfcsum;
 525         struct adapter_params params;
 526         struct t4_virt_res vres;
 527 
 528 #ifdef TCP_OFFLOAD_ENABLE
 529         struct uld_softc tom;
 530         struct tom_tunables tt;
 531 #endif
 532 
 533 #ifdef TCP_OFFLOAD_ENABLE
 534         int offload_map;
 535 #endif
 536         uint16_t linkcaps;
 537         uint16_t niccaps;
 538         uint16_t toecaps;
 539         uint16_t rdmacaps;
 540         uint16_t iscsicaps;
 541         uint16_t fcoecaps;
 542 
 543         fw_msg_handler_t fw_msg_handler[5]; /* NUM_FW6_TYPES */
 544         cpl_handler_t cpl_handler[0xef]; /* NUM_CPL_CMDS */
 545 
 546         kmutex_t lock;
 547         kcondvar_t cv;
 548 
 549         /* Starving free lists */
 550         kmutex_t sfl_lock;      /* same cache-line as sc_lock? but that's ok */
 551         TAILQ_HEAD(, sge_fl) sfl;
 552         timeout_id_t sfl_timer;
 553 };
 554 
 555 enum {
 556         NIC_H = 0,
 557         TOM_H,
 558         IW_H,
 559         ISCSI_H
 560 };
 561 
 562 struct memwin {
 563         uint32_t base;
 564         uint32_t aperture;
 565 };
 566 
 567 #define ADAPTER_LOCK(sc)                mutex_enter(&(sc)->lock)
 568 #define ADAPTER_UNLOCK(sc)              mutex_exit(&(sc)->lock)
 569 #define ADAPTER_LOCK_ASSERT_OWNED(sc)   ASSERT(mutex_owned(&(sc)->lock))
 570 #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) ASSERT(!mutex_owned(&(sc)->lock))
 571 
 572 #define PORT_LOCK(pi)                   mutex_enter(&(pi)->lock)
 573 #define PORT_UNLOCK(pi)                 mutex_exit(&(pi)->lock)
 574 #define PORT_LOCK_ASSERT_OWNED(pi)      ASSERT(mutex_owned(&(pi)->lock))
 575 #define PORT_LOCK_ASSERT_NOTOWNED(pi)   ASSERT(!mutex_owned(&(pi)->lock))
 576 
 577 #define IQ_LOCK(iq)                     mutex_enter(&(iq)->lock)
 578 #define IQ_UNLOCK(iq)                   mutex_exit(&(iq)->lock)
 579 #define IQ_LOCK_ASSERT_OWNED(iq)        ASSERT(mutex_owned(&(iq)->lock))
 580 #define IQ_LOCK_ASSERT_NOTOWNED(iq)     ASSERT(!mutex_owned(&(iq)->lock))
 581 
 582 #define FL_LOCK(fl)                     mutex_enter(&(fl)->lock)
 583 #define FL_UNLOCK(fl)                   mutex_exit(&(fl)->lock)
 584 #define FL_LOCK_ASSERT_OWNED(fl)        ASSERT(mutex_owned(&(fl)->lock))
 585 #define FL_LOCK_ASSERT_NOTOWNED(fl)     ASSERT(!mutex_owned(&(fl)->lock))
 586 
 587 #define RXQ_LOCK(rxq)                   IQ_LOCK(&(rxq)->iq)
 588 #define RXQ_UNLOCK(rxq)                 IQ_UNLOCK(&(rxq)->iq)
 589 #define RXQ_LOCK_ASSERT_OWNED(rxq)      IQ_LOCK_ASSERT_OWNED(&(rxq)->iq)
 590 #define RXQ_LOCK_ASSERT_NOTOWNED(rxq)   IQ_LOCK_ASSERT_NOTOWNED(&(rxq)->iq)
 591 
 592 #define RXQ_FL_LOCK(rxq)                FL_LOCK(&(rxq)->fl)
 593 #define RXQ_FL_UNLOCK(rxq)              FL_UNLOCK(&(rxq)->fl)
 594 #define RXQ_FL_LOCK_ASSERT_OWNED(rxq)   FL_LOCK_ASSERT_OWNED(&(rxq)->fl)
 595 #define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl)
 596 
 597 #define EQ_LOCK(eq)                     mutex_enter(&(eq)->lock)
 598 #define EQ_UNLOCK(eq)                   mutex_exit(&(eq)->lock)
 599 #define EQ_LOCK_ASSERT_OWNED(eq)        ASSERT(mutex_owned(&(eq)->lock))
 600 #define EQ_LOCK_ASSERT_NOTOWNED(eq)     ASSERT(!mutex_owned(&(eq)->lock))
 601 
 602 #define TXQ_LOCK(txq)                   EQ_LOCK(&(txq)->eq)
 603 #define TXQ_UNLOCK(txq)                 EQ_UNLOCK(&(txq)->eq)
 604 #define TXQ_LOCK_ASSERT_OWNED(txq)      EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
 605 #define TXQ_LOCK_ASSERT_NOTOWNED(txq)   EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
 606 
 607 #define for_each_txq(pi, iter, txq) \
 608         txq = &pi->adapter->sge.txq[pi->first_txq]; \
 609         for (iter = 0; iter < pi->ntxq; ++iter, ++txq)
 610 #define for_each_rxq(pi, iter, rxq) \
 611         rxq = &pi->adapter->sge.rxq[pi->first_rxq]; \
 612         for (iter = 0; iter < pi->nrxq; ++iter, ++rxq)
 613 #define for_each_ofld_txq(pi, iter, ofld_txq) \
 614         ofld_txq = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq]; \
 615         for (iter = 0; iter < pi->nofldtxq; ++iter, ++ofld_txq)
 616 #define for_each_ofld_rxq(pi, iter, ofld_rxq) \
 617         ofld_rxq = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq]; \
 618         for (iter = 0; iter < pi->nofldrxq; ++iter, ++ofld_rxq)
 619 
 620 #define NFIQ(sc) ((sc)->intr_count > 1 ? (sc)->intr_count - 1 : 1)
 621 
 622 /* One for errors, one for firmware events */
 623 #define T4_EXTRA_INTR 2
 624 
 625 /* Presently disabling locking around  mbox access
 626  * We may need to reenable it later
 627  */
 628 typedef int t4_os_lock_t;
 629 static inline void t4_os_lock(t4_os_lock_t *lock)
 630 {
 631 
 632 }
 633 static inline void t4_os_unlock(t4_os_lock_t *lock)
 634 {
 635 
 636 }
 637 
 638 static inline uint32_t
 639 t4_read_reg(struct adapter *sc, uint32_t reg)
 640 {
 641         /* LINTED: E_BAD_PTR_CAST_ALIGN */
 642         return (ddi_get32(sc->regh, (uint32_t *)(sc->regp + reg)));
 643 }
 644 
 645 static inline void
 646 t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
 647 {
 648         /* LINTED: E_BAD_PTR_CAST_ALIGN */
 649         ddi_put32(sc->regh, (uint32_t *)(sc->regp + reg), val);
 650 }
 651 
 652 static inline void
 653 t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val)
 654 {
 655         *val = pci_config_get8(sc->pci_regh, reg);
 656 }
 657 
 658 static inline void
 659 t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val)
 660 {
 661         pci_config_put8(sc->pci_regh, reg, val);
 662 }
 663 
 664 static inline void
 665 t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val)
 666 {
 667         *val = pci_config_get16(sc->pci_regh, reg);
 668 }
 669 
 670 static inline void
 671 t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val)
 672 {
 673         pci_config_put16(sc->pci_regh, reg, val);
 674 }
 675 
 676 static inline void
 677 t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val)
 678 {
 679         *val = pci_config_get32(sc->pci_regh, reg);
 680 }
 681 
 682 static inline void
 683 t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val)
 684 {
 685         pci_config_put32(sc->pci_regh, reg, val);
 686 }
 687 
 688 static inline uint64_t
 689 t4_read_reg64(struct adapter *sc, uint32_t reg)
 690 {
 691         /* LINTED: E_BAD_PTR_CAST_ALIGN */
 692         return (ddi_get64(sc->regh, (uint64_t *)(sc->regp + reg)));
 693 }
 694 
 695 static inline void
 696 t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val)
 697 {
 698         /* LINTED: E_BAD_PTR_CAST_ALIGN */
 699         ddi_put64(sc->regh, (uint64_t *)(sc->regp + reg), val);
 700 }
 701 
 702 static inline struct port_info *
 703 adap2pinfo(struct adapter *sc, int idx)
 704 {
 705         return (sc->port[idx]);
 706 }
 707 
 708 static inline void
 709 t4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[])
 710 {
 711         bcopy(hw_addr, sc->port[idx]->hw_addr, ETHERADDRL);
 712 }
 713 
 714 static inline bool
 715 is_10G_port(const struct port_info *pi)
 716 {
 717         return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0);
 718 }
 719 
 720 static inline struct sge_rxq *
 721 iq_to_rxq(struct sge_iq *iq)
 722 {
 723         return (__containerof(iq, struct sge_rxq, iq));
 724 }
 725 
 726 static inline bool
 727 is_25G_port(const struct port_info *pi)
 728 {
 729         return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_25G) != 0);
 730 }
 731 
 732 static inline bool
 733 is_40G_port(const struct port_info *pi)
 734 {
 735         return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) != 0);
 736 }
 737 
 738 static inline bool
 739 is_100G_port(const struct port_info *pi)
 740 {
 741         return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_100G) != 0);
 742 }
 743 
 744 static inline bool
 745 is_10XG_port(const struct port_info *pi)
 746 {
 747         return (is_10G_port(pi) || is_40G_port(pi) ||
 748                 is_25G_port(pi) || is_100G_port(pi));
 749 }
 750 
 751 static inline char *
 752 print_port_speed(const struct port_info *pi)
 753 {
 754         if (!pi)
 755                 return "-";
 756 
 757         if (is_100G_port(pi))
 758                 return "100G";
 759         else if (is_40G_port(pi))
 760                 return "40G";
 761         else if (is_25G_port(pi))
 762                 return "25G";
 763         else if (is_10G_port(pi))
 764                 return "10G";
 765         else
 766                 return "1G";
 767 }
 768 
 769 #ifdef TCP_OFFLOAD_ENABLE
 770 int t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0);
 771 
 772 static inline int
 773 t4_wrq_tx(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m)
 774 {
 775         int rc;
 776 
 777         TXQ_LOCK(wrq);
 778         rc = t4_wrq_tx_locked(sc, wrq, m);
 779         TXQ_UNLOCK(wrq);
 780         return (rc);
 781 }
 782 #endif
 783 
 784 /**
 785  * t4_os_pci_read_seeprom - read four bytes of SEEPROM/VPD contents
 786  * @adapter: the adapter
 787  * @addr: SEEPROM/VPD Address to read
 788  * @valp: where to store the value read
 789  *
 790  * Read a 32-bit value from the given address in the SEEPROM/VPD.  The address
 791  * must be four-byte aligned.  Returns 0 on success, a negative erro number
 792  * on failure.
 793  */
 794 static inline int t4_os_pci_read_seeprom(adapter_t *adapter,
 795                                          int addr, u32 *valp)
 796 {
 797         int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data);
 798         int ret;
 799 
 800         ret = t4_seeprom_read(adapter, addr, valp);
 801 
 802         return ret >= 0 ? 0 : ret;
 803 }
 804 
 805 /**
 806  * t4_os_pci_write_seeprom - write four bytes of SEEPROM/VPD contents
 807  * @adapter: the adapter
 808  * @addr: SEEPROM/VPD Address to write
 809  * @val: the value write
 810  *
 811  * Write a 32-bit value to the given address in the SEEPROM/VPD.  The address
 812  * must be four-byte aligned.  Returns 0 on success, a negative erro number
 813  * on failure.
 814  */
 815 static inline int t4_os_pci_write_seeprom(adapter_t *adapter,
 816                                           int addr, u32 val)
 817 {
 818         int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data);
 819         int ret;
 820 
 821         ret = t4_seeprom_write(adapter, addr, val);
 822 
 823         return ret >= 0 ? 0 : ret;
 824 }
 825 
 826 static inline int t4_os_pci_set_vpd_size(struct adapter *adapter, size_t len)
 827 {
 828         return 0;
 829 }
 830 
 831 static inline unsigned int t4_use_ldst(struct adapter *adap)
 832 {
 833         return (adap->flags & FW_OK);
 834 }
 835 #define t4_os_alloc(_size)      kmem_alloc(_size, KM_SLEEP)
 836 
 837 static inline void t4_db_full(struct adapter *adap) {}
 838 static inline void t4_db_dropped(struct adapter *adap) {}
 839 
 840 /* t4_nexus.c */
 841 int t4_os_find_pci_capability(struct adapter *sc, int cap);
 842 void t4_os_portmod_changed(const struct adapter *sc, int idx);
 843 int adapter_full_init(struct adapter *sc);
 844 int adapter_full_uninit(struct adapter *sc);
 845 int port_full_init(struct port_info *pi);
 846 int port_full_uninit(struct port_info *pi);
 847 void enable_port_queues(struct port_info *pi);
 848 void disable_port_queues(struct port_info *pi);
 849 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h);
 850 int t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t);
 851 void t4_iterate(void (*func)(int, void *), void *arg);
 852 
 853 /* t4_sge.c */
 854 void t4_sge_init(struct adapter *sc);
 855 int t4_setup_adapter_queues(struct adapter *sc);
 856 int t4_teardown_adapter_queues(struct adapter *sc);
 857 int t4_setup_port_queues(struct port_info *pi);
 858 int t4_teardown_port_queues(struct port_info *pi);
 859 uint_t t4_intr_all(caddr_t arg1, caddr_t arg2);
 860 uint_t t4_intr(caddr_t arg1, caddr_t arg2);
 861 uint_t t4_intr_err(caddr_t arg1, caddr_t arg2);
 862 int t4_mgmt_tx(struct adapter *sc, mblk_t *m);
 863 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 864 uint32_t position_memwin(struct adapter *, int, uint32_t);
 865 
 866 mblk_t *t4_eth_tx(void *, mblk_t *);
 867 mblk_t *t4_mc_tx(void *arg, mblk_t *m);
 868 mblk_t *t4_ring_rx(struct sge_rxq *rxq, int poll_bytes);
 869 int t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps,  int count,
 870     int flags);
 871 
 872 /* t4_mac.c */
 873 void t4_mc_init(struct port_info *pi);
 874 void t4_mc_cb_init(struct port_info *);
 875 void t4_os_link_changed(struct adapter *sc, int idx, int link_stat);
 876 void t4_mac_rx(struct port_info *pi, struct sge_rxq *rxq, mblk_t *m);
 877 void t4_mac_tx_update(struct port_info *pi, struct sge_txq *txq);
 878 int t4_addmac(void *arg, const uint8_t *ucaddr);
 879 
 880 /* t4_ioctl.c */
 881 int t4_ioctl(struct adapter *sc, int cmd, void *data, int mode);
 882 
 883 struct l2t_data *t4_init_l2t(struct adapter *sc);
 884 #endif /* __CXGBE_ADAPTER_H */