1 /*
   2  * Copyright (c) 2008-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30 
  31 #ifndef _SYS_SFXGE_H
  32 #define _SYS_SFXGE_H
  33 
  34 #ifdef  __cplusplus
  35 extern "C" {
  36 #endif
  37 
  38 #include <sys/types.h>
  39 #include <sys/ddi.h>
  40 #include <sys/sunddi.h>
  41 #include <sys/stream.h>
  42 #include <sys/ethernet.h>
  43 #include <sys/cpuvar.h>
  44 #include <sys/id_space.h>
  45 
  46 #include <sys/mac.h>
  47 #include <sys/mac_ether.h>
  48 #include <sys/mac_provider.h>
  49 
  50 #include "sfxge_ioc.h"
  51 #include "sfxge_debug.h"
  52 
  53 #include "efx.h"
  54 #include "efx_regs.h"
  55 
  56 #ifdef  _KERNEL
  57 
  58 #define SFXGE_DRIVER_NAME "sfxge"
  59 
  60 #define SFXGE_CPU_CACHE_SIZE    64
  61 
  62 typedef struct sfxge_s  sfxge_t;
  63 
  64 typedef enum sfxge_intr_state_e {
  65         SFXGE_INTR_UNINITIALIZED = 0,
  66         SFXGE_INTR_INITIALIZED,
  67         SFXGE_INTR_TESTING,
  68         SFXGE_INTR_STARTED
  69 } sfxge_intr_state_t;
  70 
  71 typedef struct sfxge_intr_s {
  72         ddi_intr_handle_t       *si_table;
  73         int                     si_table_size;
  74         int                     si_nalloc;
  75         int                     si_type;
  76         int                     si_cap;
  77         efsys_mem_t             si_mem;
  78         uint64_t                si_mask;
  79         sfxge_intr_state_t      si_state;
  80         uint32_t                si_zero_count;
  81         int                     si_intr_pri;
  82 } sfxge_intr_t;
  83 
  84 typedef enum sfxge_promisc_type_e {
  85         SFXGE_PROMISC_OFF = 0,
  86         SFXGE_PROMISC_ALL_MULTI,
  87         SFXGE_PROMISC_ALL_PHYS
  88 } sfxge_promisc_type_t;
  89 
  90 typedef enum sfxge_link_duplex_e {
  91         SFXGE_LINK_DUPLEX_UNKNOWN = 0,
  92         SFXGE_LINK_DUPLEX_HALF,
  93         SFXGE_LINK_DUPLEX_FULL
  94 } sfxge_link_duplex_t;
  95 
  96 typedef enum sfxge_unicst_type_e {
  97         SFXGE_UNICST_BIA = 0,
  98         SFXGE_UNICST_LAA,
  99         SFXGE_UNICST_NTYPES
 100 } sfxge_unicst_type_t;
 101 
 102 typedef struct sfxge_phy_s {
 103                 kstat_t                 *sp_ksp;
 104                 kstat_named_t           *sp_stat;
 105                 uint32_t                *sp_statbuf;
 106                 efsys_mem_t             sp_mem;
 107 } sfxge_phy_t;
 108 
 109 typedef enum sfxge_mac_state_e {
 110         SFXGE_MAC_UNINITIALIZED = 0,
 111         SFXGE_MAC_INITIALIZED,
 112         SFXGE_MAC_STARTED
 113 } sfxge_mac_state_t;
 114 
 115 typedef struct sfxge_mac_s {
 116         sfxge_t                 *sm_sp;
 117         efsys_mem_t             sm_mem;
 118         kstat_t                 *sm_ksp;
 119         kstat_named_t           *sm_stat;
 120         uint8_t                 sm_bia[ETHERADDRL];
 121         uint8_t                 sm_laa[ETHERADDRL];
 122         boolean_t               sm_laa_valid;
 123         unsigned int            sm_fcntl;
 124         sfxge_promisc_type_t    sm_promisc;
 125         uint8_t                 sm_mcast_addr[EFX_MAC_MULTICAST_LIST_MAX *
 126             ETHERADDRL]; /* List of multicast addresses to filter on */
 127         int                     sm_mcast_count;
 128         clock_t                 sm_lbolt;
 129         kmutex_t                sm_lock;
 130         efx_link_mode_t         sm_link_mode;
 131         unsigned int            sm_link_speed;
 132         sfxge_link_duplex_t     sm_link_duplex;
 133         boolean_t               sm_link_up;
 134         boolean_t               sm_link_poll_reqd;
 135         kcondvar_t              sm_link_poll_kv;
 136         boolean_t               sm_mac_stats_timer_reqd;
 137         boolean_t               sm_mac_stats_pend;
 138         ddi_taskq_t             *sm_tqp;
 139         sfxge_mac_state_t       sm_state;
 140         sfxge_phy_t             sm_phy;
 141         uint32_t                sm_phy_cap_to_set;
 142         uint32_t                sm_phy_cap_to_unset;
 143 } sfxge_mac_t;
 144 
 145 typedef enum sfxge_mon_state_e {
 146         SFXGE_MON_UNINITIALIZED = 0,
 147         SFXGE_MON_INITIALIZED,
 148         SFXGE_MON_STARTED
 149 } sfxge_mon_state_t;
 150 
 151 typedef struct sfxge_mon_s {
 152         sfxge_t                 *sm_sp;
 153         efx_mon_type_t          sm_type;
 154         unsigned int            sm_devid;
 155         kstat_t                 *sm_ksp;
 156         kstat_named_t           *sm_stat;
 157         efx_mon_stat_value_t    *sm_statbuf;
 158         kmutex_t                sm_lock;
 159         sfxge_mon_state_t       sm_state;
 160         efsys_mem_t             sm_mem;
 161         int                     sm_polling;
 162 } sfxge_mon_t;
 163 
 164 typedef enum sfxge_sram_state_e {
 165         SFXGE_SRAM_UNINITIALIZED = 0,
 166         SFXGE_SRAM_INITIALIZED,
 167         SFXGE_SRAM_STARTED
 168 } sfxge_sram_state_t;
 169 
 170 typedef struct sfxge_sram_s {
 171         sfxge_t                 *ss_sp;
 172         kmutex_t                ss_lock;
 173         id_space_t              *ss_buf_tbl_ids;
 174         unsigned int            ss_count;
 175         sfxge_sram_state_t      ss_state;
 176 } sfxge_sram_t;
 177 
 178 typedef enum sfxge_mcdi_state_e {
 179         SFXGE_MCDI_UNINITIALIZED = 0,
 180         SFXGE_MCDI_INITIALIZED,
 181         SFXGE_MCDI_BUSY,
 182         SFXGE_MCDI_COMPLETED
 183 } sfxge_mcdi_state_t;
 184 
 185 typedef struct sfxge_mcdi_s {
 186         sfxge_t                 *sm_sp;
 187         kmutex_t                sm_lock;
 188         sfxge_mcdi_state_t      sm_state;
 189         efx_mcdi_transport_t    sm_emt;
 190         efsys_mem_t             sm_mem;
 191         kcondvar_t              sm_kv;          /* MCDI poll complete */
 192 } sfxge_mcdi_t;
 193 
 194 #define SFXGE_NEVS                      4096
 195 #define SFXGE_RX_NDESCS                 1024
 196 #define SFXGE_TX_NDESCS                 1024
 197 #define SFXGE_TX_NLABELS                EFX_EV_TX_NLABELS
 198 
 199 #define SFXGE_DEFAULT_RXQ_SIZE          1024
 200 #define SFXGE_DEFAULT_MODERATION        30
 201 
 202 typedef enum sfxge_evq_state_e {
 203         SFXGE_EVQ_UNINITIALIZED = 0,
 204         SFXGE_EVQ_INITIALIZED,
 205         SFXGE_EVQ_STARTING,
 206         SFXGE_EVQ_STARTED
 207 } sfxge_evq_state_t;
 208 
 209 #define SFXGE_EV_BATCH  (SFXGE_NEVS / 4)
 210 
 211 typedef struct sfxge_txq_s      sfxge_txq_t;
 212 
 213 typedef struct sfxge_evq_s {
 214         union {
 215                 struct {
 216                         sfxge_t                 *__se_sp;
 217                         unsigned int            __se_index;
 218                         efsys_mem_t             __se_mem;
 219                         unsigned int            __se_id;
 220                         kstat_t                 *__se_ksp;
 221                         kstat_named_t           *__se_stat;
 222                         efx_ev_callbacks_t      __se_eec;
 223                         sfxge_evq_state_t       __se_state;
 224                         boolean_t               __se_exception;
 225                 } __se_s1;
 226                 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
 227         } __se_u1;
 228         union {
 229                 struct {
 230                         kmutex_t                __se_lock;
 231                         kcondvar_t              __se_init_kv;
 232                         efx_evq_t               *__se_eep;
 233                         unsigned int            __se_count;
 234                         unsigned int            __se_rx;
 235                         unsigned int            __se_tx;
 236                         sfxge_txq_t             *__se_stp;
 237                         sfxge_txq_t             **__se_stpp;
 238                         processorid_t           __se_cpu_id;
 239                         uint16_t                __se_ev_batch;
 240                 } __se_s2;
 241                 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE];
 242         } __se_u2;
 243         union {
 244                 struct {
 245                         sfxge_txq_t     *__se_label_stp[SFXGE_TX_NLABELS];
 246                 } __se_s3;
 247                 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
 248         } __se_u3;
 249 } sfxge_evq_t;
 250 
 251 #define se_sp           __se_u1.__se_s1.__se_sp
 252 #define se_index        __se_u1.__se_s1.__se_index
 253 #define se_mem          __se_u1.__se_s1.__se_mem
 254 #define se_id           __se_u1.__se_s1.__se_id
 255 #define se_ksp          __se_u1.__se_s1.__se_ksp
 256 #define se_stat         __se_u1.__se_s1.__se_stat
 257 #define se_eec          __se_u1.__se_s1.__se_eec
 258 #define se_state        __se_u1.__se_s1.__se_state
 259 #define se_exception    __se_u1.__se_s1.__se_exception
 260 
 261 #define se_lock         __se_u2.__se_s2.__se_lock
 262 #define se_init_kv      __se_u2.__se_s2.__se_init_kv
 263 #define se_eep          __se_u2.__se_s2.__se_eep
 264 #define se_count        __se_u2.__se_s2.__se_count
 265 #define se_rx           __se_u2.__se_s2.__se_rx
 266 #define se_tx           __se_u2.__se_s2.__se_tx
 267 #define se_stp          __se_u2.__se_s2.__se_stp
 268 #define se_stpp         __se_u2.__se_s2.__se_stpp
 269 #define se_cpu_id       __se_u2.__se_s2.__se_cpu_id
 270 #define se_ev_batch     __se_u2.__se_s2.__se_ev_batch
 271 
 272 #define se_label_stp    __se_u3.__se_s3.__se_label_stp
 273 
 274 
 275 #define SFXGE_MAGIC_RESERVED    0x8000
 276 
 277 #define SFXGE_MAGIC_DMAQ_LABEL_WIDTH  5
 278 #define SFXGE_MAGIC_DMAQ_LABEL_MASK   ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1)
 279 
 280 #define SFXGE_MAGIC_RX_QFLUSH_DONE                                      \
 281         (SFXGE_MAGIC_RESERVED | (1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
 282 
 283 #define SFXGE_MAGIC_RX_QFLUSH_FAILED                                    \
 284         (SFXGE_MAGIC_RESERVED | (2 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
 285 
 286 #define SFXGE_MAGIC_RX_QFPP_TRIM                                        \
 287         (SFXGE_MAGIC_RESERVED | (3 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
 288 
 289 #define SFXGE_MAGIC_TX_QFLUSH_DONE                                      \
 290         (SFXGE_MAGIC_RESERVED | (4 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
 291 
 292 typedef struct sfxge_rxq_s              sfxge_rxq_t;
 293 
 294 #define SFXGE_ETHERTYPE_LOOPBACK        0x9000  /* Xerox loopback */
 295 
 296 typedef struct sfxge_rx_packet_s        sfxge_rx_packet_t;
 297 
 298 struct sfxge_rx_packet_s {
 299         union {
 300                 struct {
 301                         frtn_t                  __srp_free;
 302                         uint16_t                __srp_flags;
 303                         uint16_t                __srp_size;
 304                         mblk_t                  *__srp_mp;
 305                         struct ether_header     *__srp_etherhp;
 306                         struct ip               *__srp_iphp;
 307                         struct tcphdr           *__srp_thp;
 308                         size_t                  __srp_off;
 309                 } __srp_s1;
 310                 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE];
 311         } __srp_u1;
 312         union {
 313                 struct {
 314                         sfxge_rxq_t             *__srp_srp;
 315                         ddi_dma_handle_t        __srp_dma_handle;
 316                         ddi_acc_handle_t        __srp_acc_handle;
 317                         unsigned char           *__srp_base;
 318                         size_t                  __srp_mblksize;
 319                         uint64_t                __srp_addr;
 320                         boolean_t               __srp_recycle;
 321                         caddr_t                 __srp_putp;
 322                 } __srp_s2;
 323                 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE * 2];
 324         } __srp_u2;
 325 };
 326 
 327 #define srp_free        __srp_u1.__srp_s1.__srp_free
 328 #define srp_flags       __srp_u1.__srp_s1.__srp_flags
 329 #define srp_size        __srp_u1.__srp_s1.__srp_size
 330 #define srp_mp          __srp_u1.__srp_s1.__srp_mp
 331 #define srp_etherhp     __srp_u1.__srp_s1.__srp_etherhp
 332 #define srp_iphp        __srp_u1.__srp_s1.__srp_iphp
 333 #define srp_thp         __srp_u1.__srp_s1.__srp_thp
 334 #define srp_off         __srp_u1.__srp_s1.__srp_off
 335 
 336 #define srp_srp         __srp_u2.__srp_s2.__srp_srp
 337 #define srp_dma_handle  __srp_u2.__srp_s2.__srp_dma_handle
 338 #define srp_acc_handle  __srp_u2.__srp_s2.__srp_acc_handle
 339 #define srp_base        __srp_u2.__srp_s2.__srp_base
 340 #define srp_mblksize    __srp_u2.__srp_s2.__srp_mblksize
 341 #define srp_addr        __srp_u2.__srp_s2.__srp_addr
 342 #define srp_recycle     __srp_u2.__srp_s2.__srp_recycle
 343 #define srp_putp        __srp_u2.__srp_s2.__srp_putp
 344 
 345 #define SFXGE_RX_FPP_NSLOTS     8
 346 #define SFXGE_RX_FPP_MASK       (SFXGE_RX_FPP_NSLOTS - 1)
 347 
 348 /* Free packet pool putlist (dynamically allocated) */
 349 typedef struct sfxge_rx_fpp_putlist_s {
 350         kmutex_t                srfpl_lock;
 351         unsigned int            srfpl_count;
 352         mblk_t                  *srfpl_putp;
 353         mblk_t                  **srfpl_putpp;
 354 } sfxge_rx_fpp_putlist_t;
 355 
 356 /* Free packet pool */
 357 typedef struct sfxge_rx_fpp_s {
 358         caddr_t         srfpp_putp;
 359         unsigned int    srfpp_loaned;
 360         mblk_t          *srfpp_get;
 361         unsigned int    srfpp_count;
 362         unsigned int    srfpp_min;
 363         /* Low water mark: Don't trim to below this */
 364         unsigned int    srfpp_lowat;
 365 } sfxge_rx_fpp_t;
 366 
 367 typedef struct sfxge_rx_flow_s  sfxge_rx_flow_t;
 368 
 369 struct sfxge_rx_flow_s {
 370         uint32_t                srf_tag;
 371         /* in-order segment count */
 372         unsigned int            srf_count;
 373         uint16_t                srf_tci;
 374         uint32_t                srf_saddr;
 375         uint32_t                srf_daddr;
 376         uint16_t                srf_sport;
 377         uint16_t                srf_dport;
 378         /* sequence number */
 379         uint32_t                srf_seq;
 380         clock_t                 srf_lbolt;
 381         mblk_t                  *srf_mp;
 382         mblk_t                  **srf_mpp;
 383         struct ether_header     *srf_etherhp;
 384         struct ip               *srf_iphp;
 385         struct tcphdr           *srf_first_thp;
 386         struct tcphdr           *srf_last_thp;
 387         size_t                  srf_len;
 388         sfxge_rx_flow_t         *srf_next;
 389 };
 390 
 391 #define SFXGE_MAX_FLOW          1024
 392 #define SFXGE_SLOW_START        20
 393 
 394 typedef enum sfxge_flush_state_e {
 395         SFXGE_FLUSH_INACTIVE = 0,
 396         SFXGE_FLUSH_DONE,
 397         SFXGE_FLUSH_PENDING,
 398         SFXGE_FLUSH_FAILED
 399 } sfxge_flush_state_t;
 400 
 401 typedef enum sfxge_rxq_state_e {
 402         SFXGE_RXQ_UNINITIALIZED = 0,
 403         SFXGE_RXQ_INITIALIZED,
 404         SFXGE_RXQ_STARTED
 405 } sfxge_rxq_state_t;
 406 
 407 
 408 #define SFXGE_RX_BATCH  128
 409 #define SFXGE_RX_NSTATS 8 /* note that *esballoc share one kstat */
 410 
 411 struct sfxge_rxq_s {
 412         union {
 413                 struct {
 414                         sfxge_t                         *__sr_sp;
 415                         unsigned int                    __sr_index;
 416                         efsys_mem_t                     __sr_mem;
 417                         unsigned int                    __sr_id;
 418                         unsigned int                    __sr_lowat;
 419                         unsigned int                    __sr_hiwat;
 420                         volatile timeout_id_t           __sr_tid;
 421                         sfxge_rxq_state_t               __sr_state;
 422                 } __sr_s1;
 423                 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
 424         } __sr_u1;
 425         union {
 426                 struct {
 427                         sfxge_rx_packet_t               **__sr_srpp;
 428                         unsigned int                    __sr_added;
 429                         unsigned int                    __sr_pushed;
 430                         unsigned int                    __sr_pending;
 431                         unsigned int                    __sr_completed;
 432                         unsigned int                    __sr_loopback;
 433                         mblk_t                          *__sr_mp;
 434                         mblk_t                          **__sr_mpp;
 435                         sfxge_rx_flow_t                 *__sr_flow;
 436                         sfxge_rx_flow_t                 *__sr_srfp;
 437                         sfxge_rx_flow_t                 **__sr_srfpp;
 438                         clock_t                         __sr_rto;
 439                 } __sr_s2;
 440                 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
 441         } __sr_u2;
 442         union {
 443                 struct {
 444                         sfxge_rx_fpp_t                  __sr_fpp;
 445                         efx_rxq_t                       *__sr_erp;
 446                         volatile sfxge_flush_state_t    __sr_flush;
 447                         kcondvar_t                      __sr_flush_kv;
 448                         kstat_t                         *__sr_ksp;
 449                 } __sr_s3;
 450                 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE];
 451         } __sr_u3;
 452         struct {
 453                 /* NB must match SFXGE_RX_NSTATS */
 454                 uint32_t    srk_rx_pkt_mem_limit;
 455                 uint32_t    srk_kcache_alloc_nomem;
 456                 uint32_t    srk_dma_alloc_nomem;
 457                 uint32_t    srk_dma_alloc_fail;
 458                 uint32_t    srk_dma_bind_nomem;
 459                 uint32_t    srk_dma_bind_fail;
 460                 uint32_t    srk_desballoc_fail;
 461                 uint32_t    srk_rxq_empty_discard;
 462         } sr_kstat;
 463 };
 464 
 465 #define sr_sp           __sr_u1.__sr_s1.__sr_sp
 466 #define sr_index        __sr_u1.__sr_s1.__sr_index
 467 #define sr_mem          __sr_u1.__sr_s1.__sr_mem
 468 #define sr_id           __sr_u1.__sr_s1.__sr_id
 469 #define sr_mrh          __sr_u1.__sr_s1.__sr_mrh
 470 #define sr_lowat        __sr_u1.__sr_s1.__sr_lowat
 471 #define sr_hiwat        __sr_u1.__sr_s1.__sr_hiwat
 472 #define sr_tid          __sr_u1.__sr_s1.__sr_tid
 473 #define sr_state        __sr_u1.__sr_s1.__sr_state
 474 
 475 #define sr_srpp         __sr_u2.__sr_s2.__sr_srpp
 476 #define sr_added        __sr_u2.__sr_s2.__sr_added
 477 #define sr_pushed       __sr_u2.__sr_s2.__sr_pushed
 478 #define sr_pending      __sr_u2.__sr_s2.__sr_pending
 479 #define sr_completed    __sr_u2.__sr_s2.__sr_completed
 480 #define sr_loopback     __sr_u2.__sr_s2.__sr_loopback
 481 #define sr_mp           __sr_u2.__sr_s2.__sr_mp
 482 #define sr_mpp          __sr_u2.__sr_s2.__sr_mpp
 483 #define sr_flow         __sr_u2.__sr_s2.__sr_flow
 484 #define sr_srfp         __sr_u2.__sr_s2.__sr_srfp
 485 #define sr_srfpp        __sr_u2.__sr_s2.__sr_srfpp
 486 #define sr_rto          __sr_u2.__sr_s2.__sr_rto
 487 
 488 #define sr_fpp          __sr_u3.__sr_s3.__sr_fpp
 489 #define sr_erp          __sr_u3.__sr_s3.__sr_erp
 490 #define sr_flush        __sr_u3.__sr_s3.__sr_flush
 491 #define sr_flush_kv     __sr_u3.__sr_s3.__sr_flush_kv
 492 #define sr_ksp          __sr_u3.__sr_s3.__sr_ksp
 493 
 494 typedef struct sfxge_tx_packet_s        sfxge_tx_packet_t;
 495 
 496 /* Packet type from parsing transmit packet */
 497 typedef enum sfxge_packet_type_e {
 498         SFXGE_PACKET_TYPE_UNKNOWN = 0,
 499         SFXGE_PACKET_TYPE_IPV4_TCP,
 500         SFXGE_PACKET_TYPE_IPV4_UDP,
 501         SFXGE_PACKET_TYPE_IPV4_SCTP,
 502         SFXGE_PACKET_TYPE_IPV4_OTHER,
 503         SFXGE_PACKET_NTYPES
 504 } sfxge_packet_type_t;
 505 
 506 struct sfxge_tx_packet_s {
 507         sfxge_tx_packet_t       *stp_next;
 508         mblk_t                  *stp_mp;
 509         struct ether_header     *stp_etherhp;
 510         struct ip               *stp_iphp;
 511         struct tcphdr           *stp_thp;
 512         size_t                  stp_off;
 513         size_t                  stp_size;
 514         size_t                  stp_mss;
 515         uint32_t                stp_dpl_put_len;
 516 };
 517 
 518 #define SFXGE_TX_FPP_MAX        64
 519 
 520 typedef struct sfxge_tx_fpp_s {
 521         sfxge_tx_packet_t       *stf_stpp;
 522         unsigned int            stf_count;
 523 } sfxge_tx_fpp_t;
 524 
 525 typedef struct sfxge_tx_mapping_s       sfxge_tx_mapping_t;
 526 
 527 #define SFXGE_TX_MAPPING_NADDR  (((1 << 16) >> 12) + 2)
 528 
 529 struct sfxge_tx_mapping_s {
 530         sfxge_tx_mapping_t      *stm_next;
 531         sfxge_t                 *stm_sp;
 532         mblk_t                  *stm_mp;
 533         ddi_dma_handle_t        stm_dma_handle;
 534         caddr_t                 stm_base;
 535         size_t                  stm_size;
 536         size_t                  stm_off;
 537         uint64_t                stm_addr[SFXGE_TX_MAPPING_NADDR];
 538 };
 539 
 540 typedef struct sfxge_tx_fmp_s {
 541         sfxge_tx_mapping_t      *stf_stmp;
 542         unsigned int            stf_count;
 543 } sfxge_tx_fmp_t;
 544 
 545 typedef struct sfxge_tx_buffer_s        sfxge_tx_buffer_t;
 546 
 547 struct sfxge_tx_buffer_s {
 548         sfxge_tx_buffer_t       *stb_next;
 549         size_t                  stb_off;
 550         efsys_mem_t             stb_esm;
 551 };
 552 
 553 #define SFXGE_TX_BUFFER_SIZE    0x400
 554 #define SFXGE_TX_HEADER_SIZE    0x100
 555 #define SFXGE_TX_COPY_THRESHOLD 0x200
 556 
 557 typedef struct sfxge_tx_fbp_s {
 558         sfxge_tx_buffer_t       *stf_stbp;
 559         unsigned int            stf_count;
 560 } sfxge_tx_fbp_t;
 561 
 562 typedef struct sfxge_tx_dpl_s {
 563         uintptr_t               std_put;
 564         sfxge_tx_packet_t       *std_get;
 565         sfxge_tx_packet_t       **std_getp;
 566         unsigned int            std_count; /* only get list count */
 567         unsigned int            get_pkt_limit;
 568         unsigned int            put_pkt_limit;
 569         unsigned int            get_full_count;
 570         unsigned int            put_full_count;
 571 } sfxge_tx_dpl_t;
 572 
 573 typedef enum sfxge_txq_state_e {
 574         SFXGE_TXQ_UNINITIALIZED = 0,
 575         SFXGE_TXQ_INITIALIZED,
 576         SFXGE_TXQ_STARTED,
 577         SFXGE_TXQ_FLUSH_PENDING,
 578         SFXGE_TXQ_FLUSH_DONE,
 579         SFXGE_TXQ_FLUSH_FAILED
 580 } sfxge_txq_state_t;
 581 
 582 typedef enum sfxge_txq_type_e {
 583         SFXGE_TXQ_NON_CKSUM = 0,
 584         SFXGE_TXQ_IP_CKSUM,
 585         SFXGE_TXQ_IP_TCP_UDP_CKSUM,
 586         SFXGE_TXQ_NTYPES
 587 } sfxge_txq_type_t;
 588 
 589 #define SFXGE_TXQ_UNBLOCK_LEVEL1        (EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) / 4)
 590 #define SFXGE_TXQ_UNBLOCK_LEVEL2        0
 591 #define SFXGE_TXQ_NOT_BLOCKED           -1
 592 
 593 #define SFXGE_TX_BATCH  64
 594 
 595 struct sfxge_txq_s {
 596         union {
 597                 struct {
 598                         sfxge_t                         *__st_sp;
 599                         unsigned int                    __st_index;
 600                         unsigned int                    __st_label;
 601                         sfxge_txq_type_t                __st_type;
 602                         unsigned int                    __st_evq;
 603                         efsys_mem_t                     __st_mem;
 604                         unsigned int                    __st_id;
 605                         kstat_t                         *__st_ksp;
 606                         kstat_named_t                   *__st_stat;
 607                         sfxge_txq_state_t               __st_state;
 608                 } __st_s1;
 609                 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 2];
 610         } __st_u1;
 611         union {
 612                 struct {
 613                         sfxge_tx_dpl_t                  __st_dpl;
 614                 } __st_s2;
 615                 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
 616         } __st_u2;
 617         union {
 618                 struct {
 619                         kmutex_t                        __st_lock;
 620                         /* mapping pool - sfxge_tx_mapping_t */
 621                         sfxge_tx_fmp_t                  __st_fmp;
 622                         /* buffer pool - sfxge_tx_buffer_t */
 623                         sfxge_tx_fbp_t                  __st_fbp;
 624                         /* packet pool - sfxge_tx_packet_t */
 625                         sfxge_tx_fpp_t                  __st_fpp;
 626                         efx_buffer_t                    *__st_eb;
 627                         unsigned int                    __st_n;
 628                         efx_txq_t                       *__st_etp;
 629                         sfxge_tx_mapping_t              **__st_stmp;
 630                         sfxge_tx_buffer_t               **__st_stbp;
 631                         mblk_t                          **__st_mp;
 632                         unsigned int                    __st_added;
 633                         unsigned int                    __st_reaped;
 634                         int                             __st_unblock;
 635                 } __st_s3;
 636                 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 3];
 637         } __st_u3;
 638         union {
 639                 struct {
 640                         sfxge_txq_t                     *__st_next;
 641                         unsigned int                    __st_pending;
 642                         unsigned int                    __st_completed;
 643 
 644                 } __st_s4;
 645                 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
 646         } __st_u4;
 647 };
 648 
 649 #define st_sp           __st_u1.__st_s1.__st_sp
 650 #define st_index        __st_u1.__st_s1.__st_index
 651 #define st_label        __st_u1.__st_s1.__st_label
 652 #define st_type         __st_u1.__st_s1.__st_type
 653 #define st_evq          __st_u1.__st_s1.__st_evq
 654 #define st_mem          __st_u1.__st_s1.__st_mem
 655 #define st_id           __st_u1.__st_s1.__st_id
 656 #define st_ksp          __st_u1.__st_s1.__st_ksp
 657 #define st_stat         __st_u1.__st_s1.__st_stat
 658 #define st_state        __st_u1.__st_s1.__st_state
 659 
 660 #define st_dpl          __st_u2.__st_s2.__st_dpl
 661 
 662 #define st_lock         __st_u3.__st_s3.__st_lock
 663 #define st_fmp          __st_u3.__st_s3.__st_fmp
 664 #define st_fbp          __st_u3.__st_s3.__st_fbp
 665 #define st_fpp          __st_u3.__st_s3.__st_fpp
 666 #define st_eb           __st_u3.__st_s3.__st_eb
 667 #define st_n            __st_u3.__st_s3.__st_n
 668 #define st_etp          __st_u3.__st_s3.__st_etp
 669 #define st_stmp         __st_u3.__st_s3.__st_stmp
 670 #define st_stbp         __st_u3.__st_s3.__st_stbp
 671 #define st_mp           __st_u3.__st_s3.__st_mp
 672 #define st_added        __st_u3.__st_s3.__st_added
 673 #define st_reaped       __st_u3.__st_s3.__st_reaped
 674 #define st_unblock      __st_u3.__st_s3.__st_unblock
 675 
 676 #define st_next         __st_u4.__st_s4.__st_next
 677 #define st_pending      __st_u4.__st_s4.__st_pending
 678 #define st_completed    __st_u4.__st_s4.__st_completed
 679 
 680 typedef enum sfxge_rx_scale_state_e {
 681         SFXGE_RX_SCALE_UNINITIALIZED = 0,
 682         SFXGE_RX_SCALE_INITIALIZED,
 683         SFXGE_RX_SCALE_STARTED
 684 } sfxge_rx_scale_state_t;
 685 
 686 #define SFXGE_RX_SCALE_MAX      EFX_RSS_TBL_SIZE
 687 
 688 typedef struct sfxge_rx_scale_s {
 689         kmutex_t                srs_lock;
 690         unsigned int            *srs_cpu;
 691         unsigned int            srs_tbl[SFXGE_RX_SCALE_MAX];
 692         unsigned int            srs_count;
 693         kstat_t                 *srs_ksp;
 694         sfxge_rx_scale_state_t  srs_state;
 695 } sfxge_rx_scale_t;
 696 
 697 
 698 typedef enum sfxge_rx_coalesce_mode_e {
 699         SFXGE_RX_COALESCE_OFF = 0,
 700         SFXGE_RX_COALESCE_DISALLOW_PUSH = 1,
 701         SFXGE_RX_COALESCE_ALLOW_PUSH = 2
 702 } sfxge_rx_coalesce_mode_t;
 703 
 704 typedef enum sfxge_vpd_type_e {
 705         SFXGE_VPD_ID = 0,
 706         SFXGE_VPD_PN = 1,
 707         SFXGE_VPD_SN = 2,
 708         SFXGE_VPD_EC = 3,
 709         SFXGE_VPD_MN = 4,
 710         SFXGE_VPD_VD = 5,
 711         SFXGE_VPD_VE = 6,
 712         SFXGE_VPD_MAX = 7,
 713 } sfxge_vpd_type_t;
 714 
 715 typedef struct sfxge_vpd_kstat_s {
 716         kstat_t         *svk_ksp;
 717         kstat_named_t   svk_stat[SFXGE_VPD_MAX];
 718         efx_vpd_value_t *svk_vv;
 719 } sfxge_vpd_kstat_t;
 720 
 721 typedef struct sfxge_cfg_kstat_s {
 722         struct {
 723                 kstat_named_t   sck_mac;
 724                 kstat_named_t   sck_version;
 725         } kstat;
 726         struct {
 727                 char            sck_mac[64 + 1];
 728         } buf;
 729 } sfxge_cfg_kstat_t;
 730 
 731 typedef enum sfxge_state_e {
 732         SFXGE_UNINITIALIZED = 0,
 733         SFXGE_INITIALIZED,
 734         SFXGE_REGISTERED,
 735         SFXGE_STARTING,
 736         SFXGE_STARTED,
 737         SFXGE_STOPPING
 738 } sfxge_state_t;
 739 
 740 typedef enum sfxge_hw_err_e {
 741         SFXGE_HW_OK = 0,
 742         SFXGE_HW_ERR,
 743 } sfxge_hw_err_t;
 744 
 745 typedef enum sfxge_action_on_hw_err_e {
 746         SFXGE_RECOVER = 0,
 747         SFXGE_INVISIBLE = 1,
 748         SFXGE_LEAVE_DEAD = 2,
 749 } sfxge_action_on_hw_err_t;
 750 
 751 typedef char *sfxge_mac_priv_prop_t;
 752 
 753 #define SFXGE_TOEPLITZ_KEY_LEN 40
 754 
 755 struct sfxge_s {
 756         kmutex_t                        s_state_lock;
 757         sfxge_state_t                   s_state;
 758         dev_info_t                      *s_dip;
 759         ddi_taskq_t                     *s_tqp;
 760         ddi_acc_handle_t                s_pci_handle;
 761         uint16_t                        s_pci_venid;
 762         uint16_t                        s_pci_devid;
 763 #if EFSYS_OPT_MCDI_LOGGING
 764         unsigned int                    s_bus_addr;
 765 #endif
 766         efx_family_t                    s_family;
 767         unsigned int                    s_pcie_nlanes;
 768         unsigned int                    s_pcie_linkspeed;
 769         kmutex_t                        s_nic_lock;
 770         efsys_bar_t                     s_bar;
 771         sfxge_intr_t                    s_intr;
 772         sfxge_mac_t                     s_mac;
 773         sfxge_mon_t                     s_mon;
 774         sfxge_sram_t                    s_sram;
 775         sfxge_mcdi_t                    s_mcdi;
 776         kmem_cache_t                    *s_eq0c; /* eventQ 0 */
 777         kmem_cache_t                    *s_eqXc; /* all other eventQs */
 778         sfxge_evq_t                     *s_sep[SFXGE_RX_SCALE_MAX];
 779         unsigned int                    s_ev_moderation;
 780         kmem_cache_t                    *s_rqc;
 781         sfxge_rxq_t                     *s_srp[SFXGE_RX_SCALE_MAX];
 782         sfxge_rx_scale_t                s_rx_scale;
 783         size_t                          s_rx_prefix_size;
 784         size_t                          s_rx_buffer_size;
 785         size_t                          s_rx_buffer_align;
 786         sfxge_rx_coalesce_mode_t        s_rx_coalesce_mode;
 787         int64_t                         s_rx_pkt_mem_max;
 788         volatile uint64_t               s_rx_pkt_mem_alloc;
 789         kmem_cache_t                    *s_rpc;
 790         kmem_cache_t                    *s_tqc;
 791         unsigned int                    s_tx_scale_base[SFXGE_TXQ_NTYPES];
 792         unsigned int                    s_tx_scale_max[SFXGE_TXQ_NTYPES];
 793         int                             s_tx_qcount;
 794         sfxge_txq_t                     *s_stp[SFXGE_RX_SCALE_MAX *
 795             SFXGE_TXQ_NTYPES]; /* Sparse array */
 796         kmem_cache_t                    *s_tpc;
 797         int                             s_tx_flush_pending;
 798         kmutex_t                        s_tx_flush_lock;
 799         kcondvar_t                      s_tx_flush_kv;
 800         kmem_cache_t                    *s_tbc;
 801         kmem_cache_t                    *s_tmc;
 802         efx_nic_t                       *s_enp;
 803         sfxge_vpd_kstat_t               s_vpd_kstat;
 804         sfxge_cfg_kstat_t               s_cfg_kstat;
 805         kstat_t                         *s_cfg_ksp;
 806         size_t                          s_mtu;
 807         int                             s_rxq_poll_usec;
 808         mac_callbacks_t                 s_mc;
 809         mac_handle_t                    s_mh;
 810         sfxge_mac_priv_prop_t           *s_mac_priv_props;
 811         int                             s_mac_priv_props_alloc;
 812         volatile uint32_t               s_nested_restarts;
 813         uint32_t                        s_num_restarts;
 814         uint32_t                        s_num_restarts_hw_err;
 815         sfxge_hw_err_t                  s_hw_err;
 816         sfxge_action_on_hw_err_t        s_action_on_hw_err;
 817         uint16_t                        s_rxq_size;
 818         uint16_t                        s_evq0_size;
 819         uint16_t                        s_evqX_size;
 820 #if EFSYS_OPT_MCDI_LOGGING
 821         int                             s_mcdi_logging;
 822 #endif
 823         const uint32_t                  *s_toeplitz_cache;
 824 };
 825 
 826 typedef struct sfxge_dma_buffer_attr_s {
 827         dev_info_t              *sdba_dip;
 828         ddi_dma_attr_t          *sdba_dattrp;
 829         int                     (*sdba_callback) (caddr_t);
 830         size_t                  sdba_length;
 831         uint_t                  sdba_memflags;
 832         ddi_device_acc_attr_t   *sdba_devaccp;
 833         uint_t                  sdba_bindflags;
 834         int                     sdba_maxcookies;
 835         boolean_t               sdba_zeroinit;
 836 } sfxge_dma_buffer_attr_t;
 837 
 838 extern const char               sfxge_ident[];
 839 extern uint8_t                  sfxge_brdcst[];
 840 
 841 extern kmutex_t                 sfxge_global_lock;
 842 
 843 extern unsigned int             *sfxge_cpu;
 844 
 845 extern int                      sfxge_start(sfxge_t *, boolean_t);
 846 extern void                     sfxge_stop(sfxge_t *);
 847 extern void                     sfxge_ioctl(sfxge_t *, queue_t *, mblk_t *);
 848 extern int                      sfxge_restart_dispatch(sfxge_t *, uint_t,
 849     sfxge_hw_err_t, const char *, uint32_t);
 850 
 851 extern void                     sfxge_gld_link_update(sfxge_t *);
 852 extern void                     sfxge_gld_mtu_update(sfxge_t *);
 853 extern void                     sfxge_gld_rx_post(sfxge_t *, unsigned int,
 854     mblk_t *);
 855 extern void                     sfxge_gld_rx_push(sfxge_t *);
 856 extern int                      sfxge_gld_register(sfxge_t *);
 857 extern int                      sfxge_gld_unregister(sfxge_t *);
 858 
 859 extern int                      sfxge_dma_buffer_create(efsys_mem_t *,
 860     const sfxge_dma_buffer_attr_t *);
 861 extern void                     sfxge_dma_buffer_destroy(efsys_mem_t *);
 862 
 863 extern int                      sfxge_intr_init(sfxge_t *);
 864 extern int                      sfxge_intr_start(sfxge_t *);
 865 extern void                     sfxge_intr_stop(sfxge_t *);
 866 extern void                     sfxge_intr_fini(sfxge_t *);
 867 extern void                     sfxge_intr_fatal(sfxge_t *);
 868 
 869 extern int                      sfxge_ev_init(sfxge_t *);
 870 extern int                      sfxge_ev_start(sfxge_t *);
 871 extern void                     sfxge_ev_moderation_get(sfxge_t *,
 872     unsigned int *);
 873 extern int                      sfxge_ev_moderation_set(sfxge_t *,
 874     unsigned int);
 875 extern int                      sfxge_ev_qmoderate(sfxge_t *, unsigned int,
 876     unsigned int);
 877 extern int                      sfxge_ev_qpoll(sfxge_t *, unsigned int);
 878 extern int                      sfxge_ev_qprime(sfxge_t *, unsigned int);
 879 extern void                     sfxge_ev_stop(sfxge_t *);
 880 extern void                     sfxge_ev_fini(sfxge_t *);
 881 extern int                      sfxge_ev_txlabel_alloc(sfxge_t *sp,
 882     unsigned int evq, sfxge_txq_t *stp, unsigned int *labelp);
 883 extern int                      sfxge_ev_txlabel_free(sfxge_t *sp,
 884     unsigned int evq, sfxge_txq_t *stp, unsigned int label);
 885 
 886 extern int                      sfxge_mon_init(sfxge_t *);
 887 extern int                      sfxge_mon_start(sfxge_t *);
 888 extern void                     sfxge_mon_stop(sfxge_t *);
 889 extern void                     sfxge_mon_fini(sfxge_t *);
 890 
 891 extern int                      sfxge_mac_init(sfxge_t *);
 892 extern int                      sfxge_mac_start(sfxge_t *, boolean_t);
 893 extern void                     sfxge_mac_stat_get(sfxge_t *, unsigned int,
 894     uint64_t *);
 895 extern void                     sfxge_mac_link_check(sfxge_t *, boolean_t *);
 896 extern void                     sfxge_mac_link_speed_get(sfxge_t *,
 897     unsigned int *);
 898 extern void                     sfxge_mac_link_duplex_get(sfxge_t *,
 899     sfxge_link_duplex_t *);
 900 extern void                     sfxge_mac_fcntl_get(sfxge_t *, unsigned int *);
 901 extern int                      sfxge_mac_fcntl_set(sfxge_t *, unsigned int);
 902 extern int                      sfxge_mac_unicst_get(sfxge_t *,
 903     sfxge_unicst_type_t, uint8_t *);
 904 extern int                      sfxge_mac_unicst_set(sfxge_t *,
 905     uint8_t *);
 906 extern int                      sfxge_mac_promisc_set(sfxge_t *,
 907     sfxge_promisc_type_t);
 908 extern int                      sfxge_mac_multicst_add(sfxge_t *,
 909     uint8_t const *addr);
 910 extern int                      sfxge_mac_multicst_remove(sfxge_t *,
 911     uint8_t const *addr);
 912 extern void                     sfxge_mac_stop(sfxge_t *);
 913 extern void                     sfxge_mac_fini(sfxge_t *);
 914 extern void                     sfxge_mac_link_update(sfxge_t *sp,
 915     efx_link_mode_t mode);
 916 
 917 extern int                      sfxge_mcdi_init(sfxge_t *sp);
 918 extern void                     sfxge_mcdi_fini(sfxge_t *sp);
 919 extern int                      sfxge_mcdi_ioctl(sfxge_t *sp,
 920     sfxge_mcdi_ioc_t *smip);
 921 extern int                      sfxge_mcdi2_ioctl(sfxge_t *sp,
 922     sfxge_mcdi2_ioc_t *smip);
 923 
 924 extern int                      sfxge_phy_init(sfxge_t *);
 925 extern void                     sfxge_phy_link_mode_get(sfxge_t *,
 926     efx_link_mode_t *);
 927 extern void                     sfxge_phy_fini(sfxge_t *);
 928 extern int                      sfxge_phy_kstat_init(sfxge_t *sp);
 929 extern void                     sfxge_phy_kstat_fini(sfxge_t *sp);
 930 extern uint8_t                  sfxge_phy_lp_cap_test(sfxge_t *sp,
 931     uint32_t field);
 932 extern int                      sfxge_phy_cap_apply(sfxge_t *sp,
 933     boolean_t use_default);
 934 extern uint8_t                  sfxge_phy_cap_test(sfxge_t *sp, uint32_t flags,
 935     uint32_t field, boolean_t *mutablep);
 936 extern int                      sfxge_phy_cap_set(sfxge_t *sp, uint32_t field,
 937     int set);
 938 
 939 extern int                      sfxge_rx_init(sfxge_t *);
 940 extern int                      sfxge_rx_start(sfxge_t *);
 941 extern void                     sfxge_rx_coalesce_mode_get(sfxge_t *,
 942     sfxge_rx_coalesce_mode_t *);
 943 extern int                      sfxge_rx_coalesce_mode_set(sfxge_t *,
 944     sfxge_rx_coalesce_mode_t);
 945 extern unsigned int             sfxge_rx_scale_prop_get(sfxge_t *);
 946 extern void                     sfxge_rx_scale_update(void *);
 947 extern int                      sfxge_rx_scale_count_get(sfxge_t *,
 948     unsigned int *);
 949 extern int                      sfxge_rx_scale_count_set(sfxge_t *,
 950     unsigned int);
 951 extern void                     sfxge_rx_qcomplete(sfxge_rxq_t *, boolean_t);
 952 extern void                     sfxge_rx_qflush_done(sfxge_rxq_t *);
 953 extern void                     sfxge_rx_qflush_failed(sfxge_rxq_t *);
 954 extern void                     sfxge_rx_qfpp_trim(sfxge_rxq_t *);
 955 extern void                     sfxge_rx_stop(sfxge_t *);
 956 extern unsigned int             sfxge_rx_loaned(sfxge_t *);
 957 extern void                     sfxge_rx_fini(sfxge_t *);
 958 
 959 extern int                      sfxge_tx_init(sfxge_t *);
 960 extern int                      sfxge_tx_start(sfxge_t *);
 961 extern int                      sfxge_tx_packet_add(sfxge_t *, mblk_t *);
 962 extern void                     sfxge_tx_qcomplete(sfxge_txq_t *);
 963 extern void                     sfxge_tx_qflush_done(sfxge_txq_t *);
 964 extern void                     sfxge_tx_stop(sfxge_t *);
 965 extern void                     sfxge_tx_fini(sfxge_t *);
 966 extern void                     sfxge_tx_qdpl_flush(sfxge_txq_t *stp);
 967 
 968 extern void                     sfxge_sram_init(sfxge_t *);
 969 extern int                      sfxge_sram_buf_tbl_alloc(sfxge_t *, size_t,
 970     uint32_t *);
 971 extern int                      sfxge_sram_start(sfxge_t *);
 972 extern int                      sfxge_sram_buf_tbl_set(sfxge_t *, uint32_t,
 973     efsys_mem_t *, size_t);
 974 extern void                     sfxge_sram_buf_tbl_clear(sfxge_t *, uint32_t,
 975     size_t);
 976 extern void                     sfxge_sram_stop(sfxge_t *);
 977 extern void                     sfxge_sram_buf_tbl_free(sfxge_t *, uint32_t,
 978     size_t);
 979 extern void                     sfxge_sram_fini(sfxge_t *);
 980 
 981 extern sfxge_packet_type_t      sfxge_pkthdr_parse(mblk_t *,
 982     struct ether_header **, struct ip **, struct tcphdr **, size_t *, size_t *,
 983     uint16_t *, uint16_t *);
 984 
 985 extern int sfxge_toeplitz_hash_init(sfxge_t *);
 986 extern void sfxge_toeplitz_hash_fini(sfxge_t *);
 987 extern uint32_t sfxge_toeplitz_hash(sfxge_t *, unsigned int,
 988     uint8_t *, uint16_t, uint8_t *, uint16_t);
 989 
 990 /*
 991  * 4-tuple hash for TCP/IPv4 used for LRO, TSO and TX queue selection.
 992  * To compute the same hash value as Siena/Huntington hardware, the inputs
 993  * must be in big endian (network) byte order.
 994  */
 995 #define SFXGE_TCP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash)      \
 996         do { \
 997                 (_hash) = sfxge_toeplitz_hash(_sp, \
 998                                         sizeof (struct in_addr), \
 999                                         (uint8_t *)(_raddr), \
1000                                         (_rport), \
1001                                         (uint8_t *)(_laddr), \
1002                                         (_lport)); \
1003                 _NOTE(CONSTANTCONDITION) \
1004         } while (B_FALSE)
1005 
1006 /*
1007  * 4-tuple hash for non-TCP IPv4 packets, used for TX queue selection.
1008  * For UDP or SCTP packets, calculate a 4-tuple hash using port numbers.
1009  * For other IPv4 non-TCP packets, use zero for the port numbers.
1010  */
1011 #define SFXGE_IP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash)       \
1012         SFXGE_TCP_HASH((_sp), (_raddr), (_rport), (_laddr), (_lport), (_hash))
1013 
1014 
1015 extern int              sfxge_nvram_ioctl(sfxge_t *, sfxge_nvram_ioc_t *);
1016 
1017 extern int              sfxge_pci_init(sfxge_t *);
1018 extern void             sfxge_pcie_check_link(sfxge_t *, unsigned int,
1019     unsigned int);
1020 extern void             sfxge_pci_fini(sfxge_t *);
1021 
1022 extern int              sfxge_bar_init(sfxge_t *);
1023 extern void             sfxge_bar_fini(sfxge_t *);
1024 
1025 extern int              sfxge_vpd_ioctl(sfxge_t *, sfxge_vpd_ioc_t *);
1026 
1027 
1028 #endif /* _KERNEL */
1029 
1030 #ifdef  __cplusplus
1031 }
1032 #endif
1033 
1034 #endif  /* _SYS_SFXGE_H */