1 /*
   2  * Copyright 2007-2013 Solarflare Communications Inc.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  */
  25 
  26 #include "efsys.h"
  27 #include "efx.h"
  28 #include "efx_types.h"
  29 #include "efx_regs.h"
  30 #include "efx_impl.h"
  31 
  32 #if EFSYS_OPT_QSTATS
  33 #define EFX_EV_QSTAT_INCR(_eep, _stat)                                  \
  34         do {                                                            \
  35                 (_eep)->ee_stat[_stat]++;                            \
  36         _NOTE(CONSTANTCONDITION)                                        \
  37         } while (B_FALSE)
  38 #else
  39 #define EFX_EV_QSTAT_INCR(_eep, _stat)
  40 #endif
  41 
  42         __checkReturn   int
  43 efx_ev_init(
  44         __in            efx_nic_t *enp)
  45 {
  46         efx_oword_t oword;
  47         int rc;
  48 
  49         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
  50         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
  51 
  52         if (enp->en_mod_flags & EFX_MOD_EV) {
  53                 rc = EINVAL;
  54                 goto fail1;
  55         }
  56 
  57         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
  58 
  59         /*
  60          * Program the event queue for receive and transmit queue
  61          * flush events.
  62          */
  63         EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
  64         EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
  65         EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
  66 
  67         enp->en_mod_flags |= EFX_MOD_EV;
  68         return (0);
  69 
  70 fail1:
  71         EFSYS_PROBE1(fail1, int, rc);
  72 
  73         return (rc);
  74 }
  75 
  76 static  __checkReturn   boolean_t
  77 efx_ev_rx_not_ok(
  78         __in            efx_evq_t *eep,
  79         __in            efx_qword_t *eqp,
  80         __in            uint32_t label,
  81         __in            uint32_t id,
  82         __inout         uint16_t *flagsp)
  83 {
  84         boolean_t ignore = B_FALSE;
  85 
  86         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
  87                 EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
  88                 EFSYS_PROBE(tobe_disc);
  89                 /*
  90                  * Assume this is a unicast address mismatch, unless below
  91                  * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
  92                  * EV_RX_PAUSE_FRM_ERR is set.
  93                  */
  94                 (*flagsp) |= EFX_ADDR_MISMATCH;
  95         }
  96 
  97         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
  98                 EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
  99                 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
 100                 (*flagsp) |= EFX_DISCARD;
 101 
 102 #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
 103                 /*
 104                  * Lookout for payload queue ran dry errors and ignore them.
 105                  *
 106                  * Sadly for the header/data split cases, the descriptor
 107                  * pointer in this event refers to the header queue and
 108                  * therefore cannot be easily detected as duplicate.
 109                  * So we drop these and rely on the receive processing seeing
 110                  * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
 111                  * the partially received packet.
 112                  */
 113                 if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
 114                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
 115                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
 116                         ignore = B_TRUE;
 117 #endif  /* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
 118         }
 119 
 120         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
 121                 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
 122                 EFSYS_PROBE(crc_err);
 123                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 124                 (*flagsp) |= EFX_DISCARD;
 125         }
 126 
 127         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
 128                 EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
 129                 EFSYS_PROBE(pause_frm_err);
 130                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 131                 (*flagsp) |= EFX_DISCARD;
 132         }
 133 
 134         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
 135                 EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
 136                 EFSYS_PROBE(owner_id_err);
 137                 (*flagsp) |= EFX_DISCARD;
 138         }
 139 
 140         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
 141                 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
 142                 EFSYS_PROBE(ipv4_err);
 143                 (*flagsp) &= ~EFX_CKSUM_IPV4;
 144         }
 145 
 146         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
 147                 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
 148                 EFSYS_PROBE(udp_chk_err);
 149                 (*flagsp) &= ~EFX_CKSUM_TCPUDP;
 150         }
 151 
 152         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
 153                 EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
 154 
 155                 /*
 156                  * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
 157                  * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
 158                  * condition.
 159                  */
 160                 (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
 161         }
 162 
 163         return (ignore);
 164 }
 165 
 166 static  __checkReturn   boolean_t
 167 efx_ev_rx(
 168         __in            efx_evq_t *eep,
 169         __in            efx_qword_t *eqp,
 170         __in            const efx_ev_callbacks_t *eecp,
 171         __in_opt        void *arg)
 172 {
 173         efx_nic_t *enp = eep->ee_enp;
 174         uint32_t id;
 175         uint32_t size;
 176         uint32_t label;
 177         boolean_t ok;
 178 #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
 179         boolean_t sop;
 180         boolean_t jumbo_cont;
 181 #endif  /* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
 182         uint32_t hdr_type;
 183         boolean_t is_v6;
 184         uint16_t flags;
 185         boolean_t ignore;
 186         boolean_t should_abort;
 187 
 188         EFX_EV_QSTAT_INCR(eep, EV_RX);
 189 
 190         /* Basic packet information */
 191         id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
 192         size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
 193         label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
 194         ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
 195 
 196 #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
 197         sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
 198         jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
 199 #endif  /* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
 200 
 201         hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
 202 
 203         is_v6 = (enp->en_family != EFX_FAMILY_FALCON &&
 204                     EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
 205 
 206         /*
 207          * If packet is marked as OK and packet type is TCP/IP or
 208          * UDP/IP or other IP, then we can rely on the hardware checksums.
 209          */
 210         switch (hdr_type) {
 211         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
 212                 flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
 213                 if (is_v6) {
 214                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
 215                         flags |= EFX_PKT_IPV6;
 216                 } else {
 217                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
 218                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 219                 }
 220                 break;
 221 
 222         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
 223                 flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
 224                 if (is_v6) {
 225                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
 226                         flags |= EFX_PKT_IPV6;
 227                 } else {
 228                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
 229                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 230                 }
 231                 break;
 232 
 233         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
 234                 if (is_v6) {
 235                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
 236                         flags = EFX_PKT_IPV6;
 237                 } else {
 238                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
 239                         flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 240                 }
 241                 break;
 242 
 243         case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
 244                 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
 245                 flags = 0;
 246                 break;
 247 
 248         default:
 249                 EFSYS_ASSERT(B_FALSE);
 250                 flags = 0;
 251                 break;
 252         }
 253 
 254 #if EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT
 255         /* Report scatter and header/lookahead split buffer flags */
 256         if (sop)
 257                 flags |= EFX_PKT_START;
 258         if (jumbo_cont)
 259                 flags |= EFX_PKT_CONT;
 260 #endif  /* EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT */
 261 
 262         /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
 263         if (!ok) {
 264                 ignore = efx_ev_rx_not_ok(eep, eqp, label, id, &flags);
 265                 if (ignore) {
 266                         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 267                             uint32_t, size, uint16_t, flags);
 268 
 269                         return (B_FALSE);
 270                 }
 271         }
 272 
 273         /* If we're not discarding the packet then it is ok */
 274         if (~flags & EFX_DISCARD)
 275                 EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
 276 
 277         /* Detect multicast packets that didn't match the filter */
 278         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
 279                 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
 280 
 281                 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
 282                         EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
 283                 } else {
 284                         EFSYS_PROBE(mcast_mismatch);
 285                         flags |= EFX_ADDR_MISMATCH;
 286                 }
 287         } else {
 288                 flags |= EFX_PKT_UNICAST;
 289         }
 290 
 291         /*
 292          * The packet parser in Siena can abort parsing packets under
 293          * certain error conditions, setting the PKT_NOT_PARSED bit
 294          * (which clears PKT_OK). If this is set, then don't trust
 295          * the PKT_TYPE field.
 296          */
 297         if (enp->en_family != EFX_FAMILY_FALCON && !ok) {
 298                 uint32_t parse_err;
 299 
 300                 parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
 301                 if (parse_err != 0)
 302                         flags |= EFX_CHECK_VLAN;
 303         }
 304 
 305         if (~flags & EFX_CHECK_VLAN) {
 306                 uint32_t pkt_type;
 307 
 308                 pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
 309                 if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
 310                         flags |= EFX_PKT_VLAN_TAGGED;
 311         }
 312 
 313         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 314             uint32_t, size, uint16_t, flags);
 315 
 316         EFSYS_ASSERT(eecp->eec_rx != NULL);
 317         should_abort = eecp->eec_rx(arg, label, id, size, flags);
 318 
 319         return (should_abort);
 320 }
 321 
 322 static  __checkReturn   boolean_t
 323 efx_ev_tx(
 324         __in            efx_evq_t *eep,
 325         __in            efx_qword_t *eqp,
 326         __in            const efx_ev_callbacks_t *eecp,
 327         __in_opt        void *arg)
 328 {
 329         uint32_t id;
 330         uint32_t label;
 331         boolean_t should_abort;
 332 
 333         EFX_EV_QSTAT_INCR(eep, EV_TX);
 334 
 335         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
 336             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
 337             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
 338             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
 339 
 340                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
 341                 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
 342 
 343                 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
 344 
 345                 EFSYS_ASSERT(eecp->eec_tx != NULL);
 346                 should_abort = eecp->eec_tx(arg, label, id);
 347 
 348                 return (should_abort);
 349         }
 350 
 351         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
 352                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 353                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 354                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 355 
 356         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
 357                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
 358 
 359         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
 360                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
 361 
 362         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
 363                 EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
 364 
 365         EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
 366         return (B_FALSE);
 367 }
 368 
 369 static  __checkReturn   boolean_t
 370 efx_ev_global(
 371         __in            efx_evq_t *eep,
 372         __in            efx_qword_t *eqp,
 373         __in            const efx_ev_callbacks_t *eecp,
 374         __in_opt        void *arg)
 375 {
 376         efx_nic_t *enp = eep->ee_enp;
 377         efx_port_t *epp = &(enp->en_port);
 378         boolean_t should_abort;
 379 
 380         EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
 381         should_abort = B_FALSE;
 382 
 383         /* Check for a link management event */
 384         if (EFX_QWORD_FIELD(*eqp, FSF_BZ_GLB_EV_XG_MNT_INTR) != 0) {
 385                 EFX_EV_QSTAT_INCR(eep, EV_GLOBAL_MNT);
 386 
 387                 EFSYS_PROBE(xg_mgt);
 388 
 389                 epp->ep_mac_poll_needed = B_TRUE;
 390         }
 391 
 392         return (should_abort);
 393 }
 394 
 395 static  __checkReturn   boolean_t
 396 efx_ev_driver(
 397         __in            efx_evq_t *eep,
 398         __in            efx_qword_t *eqp,
 399         __in            const efx_ev_callbacks_t *eecp,
 400         __in_opt        void *arg)
 401 {
 402         boolean_t should_abort;
 403 
 404         EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
 405         should_abort = B_FALSE;
 406 
 407         switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
 408         case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
 409                 uint32_t label;
 410 
 411                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
 412 
 413                 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 414 
 415                 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, label);
 416 
 417                 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
 418                 should_abort = eecp->eec_txq_flush_done(arg, label);
 419 
 420                 break;
 421         }
 422         case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
 423                 uint32_t label;
 424                 uint32_t failed;
 425 
 426                 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
 427                 failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
 428 
 429                 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
 430                 EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
 431 
 432                 if (failed) {
 433                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
 434 
 435                         EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, label);
 436 
 437                         should_abort = eecp->eec_rxq_flush_failed(arg, label);
 438                 } else {
 439                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
 440 
 441                         EFSYS_PROBE1(rx_descq_fls_done, uint32_t, label);
 442 
 443                         should_abort = eecp->eec_rxq_flush_done(arg, label);
 444                 }
 445 
 446                 break;
 447         }
 448         case FSE_AZ_EVQ_INIT_DONE_EV:
 449                 EFSYS_ASSERT(eecp->eec_initialized != NULL);
 450                 should_abort = eecp->eec_initialized(arg);
 451 
 452                 break;
 453 
 454         case FSE_AZ_EVQ_NOT_EN_EV:
 455                 EFSYS_PROBE(evq_not_en);
 456                 break;
 457 
 458         case FSE_AZ_SRM_UPD_DONE_EV: {
 459                 uint32_t code;
 460 
 461                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
 462 
 463                 code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 464 
 465                 EFSYS_ASSERT(eecp->eec_sram != NULL);
 466                 should_abort = eecp->eec_sram(arg, code);
 467 
 468                 break;
 469         }
 470         case FSE_AZ_WAKE_UP_EV: {
 471                 uint32_t id;
 472 
 473                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 474 
 475                 EFSYS_ASSERT(eecp->eec_wake_up != NULL);
 476                 should_abort = eecp->eec_wake_up(arg, id);
 477 
 478                 break;
 479         }
 480         case FSE_AZ_TX_PKT_NON_TCP_UDP:
 481                 EFSYS_PROBE(tx_pkt_non_tcp_udp);
 482                 break;
 483 
 484         case FSE_AZ_TIMER_EV: {
 485                 uint32_t id;
 486 
 487                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 488 
 489                 EFSYS_ASSERT(eecp->eec_timer != NULL);
 490                 should_abort = eecp->eec_timer(arg, id);
 491 
 492                 break;
 493         }
 494         case FSE_AZ_RX_DSC_ERROR_EV:
 495                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
 496 
 497                 EFSYS_PROBE(rx_dsc_error);
 498 
 499                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 500                 should_abort = eecp->eec_exception(arg,
 501                         EFX_EXCEPTION_RX_DSC_ERROR, 0);
 502 
 503                 break;
 504 
 505         case FSE_AZ_TX_DSC_ERROR_EV:
 506                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
 507 
 508                 EFSYS_PROBE(tx_dsc_error);
 509 
 510                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 511                 should_abort = eecp->eec_exception(arg,
 512                         EFX_EXCEPTION_TX_DSC_ERROR, 0);
 513 
 514                 break;
 515 
 516         default:
 517                 break;
 518         }
 519 
 520         return (should_abort);
 521 }
 522 
 523 static  __checkReturn   boolean_t
 524 efx_ev_drv_gen(
 525         __in            efx_evq_t *eep,
 526         __in            efx_qword_t *eqp,
 527         __in            const efx_ev_callbacks_t *eecp,
 528         __in_opt        void *arg)
 529 {
 530         uint32_t data;
 531         boolean_t should_abort;
 532 
 533         EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
 534 
 535         data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
 536         if (data >= ((uint32_t)1 << 16)) {
 537                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 538                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 539                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 540                 return (B_TRUE);
 541         }
 542 
 543         EFSYS_ASSERT(eecp->eec_software != NULL);
 544         should_abort = eecp->eec_software(arg, (uint16_t)data);
 545 
 546         return (should_abort);
 547 }
 548 
 549 #if EFSYS_OPT_MCDI
 550 
 551 static  __checkReturn   boolean_t
 552 efx_ev_mcdi(
 553         __in            efx_evq_t *eep,
 554         __in            efx_qword_t *eqp,
 555         __in            const efx_ev_callbacks_t *eecp,
 556         __in_opt        void *arg)
 557 {
 558         efx_nic_t *enp = eep->ee_enp;
 559         unsigned code;
 560         boolean_t should_abort = B_FALSE;
 561 
 562         EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
 563 
 564         if (enp->en_family != EFX_FAMILY_SIENA)
 565                 goto out;
 566 
 567         EFSYS_ASSERT(eecp->eec_link_change != NULL);
 568         EFSYS_ASSERT(eecp->eec_exception != NULL);
 569 #if EFSYS_OPT_MON_STATS
 570         EFSYS_ASSERT(eecp->eec_monitor != NULL);
 571 #endif
 572 
 573         EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
 574 
 575         code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
 576         switch (code) {
 577         case MCDI_EVENT_CODE_BADSSERT:
 578                 efx_mcdi_ev_death(enp, EINTR);
 579                 break;
 580 
 581         case MCDI_EVENT_CODE_CMDDONE:
 582                 efx_mcdi_ev_cpl(enp,
 583                                 MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
 584                                 MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
 585                                 MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
 586                 break;
 587 
 588         case MCDI_EVENT_CODE_LINKCHANGE: {
 589                 efx_link_mode_t link_mode;
 590 
 591                 siena_phy_link_ev(enp, eqp, &link_mode);
 592                 should_abort = eecp->eec_link_change(arg, link_mode);
 593                 break;
 594         }
 595         case MCDI_EVENT_CODE_SENSOREVT: {
 596 #if EFSYS_OPT_MON_STATS
 597                 efx_mon_stat_t id;
 598                 efx_mon_stat_value_t value;
 599                 int rc;
 600 
 601                 if ((rc = siena_mon_ev(enp, eqp, &id, &value)) == 0)
 602                         should_abort = eecp->eec_monitor(arg, id, value);
 603                 else if (rc == ENOTSUP) {
 604                         should_abort = eecp->eec_exception(arg,
 605                                 EFX_EXCEPTION_UNKNOWN_SENSOREVT,
 606                                 MCDI_EV_FIELD(eqp, DATA));
 607                 } else
 608                         EFSYS_ASSERT(rc == ENODEV);     /* Wrong port */
 609 #else
 610                 should_abort = B_FALSE;
 611 #endif
 612                 break;
 613         }
 614         case MCDI_EVENT_CODE_SCHEDERR:
 615                 /* Informational only */
 616                 break;
 617 
 618         case MCDI_EVENT_CODE_REBOOT:
 619                 efx_mcdi_ev_death(enp, EIO);
 620                 break;
 621 
 622         case MCDI_EVENT_CODE_MAC_STATS_DMA:
 623 #if EFSYS_OPT_MAC_STATS
 624                 if (eecp->eec_mac_stats != NULL) {
 625                         eecp->eec_mac_stats(arg,
 626                             MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
 627                 }
 628 #endif
 629                 break;
 630 
 631         case MCDI_EVENT_CODE_FWALERT: {
 632                 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
 633 
 634                 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
 635                         should_abort = eecp->eec_exception(arg,
 636                                 EFX_EXCEPTION_FWALERT_SRAM,
 637                                 MCDI_EV_FIELD(eqp, FWALERT_DATA));
 638                 else
 639                         should_abort = eecp->eec_exception(arg,
 640                                 EFX_EXCEPTION_UNKNOWN_FWALERT,
 641                                 MCDI_EV_FIELD(eqp, DATA));
 642                 break;
 643         }
 644 
 645         default:
 646                 EFSYS_PROBE1(mc_pcol_error, int, code);
 647                 break;
 648         }
 649 
 650 out:
 651         return (should_abort);
 652 }
 653 
 654 #endif  /* EFSYS_OPT_MCDI */
 655 
 656         __checkReturn   int
 657 efx_ev_qprime(
 658         __in            efx_evq_t *eep,
 659         __in            unsigned int count)
 660 {
 661         efx_nic_t *enp = eep->ee_enp;
 662         uint32_t rptr;
 663         efx_dword_t dword;
 664         int rc;
 665 
 666         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 667 
 668         if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
 669                 rc = EINVAL;
 670                 goto fail1;
 671         }
 672 
 673         rptr = count & eep->ee_mask;
 674 
 675         EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
 676 
 677         EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
 678                             &dword, B_FALSE);
 679 
 680         return (0);
 681 
 682 fail1:
 683         EFSYS_PROBE1(fail1, int, rc);
 684 
 685         return (rc);
 686 }
 687 
 688         __checkReturn   boolean_t
 689 efx_ev_qpending(
 690         __in            efx_evq_t *eep,
 691         __in            unsigned int count)
 692 {
 693         size_t offset;
 694         efx_qword_t qword;
 695 
 696         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 697 
 698         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 699         EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
 700 
 701         return (EFX_QWORD_FIELD(qword, EFX_DWORD_0) != 0xffffffff &&
 702                 EFX_QWORD_FIELD(qword, EFX_DWORD_1) != 0xffffffff);
 703 }
 704 
 705 #if EFSYS_OPT_EV_PREFETCH
 706 
 707                         void
 708 efx_ev_qprefetch(
 709         __in            efx_evq_t *eep,
 710         __in            unsigned int count)
 711 {
 712         unsigned int offset;
 713 
 714         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 715 
 716         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 717         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 718 }
 719 
 720 #endif  /* EFSYS_OPT_EV_PREFETCH */
 721 
 722 #define EFX_EV_BATCH    8
 723 
 724 #define EFX_EV_PRESENT(_qword)                                          \
 725         (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&        \
 726         EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
 727 
 728                         void
 729 efx_ev_qpoll(
 730         __in            efx_evq_t *eep,
 731         __inout         unsigned int *countp,
 732         __in            const efx_ev_callbacks_t *eecp,
 733         __in_opt        void *arg)
 734 {
 735         efx_qword_t ev[EFX_EV_BATCH];
 736         unsigned int batch;
 737         unsigned int total;
 738         unsigned int count;
 739         unsigned int index;
 740         size_t offset;
 741 
 742         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 743         EFSYS_ASSERT(countp != NULL);
 744         EFSYS_ASSERT(eecp != NULL);
 745 
 746         count = *countp;
 747         do {
 748                 /* Read up until the end of the batch period */
 749                 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
 750                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 751                 for (total = 0; total < batch; ++total) {
 752                         EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
 753 
 754                         if (!EFX_EV_PRESENT(ev[total]))
 755                                 break;
 756 
 757                         EFSYS_PROBE3(event, unsigned int, eep->ee_index,
 758                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
 759                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
 760 
 761                         offset += sizeof (efx_qword_t);
 762                 }
 763 
 764 #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
 765                 /*
 766                  * Prefetch the next batch when we get within PREFETCH_PERIOD
 767                  * of a completed batch. If the batch is smaller, then prefetch
 768                  * immediately.
 769                  */
 770                 if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
 771                         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 772 #endif  /* EFSYS_OPT_EV_PREFETCH */
 773 
 774                 /* Process the batch of events */
 775                 for (index = 0; index < total; ++index) {
 776                         boolean_t should_abort;
 777                         uint32_t code;
 778                         efx_ev_handler_t handler;
 779 
 780 #if EFSYS_OPT_EV_PREFETCH
 781                         /* Prefetch if we've now reached the batch period */
 782                         if (total == batch &&
 783                             index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
 784                                 offset = (count + batch) & eep->ee_mask;
 785                                 offset *= sizeof (efx_qword_t);
 786 
 787                                 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 788                         }
 789 #endif  /* EFSYS_OPT_EV_PREFETCH */
 790 
 791                         EFX_EV_QSTAT_INCR(eep, EV_ALL);
 792 
 793                         code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
 794                         handler = eep->ee_handler[code];
 795                         EFSYS_ASSERT(handler != NULL);
 796                         should_abort = handler(eep, &(ev[index]), eecp, arg);
 797                         if (should_abort) {
 798                                 /* Ignore subsequent events */
 799                                 total = index + 1;
 800                                 break;
 801                         }
 802                 }
 803 
 804                 /*
 805                  * Now that the hardware has most likely moved onto dma'ing
 806                  * into the next cache line, clear the processed events. Take
 807                  * care to only clear out events that we've processed
 808                  */
 809                 EFX_SET_QWORD(ev[0]);
 810                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 811                 for (index = 0; index < total; ++index) {
 812                         EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
 813                         offset += sizeof (efx_qword_t);
 814                 }
 815 
 816                 count += total;
 817 
 818         } while (total == batch);
 819 
 820         *countp = count;
 821 }
 822 
 823                 void
 824 efx_ev_qpost(
 825         __in    efx_evq_t *eep,
 826         __in    uint16_t data)
 827 {
 828         efx_nic_t *enp = eep->ee_enp;
 829         efx_qword_t ev;
 830         efx_oword_t oword;
 831 
 832         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 833 
 834         EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
 835             FSF_AZ_EV_DATA_DW0, (uint32_t)data);
 836 
 837         EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
 838             EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
 839             EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
 840 
 841         EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
 842 }
 843 
 844         __checkReturn   int
 845 efx_ev_qmoderate(
 846         __in            efx_evq_t *eep,
 847         __in            unsigned int us)
 848 {
 849         efx_nic_t *enp = eep->ee_enp;
 850         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 851         unsigned int locked;
 852         efx_dword_t dword;
 853         int rc;
 854 
 855         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 856 
 857         if (us > encp->enc_evq_timer_max_us) {
 858                 rc = EINVAL;
 859                 goto fail1;
 860         }
 861 
 862         /* If the value is zero then disable the timer */
 863         if (us == 0) {
 864                 if (enp->en_family == EFX_FAMILY_FALCON)
 865                         EFX_POPULATE_DWORD_2(dword,
 866                             FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_DIS,
 867                             FRF_AB_TC_TIMER_VAL, 0);
 868                 else
 869                         EFX_POPULATE_DWORD_2(dword,
 870                             FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
 871                             FRF_CZ_TC_TIMER_VAL, 0);
 872         } else {
 873                 uint32_t timer_val;
 874 
 875                 /* Calculate the timer value in quanta */
 876                 timer_val = us * 1000 / encp->enc_evq_timer_quantum_ns;
 877 
 878                 /* Moderation value is base 0 so we need to deduct 1 */
 879                 if (timer_val > 0)
 880                         timer_val--;
 881 
 882                 if (enp->en_family == EFX_FAMILY_FALCON)
 883                         EFX_POPULATE_DWORD_2(dword,
 884                             FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_INT_HLDOFF,
 885                             FRF_AB_TIMER_VAL, timer_val);
 886                 else
 887                         EFX_POPULATE_DWORD_2(dword,
 888                             FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
 889                             FRF_CZ_TC_TIMER_VAL, timer_val);
 890         }
 891 
 892         locked = (eep->ee_index == 0) ? 1 : 0;
 893 
 894         EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
 895             eep->ee_index, &dword, locked);
 896 
 897         return (0);
 898 
 899 fail1:
 900         EFSYS_PROBE1(fail1, int, rc);
 901 
 902         return (rc);
 903 }
 904 
 905         __checkReturn   int
 906 efx_ev_qcreate(
 907         __in            efx_nic_t *enp,
 908         __in            unsigned int index,
 909         __in            efsys_mem_t *esmp,
 910         __in            size_t n,
 911         __in            uint32_t id,
 912         __deref_out     efx_evq_t **eepp)
 913 {
 914         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 915         uint32_t size;
 916         efx_evq_t *eep;
 917         efx_oword_t oword;
 918         int rc;
 919 
 920         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 921         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 922 
 923         EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit);
 924 
 925         if (!ISP2(n) || !(n & EFX_EVQ_NEVS_MASK)) {
 926                 rc = EINVAL;
 927                 goto fail1;
 928         }
 929         if (index >= encp->enc_evq_limit) {
 930                 rc = EINVAL;
 931                 goto fail2;
 932         }
 933 #if EFSYS_OPT_RX_SCALE
 934         if (enp->en_intr.ei_type == EFX_INTR_LINE &&
 935             index >= EFX_MAXRSS_LEGACY) {
 936                 rc = EINVAL;
 937                 goto fail3;
 938         }
 939 #endif
 940         for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS);
 941             size++)
 942                 if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS))
 943                         break;
 944         if (id + (1 << size) >= encp->enc_buftbl_limit) {
 945                 rc = EINVAL;
 946                 goto fail4;
 947         }
 948 
 949         /* Allocate an EVQ object */
 950         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
 951         if (eep == NULL) {
 952                 rc = ENOMEM;
 953                 goto fail5;
 954         }
 955 
 956         eep->ee_magic = EFX_EVQ_MAGIC;
 957         eep->ee_enp = enp;
 958         eep->ee_index = index;
 959         eep->ee_mask = n - 1;
 960         eep->ee_esmp = esmp;
 961 
 962         /* Set up the handler table */
 963         eep->ee_handler[FSE_AZ_EV_CODE_RX_EV] = efx_ev_rx;
 964         eep->ee_handler[FSE_AZ_EV_CODE_TX_EV] = efx_ev_tx;
 965         eep->ee_handler[FSE_AZ_EV_CODE_DRIVER_EV] = efx_ev_driver;
 966         eep->ee_handler[FSE_AZ_EV_CODE_GLOBAL_EV] = efx_ev_global;
 967         eep->ee_handler[FSE_AZ_EV_CODE_DRV_GEN_EV] = efx_ev_drv_gen;
 968 #if EFSYS_OPT_MCDI
 969         eep->ee_handler[FSE_AZ_EV_CODE_MCDI_EVRESPONSE] = efx_ev_mcdi;
 970 #endif  /* EFSYS_OPT_MCDI */
 971 
 972         /* Set up the new event queue */
 973         if (enp->en_family != EFX_FAMILY_FALCON) {
 974                 EFX_POPULATE_OWORD_1(oword, FRF_CZ_TIMER_Q_EN, 1);
 975                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword);
 976         }
 977 
 978         EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
 979             FRF_AZ_EVQ_BUF_BASE_ID, id);
 980 
 981         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword);
 982 
 983         enp->en_ev_qcount++;
 984         *eepp = eep;
 985         return (0);
 986 
 987 fail5:
 988         EFSYS_PROBE(fail5);
 989 fail4:
 990         EFSYS_PROBE(fail4);
 991 #if EFSYS_OPT_RX_SCALE
 992 fail3:
 993         EFSYS_PROBE(fail3);
 994 #endif
 995 fail2:
 996         EFSYS_PROBE(fail2);
 997 fail1:
 998         EFSYS_PROBE1(fail1, int, rc);
 999 
1000         return (rc);
1001 }
1002 
1003 #if EFSYS_OPT_NAMES
1004 /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock 67e9bdcd920059bd */
1005 static const char       __cs * __cs __efx_ev_qstat_name[] = {
1006         "all",
1007         "rx",
1008         "rx_ok",
1009         "rx_recovery",
1010         "rx_frm_trunc",
1011         "rx_tobe_disc",
1012         "rx_pause_frm_err",
1013         "rx_buf_owner_id_err",
1014         "rx_ipv4_hdr_chksum_err",
1015         "rx_tcp_udp_chksum_err",
1016         "rx_eth_crc_err",
1017         "rx_ip_frag_err",
1018         "rx_mcast_pkt",
1019         "rx_mcast_hash_match",
1020         "rx_tcp_ipv4",
1021         "rx_tcp_ipv6",
1022         "rx_udp_ipv4",
1023         "rx_udp_ipv6",
1024         "rx_other_ipv4",
1025         "rx_other_ipv6",
1026         "rx_non_ip",
1027         "rx_overrun",
1028         "tx",
1029         "tx_wq_ff_full",
1030         "tx_pkt_err",
1031         "tx_pkt_too_big",
1032         "tx_unexpected",
1033         "global",
1034         "global_phy",
1035         "global_mnt",
1036         "global_rx_recovery",
1037         "driver",
1038         "driver_srm_upd_done",
1039         "driver_tx_descq_fls_done",
1040         "driver_rx_descq_fls_done",
1041         "driver_rx_descq_fls_failed",
1042         "driver_rx_dsc_error",
1043         "driver_tx_dsc_error",
1044         "drv_gen",
1045         "mcdi_response",
1046 };
1047 /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
1048 
1049                 const char __cs *
1050 efx_ev_qstat_name(
1051         __in    efx_nic_t *enp,
1052         __in    unsigned int id)
1053 {
1054         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1055         EFSYS_ASSERT3U(id, <, EV_NQSTATS);
1056 
1057         return (__efx_ev_qstat_name[id]);
1058 }
1059 #endif  /* EFSYS_OPT_NAMES */
1060 
1061 #if EFSYS_OPT_QSTATS
1062                                         void
1063 efx_ev_qstats_update(
1064         __in                            efx_evq_t *eep,
1065         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
1066 {
1067         unsigned int id;
1068 
1069         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
1070 
1071         for (id = 0; id < EV_NQSTATS; id++) {
1072                 efsys_stat_t *essp = &stat[id];
1073 
1074                 EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
1075                 eep->ee_stat[id] = 0;
1076         }
1077 }
1078 #endif  /* EFSYS_OPT_QSTATS */
1079 
1080                 void
1081 efx_ev_qdestroy(
1082         __in    efx_evq_t *eep)
1083 {
1084         efx_nic_t *enp = eep->ee_enp;
1085         efx_oword_t oword;
1086 
1087         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
1088 
1089         EFSYS_ASSERT(enp->en_ev_qcount != 0);
1090         --enp->en_ev_qcount;
1091 
1092         /* Purge event queue */
1093         EFX_ZERO_OWORD(oword);
1094 
1095         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
1096             eep->ee_index, &oword);
1097 
1098         if (enp->en_family != EFX_FAMILY_FALCON) {
1099                 EFX_ZERO_OWORD(oword);
1100                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL,
1101                     eep->ee_index, &oword);
1102         }
1103 
1104         /* Free the EVQ object */
1105         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
1106 }
1107 
1108                 void
1109 efx_ev_fini(
1110         __in    efx_nic_t *enp)
1111 {
1112         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1113         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
1114         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
1115         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
1116         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
1117         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
1118 
1119         enp->en_mod_flags &= ~EFX_MOD_EV;
1120 }