1 /*
   2  * Copyright (c) 2007-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30 
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33 #if EFSYS_OPT_MON_MCDI
  34 #include "mcdi_mon.h"
  35 #endif
  36 
  37 #if EFSYS_OPT_QSTATS
  38 #define EFX_EV_QSTAT_INCR(_eep, _stat)                                  \
  39         do {                                                            \
  40                 (_eep)->ee_stat[_stat]++;                            \
  41         _NOTE(CONSTANTCONDITION)                                        \
  42         } while (B_FALSE)
  43 #else
  44 #define EFX_EV_QSTAT_INCR(_eep, _stat)
  45 #endif
  46 
  47 #define EFX_EV_PRESENT(_qword)                                          \
  48         (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&        \
  49         EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
  50 
  51 
  52 
  53 #if EFSYS_OPT_SIENA
  54 
  55 static  __checkReturn   efx_rc_t
  56 siena_ev_init(
  57         __in            efx_nic_t *enp);
  58 
  59 static                  void
  60 siena_ev_fini(
  61         __in            efx_nic_t *enp);
  62 
  63 static  __checkReturn   efx_rc_t
  64 siena_ev_qcreate(
  65         __in            efx_nic_t *enp,
  66         __in            unsigned int index,
  67         __in            efsys_mem_t *esmp,
  68         __in            size_t n,
  69         __in            uint32_t id,
  70         __in            efx_evq_t *eep);
  71 
  72 static                  void
  73 siena_ev_qdestroy(
  74         __in            efx_evq_t *eep);
  75 
  76 static  __checkReturn   efx_rc_t
  77 siena_ev_qprime(
  78         __in            efx_evq_t *eep,
  79         __in            unsigned int count);
  80 
  81 static                  void
  82 siena_ev_qpoll(
  83         __in            efx_evq_t *eep,
  84         __inout         unsigned int *countp,
  85         __in            const efx_ev_callbacks_t *eecp,
  86         __in_opt        void *arg);
  87 
  88 static                  void
  89 siena_ev_qpost(
  90         __in    efx_evq_t *eep,
  91         __in    uint16_t data);
  92 
  93 static  __checkReturn   efx_rc_t
  94 siena_ev_qmoderate(
  95         __in            efx_evq_t *eep,
  96         __in            unsigned int us);
  97 
  98 #if EFSYS_OPT_QSTATS
  99 static                  void
 100 siena_ev_qstats_update(
 101         __in                            efx_evq_t *eep,
 102         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat);
 103 
 104 #endif
 105 
 106 #endif /* EFSYS_OPT_SIENA */
 107 
 108 #if EFSYS_OPT_SIENA
 109 static const efx_ev_ops_t       __efx_ev_siena_ops = {
 110         siena_ev_init,                          /* eevo_init */
 111         siena_ev_fini,                          /* eevo_fini */
 112         siena_ev_qcreate,                       /* eevo_qcreate */
 113         siena_ev_qdestroy,                      /* eevo_qdestroy */
 114         siena_ev_qprime,                        /* eevo_qprime */
 115         siena_ev_qpost,                         /* eevo_qpost */
 116         siena_ev_qmoderate,                     /* eevo_qmoderate */
 117 #if EFSYS_OPT_QSTATS
 118         siena_ev_qstats_update,                 /* eevo_qstats_update */
 119 #endif
 120 };
 121 #endif /* EFSYS_OPT_SIENA */
 122 
 123 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 124 static const efx_ev_ops_t       __efx_ev_ef10_ops = {
 125         ef10_ev_init,                           /* eevo_init */
 126         ef10_ev_fini,                           /* eevo_fini */
 127         ef10_ev_qcreate,                        /* eevo_qcreate */
 128         ef10_ev_qdestroy,                       /* eevo_qdestroy */
 129         ef10_ev_qprime,                         /* eevo_qprime */
 130         ef10_ev_qpost,                          /* eevo_qpost */
 131         ef10_ev_qmoderate,                      /* eevo_qmoderate */
 132 #if EFSYS_OPT_QSTATS
 133         ef10_ev_qstats_update,                  /* eevo_qstats_update */
 134 #endif
 135 };
 136 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 137 
 138 
 139         __checkReturn   efx_rc_t
 140 efx_ev_init(
 141         __in            efx_nic_t *enp)
 142 {
 143         const efx_ev_ops_t *eevop;
 144         efx_rc_t rc;
 145 
 146         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 147         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
 148 
 149         if (enp->en_mod_flags & EFX_MOD_EV) {
 150                 rc = EINVAL;
 151                 goto fail1;
 152         }
 153 
 154         switch (enp->en_family) {
 155 #if EFSYS_OPT_SIENA
 156         case EFX_FAMILY_SIENA:
 157                 eevop = &__efx_ev_siena_ops;
 158                 break;
 159 #endif /* EFSYS_OPT_SIENA */
 160 
 161 #if EFSYS_OPT_HUNTINGTON
 162         case EFX_FAMILY_HUNTINGTON:
 163                 eevop = &__efx_ev_ef10_ops;
 164                 break;
 165 #endif /* EFSYS_OPT_HUNTINGTON */
 166 
 167 #if EFSYS_OPT_MEDFORD
 168         case EFX_FAMILY_MEDFORD:
 169                 eevop = &__efx_ev_ef10_ops;
 170                 break;
 171 #endif /* EFSYS_OPT_MEDFORD */
 172 
 173         default:
 174                 EFSYS_ASSERT(0);
 175                 rc = ENOTSUP;
 176                 goto fail1;
 177         }
 178 
 179         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
 180 
 181         if ((rc = eevop->eevo_init(enp)) != 0)
 182                 goto fail2;
 183 
 184         enp->en_eevop = eevop;
 185         enp->en_mod_flags |= EFX_MOD_EV;
 186         return (0);
 187 
 188 fail2:
 189         EFSYS_PROBE(fail2);
 190 
 191 fail1:
 192         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 193 
 194         enp->en_eevop = NULL;
 195         enp->en_mod_flags &= ~EFX_MOD_EV;
 196         return (rc);
 197 }
 198 
 199                 void
 200 efx_ev_fini(
 201         __in    efx_nic_t *enp)
 202 {
 203         const efx_ev_ops_t *eevop = enp->en_eevop;
 204 
 205         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 206         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
 207         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 208         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
 209         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
 210         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
 211 
 212         eevop->eevo_fini(enp);
 213 
 214         enp->en_eevop = NULL;
 215         enp->en_mod_flags &= ~EFX_MOD_EV;
 216 }
 217 
 218 
 219         __checkReturn   efx_rc_t
 220 efx_ev_qcreate(
 221         __in            efx_nic_t *enp,
 222         __in            unsigned int index,
 223         __in            efsys_mem_t *esmp,
 224         __in            size_t n,
 225         __in            uint32_t id,
 226         __deref_out     efx_evq_t **eepp)
 227 {
 228         const efx_ev_ops_t *eevop = enp->en_eevop;
 229         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 230         efx_evq_t *eep;
 231         efx_rc_t rc;
 232 
 233         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 234         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 235 
 236         EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit);
 237 
 238         /* Allocate an EVQ object */
 239         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
 240         if (eep == NULL) {
 241                 rc = ENOMEM;
 242                 goto fail1;
 243         }
 244 
 245         eep->ee_magic = EFX_EVQ_MAGIC;
 246         eep->ee_enp = enp;
 247         eep->ee_index = index;
 248         eep->ee_mask = n - 1;
 249         eep->ee_esmp = esmp;
 250 
 251         if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, eep)) != 0)
 252                 goto fail2;
 253 
 254         enp->en_ev_qcount++;
 255         *eepp = eep;
 256 
 257         return (0);
 258 
 259 fail2:
 260         EFSYS_PROBE(fail2);
 261         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
 262 fail1:
 263         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 264         return (rc);
 265 }
 266 
 267                 void
 268 efx_ev_qdestroy(
 269         __in    efx_evq_t *eep)
 270 {
 271         efx_nic_t *enp = eep->ee_enp;
 272         const efx_ev_ops_t *eevop = enp->en_eevop;
 273 
 274         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 275 
 276         EFSYS_ASSERT(enp->en_ev_qcount != 0);
 277         --enp->en_ev_qcount;
 278 
 279         eevop->eevo_qdestroy(eep);
 280 
 281         /* Free the EVQ object */
 282         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
 283 }
 284 
 285         __checkReturn   efx_rc_t
 286 efx_ev_qprime(
 287         __in            efx_evq_t *eep,
 288         __in            unsigned int count)
 289 {
 290         efx_nic_t *enp = eep->ee_enp;
 291         const efx_ev_ops_t *eevop = enp->en_eevop;
 292         efx_rc_t rc;
 293 
 294         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 295 
 296         if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
 297                 rc = EINVAL;
 298                 goto fail1;
 299         }
 300 
 301         if ((rc = eevop->eevo_qprime(eep, count)) != 0)
 302                 goto fail2;
 303 
 304         return (0);
 305 
 306 fail2:
 307         EFSYS_PROBE(fail2);
 308 fail1:
 309         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 310         return (rc);
 311 }
 312 
 313         __checkReturn   boolean_t
 314 efx_ev_qpending(
 315         __in            efx_evq_t *eep,
 316         __in            unsigned int count)
 317 {
 318         size_t offset;
 319         efx_qword_t qword;
 320 
 321         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 322 
 323         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 324         EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
 325 
 326         return (EFX_EV_PRESENT(qword));
 327 }
 328 
 329 #if EFSYS_OPT_EV_PREFETCH
 330 
 331                         void
 332 efx_ev_qprefetch(
 333         __in            efx_evq_t *eep,
 334         __in            unsigned int count)
 335 {
 336         efx_nic_t *enp = eep->ee_enp;
 337         unsigned int offset;
 338 
 339         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 340 
 341         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 342         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 343 }
 344 
 345 #endif  /* EFSYS_OPT_EV_PREFETCH */
 346 
 347                         void
 348 efx_ev_qpoll(
 349         __in            efx_evq_t *eep,
 350         __inout         unsigned int *countp,
 351         __in            const efx_ev_callbacks_t *eecp,
 352         __in_opt        void *arg)
 353 {
 354         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 355 
 356         /*
 357          * FIXME: Huntington will require support for hardware event batching
 358          * and merging, which will need a different ev_qpoll implementation.
 359          *
 360          * Without those features the Falcon/Siena code can be used unchanged.
 361          */
 362         EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
 363         EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
 364 
 365         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
 366         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
 367         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
 368         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
 369             FSE_AZ_EV_CODE_DRV_GEN_EV);
 370 #if EFSYS_OPT_MCDI
 371         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
 372             FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
 373 #endif
 374         siena_ev_qpoll(eep, countp, eecp, arg);
 375 }
 376 
 377                         void
 378 efx_ev_qpost(
 379         __in    efx_evq_t *eep,
 380         __in    uint16_t data)
 381 {
 382         efx_nic_t *enp = eep->ee_enp;
 383         const efx_ev_ops_t *eevop = enp->en_eevop;
 384 
 385         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 386 
 387         EFSYS_ASSERT(eevop != NULL &&
 388             eevop->eevo_qpost != NULL);
 389 
 390         eevop->eevo_qpost(eep, data);
 391 }
 392 
 393         __checkReturn   efx_rc_t
 394 efx_ev_qmoderate(
 395         __in            efx_evq_t *eep,
 396         __in            unsigned int us)
 397 {
 398         efx_nic_t *enp = eep->ee_enp;
 399         const efx_ev_ops_t *eevop = enp->en_eevop;
 400         efx_rc_t rc;
 401 
 402         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 403 
 404         if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
 405                 goto fail1;
 406 
 407         return (0);
 408 
 409 fail1:
 410         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 411         return (rc);
 412 }
 413 
 414 #if EFSYS_OPT_QSTATS
 415                                         void
 416 efx_ev_qstats_update(
 417         __in                            efx_evq_t *eep,
 418         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
 419 
 420 {       efx_nic_t *enp = eep->ee_enp;
 421         const efx_ev_ops_t *eevop = enp->en_eevop;
 422 
 423         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 424 
 425         eevop->eevo_qstats_update(eep, stat);
 426 }
 427 
 428 #endif  /* EFSYS_OPT_QSTATS */
 429 
 430 #if EFSYS_OPT_SIENA
 431 
 432 static  __checkReturn   efx_rc_t
 433 siena_ev_init(
 434         __in            efx_nic_t *enp)
 435 {
 436         efx_oword_t oword;
 437 
 438         /*
 439          * Program the event queue for receive and transmit queue
 440          * flush events.
 441          */
 442         EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
 443         EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
 444         EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
 445 
 446         return (0);
 447 
 448 }
 449 
 450 static  __checkReturn   boolean_t
 451 siena_ev_rx_not_ok(
 452         __in            efx_evq_t *eep,
 453         __in            efx_qword_t *eqp,
 454         __in            uint32_t label,
 455         __in            uint32_t id,
 456         __inout         uint16_t *flagsp)
 457 {
 458         boolean_t ignore = B_FALSE;
 459 
 460         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
 461                 EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
 462                 EFSYS_PROBE(tobe_disc);
 463                 /*
 464                  * Assume this is a unicast address mismatch, unless below
 465                  * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
 466                  * EV_RX_PAUSE_FRM_ERR is set.
 467                  */
 468                 (*flagsp) |= EFX_ADDR_MISMATCH;
 469         }
 470 
 471         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
 472                 EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
 473                 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
 474                 (*flagsp) |= EFX_DISCARD;
 475 
 476 #if EFSYS_OPT_RX_SCATTER
 477                 /*
 478                  * Lookout for payload queue ran dry errors and ignore them.
 479                  *
 480                  * Sadly for the header/data split cases, the descriptor
 481                  * pointer in this event refers to the header queue and
 482                  * therefore cannot be easily detected as duplicate.
 483                  * So we drop these and rely on the receive processing seeing
 484                  * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
 485                  * the partially received packet.
 486                  */
 487                 if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
 488                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
 489                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
 490                         ignore = B_TRUE;
 491 #endif  /* EFSYS_OPT_RX_SCATTER */
 492         }
 493 
 494         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
 495                 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
 496                 EFSYS_PROBE(crc_err);
 497                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 498                 (*flagsp) |= EFX_DISCARD;
 499         }
 500 
 501         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
 502                 EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
 503                 EFSYS_PROBE(pause_frm_err);
 504                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 505                 (*flagsp) |= EFX_DISCARD;
 506         }
 507 
 508         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
 509                 EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
 510                 EFSYS_PROBE(owner_id_err);
 511                 (*flagsp) |= EFX_DISCARD;
 512         }
 513 
 514         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
 515                 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
 516                 EFSYS_PROBE(ipv4_err);
 517                 (*flagsp) &= ~EFX_CKSUM_IPV4;
 518         }
 519 
 520         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
 521                 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
 522                 EFSYS_PROBE(udp_chk_err);
 523                 (*flagsp) &= ~EFX_CKSUM_TCPUDP;
 524         }
 525 
 526         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
 527                 EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
 528 
 529                 /*
 530                  * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
 531                  * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
 532                  * condition.
 533                  */
 534                 (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
 535         }
 536 
 537         return (ignore);
 538 }
 539 
 540 static  __checkReturn   boolean_t
 541 siena_ev_rx(
 542         __in            efx_evq_t *eep,
 543         __in            efx_qword_t *eqp,
 544         __in            const efx_ev_callbacks_t *eecp,
 545         __in_opt        void *arg)
 546 {
 547         uint32_t id;
 548         uint32_t size;
 549         uint32_t label;
 550         boolean_t ok;
 551 #if EFSYS_OPT_RX_SCATTER
 552         boolean_t sop;
 553         boolean_t jumbo_cont;
 554 #endif  /* EFSYS_OPT_RX_SCATTER */
 555         uint32_t hdr_type;
 556         boolean_t is_v6;
 557         uint16_t flags;
 558         boolean_t ignore;
 559         boolean_t should_abort;
 560 
 561         EFX_EV_QSTAT_INCR(eep, EV_RX);
 562 
 563         /* Basic packet information */
 564         id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
 565         size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
 566         label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
 567         ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
 568 
 569 #if EFSYS_OPT_RX_SCATTER
 570         sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
 571         jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
 572 #endif  /* EFSYS_OPT_RX_SCATTER */
 573 
 574         hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
 575 
 576         is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
 577 
 578         /*
 579          * If packet is marked as OK and packet type is TCP/IP or
 580          * UDP/IP or other IP, then we can rely on the hardware checksums.
 581          */
 582         switch (hdr_type) {
 583         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
 584                 flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
 585                 if (is_v6) {
 586                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
 587                         flags |= EFX_PKT_IPV6;
 588                 } else {
 589                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
 590                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 591                 }
 592                 break;
 593 
 594         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
 595                 flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
 596                 if (is_v6) {
 597                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
 598                         flags |= EFX_PKT_IPV6;
 599                 } else {
 600                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
 601                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 602                 }
 603                 break;
 604 
 605         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
 606                 if (is_v6) {
 607                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
 608                         flags = EFX_PKT_IPV6;
 609                 } else {
 610                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
 611                         flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 612                 }
 613                 break;
 614 
 615         case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
 616                 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
 617                 flags = 0;
 618                 break;
 619 
 620         default:
 621                 EFSYS_ASSERT(B_FALSE);
 622                 flags = 0;
 623                 break;
 624         }
 625 
 626 #if EFSYS_OPT_RX_SCATTER
 627         /* Report scatter and header/lookahead split buffer flags */
 628         if (sop)
 629                 flags |= EFX_PKT_START;
 630         if (jumbo_cont)
 631                 flags |= EFX_PKT_CONT;
 632 #endif  /* EFSYS_OPT_RX_SCATTER */
 633 
 634         /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
 635         if (!ok) {
 636                 ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags);
 637                 if (ignore) {
 638                         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 639                             uint32_t, size, uint16_t, flags);
 640 
 641                         return (B_FALSE);
 642                 }
 643         }
 644 
 645         /* If we're not discarding the packet then it is ok */
 646         if (~flags & EFX_DISCARD)
 647                 EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
 648 
 649         /* Detect multicast packets that didn't match the filter */
 650         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
 651                 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
 652 
 653                 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
 654                         EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
 655                 } else {
 656                         EFSYS_PROBE(mcast_mismatch);
 657                         flags |= EFX_ADDR_MISMATCH;
 658                 }
 659         } else {
 660                 flags |= EFX_PKT_UNICAST;
 661         }
 662 
 663         /*
 664          * The packet parser in Siena can abort parsing packets under
 665          * certain error conditions, setting the PKT_NOT_PARSED bit
 666          * (which clears PKT_OK). If this is set, then don't trust
 667          * the PKT_TYPE field.
 668          */
 669         if (!ok) {
 670                 uint32_t parse_err;
 671 
 672                 parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
 673                 if (parse_err != 0)
 674                         flags |= EFX_CHECK_VLAN;
 675         }
 676 
 677         if (~flags & EFX_CHECK_VLAN) {
 678                 uint32_t pkt_type;
 679 
 680                 pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
 681                 if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
 682                         flags |= EFX_PKT_VLAN_TAGGED;
 683         }
 684 
 685         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 686             uint32_t, size, uint16_t, flags);
 687 
 688         EFSYS_ASSERT(eecp->eec_rx != NULL);
 689         should_abort = eecp->eec_rx(arg, label, id, size, flags);
 690 
 691         return (should_abort);
 692 }
 693 
 694 static  __checkReturn   boolean_t
 695 siena_ev_tx(
 696         __in            efx_evq_t *eep,
 697         __in            efx_qword_t *eqp,
 698         __in            const efx_ev_callbacks_t *eecp,
 699         __in_opt        void *arg)
 700 {
 701         uint32_t id;
 702         uint32_t label;
 703         boolean_t should_abort;
 704 
 705         EFX_EV_QSTAT_INCR(eep, EV_TX);
 706 
 707         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
 708             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
 709             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
 710             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
 711 
 712                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
 713                 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
 714 
 715                 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
 716 
 717                 EFSYS_ASSERT(eecp->eec_tx != NULL);
 718                 should_abort = eecp->eec_tx(arg, label, id);
 719 
 720                 return (should_abort);
 721         }
 722 
 723         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
 724                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 725                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 726                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 727 
 728         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
 729                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
 730 
 731         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
 732                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
 733 
 734         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
 735                 EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
 736 
 737         EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
 738         return (B_FALSE);
 739 }
 740 
 741 static  __checkReturn   boolean_t
 742 siena_ev_global(
 743         __in            efx_evq_t *eep,
 744         __in            efx_qword_t *eqp,
 745         __in            const efx_ev_callbacks_t *eecp,
 746         __in_opt        void *arg)
 747 {
 748         _NOTE(ARGUNUSED(eqp, eecp, arg))
 749 
 750         EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
 751 
 752         return (B_FALSE);
 753 }
 754 
 755 static  __checkReturn   boolean_t
 756 siena_ev_driver(
 757         __in            efx_evq_t *eep,
 758         __in            efx_qword_t *eqp,
 759         __in            const efx_ev_callbacks_t *eecp,
 760         __in_opt        void *arg)
 761 {
 762         boolean_t should_abort;
 763 
 764         EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
 765         should_abort = B_FALSE;
 766 
 767         switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
 768         case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
 769                 uint32_t txq_index;
 770 
 771                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
 772 
 773                 txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 774 
 775                 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
 776 
 777                 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
 778                 should_abort = eecp->eec_txq_flush_done(arg, txq_index);
 779 
 780                 break;
 781         }
 782         case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
 783                 uint32_t rxq_index;
 784                 uint32_t failed;
 785 
 786                 rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
 787                 failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
 788 
 789                 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
 790                 EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
 791 
 792                 if (failed) {
 793                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
 794 
 795                         EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
 796 
 797                         should_abort = eecp->eec_rxq_flush_failed(arg,
 798                                                                     rxq_index);
 799                 } else {
 800                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
 801 
 802                         EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
 803 
 804                         should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
 805                 }
 806 
 807                 break;
 808         }
 809         case FSE_AZ_EVQ_INIT_DONE_EV:
 810                 EFSYS_ASSERT(eecp->eec_initialized != NULL);
 811                 should_abort = eecp->eec_initialized(arg);
 812 
 813                 break;
 814 
 815         case FSE_AZ_EVQ_NOT_EN_EV:
 816                 EFSYS_PROBE(evq_not_en);
 817                 break;
 818 
 819         case FSE_AZ_SRM_UPD_DONE_EV: {
 820                 uint32_t code;
 821 
 822                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
 823 
 824                 code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 825 
 826                 EFSYS_ASSERT(eecp->eec_sram != NULL);
 827                 should_abort = eecp->eec_sram(arg, code);
 828 
 829                 break;
 830         }
 831         case FSE_AZ_WAKE_UP_EV: {
 832                 uint32_t id;
 833 
 834                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 835 
 836                 EFSYS_ASSERT(eecp->eec_wake_up != NULL);
 837                 should_abort = eecp->eec_wake_up(arg, id);
 838 
 839                 break;
 840         }
 841         case FSE_AZ_TX_PKT_NON_TCP_UDP:
 842                 EFSYS_PROBE(tx_pkt_non_tcp_udp);
 843                 break;
 844 
 845         case FSE_AZ_TIMER_EV: {
 846                 uint32_t id;
 847 
 848                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 849 
 850                 EFSYS_ASSERT(eecp->eec_timer != NULL);
 851                 should_abort = eecp->eec_timer(arg, id);
 852 
 853                 break;
 854         }
 855         case FSE_AZ_RX_DSC_ERROR_EV:
 856                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
 857 
 858                 EFSYS_PROBE(rx_dsc_error);
 859 
 860                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 861                 should_abort = eecp->eec_exception(arg,
 862                         EFX_EXCEPTION_RX_DSC_ERROR, 0);
 863 
 864                 break;
 865 
 866         case FSE_AZ_TX_DSC_ERROR_EV:
 867                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
 868 
 869                 EFSYS_PROBE(tx_dsc_error);
 870 
 871                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 872                 should_abort = eecp->eec_exception(arg,
 873                         EFX_EXCEPTION_TX_DSC_ERROR, 0);
 874 
 875                 break;
 876 
 877         default:
 878                 break;
 879         }
 880 
 881         return (should_abort);
 882 }
 883 
 884 static  __checkReturn   boolean_t
 885 siena_ev_drv_gen(
 886         __in            efx_evq_t *eep,
 887         __in            efx_qword_t *eqp,
 888         __in            const efx_ev_callbacks_t *eecp,
 889         __in_opt        void *arg)
 890 {
 891         uint32_t data;
 892         boolean_t should_abort;
 893 
 894         EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
 895 
 896         data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
 897         if (data >= ((uint32_t)1 << 16)) {
 898                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 899                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 900                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 901                 return (B_TRUE);
 902         }
 903 
 904         EFSYS_ASSERT(eecp->eec_software != NULL);
 905         should_abort = eecp->eec_software(arg, (uint16_t)data);
 906 
 907         return (should_abort);
 908 }
 909 
 910 #if EFSYS_OPT_MCDI
 911 
 912 static  __checkReturn   boolean_t
 913 siena_ev_mcdi(
 914         __in            efx_evq_t *eep,
 915         __in            efx_qword_t *eqp,
 916         __in            const efx_ev_callbacks_t *eecp,
 917         __in_opt        void *arg)
 918 {
 919         efx_nic_t *enp = eep->ee_enp;
 920         unsigned code;
 921         boolean_t should_abort = B_FALSE;
 922 
 923         EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
 924 
 925         if (enp->en_family != EFX_FAMILY_SIENA)
 926                 goto out;
 927 
 928         EFSYS_ASSERT(eecp->eec_link_change != NULL);
 929         EFSYS_ASSERT(eecp->eec_exception != NULL);
 930 #if EFSYS_OPT_MON_STATS
 931         EFSYS_ASSERT(eecp->eec_monitor != NULL);
 932 #endif
 933 
 934         EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
 935 
 936         code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
 937         switch (code) {
 938         case MCDI_EVENT_CODE_BADSSERT:
 939                 efx_mcdi_ev_death(enp, EINTR);
 940                 break;
 941 
 942         case MCDI_EVENT_CODE_CMDDONE:
 943                 efx_mcdi_ev_cpl(enp,
 944                     MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
 945                     MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
 946                     MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
 947                 break;
 948 
 949         case MCDI_EVENT_CODE_LINKCHANGE: {
 950                 efx_link_mode_t link_mode;
 951 
 952                 siena_phy_link_ev(enp, eqp, &link_mode);
 953                 should_abort = eecp->eec_link_change(arg, link_mode);
 954                 break;
 955         }
 956         case MCDI_EVENT_CODE_SENSOREVT: {
 957 #if EFSYS_OPT_MON_STATS
 958                 efx_mon_stat_t id;
 959                 efx_mon_stat_value_t value;
 960                 efx_rc_t rc;
 961 
 962                 if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0)
 963                         should_abort = eecp->eec_monitor(arg, id, value);
 964                 else if (rc == ENOTSUP) {
 965                         should_abort = eecp->eec_exception(arg,
 966                                 EFX_EXCEPTION_UNKNOWN_SENSOREVT,
 967                                 MCDI_EV_FIELD(eqp, DATA));
 968                 } else
 969                         EFSYS_ASSERT(rc == ENODEV);     /* Wrong port */
 970 #else
 971                 should_abort = B_FALSE;
 972 #endif
 973                 break;
 974         }
 975         case MCDI_EVENT_CODE_SCHEDERR:
 976                 /* Informational only */
 977                 break;
 978 
 979         case MCDI_EVENT_CODE_REBOOT:
 980                 efx_mcdi_ev_death(enp, EIO);
 981                 break;
 982 
 983         case MCDI_EVENT_CODE_MAC_STATS_DMA:
 984 #if EFSYS_OPT_MAC_STATS
 985                 if (eecp->eec_mac_stats != NULL) {
 986                         eecp->eec_mac_stats(arg,
 987                             MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
 988                 }
 989 #endif
 990                 break;
 991 
 992         case MCDI_EVENT_CODE_FWALERT: {
 993                 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
 994 
 995                 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
 996                         should_abort = eecp->eec_exception(arg,
 997                                 EFX_EXCEPTION_FWALERT_SRAM,
 998                                 MCDI_EV_FIELD(eqp, FWALERT_DATA));
 999                 else
1000                         should_abort = eecp->eec_exception(arg,
1001                                 EFX_EXCEPTION_UNKNOWN_FWALERT,
1002                                 MCDI_EV_FIELD(eqp, DATA));
1003                 break;
1004         }
1005 
1006         default:
1007                 EFSYS_PROBE1(mc_pcol_error, int, code);
1008                 break;
1009         }
1010 
1011 out:
1012         return (should_abort);
1013 }
1014 
1015 #endif  /* EFSYS_OPT_MCDI */
1016 
1017 static  __checkReturn   efx_rc_t
1018 siena_ev_qprime(
1019         __in            efx_evq_t *eep,
1020         __in            unsigned int count)
1021 {
1022         efx_nic_t *enp = eep->ee_enp;
1023         uint32_t rptr;
1024         efx_dword_t dword;
1025 
1026         rptr = count & eep->ee_mask;
1027 
1028         EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
1029 
1030         EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
1031                             &dword, B_FALSE);
1032 
1033         return (0);
1034 }
1035 
1036 #define EFX_EV_BATCH    8
1037 
1038 static                  void
1039 siena_ev_qpoll(
1040         __in            efx_evq_t *eep,
1041         __inout         unsigned int *countp,
1042         __in            const efx_ev_callbacks_t *eecp,
1043         __in_opt        void *arg)
1044 {
1045         efx_qword_t ev[EFX_EV_BATCH];
1046         unsigned int batch;
1047         unsigned int total;
1048         unsigned int count;
1049         unsigned int index;
1050         size_t offset;
1051 
1052         EFSYS_ASSERT(countp != NULL);
1053         EFSYS_ASSERT(eecp != NULL);
1054 
1055         count = *countp;
1056         do {
1057                 /* Read up until the end of the batch period */
1058                 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
1059                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1060                 for (total = 0; total < batch; ++total) {
1061                         EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
1062 
1063                         if (!EFX_EV_PRESENT(ev[total]))
1064                                 break;
1065 
1066                         EFSYS_PROBE3(event, unsigned int, eep->ee_index,
1067                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
1068                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
1069 
1070                         offset += sizeof (efx_qword_t);
1071                 }
1072 
1073 #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
1074                 /*
1075                  * Prefetch the next batch when we get within PREFETCH_PERIOD
1076                  * of a completed batch. If the batch is smaller, then prefetch
1077                  * immediately.
1078                  */
1079                 if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
1080                         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1081 #endif  /* EFSYS_OPT_EV_PREFETCH */
1082 
1083                 /* Process the batch of events */
1084                 for (index = 0; index < total; ++index) {
1085                         boolean_t should_abort;
1086                         uint32_t code;
1087 
1088 #if EFSYS_OPT_EV_PREFETCH
1089                         /* Prefetch if we've now reached the batch period */
1090                         if (total == batch &&
1091                             index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
1092                                 offset = (count + batch) & eep->ee_mask;
1093                                 offset *= sizeof (efx_qword_t);
1094 
1095                                 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1096                         }
1097 #endif  /* EFSYS_OPT_EV_PREFETCH */
1098 
1099                         EFX_EV_QSTAT_INCR(eep, EV_ALL);
1100 
1101                         code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
1102                         switch (code) {
1103                         case FSE_AZ_EV_CODE_RX_EV:
1104                                 should_abort = eep->ee_rx(eep,
1105                                     &(ev[index]), eecp, arg);
1106                                 break;
1107                         case FSE_AZ_EV_CODE_TX_EV:
1108                                 should_abort = eep->ee_tx(eep,
1109                                     &(ev[index]), eecp, arg);
1110                                 break;
1111                         case FSE_AZ_EV_CODE_DRIVER_EV:
1112                                 should_abort = eep->ee_driver(eep,
1113                                     &(ev[index]), eecp, arg);
1114                                 break;
1115                         case FSE_AZ_EV_CODE_DRV_GEN_EV:
1116                                 should_abort = eep->ee_drv_gen(eep,
1117                                     &(ev[index]), eecp, arg);
1118                                 break;
1119 #if EFSYS_OPT_MCDI
1120                         case FSE_AZ_EV_CODE_MCDI_EVRESPONSE:
1121                                 should_abort = eep->ee_mcdi(eep,
1122                                     &(ev[index]), eecp, arg);
1123                                 break;
1124 #endif
1125                         case FSE_AZ_EV_CODE_GLOBAL_EV:
1126                                 if (eep->ee_global) {
1127                                         should_abort = eep->ee_global(eep,
1128                                             &(ev[index]), eecp, arg);
1129                                         break;
1130                                 }
1131                                 /* else fallthrough */
1132                         default:
1133                                 EFSYS_PROBE3(bad_event,
1134                                     unsigned int, eep->ee_index,
1135                                     uint32_t,
1136                                     EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
1137                                     uint32_t,
1138                                     EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
1139 
1140                                 EFSYS_ASSERT(eecp->eec_exception != NULL);
1141                                 (void) eecp->eec_exception(arg,
1142                                         EFX_EXCEPTION_EV_ERROR, code);
1143                                 should_abort = B_TRUE;
1144                         }
1145                         if (should_abort) {
1146                                 /* Ignore subsequent events */
1147                                 total = index + 1;
1148                                 break;
1149                         }
1150                 }
1151 
1152                 /*
1153                  * Now that the hardware has most likely moved onto dma'ing
1154                  * into the next cache line, clear the processed events. Take
1155                  * care to only clear out events that we've processed
1156                  */
1157                 EFX_SET_QWORD(ev[0]);
1158                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1159                 for (index = 0; index < total; ++index) {
1160                         EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
1161                         offset += sizeof (efx_qword_t);
1162                 }
1163 
1164                 count += total;
1165 
1166         } while (total == batch);
1167 
1168         *countp = count;
1169 }
1170 
1171 static          void
1172 siena_ev_qpost(
1173         __in    efx_evq_t *eep,
1174         __in    uint16_t data)
1175 {
1176         efx_nic_t *enp = eep->ee_enp;
1177         efx_qword_t ev;
1178         efx_oword_t oword;
1179 
1180         EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
1181             FSF_AZ_EV_DATA_DW0, (uint32_t)data);
1182 
1183         EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
1184             EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
1185             EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
1186 
1187         EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
1188 }
1189 
1190 static  __checkReturn   efx_rc_t
1191 siena_ev_qmoderate(
1192         __in            efx_evq_t *eep,
1193         __in            unsigned int us)
1194 {
1195         efx_nic_t *enp = eep->ee_enp;
1196         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1197         unsigned int locked;
1198         efx_dword_t dword;
1199         efx_rc_t rc;
1200 
1201         if (us > encp->enc_evq_timer_max_us) {
1202                 rc = EINVAL;
1203                 goto fail1;
1204         }
1205 
1206         /* If the value is zero then disable the timer */
1207         if (us == 0) {
1208                 EFX_POPULATE_DWORD_2(dword,
1209                     FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
1210                     FRF_CZ_TC_TIMER_VAL, 0);
1211         } else {
1212                 uint32_t timer_val;
1213 
1214                 /* Calculate the timer value in quanta */
1215                 timer_val = us * 1000 / encp->enc_evq_timer_quantum_ns;
1216 
1217                 /* Moderation value is base 0 so we need to deduct 1 */
1218                 if (timer_val > 0)
1219                         timer_val--;
1220 
1221                 EFX_POPULATE_DWORD_2(dword,
1222                     FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
1223                     FRF_CZ_TC_TIMER_VAL, timer_val);
1224         }
1225 
1226         locked = (eep->ee_index == 0) ? 1 : 0;
1227 
1228         EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
1229             eep->ee_index, &dword, locked);
1230 
1231         return (0);
1232 
1233 fail1:
1234         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1235 
1236         return (rc);
1237 }
1238 
1239 static  __checkReturn   efx_rc_t
1240 siena_ev_qcreate(
1241         __in            efx_nic_t *enp,
1242         __in            unsigned int index,
1243         __in            efsys_mem_t *esmp,
1244         __in            size_t n,
1245         __in            uint32_t id,
1246         __in            efx_evq_t *eep)
1247 {
1248         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1249         uint32_t size;
1250         efx_oword_t oword;
1251         efx_rc_t rc;
1252 
1253         _NOTE(ARGUNUSED(esmp));
1254 
1255         EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS));
1256         EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS));
1257 
1258         if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) {
1259                 rc = EINVAL;
1260                 goto fail1;
1261         }
1262         if (index >= encp->enc_evq_limit) {
1263                 rc = EINVAL;
1264                 goto fail2;
1265         }
1266 #if EFSYS_OPT_RX_SCALE
1267         if (enp->en_intr.ei_type == EFX_INTR_LINE &&
1268             index >= EFX_MAXRSS_LEGACY) {
1269                 rc = EINVAL;
1270                 goto fail3;
1271         }
1272 #endif
1273         for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS);
1274             size++)
1275                 if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS))
1276                         break;
1277         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1278                 rc = EINVAL;
1279                 goto fail4;
1280         }
1281 
1282         /* Set up the handler table */
1283         eep->ee_rx   = siena_ev_rx;
1284         eep->ee_tx   = siena_ev_tx;
1285         eep->ee_driver       = siena_ev_driver;
1286         eep->ee_global       = siena_ev_global;
1287         eep->ee_drv_gen      = siena_ev_drv_gen;
1288 #if EFSYS_OPT_MCDI
1289         eep->ee_mcdi = siena_ev_mcdi;
1290 #endif  /* EFSYS_OPT_MCDI */
1291 
1292         /* Set up the new event queue */
1293         EFX_POPULATE_OWORD_1(oword, FRF_CZ_TIMER_Q_EN, 1);
1294         EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE);
1295 
1296         EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
1297             FRF_AZ_EVQ_BUF_BASE_ID, id);
1298 
1299         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE);
1300 
1301         return (0);
1302 
1303 fail4:
1304         EFSYS_PROBE(fail4);
1305 #if EFSYS_OPT_RX_SCALE
1306 fail3:
1307         EFSYS_PROBE(fail3);
1308 #endif
1309 fail2:
1310         EFSYS_PROBE(fail2);
1311 fail1:
1312         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1313 
1314         return (rc);
1315 }
1316 
1317 #endif /* EFSYS_OPT_SIENA */
1318 
1319 #if EFSYS_OPT_QSTATS
1320 #if EFSYS_OPT_NAMES
1321 /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock b693ddf85aee1bfd */
1322 static const char       *__efx_ev_qstat_name[] = {
1323         "all",
1324         "rx",
1325         "rx_ok",
1326         "rx_frm_trunc",
1327         "rx_tobe_disc",
1328         "rx_pause_frm_err",
1329         "rx_buf_owner_id_err",
1330         "rx_ipv4_hdr_chksum_err",
1331         "rx_tcp_udp_chksum_err",
1332         "rx_eth_crc_err",
1333         "rx_ip_frag_err",
1334         "rx_mcast_pkt",
1335         "rx_mcast_hash_match",
1336         "rx_tcp_ipv4",
1337         "rx_tcp_ipv6",
1338         "rx_udp_ipv4",
1339         "rx_udp_ipv6",
1340         "rx_other_ipv4",
1341         "rx_other_ipv6",
1342         "rx_non_ip",
1343         "rx_batch",
1344         "tx",
1345         "tx_wq_ff_full",
1346         "tx_pkt_err",
1347         "tx_pkt_too_big",
1348         "tx_unexpected",
1349         "global",
1350         "global_mnt",
1351         "driver",
1352         "driver_srm_upd_done",
1353         "driver_tx_descq_fls_done",
1354         "driver_rx_descq_fls_done",
1355         "driver_rx_descq_fls_failed",
1356         "driver_rx_dsc_error",
1357         "driver_tx_dsc_error",
1358         "drv_gen",
1359         "mcdi_response",
1360 };
1361 /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
1362 
1363                 const char *
1364 efx_ev_qstat_name(
1365         __in    efx_nic_t *enp,
1366         __in    unsigned int id)
1367 {
1368         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1369         EFSYS_ASSERT3U(id, <, EV_NQSTATS);
1370 
1371         return (__efx_ev_qstat_name[id]);
1372 }
1373 #endif  /* EFSYS_OPT_NAMES */
1374 #endif  /* EFSYS_OPT_QSTATS */
1375 
1376 #if EFSYS_OPT_SIENA
1377 
1378 #if EFSYS_OPT_QSTATS
1379 static                                  void
1380 siena_ev_qstats_update(
1381         __in                            efx_evq_t *eep,
1382         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
1383 {
1384         unsigned int id;
1385 
1386         for (id = 0; id < EV_NQSTATS; id++) {
1387                 efsys_stat_t *essp = &stat[id];
1388 
1389                 EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
1390                 eep->ee_stat[id] = 0;
1391         }
1392 }
1393 #endif  /* EFSYS_OPT_QSTATS */
1394 
1395 static          void
1396 siena_ev_qdestroy(
1397         __in    efx_evq_t *eep)
1398 {
1399         efx_nic_t *enp = eep->ee_enp;
1400         efx_oword_t oword;
1401 
1402         /* Purge event queue */
1403         EFX_ZERO_OWORD(oword);
1404 
1405         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
1406             eep->ee_index, &oword, B_TRUE);
1407 
1408         EFX_ZERO_OWORD(oword);
1409         EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE);
1410 }
1411 
1412 static          void
1413 siena_ev_fini(
1414         __in    efx_nic_t *enp)
1415 {
1416         _NOTE(ARGUNUSED(enp))
1417 }
1418 
1419 #endif /* EFSYS_OPT_SIENA */